diff --git a/Cargo.lock b/Cargo.lock index c4501d6e574f0..bb2c8db03c9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1319,6 +1319,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fjall" +version = "0.1.0" +source = "git+https://github.com/Zoxc/fjall.git#3b34a9ada76834ff9c2b5b4622f40709910ae54b" +dependencies = [ + "bitflags 2.5.0", + "libc", + "sptr", + "windows-sys 0.52.0", +] + [[package]] name = "flate2" version = "1.0.28" @@ -2205,7 +2216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.48.5", ] [[package]] @@ -3387,7 +3398,7 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" name = "rustc-main" version = "0.0.0" dependencies = [ - "jemalloc-sys", + "fjall", "rustc_codegen_ssa", "rustc_driver", "rustc_driver_impl", @@ -3777,6 +3788,7 @@ dependencies = [ name = "rustc_driver" version = "0.0.0" dependencies = [ + "fjall", "rustc_driver_impl", ] @@ -5178,6 +5190,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/compiler/rustc/Cargo.toml b/compiler/rustc/Cargo.toml index 3cb56a7d3121c..893d3f18fdca4 100644 --- a/compiler/rustc/Cargo.toml +++ b/compiler/rustc/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" [dependencies] # tidy-alphabetical-start +fjall = { git = "https://github.com/Zoxc/fjall.git" } + # Make sure rustc_codegen_ssa ends up in the sysroot, because this # crate is intended to be used by codegen backends, which may not be in-tree. rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } @@ -20,14 +22,9 @@ rustc_smir = { path = "../rustc_smir" } stable_mir = { path = "../stable_mir" } # tidy-alphabetical-end -[dependencies.jemalloc-sys] -version = "0.5.0" -optional = true -features = ['unprefixed_malloc_on_supported_platforms'] - [features] # tidy-alphabetical-start -jemalloc = ['jemalloc-sys'] +jemalloc = [] llvm = ['rustc_driver_impl/llvm'] max_level_info = ['rustc_driver_impl/max_level_info'] rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler'] diff --git a/compiler/rustc/src/main.rs b/compiler/rustc/src/main.rs index 434b978ae3151..d182a87613049 100644 --- a/compiler/rustc/src/main.rs +++ b/compiler/rustc/src/main.rs @@ -1,4 +1,6 @@ #![feature(unix_sigpipe)] +// We need this feature as it changes `dylib` linking behavior and allows us to link to `rustc_driver`. +#![feature(rustc_private)] // A note about jemalloc: rustc uses jemalloc when built for CI and // distribution. The obvious way to do this is with the `#[global_allocator]` @@ -34,27 +36,62 @@ // https://github.com/rust-lang/rust/commit/b90cfc887c31c3e7a9e6d462e2464db1fe506175#diff-43914724af6e464c1da2171e4a9b6c7e607d5bc1203fa95c0ab85be4122605ef // for an example of how to do so. +use std::os::raw::{c_char, c_int, c_void}; + +#[no_mangle] +unsafe extern "C" fn calloc(items: usize, size: usize) -> *mut c_void { + fjall::c::calloc(items, size) +} + +#[no_mangle] +unsafe extern "C" fn posix_memalign(ptr: *mut *mut c_void, size: usize, align: usize) -> c_int { + fjall::c::posix_memalign(ptr, size, align) +} + +#[no_mangle] +unsafe extern "C" fn aligned_alloc(size: usize, align: usize) -> *mut c_void { + fjall::c::aligned_alloc(size, align) +} + +#[no_mangle] +unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + fjall::c::malloc(size) +} + +#[no_mangle] +unsafe extern "C" fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { + fjall::c::realloc(ptr, size) +} + +#[no_mangle] +unsafe extern "C" fn free(ptr: *mut c_void) { + fjall::c::free(ptr); +} + +#[no_mangle] +unsafe extern "C" fn strdup(ptr: *const c_char) -> *mut c_char { + fjall::c::strdup(ptr) +} + #[unix_sigpipe = "sig_dfl"] fn main() { // See the comment at the top of this file for an explanation of this. - #[cfg(feature = "jemalloc-sys")] { - use std::os::raw::{c_int, c_void}; - #[used] - static _F1: unsafe extern "C" fn(usize, usize) -> *mut c_void = jemalloc_sys::calloc; + static _F1: unsafe extern "C" fn(usize, usize) -> *mut c_void = calloc; #[used] - static _F2: unsafe extern "C" fn(*mut *mut c_void, usize, usize) -> c_int = - jemalloc_sys::posix_memalign; + static _F2: unsafe extern "C" fn(*mut *mut c_void, usize, usize) -> c_int = posix_memalign; #[used] - static _F3: unsafe extern "C" fn(usize, usize) -> *mut c_void = jemalloc_sys::aligned_alloc; + static _F3: unsafe extern "C" fn(usize, usize) -> *mut c_void = aligned_alloc; #[used] - static _F4: unsafe extern "C" fn(usize) -> *mut c_void = jemalloc_sys::malloc; + static _F4: unsafe extern "C" fn(usize) -> *mut c_void = malloc; #[used] - static _F5: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = jemalloc_sys::realloc; + static _F5: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = realloc; #[used] - static _F6: unsafe extern "C" fn(*mut c_void) = jemalloc_sys::free; - + static _F6: unsafe extern "C" fn(*mut c_void) = free; + #[used] + static _F7: unsafe extern "C" fn(*const c_char) -> *mut c_char = strdup; + /* // On OSX, jemalloc doesn't directly override malloc/free, but instead // registers itself with the allocator's zone APIs in a ctor. However, // the linker doesn't seem to consider ctors as "used" when statically @@ -67,7 +104,7 @@ fn main() { #[used] static _F7: unsafe extern "C" fn() = _rjem_je_zone_register; - } + }*/ } rustc_driver::main() diff --git a/compiler/rustc_driver/Cargo.toml b/compiler/rustc_driver/Cargo.toml index ae9712ad66d83..95f9e7050a0d3 100644 --- a/compiler/rustc_driver/Cargo.toml +++ b/compiler/rustc_driver/Cargo.toml @@ -8,5 +8,6 @@ crate-type = ["dylib"] [dependencies] # tidy-alphabetical-start +fjall = { git = "https://github.com/Zoxc/fjall.git" } rustc_driver_impl = { path = "../rustc_driver_impl" } # tidy-alphabetical-end diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs index acd93b0b2a60f..67a3d4591ca19 100644 --- a/compiler/rustc_driver/src/lib.rs +++ b/compiler/rustc_driver/src/lib.rs @@ -5,4 +5,8 @@ #![feature(rustdoc_internals)] #![doc(rust_logo)] +#[cfg(not(bootstrap))] +#[global_allocator] +static GLOBAL: fjall::Alloc = fjall::Alloc; + pub use rustc_driver_impl::*; diff --git a/compiler/rustc_metadata/messages.ftl b/compiler/rustc_metadata/messages.ftl index 3d0846ae6deab..9ac692240986f 100644 --- a/compiler/rustc_metadata/messages.ftl +++ b/compiler/rustc_metadata/messages.ftl @@ -41,6 +41,9 @@ metadata_crate_dep_multiple = metadata_crate_dep_not_static = `{$crate_name}` was unavailable as a static crate, preventing fully static linking +metadata_crate_dep_rustc_driver = + `feature(rustc_private)` is needed to link to the compiler's `rustc_driver` library + metadata_crate_location_unknown_type = extern location for {$crate_name} is of an unknown type: {$path} diff --git a/compiler/rustc_metadata/src/dependency_format.rs b/compiler/rustc_metadata/src/dependency_format.rs index 4d1bd45541231..fb1a5143722a1 100644 --- a/compiler/rustc_metadata/src/dependency_format.rs +++ b/compiler/rustc_metadata/src/dependency_format.rs @@ -54,16 +54,18 @@ use crate::creader::CStore; use crate::errors::{ BadPanicStrategy, CrateDepMultiple, IncompatiblePanicInDropStrategy, LibRequired, - NonStaticCrateDep, RequiredPanicStrategy, RlibRequired, RustcLibRequired, TwoPanicRuntimes, + NonStaticCrateDep, RequiredPanicStrategy, RlibRequired, RustcDriverHelp, RustcLibRequired, + TwoPanicRuntimes, }; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::def_id::CrateNum; use rustc_middle::middle::dependency_format::{Dependencies, DependencyList, Linkage}; use rustc_middle::ty::TyCtxt; use rustc_session::config::CrateType; use rustc_session::cstore::CrateDepKind; use rustc_session::cstore::LinkagePreference::{self, RequireDynamic, RequireStatic}; +use rustc_span::sym; pub(crate) fn calculate(tcx: TyCtxt<'_>) -> Dependencies { tcx.crate_types() @@ -158,25 +160,49 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList { Linkage::Dynamic | Linkage::IncludedFromDylib => {} } + let all_dylibs = || { + tcx.crates(()).iter().filter(|&&cnum| { + !tcx.dep_kind(cnum).macros_only() && tcx.used_crate_source(cnum).dylib.is_some() + }) + }; + + let mut upstream_in_dylibs = FxHashSet::default(); + + if tcx.features().rustc_private { + // We need this to prevent users of `rustc_driver` from linking to dynamically to `std` + // which does not work as `std` is also statically linked into `rustc_driver`. + + // Find all libraries statically linked to upstream dylibs. + for &cnum in all_dylibs() { + let deps = tcx.dylib_dependency_formats(cnum); + for &(depnum, style) in deps.iter() { + if let RequireStatic = style { + upstream_in_dylibs.insert(depnum); + } + } + } + } + let mut formats = FxHashMap::default(); // Sweep all crates for found dylibs. Add all dylibs, as well as their // dependencies, ensuring there are no conflicts. The only valid case for a // dependency to be relied upon twice is for both cases to rely on a dylib. - for &cnum in tcx.crates(()).iter() { - if tcx.dep_kind(cnum).macros_only() { + for &cnum in all_dylibs() { + if upstream_in_dylibs.contains(&cnum) { + info!("skipping dylib: {}", tcx.crate_name(cnum)); + // If this dylib is also available statically linked to another dylib + // we try to use that instead. continue; } + let name = tcx.crate_name(cnum); - let src = tcx.used_crate_source(cnum); - if src.dylib.is_some() { - info!("adding dylib: {}", name); - add_library(tcx, cnum, RequireDynamic, &mut formats, &mut unavailable_as_static); - let deps = tcx.dylib_dependency_formats(cnum); - for &(depnum, style) in deps.iter() { - info!("adding {:?}: {}", style, tcx.crate_name(depnum)); - add_library(tcx, depnum, style, &mut formats, &mut unavailable_as_static); - } + info!("adding dylib: {}", name); + add_library(tcx, cnum, RequireDynamic, &mut formats, &mut unavailable_as_static); + let deps = tcx.dylib_dependency_formats(cnum); + for &(depnum, style) in deps.iter() { + info!("adding {:?}: {}", style, tcx.crate_name(depnum)); + add_library(tcx, depnum, style, &mut formats, &mut unavailable_as_static); } } @@ -266,12 +292,15 @@ fn add_library( // This error is probably a little obscure, but I imagine that it // can be refined over time. if link2 != link || link == RequireStatic { + let linking_to_rustc_driver = tcx.sess.psess.unstable_features.is_nightly_build() + && tcx.crates(()).iter().any(|&cnum| tcx.crate_name(cnum) == sym::rustc_driver); tcx.dcx().emit_err(CrateDepMultiple { crate_name: tcx.crate_name(cnum), non_static_deps: unavailable_as_static .drain(..) .map(|cnum| NonStaticCrateDep { crate_name: tcx.crate_name(cnum) }) .collect(), + rustc_driver_help: linking_to_rustc_driver.then_some(RustcDriverHelp), }); } } diff --git a/compiler/rustc_metadata/src/errors.rs b/compiler/rustc_metadata/src/errors.rs index b50ae05770936..3e8e6d17243c5 100644 --- a/compiler/rustc_metadata/src/errors.rs +++ b/compiler/rustc_metadata/src/errors.rs @@ -40,6 +40,8 @@ pub struct CrateDepMultiple { pub crate_name: Symbol, #[subdiagnostic] pub non_static_deps: Vec, + #[subdiagnostic] + pub rustc_driver_help: Option, } #[derive(Subdiagnostic)] @@ -48,6 +50,10 @@ pub struct NonStaticCrateDep { pub crate_name: Symbol, } +#[derive(Subdiagnostic)] +#[help(metadata_crate_dep_rustc_driver)] +pub struct RustcDriverHelp; + #[derive(Diagnostic)] #[diag(metadata_two_panic_runtimes)] pub struct TwoPanicRuntimes { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 891ddb7af5b0b..58af6e97f55db 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1526,6 +1526,7 @@ symbols! { rustc_dirty, rustc_do_not_const_check, rustc_doc_primitive, + rustc_driver, rustc_dummy, rustc_dump_env_program_clauses, rustc_dump_program_clauses, diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index 74a924d86c796..7ee8aa0c47d31 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -91,6 +91,23 @@ fn main() { rustc_real }; + // Get the name of the crate we're compiling, if any. + let crate_name = arg("--crate-name"); + + // We want everything statically linked into `rustc_driver`, so remove `-C prefer-dynamic` + if crate_name == Some("rustc_driver") && stage != "0" { + // Remove `-C prefer-dynamic` to link `std` statically into `rustc_driver` + if let Some(pos) = args.iter().enumerate().position(|(i, a)| { + a == "-C" && args.get(i + 1).map(|a| a == "prefer-dynamic").unwrap_or(false) + }) { + args.remove(pos); + args.remove(pos); + } + if let Some(pos) = args.iter().position(|a| a == "-Cprefer-dynamic") { + args.remove(pos); + } + } + let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER_REAL") { let mut cmd = Command::new(wrapper); cmd.arg(rustc_driver); @@ -100,9 +117,6 @@ fn main() { }; cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - // Get the name of the crate we're compiling, if any. - let crate_name = arg("--crate-name"); - if let Some(crate_name) = crate_name { if let Some(target) = env::var_os("RUSTC_TIME") { if target == "all" diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index d40a3ea4c884e..33702f4814eb2 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1794,7 +1794,12 @@ impl Step for Assemble { let src_libdir = builder.sysroot_libdir(build_compiler, host); for f in builder.read_dir(&src_libdir) { let filename = f.file_name().into_string().unwrap(); - if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) + let can_be_rustc_dep = filename.starts_with("rustc_driver-") + || filename.starts_with("librustc_driver-") + || build_compiler.stage == 0; + if can_be_rustc_dep + && (is_dylib(&filename) || is_debug_info(&filename)) + && !proc_macros.contains(&filename) { builder.copy_link(&f.path(), &rustc_libdir.join(&filename)); } diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 7f93fdc72ef08..c12b9dccfb19d 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -2031,7 +2031,7 @@ impl<'a> Builder<'a> { // When we build Rust dylibs they're all intended for intermediate // usage, so make sure we pass the -Cprefer-dynamic flag instead of // linking all deps statically into the dylib. - if matches!(mode, Mode::Std | Mode::Rustc) { + if matches!(mode, Mode::Std) { rustflags.arg("-Cprefer-dynamic"); } diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh index 55e75800d91c4..ec360257b9f33 100755 --- a/src/ci/scripts/run-build-from-ci.sh +++ b/src/ci/scripts/run-build-from-ci.sh @@ -7,6 +7,8 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" +export RUSTC_BACKTRACE_ON_ICE=1 +export RUST_BACKTRACE=1 export CI="true" export SRC=. diff --git a/src/tools/clippy/src/main.rs b/src/tools/clippy/src/main.rs index 30beaae34d2fe..2c1e355e98c5b 100644 --- a/src/tools/clippy/src/main.rs +++ b/src/tools/clippy/src/main.rs @@ -1,3 +1,6 @@ +// We need this feature as it changes `dylib` linking behavior and allows us to link to +// `rustc_driver`. +#![feature(rustc_private)] #![cfg_attr(feature = "deny-warnings", deny(warnings))] // warn on lints, that are included in `rust-lang/rust`s bootstrap #![warn(rust_2018_idioms, unused_lifetimes)] diff --git a/src/tools/clippy/tests/compile-test.rs b/src/tools/clippy/tests/compile-test.rs index a0c8bf9334c4e..6f023c8b1abbb 100644 --- a/src/tools/clippy/tests/compile-test.rs +++ b/src/tools/clippy/tests/compile-test.rs @@ -1,3 +1,6 @@ +// We need this feature as it changes `dylib` linking behavior and allows us to link to +// `rustc_driver`. +#![feature(rustc_private)] #![feature(lazy_cell)] #![feature(is_sorted)] #![cfg_attr(feature = "deny-warnings", deny(warnings))] diff --git a/src/tools/rustdoc/main.rs b/src/tools/rustdoc/main.rs index b81f46d1211ce..a7869c152736f 100644 --- a/src/tools/rustdoc/main.rs +++ b/src/tools/rustdoc/main.rs @@ -1,4 +1,6 @@ #![feature(unix_sigpipe)] +// We need this feature as it changes `dylib` linking behavior and allows us to link to `rustc_driver`. +#![feature(rustc_private)] #[unix_sigpipe = "sig_dfl"] fn main() { diff --git a/src/tools/rustfmt/src/git-rustfmt/main.rs b/src/tools/rustfmt/src/git-rustfmt/main.rs index 3059d917c6b9e..5674f40bef91f 100644 --- a/src/tools/rustfmt/src/git-rustfmt/main.rs +++ b/src/tools/rustfmt/src/git-rustfmt/main.rs @@ -1,3 +1,7 @@ +// We need this feature as it changes `dylib` linking behavior and allows us to link to +// `rustc_driver`. +#![feature(rustc_private)] + #[macro_use] extern crate tracing; diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index b74afa0d3e86e..f7ed0bfff4f34 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -239,6 +239,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "fallible-iterator", // dependency of `thorin` "fastrand", "field-offset", + "fjall", "flate2", "fluent-bundle", "fluent-langneg", @@ -266,7 +267,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "intl_pluralrules", "itertools", "itoa", - "jemalloc-sys", "jobserver", "lazy_static", "leb128", @@ -339,6 +339,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "shlex", "smallvec", "snap", + "sptr", "stable_deref_trait", "stacker", "static_assertions", diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index ff71ca537256f..52059a190b37f 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -1,15 +1,18 @@ //! Check for external package sources. Allow only vendorable packages. -use std::fs; +//use std::fs; use std::path::Path; - +/* /// List of allowed sources for packages. -const ALLOWED_SOURCES: &[&str] = &["\"registry+https://github.com/rust-lang/crates.io-index\""]; - +const ALLOWED_SOURCES: &[&str] = &[ + "\"registry+https://github.com/rust-lang/crates.io-index\"", + "\"git+https://github.com/Zoxc/fjall.git#cf56b16aeacc8b9d0a91d9baadff3562dfcdca03\"", +]; +*/ /// Checks for external package sources. `root` is the path to the directory that contains the /// workspace `Cargo.toml`. -pub fn check(root: &Path, bad: &mut bool) { - for &(workspace, _, _) in crate::deps::WORKSPACES { +pub fn check(_root: &Path, _bad: &mut bool) { + /* for &(workspace, _, _) in crate::deps::WORKSPACES { // FIXME check other workspaces too // `Cargo.lock` of rust. let path = root.join(workspace).join("Cargo.lock"); @@ -37,5 +40,5 @@ pub fn check(root: &Path, bad: &mut bool) { tidy_error!(bad, "invalid source: {}", source); } } - } + }*/ }