Rollup merge of #101339 - the8472:ci-randomize-debug, r=Mark-Simulacrum

enable -Zrandomize-layout in debug CI builds

This builds rustc/libs/tools with `-Zrandomize-layout` on *-debug CI runners.

Only a handful of tests and asserts break with that enabled, which is promising. One test was fixable, the rest is dealt with by disabling them through new cargo features or compiletest directives.

The config.toml flag `rust.randomize-layout` defaults to false, so it has to be explicitly enabled for now.

Author: matthiaskrgr

Cargo.lock

@@ -3569,6 +3569,7 @@ dependencies = [
  "rustc_hir_pretty",
  "rustc_hir_typeck",
  "rustc_incremental",
+ "rustc_index",
  "rustc_infer",
  "rustc_interface",
  "rustc_lint",

compiler/rustc/Cargo.toml

@@ -30,5 +30,6 @@ features = ['unprefixed_malloc_on_supported_platforms']
 jemalloc = ['dep:jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
+rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
 # tidy-alphabetical-end

compiler/rustc_abi/src/layout.rs

@@ -968,8 +968,8 @@ fn univariant<
     let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
     let mut max_repr_align = repr.align;
     let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
-    let optimize = !repr.inhibit_struct_field_reordering();
-    if optimize && fields.len() > 1 {
+    let optimize_field_order = !repr.inhibit_struct_field_reordering();
+    if optimize_field_order && fields.len() > 1 {
         let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
         let optimizing = &mut inverse_memory_index.raw[..end];
         let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
     // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
     // Field 5 would be the first element, so memory_index is i:
     // Note: if we didn't optimize, it's already right.
-    let memory_index = if optimize {
+    let memory_index = if optimize_field_order {
         inverse_memory_index.invert_bijective_mapping()
     } else {
         debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
     }
     let mut layout_of_single_non_zst_field = None;
     let mut abi = Abi::Aggregate { sized };
+
+    let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
     // Try to make this a Scalar/ScalarPair.
     if sized && size.bytes() > 0 {
         // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
                     match field.abi {
                         // For plain scalars, or vectors of them, we can't unpack
                         // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                        Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
+                        Abi::Scalar(_) | Abi::Vector { .. } if optimize_abi => {
                             abi = field.abi;
                         }
                         // But scalar pairs are Rust-specific and get

compiler/rustc_abi/src/lib.rs

@@ -43,14 +43,17 @@ bitflags! {
         const IS_SIMD            = 1 << 1;
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
+        // On its own it does not prevent ABI optimizations.
         const IS_LINEAR          = 1 << 3;
-        // If true, the type's layout can be randomized using
-        // the seed stored in `ReprOptions.field_shuffle_seed`
+        // If true, the type's crate has opted into layout randomization.
+        // Other flags can still inhibit reordering and thus randomization.
+        // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
-        const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits()
+        const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_LINEAR.bits();
+        const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
         self.c() || self.int.is_some()
     }
 
+    pub fn inhibit_newtype_abi_optimization(&self) -> bool {
+        self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+    }
+
     /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
     /// e.g. `repr(C)` or `repr(<int>)`.
     pub fn inhibit_struct_field_reordering(&self) -> bool {
-        self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+        self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
     }
 
     /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`

compiler/rustc_driver_impl/Cargo.toml

@@ -23,6 +23,7 @@ rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_hir_typeck = { path = "../rustc_hir_typeck" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_interface = { path = "../rustc_interface" }
 rustc_lint = { path = "../rustc_lint" }
@@ -72,6 +73,10 @@ ctrlc = "3.4.4"
 # tidy-alphabetical-start
 llvm = ['rustc_interface/llvm']
 max_level_info = ['rustc_log/max_level_info']
+rustc_randomized_layouts = [
+    'rustc_index/rustc_randomized_layouts',
+    'rustc_middle/rustc_randomized_layouts'
+]
 rustc_use_parallel_compiler = [
     'rustc_data_structures/rustc_use_parallel_compiler',
     'rustc_interface/rustc_use_parallel_compiler',

compiler/rustc_index/Cargo.toml

@@ -20,4 +20,5 @@ nightly = [
     "dep:rustc_macros",
     "rustc_index_macros/nightly",
 ]
+rustc_randomized_layouts = []
 # tidy-alphabetical-end

compiler/rustc_index/src/lib.rs

@@ -33,8 +33,19 @@ pub use vec::IndexVec;
 ///
 /// </div>
 #[macro_export]
+#[cfg(not(feature = "rustc_randomized_layouts"))]
 macro_rules! static_assert_size {
     ($ty:ty, $size:expr) => {
         const _: [(); $size] = [(); ::std::mem::size_of::<$ty>()];
     };
 }
+
+#[macro_export]
+#[cfg(feature = "rustc_randomized_layouts")]
+macro_rules! static_assert_size {
+    ($ty:ty, $size:expr) => {
+        // no effect other than using the statements.
+        // struct sizes are not deterministic under randomized layouts
+        const _: (usize, usize) = ($size, ::std::mem::size_of::<$ty>());
+    };
+}

compiler/rustc_middle/Cargo.toml

@@ -40,5 +40,6 @@ tracing = "0.1"
 
 [features]
 # tidy-alphabetical-start
+rustc_randomized_layouts = []
 rustc_use_parallel_compiler = ["dep:rustc-rayon-core"]
 # tidy-alphabetical-end

compiler/rustc_middle/src/query/plumbing.rs

@@ -337,6 +337,7 @@ macro_rules! define_callbacks {
                 // Ensure that values grow no larger than 64 bytes by accident.
                 // Increase this limit if necessary, but do try to keep the size low if possible
                 #[cfg(target_pointer_width = "64")]
+                #[cfg(not(feature = "rustc_randomized_layouts"))]
                 const _: () = {
                     if mem::size_of::<Value<'static>>() > 64 {
                         panic!("{}", concat!(

compiler/rustc_middle/src/ty/mod.rs

@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
     extension, Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
         }
 
+        // box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+        // always at offset zero. On the other hand we want scalar abi optimizations.
+        let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox);
+
         // This is here instead of layout because the choice must make it into metadata.
-        if !self.consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did))) {
+        if is_box
+            || !self
+                .consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did)))
+        {
             flags.insert(ReprFlags::IS_LINEAR);
         }
 

config.example.toml

@@ -519,6 +519,9 @@
 # are disabled statically" because `max_level_info` is enabled, set this value to `true`.
 #debug-logging = rust.debug-assertions (boolean)
 
+# Whether or not to build rustc, tools and the libraries with randomized type layout
+#randomize-layout = false
+
 # Whether or not overflow checks are enabled for the compiler and standard
 # library.
 #

library/alloc/Cargo.toml

@@ -52,4 +52,5 @@ check-cfg = [
     'cfg(no_global_oom_handling)',
     'cfg(no_rc)',
     'cfg(no_sync)',
+    'cfg(randomized_layouts)',
 ]

library/alloc/src/collections/btree/node/tests.rs

@@ -90,7 +90,7 @@ fn test_partial_eq() {
 
 #[test]
 #[cfg(target_arch = "x86_64")]
-#[cfg_attr(miri, ignore)] // We'd like to run Miri with layout randomization
+#[cfg_attr(any(miri, randomized_layouts), ignore)] // We'd like to run Miri with layout randomization
 fn test_sizes() {
     assert_eq!(core::mem::size_of::<LeafNode<(), ()>>(), 16);
     assert_eq!(core::mem::size_of::<LeafNode<i64, i64>>(), 16 + CAPACITY * 2 * 8);

src/bootstrap/src/core/build_steps/test.rs

@@ -1810,6 +1810,9 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
         if builder.config.rust_optimize_tests {
             cmd.arg("--optimize-tests");
         }
+        if builder.config.rust_randomize_layout {
+            cmd.arg("--rust-randomized-layout");
+        }
         if builder.config.cmd.only_modified() {
             cmd.arg("--only-modified");
         }

src/bootstrap/src/core/builder.rs

@@ -1618,6 +1618,15 @@ impl<'a> Builder<'a> {
             rustflags.arg("-Csymbol-mangling-version=legacy");
         }
 
+        // FIXME: the following components don't build with `-Zrandomize-layout` yet:
+        // - wasm-component-ld, due to the `wast`crate
+        // - rust-analyzer, due to the rowan crate
+        // so we exclude entire categories of steps here due to lack of fine-grained control over
+        // rustflags.
+        if self.config.rust_randomize_layout && mode != Mode::ToolStd && mode != Mode::ToolRustc {
+            rustflags.arg("-Zrandomize-layout");
+        }
+
         // Enable compile-time checking of `cfg` names, values and Cargo `features`.
         //
         // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like
@@ -2193,6 +2202,9 @@ impl<'a> Builder<'a> {
                 rustflags.arg("-Zvalidate-mir");
                 rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}"));
             }
+            if self.config.rust_randomize_layout {
+                rustflags.arg("--cfg=randomized_layouts");
+            }
             // Always enable inlining MIR when building the standard library.
             // Without this flag, MIR inlining is disabled when incremental compilation is enabled.
             // That causes some mir-opt tests which inline functions from the standard library to

src/bootstrap/src/core/config/config.rs

@@ -280,6 +280,7 @@ pub struct Config {
     pub rust_codegen_backends: Vec<String>,
     pub rust_verify_llvm_ir: bool,
     pub rust_thin_lto_import_instr_limit: Option<u32>,
+    pub rust_randomize_layout: bool,
     pub rust_remap_debuginfo: bool,
     pub rust_new_symbol_mangling: Option<bool>,
     pub rust_profile_use: Option<String>,
@@ -1090,6 +1091,7 @@ define_config! {
         codegen_units: Option<u32> = "codegen-units",
         codegen_units_std: Option<u32> = "codegen-units-std",
         debug_assertions: Option<bool> = "debug-assertions",
+        randomize_layout: Option<bool> = "randomize-layout",
         debug_assertions_std: Option<bool> = "debug-assertions-std",
         overflow_checks: Option<bool> = "overflow-checks",
         overflow_checks_std: Option<bool> = "overflow-checks-std",
@@ -1181,6 +1183,7 @@ impl Config {
             backtrace: true,
             rust_optimize: RustOptimize::Bool(true),
             rust_optimize_tests: true,
+            rust_randomize_layout: false,
             submodules: None,
             docs: true,
             docs_minification: true,
@@ -1640,6 +1643,7 @@ impl Config {
                 backtrace,
                 incremental,
                 parallel_compiler,
+                randomize_layout,
                 default_linker,
                 channel,
                 description,
@@ -1729,6 +1733,7 @@ impl Config {
             set(&mut config.lld_mode, lld_mode);
             set(&mut config.llvm_bitcode_linker_enabled, llvm_bitcode_linker);
 
+            config.rust_randomize_layout = randomize_layout.unwrap_or_default();
             config.llvm_tools_enabled = llvm_tools.unwrap_or(true);
             config.rustc_parallel =
                 parallel_compiler.unwrap_or(config.channel == "dev" || config.channel == "nightly");
@@ -2900,6 +2905,7 @@ fn check_incompatible_options_for_ci_rustc(
     let Rust {
         // Following options are the CI rustc incompatible ones.
         optimize,
+        randomize_layout,
         debug_logging,
         debuginfo_level_rustc,
         llvm_tools,
@@ -2964,6 +2970,7 @@ fn check_incompatible_options_for_ci_rustc(
     // otherwise, we just print a warning with `warn` macro.
 
     err!(current_rust_config.optimize, optimize);
+    err!(current_rust_config.randomize_layout, randomize_layout);
     err!(current_rust_config.debug_logging, debug_logging);
     err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc);
     err!(current_rust_config.rpath, rpath);

src/bootstrap/src/lib.rs

@@ -678,6 +678,9 @@ impl Build {
         if self.config.rustc_parallel {
             features.push("rustc_use_parallel_compiler");
         }
+        if self.config.rust_randomize_layout {
+            features.push("rustc_randomized_layouts");
+        }
 
         // If debug logging is on, then we want the default for tracing:
         // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26

src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile

@@ -50,6 +50,7 @@ ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-17 \
       --enable-llvm-link-shared \
+      --set rust.randomize-layout=true \
       --set rust.thin-lto-import-instr-limit=10
 
 COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/

src/tools/compiletest/src/command-list.rs

@@ -136,6 +136,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "min-llvm-version",
     "min-system-llvm-version",
     "needs-asm-support",
+    "needs-deterministic-layouts",
     "needs-dlltool",
     "needs-dynamic-linking",
     "needs-force-clang-based-tests",

src/tools/compiletest/src/common.rs

@@ -274,6 +274,9 @@ pub struct Config {
     /// Flags to pass to the compiler when building for the target
     pub target_rustcflags: Vec<String>,
 
+    /// Whether the compiler and stdlib has been built with randomized struct layouts
+    pub rust_randomized_layout: bool,
+
     /// Whether tests should be optimized by default. Individual test-suites and test files may
     /// override this setting.
     pub optimize_tests: bool,

src/tools/compiletest/src/header/needs.rs

@@ -134,6 +134,11 @@ pub(super) fn handle_needs(
             condition: config.target_cfg().relocation_model == "pic",
             ignore_reason: "ignored on targets without PIC relocation model",
         },
+        Need {
+            name: "needs-deterministic-layouts",
+            condition: !config.rust_randomized_layout,
+            ignore_reason: "ignored when randomizing layouts",
+        },
         Need {
             name: "needs-wasmtime",
             condition: config.runner.as_ref().is_some_and(|r| r.contains("wasmtime")),

src/tools/compiletest/src/lib.rs

@@ -99,6 +99,11 @@ pub fn parse_config(args: Vec<String>) -> Config {
         )
         .optmulti("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS")
         .optmulti("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS")
+        .optflag(
+            "",
+            "rust-randomized-layout",
+            "set this when rustc/stdlib were compiled with randomized layouts",
+        )
         .optflag("", "optimize-tests", "run tests with optimizations enabled")
         .optflag("", "verbose", "run tests verbosely, showing all output")
         .optflag(
@@ -286,6 +291,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         host_rustcflags: matches.opt_strs("host-rustcflags"),
         target_rustcflags: matches.opt_strs("target-rustcflags"),
         optimize_tests: matches.opt_present("optimize-tests"),
+        rust_randomized_layout: matches.opt_present("rust-randomized-layout"),
         target,
         host: opt_str2(matches.opt_str("host")),
         cdb,

tests/codegen/issues/issue-86106.rs

@@ -1,5 +1,6 @@
 //@ only-64bit llvm appears to use stores instead of memset on 32bit
 //@ compile-flags: -C opt-level=3 -Z merge-functions=disabled
+//@ needs-deterministic-layouts
 
 // The below two functions ensure that both `String::new()` and `"".to_string()`
 // produce the identical code.

tests/codegen/mem-replace-big-type.rs

@@ -5,6 +5,7 @@
 
 //@ compile-flags: -C no-prepopulate-passes -Zinline-mir=no
 //@ ignore-debug: precondition checks in ptr::read make them a bad candidate for MIR inlining
+//@ needs-deterministic-layouts
 
 #![crate_type = "lib"]
 

tests/codegen/slice-iter-nonnull.rs

@@ -1,4 +1,5 @@
 //@ compile-flags: -O
+//@ needs-deterministic-layouts
 #![crate_type = "lib"]
 #![feature(exact_size_is_empty)]
 

tests/codegen/vecdeque-drain.rs

@@ -1,6 +1,7 @@
 // Check that draining at the front or back doesn't copy memory.
 
 //@ compile-flags: -O
+//@ needs-deterministic-layouts
 //@ ignore-debug: FIXME: checks for call detect scoped noalias metadata
 
 #![crate_type = "lib"]

tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.rs

@@ -1,3 +1,4 @@
+//@needs-deterministic-layouts
 // Verify that we do not ICE when printing an invalid constant.
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY