[NOT READY] Add support for downloading GCC artifacts

config.example.toml

@@ -164,6 +164,12 @@
 # Custom CMake defines to set when building LLVM.
 #build-config = {}
 
+# =============================================================================
+# General configuration options for the GCC backend
+# =============================================================================
+[gcc]
+# download-ci-gcc = true
+
 # =============================================================================
 # General build configuration options
 # =============================================================================

src/bootstrap/src/core/build_steps/gcc.rs

@@ -12,10 +12,14 @@ use std::fs;
 use std::path::PathBuf;
 use std::sync::OnceLock;
 
+use build_helper::ci::CiEnv;
+use build_helper::git::get_closest_merge_commit;
+
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
-use crate::core::config::TargetSelection;
+use crate::core::config::{Config, TargetSelection};
+use crate::utils::channel;
 use crate::utils::exec::command;
-use crate::utils::helpers::{self, HashStamp, t};
+use crate::utils::helpers::{self, HashStamp, output, t};
 use crate::{Kind, generate_smart_stamp_hash};
 
 pub struct Meta {
@@ -38,8 +42,7 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
     // Initialize the gcc submodule if not initialized already.
     builder.config.update_submodule("src/gcc");
 
-    // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI.
-    // builder.config.maybe_download_ci_gcc();
+    builder.config.maybe_download_ci_gcc();
 
     let root = builder.src.join("src/gcc");
     let out_dir = builder.gcc_out(target).join("build");
@@ -74,6 +77,103 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
     GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root })
 }
 
+/// This retrieves the GCC sha we *want* to use, according to git history.
+pub(crate) fn detect_gcc_sha(config: &Config, is_git: bool) -> String {
+    let gcc_sha = if is_git {
+        get_closest_merge_commit(Some(&config.src), &config.git_config(), &[
+            config.src.join("src/gcc"),
+            config.src.join("src/bootstrap/download-ci-gcc-stamp"),
+            // the GCC shared object file is named `GCC-rust-{version}-nightly`
+            config.src.join("src/version"),
+        ])
+        .unwrap()
+    } else if let Some(info) = channel::read_commit_info_file(&config.src) {
+        info.sha.trim().to_owned()
+    } else {
+        "".to_owned()
+    };
+
+    if gcc_sha.is_empty() {
+        eprintln!("error: could not find commit hash for downloading GCC");
+        eprintln!("HELP: maybe your repository history is too shallow?");
+        eprintln!("HELP: consider disabling `download-ci-gcc`");
+        eprintln!("HELP: or fetch enough history to include one upstream commit");
+        panic!();
+    }
+
+    gcc_sha
+}
+
+/// Returns whether the CI-found GCC is currently usable.
+///
+/// This checks both the build triple platform to confirm we're usable at all,
+/// and then verifies if the current HEAD matches the detected GCC SHA head,
+/// in which case GCC is indicated as not available.
+pub(crate) fn is_ci_gcc_available(config: &Config, asserts: bool) -> bool {
+    // This is currently all tier 1 targets and tier 2 targets with host tools
+    // (since others may not have CI artifacts)
+    // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
+    let supported_platforms = [
+        // tier 1
+        ("aarch64-unknown-linux-gnu", false),
+        ("aarch64-apple-darwin", false),
+        ("i686-pc-windows-gnu", false),
+        ("i686-pc-windows-msvc", false),
+        ("i686-unknown-linux-gnu", false),
+        ("x86_64-unknown-linux-gnu", true),
+        ("x86_64-apple-darwin", true),
+        ("x86_64-pc-windows-gnu", true),
+        ("x86_64-pc-windows-msvc", true),
+        // tier 2 with host tools
+        ("aarch64-pc-windows-msvc", false),
+        ("aarch64-unknown-linux-musl", false),
+        ("arm-unknown-linux-gnueabi", false),
+        ("arm-unknown-linux-gnueabihf", false),
+        ("armv7-unknown-linux-gnueabihf", false),
+        ("loongarch64-unknown-linux-gnu", false),
+        ("loongarch64-unknown-linux-musl", false),
+        ("mips-unknown-linux-gnu", false),
+        ("mips64-unknown-linux-gnuabi64", false),
+        ("mips64el-unknown-linux-gnuabi64", false),
+        ("mipsel-unknown-linux-gnu", false),
+        ("powerpc-unknown-linux-gnu", false),
+        ("powerpc64-unknown-linux-gnu", false),
+        ("powerpc64le-unknown-linux-gnu", false),
+        ("riscv64gc-unknown-linux-gnu", false),
+        ("s390x-unknown-linux-gnu", false),
+        ("x86_64-unknown-freebsd", false),
+        ("x86_64-unknown-illumos", false),
+        ("x86_64-unknown-linux-musl", false),
+        ("x86_64-unknown-netbsd", false),
+    ];
+
+    if !supported_platforms.contains(&(&*config.build.triple, asserts))
+        && (asserts || !supported_platforms.contains(&(&*config.build.triple, true)))
+    {
+        return false;
+    }
+
+    if is_ci_gcc_modified(config) {
+        eprintln!("Detected GCC as non-available: running in CI and modified GCC in this change");
+        return false;
+    }
+
+    true
+}
+
+/// Returns true if we're running in CI with modified LLVM (and thus can't download it)
+pub(crate) fn is_ci_gcc_modified(config: &Config) -> bool {
+    CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && {
+        // We assume we have access to git, so it's okay to unconditionally pass
+        // `true` here.
+        let gcc_sha = detect_gcc_sha(config, true);
+        let head_sha =
+            output(helpers::git(Some(&config.src)).arg("rev-parse").arg("HEAD").as_command_mut());
+        let head_sha = head_sha.trim();
+        gcc_sha == head_sha
+    }
+}
+
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Gcc {
     pub target: TargetSelection,

src/bootstrap/src/core/builder.rs

@@ -1258,6 +1258,11 @@ impl<'a> Builder<'a> {
             let ci_llvm_lib = self.out.join(compiler.host).join("ci-llvm").join("lib");
             dylib_dirs.push(ci_llvm_lib);
         }
+        // Ensure that the downloaded GCC libraries can be found.
+        if self.config.gcc_from_ci {
+            let ci_gcc_lib = self.out.join(compiler.host).join("ci-gcc").join("lib");
+            dylib_dirs.push(ci_gcc_lib);
+        }
 
         dylib_dirs
     }

src/bootstrap/src/core/config/config.rs

@@ -19,7 +19,7 @@ use serde::{Deserialize, Deserializer};
 use serde_derive::Deserialize;
 
 use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX;
-use crate::core::build_steps::llvm;
+use crate::core::build_steps::{gcc, llvm};
 pub use crate::core::config::flags::Subcommand;
 use crate::core::config::flags::{Color, Flags, Warnings};
 use crate::utils::cache::{INTERNER, Interned};
@@ -256,6 +256,8 @@ pub struct Config {
     pub llvm_ldflags: Option<String>,
     pub llvm_use_libcxx: bool,
 
+    pub gcc_from_ci: bool,
+
     // rust codegen options
     pub rust_optimize: RustOptimize,
     pub rust_codegen_units: Option<u32>,
@@ -616,6 +618,7 @@ pub(crate) struct TomlConfig {
     build: Option<Build>,
     install: Option<Install>,
     llvm: Option<Llvm>,
+    gcc: Option<Gcc>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
     dist: Option<Dist>,
@@ -650,7 +653,7 @@ trait Merge {
 impl Merge for TomlConfig {
     fn merge(
         &mut self,
-        TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id }: Self,
+        TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id, gcc }: Self,
         replace: ReplaceOpt,
     ) {
         fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
@@ -666,6 +669,7 @@ impl Merge for TomlConfig {
         do_merge(&mut self.build, build, replace);
         do_merge(&mut self.install, install, replace);
         do_merge(&mut self.llvm, llvm, replace);
+        do_merge(&mut self.gcc, gcc, replace);
         do_merge(&mut self.rust, rust, replace);
         do_merge(&mut self.dist, dist, replace);
 
@@ -938,6 +942,13 @@ define_config! {
     }
 }
 
+define_config! {
+    /// TOML representation of how the GCC backend is configured.
+    struct Gcc {
+        download_ci_gcc: Option<StringOrBool> = "download-ci-gcc",
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
 #[serde(untagged)]
 pub enum StringOrBool {
@@ -1816,6 +1827,17 @@ impl Config {
             ci_channel.clone_into(&mut config.channel);
         }
 
+        if let Some(gcc) = toml.gcc {
+            let Gcc { download_ci_gcc } = gcc;
+            config.gcc_from_ci = config.parse_download_ci_gcc(download_ci_gcc, false);
+        } else {
+            config.gcc_from_ci = config.parse_download_ci_gcc(None, false);
+        }
+
+        if config.gcc_enabled(config.build) {
+            config.maybe_download_ci_gcc();
+        }
+
         if let Some(llvm) = toml.llvm {
             let Llvm {
                 optimize: optimize_toml,
@@ -2312,6 +2334,12 @@ impl Config {
         self.out.join(self.build).join("ci-llvm")
     }
 
+    /// The absolute path to the downloaded GCC artifacts.
+    pub(crate) fn ci_gcc_root(&self) -> PathBuf {
+        assert!(self.gcc_from_ci);
+        self.out.join(self.build).join("ci-gcc")
+    }
+
     /// Directory where the extracted `rustc-dev` component is stored.
     pub(crate) fn ci_rustc_dir(&self) -> PathBuf {
         assert!(self.download_rustc());
@@ -2476,6 +2504,10 @@ impl Config {
         self.codegen_backends(target).contains(&"llvm".to_owned())
     }
 
+    pub fn gcc_enabled(&self, target: TargetSelection) -> bool {
+        self.codegen_backends(target).contains(&"gcc".to_owned())
+    }
+
     pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind {
         self.target_config
             .get(&target)
@@ -2777,6 +2809,48 @@ impl Config {
         }
     }
 
+    fn parse_download_ci_gcc(&self, download_ci_gcc: Option<StringOrBool>, asserts: bool) -> bool {
+        let download_ci_gcc = download_ci_gcc.unwrap_or(StringOrBool::Bool(true));
+
+        let if_unchanged = || {
+            if self.rust_info.is_from_tarball() {
+                // Git is needed for running "if-unchanged" logic.
+                println!(
+                    "WARNING: 'if-unchanged' has no effect on tarball sources; ignoring `download-ci-gcc`."
+                );
+                return false;
+            }
+
+            // Fetching the GCC submodule is unnecessary for self-tests.
+            #[cfg(not(feature = "bootstrap-self-test"))]
+            self.update_submodule("src/gcc");
+
+            // Check for untracked changes in `src/gcc`.
+            let has_changes =
+                self.last_modified_commit(&["src/gcc"], "download-ci-gcc", true).is_none();
+
+            // Return false if there are untracked changes, otherwise check if CI GCC is available.
+            if has_changes { false } else { gcc::is_ci_gcc_available(self, asserts) }
+        };
+
+        match download_ci_gcc {
+            StringOrBool::Bool(b) => {
+                if !b && self.download_rustc_commit.is_some() {
+                    panic!(
+                        "`gcc.download-ci-gcc` cannot be set to `false` if `rust.download-rustc` is set to `true` or `if-unchanged`."
+                    );
+                }
+
+                // If download-ci-gcc=true we also want to check that CI gcc is available
+                b && gcc::is_ci_gcc_available(self, asserts)
+            }
+            StringOrBool::String(s) if s == "if-unchanged" => if_unchanged(),
+            StringOrBool::String(other) => {
+                panic!("unrecognized option for download-ci-gcc: {:?}", other)
+            }
+        }
+    }
+
     /// Returns the last commit in which any of `modified_paths` were changed,
     /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true.
     pub fn last_modified_commit(

src/bootstrap/src/core/download.rs

@@ -826,6 +826,65 @@ download-rustc = false
         let llvm_root = self.ci_llvm_root();
         self.unpack(&tarball, &llvm_root, "rust-dev");
     }
+
+    #[cfg(feature = "bootstrap-self-test")]
+    pub(crate) fn maybe_download_ci_gcc(&self) {}
+
+    #[cfg(not(feature = "bootstrap-self-test"))]
+    pub(crate) fn maybe_download_ci_gcc(&self) {
+        use crate::core::build_steps::gcc::detect_gcc_sha;
+
+        if !self.gcc_from_ci {
+            return;
+        }
+
+        let gcc_root = self.ci_gcc_root();
+        let gcc_stamp = gcc_root.join(".gcc-stamp");
+        let gcc_sha = detect_gcc_sha(self, self.rust_info.is_managed_git_subrepository());
+        let key = gcc_sha.to_string();
+        if program_out_of_date(&gcc_stamp, &key) && !self.dry_run() {
+            self.download_ci_gcc(&gcc_sha);
+
+            if self.should_fix_bins_and_dylibs() {
+                for entry in t!(fs::read_dir(gcc_root.join("bin"))) {
+                    self.fix_bin_or_dylib(&t!(entry).path());
+                }
+            }
+
+            t!(fs::write(gcc_stamp, key));
+        }
+    }
+
+    #[cfg(not(feature = "bootstrap-self-test"))]
+    fn download_ci_gcc(&self, gcc_sha: &str) {
+        let cache_prefix = format!("gcc-{gcc_sha}");
+        let cache_dst =
+            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
+
+        let rustc_cache = cache_dst.join(cache_prefix);
+        if !rustc_cache.exists() {
+            t!(fs::create_dir_all(&rustc_cache));
+        }
+        let base = &self.stage0_metadata.config.artifacts_server;
+        let version = self.artifact_version_part(gcc_sha);
+        let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple);
+        let tarball = rustc_cache.join(&filename);
+        if !tarball.exists() {
+            let help_on_error = "ERROR: failed to download GCC from ci
+
+    HELP: There could be two reasons behind this:
+        1) The host triple is not supported for `download-ci-gcc`.
+        2) Old builds get deleted after a certain time.
+    HELP: In either case, disable `download-ci-gcc` in your config.toml:
+
+    [gcc]
+    download-ci-gcc = false
+    ";
+            self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error);
+        }
+        let gcc_root = self.ci_gcc_root();
+        self.unpack(&tarball, &gcc_root, "rust-dev");
+    }
 }
 
 fn path_is_dylib(path: &Path) -> bool {