From 2d8d9f8fd4bcba5f7bba28685192d4ac8281c0a5 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 23 Sep 2024 14:39:47 +0200 Subject: [PATCH] Add support for downloading GCC artifacts --- config.example.toml | 6 ++ src/bootstrap/src/core/build_steps/gcc.rs | 108 +++++++++++++++++++++- src/bootstrap/src/core/builder.rs | 5 + src/bootstrap/src/core/config/config.rs | 78 +++++++++++++++- src/bootstrap/src/core/download.rs | 59 ++++++++++++ 5 files changed, 250 insertions(+), 6 deletions(-) diff --git a/config.example.toml b/config.example.toml index 47ebb20d8fa82..df50356218e5a 100644 --- a/config.example.toml +++ b/config.example.toml @@ -164,6 +164,12 @@ # Custom CMake defines to set when building LLVM. #build-config = {} +# ============================================================================= +# General configuration options for the GCC backend +# ============================================================================= +[gcc] +# download-ci-gcc = true + # ============================================================================= # General build configuration options # ============================================================================= diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index b950bec11fd01..2e014ccf9bf84 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -12,10 +12,14 @@ use std::fs; use std::path::PathBuf; use std::sync::OnceLock; +use build_helper::ci::CiEnv; +use build_helper::git::get_closest_merge_commit; + use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::core::config::TargetSelection; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::channel; use crate::utils::exec::command; -use crate::utils::helpers::{self, HashStamp, t}; +use crate::utils::helpers::{self, HashStamp, output, t}; use crate::{Kind, generate_smart_stamp_hash}; pub struct Meta { @@ -38,8 +42,7 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc // Initialize the gcc submodule if not initialized already. builder.config.update_submodule("src/gcc"); - // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI. - // builder.config.maybe_download_ci_gcc(); + builder.config.maybe_download_ci_gcc(); let root = builder.src.join("src/gcc"); let out_dir = builder.gcc_out(target).join("build"); @@ -74,6 +77,103 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root }) } +/// This retrieves the GCC sha we *want* to use, according to git history. +pub(crate) fn detect_gcc_sha(config: &Config, is_git: bool) -> String { + let gcc_sha = if is_git { + get_closest_merge_commit(Some(&config.src), &config.git_config(), &[ + config.src.join("src/gcc"), + config.src.join("src/bootstrap/download-ci-gcc-stamp"), + // the GCC shared object file is named `GCC-rust-{version}-nightly` + config.src.join("src/version"), + ]) + .unwrap() + } else if let Some(info) = channel::read_commit_info_file(&config.src) { + info.sha.trim().to_owned() + } else { + "".to_owned() + }; + + if gcc_sha.is_empty() { + eprintln!("error: could not find commit hash for downloading GCC"); + eprintln!("HELP: maybe your repository history is too shallow?"); + eprintln!("HELP: consider disabling `download-ci-gcc`"); + eprintln!("HELP: or fetch enough history to include one upstream commit"); + panic!(); + } + + gcc_sha +} + +/// Returns whether the CI-found GCC is currently usable. +/// +/// This checks both the build triple platform to confirm we're usable at all, +/// and then verifies if the current HEAD matches the detected GCC SHA head, +/// in which case GCC is indicated as not available. +pub(crate) fn is_ci_gcc_available(config: &Config, asserts: bool) -> bool { + // This is currently all tier 1 targets and tier 2 targets with host tools + // (since others may not have CI artifacts) + // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 + let supported_platforms = [ + // tier 1 + ("aarch64-unknown-linux-gnu", false), + ("aarch64-apple-darwin", false), + ("i686-pc-windows-gnu", false), + ("i686-pc-windows-msvc", false), + ("i686-unknown-linux-gnu", false), + ("x86_64-unknown-linux-gnu", true), + ("x86_64-apple-darwin", true), + ("x86_64-pc-windows-gnu", true), + ("x86_64-pc-windows-msvc", true), + // tier 2 with host tools + ("aarch64-pc-windows-msvc", false), + ("aarch64-unknown-linux-musl", false), + ("arm-unknown-linux-gnueabi", false), + ("arm-unknown-linux-gnueabihf", false), + ("armv7-unknown-linux-gnueabihf", false), + ("loongarch64-unknown-linux-gnu", false), + ("loongarch64-unknown-linux-musl", false), + ("mips-unknown-linux-gnu", false), + ("mips64-unknown-linux-gnuabi64", false), + ("mips64el-unknown-linux-gnuabi64", false), + ("mipsel-unknown-linux-gnu", false), + ("powerpc-unknown-linux-gnu", false), + ("powerpc64-unknown-linux-gnu", false), + ("powerpc64le-unknown-linux-gnu", false), + ("riscv64gc-unknown-linux-gnu", false), + ("s390x-unknown-linux-gnu", false), + ("x86_64-unknown-freebsd", false), + ("x86_64-unknown-illumos", false), + ("x86_64-unknown-linux-musl", false), + ("x86_64-unknown-netbsd", false), + ]; + + if !supported_platforms.contains(&(&*config.build.triple, asserts)) + && (asserts || !supported_platforms.contains(&(&*config.build.triple, true))) + { + return false; + } + + if is_ci_gcc_modified(config) { + eprintln!("Detected GCC as non-available: running in CI and modified GCC in this change"); + return false; + } + + true +} + +/// Returns true if we're running in CI with modified LLVM (and thus can't download it) +pub(crate) fn is_ci_gcc_modified(config: &Config) -> bool { + CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { + // We assume we have access to git, so it's okay to unconditionally pass + // `true` here. + let gcc_sha = detect_gcc_sha(config, true); + let head_sha = + output(helpers::git(Some(&config.src)).arg("rev-parse").arg("HEAD").as_command_mut()); + let head_sha = head_sha.trim(); + gcc_sha == head_sha + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Gcc { pub target: TargetSelection, diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 47420f8fe72fb..e7e8dfd882fe1 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -1258,6 +1258,11 @@ impl<'a> Builder<'a> { let ci_llvm_lib = self.out.join(compiler.host).join("ci-llvm").join("lib"); dylib_dirs.push(ci_llvm_lib); } + // Ensure that the downloaded GCC libraries can be found. + if self.config.gcc_from_ci { + let ci_gcc_lib = self.out.join(compiler.host).join("ci-gcc").join("lib"); + dylib_dirs.push(ci_gcc_lib); + } dylib_dirs } diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 77e0ece31047f..4654be26e50ed 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; -use crate::core::build_steps::llvm; +use crate::core::build_steps::{gcc, llvm}; pub use crate::core::config::flags::Subcommand; use crate::core::config::flags::{Color, Flags, Warnings}; use crate::utils::cache::{INTERNER, Interned}; @@ -256,6 +256,8 @@ pub struct Config { pub llvm_ldflags: Option, pub llvm_use_libcxx: bool, + pub gcc_from_ci: bool, + // rust codegen options pub rust_optimize: RustOptimize, pub rust_codegen_units: Option, @@ -616,6 +618,7 @@ pub(crate) struct TomlConfig { build: Option, install: Option, llvm: Option, + gcc: Option, rust: Option, target: Option>, dist: Option, @@ -650,7 +653,7 @@ trait Merge { impl Merge for TomlConfig { fn merge( &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id }: Self, + TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id, gcc }: Self, replace: ReplaceOpt, ) { fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { @@ -666,6 +669,7 @@ impl Merge for TomlConfig { do_merge(&mut self.build, build, replace); do_merge(&mut self.install, install, replace); do_merge(&mut self.llvm, llvm, replace); + do_merge(&mut self.gcc, gcc, replace); do_merge(&mut self.rust, rust, replace); do_merge(&mut self.dist, dist, replace); @@ -938,6 +942,13 @@ define_config! { } } +define_config! { + /// TOML representation of how the GCC backend is configured. + struct Gcc { + download_ci_gcc: Option = "download-ci-gcc", + } +} + #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] #[serde(untagged)] pub enum StringOrBool { @@ -1816,6 +1827,17 @@ impl Config { ci_channel.clone_into(&mut config.channel); } + if let Some(gcc) = toml.gcc { + let Gcc { download_ci_gcc } = gcc; + config.gcc_from_ci = config.parse_download_ci_gcc(download_ci_gcc, false); + } else { + config.gcc_from_ci = config.parse_download_ci_gcc(None, false); + } + + if config.gcc_enabled(config.build) { + config.maybe_download_ci_gcc(); + } + if let Some(llvm) = toml.llvm { let Llvm { optimize: optimize_toml, @@ -2312,6 +2334,12 @@ impl Config { self.out.join(self.build).join("ci-llvm") } + /// The absolute path to the downloaded GCC artifacts. + pub(crate) fn ci_gcc_root(&self) -> PathBuf { + assert!(self.gcc_from_ci); + self.out.join(self.build).join("ci-gcc") + } + /// Directory where the extracted `rustc-dev` component is stored. pub(crate) fn ci_rustc_dir(&self) -> PathBuf { assert!(self.download_rustc()); @@ -2476,6 +2504,10 @@ impl Config { self.codegen_backends(target).contains(&"llvm".to_owned()) } + pub fn gcc_enabled(&self, target: TargetSelection) -> bool { + self.codegen_backends(target).contains(&"gcc".to_owned()) + } + pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { self.target_config .get(&target) @@ -2777,6 +2809,48 @@ impl Config { } } + fn parse_download_ci_gcc(&self, download_ci_gcc: Option, asserts: bool) -> bool { + let download_ci_gcc = download_ci_gcc.unwrap_or(StringOrBool::Bool(true)); + + let if_unchanged = || { + if self.rust_info.is_from_tarball() { + // Git is needed for running "if-unchanged" logic. + println!( + "WARNING: 'if-unchanged' has no effect on tarball sources; ignoring `download-ci-gcc`." + ); + return false; + } + + // Fetching the GCC submodule is unnecessary for self-tests. + #[cfg(not(feature = "bootstrap-self-test"))] + self.update_submodule("src/gcc"); + + // Check for untracked changes in `src/gcc`. + let has_changes = + self.last_modified_commit(&["src/gcc"], "download-ci-gcc", true).is_none(); + + // Return false if there are untracked changes, otherwise check if CI GCC is available. + if has_changes { false } else { gcc::is_ci_gcc_available(self, asserts) } + }; + + match download_ci_gcc { + StringOrBool::Bool(b) => { + if !b && self.download_rustc_commit.is_some() { + panic!( + "`gcc.download-ci-gcc` cannot be set to `false` if `rust.download-rustc` is set to `true` or `if-unchanged`." + ); + } + + // If download-ci-gcc=true we also want to check that CI gcc is available + b && gcc::is_ci_gcc_available(self, asserts) + } + StringOrBool::String(s) if s == "if-unchanged" => if_unchanged(), + StringOrBool::String(other) => { + panic!("unrecognized option for download-ci-gcc: {:?}", other) + } + } + } + /// Returns the last commit in which any of `modified_paths` were changed, /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. pub fn last_modified_commit( diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index 444b75876f248..bfce4fb332cef 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -826,6 +826,65 @@ download-rustc = false let llvm_root = self.ci_llvm_root(); self.unpack(&tarball, &llvm_root, "rust-dev"); } + + #[cfg(feature = "bootstrap-self-test")] + pub(crate) fn maybe_download_ci_gcc(&self) {} + + #[cfg(not(feature = "bootstrap-self-test"))] + pub(crate) fn maybe_download_ci_gcc(&self) { + use crate::core::build_steps::gcc::detect_gcc_sha; + + if !self.gcc_from_ci { + return; + } + + let gcc_root = self.ci_gcc_root(); + let gcc_stamp = gcc_root.join(".gcc-stamp"); + let gcc_sha = detect_gcc_sha(self, self.rust_info.is_managed_git_subrepository()); + let key = gcc_sha.to_string(); + if program_out_of_date(&gcc_stamp, &key) && !self.dry_run() { + self.download_ci_gcc(&gcc_sha); + + if self.should_fix_bins_and_dylibs() { + for entry in t!(fs::read_dir(gcc_root.join("bin"))) { + self.fix_bin_or_dylib(&t!(entry).path()); + } + } + + t!(fs::write(gcc_stamp, key)); + } + } + + #[cfg(not(feature = "bootstrap-self-test"))] + fn download_ci_gcc(&self, gcc_sha: &str) { + let cache_prefix = format!("gcc-{gcc_sha}"); + let cache_dst = + self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache")); + + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = &self.stage0_metadata.config.artifacts_server; + let version = self.artifact_version_part(gcc_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "ERROR: failed to download GCC from ci + + HELP: There could be two reasons behind this: + 1) The host triple is not supported for `download-ci-gcc`. + 2) Old builds get deleted after a certain time. + HELP: In either case, disable `download-ci-gcc` in your config.toml: + + [gcc] + download-ci-gcc = false + "; + self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error); + } + let gcc_root = self.ci_gcc_root(); + self.unpack(&tarball, &gcc_root, "rust-dev"); + } } fn path_is_dylib(path: &Path) -> bool {