Skip to content

Refactor git change detection in bootstrap #138591

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Apr 23, 2025
Merged

Refactor git change detection in bootstrap #138591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
082ab9c
Implement a new unified function for figuring out how if a set of pat…
Kobzol Mar 16, 2025
1d1f248
Use `check_path_modifications` for detecting local GCC changes
Kobzol Mar 16, 2025
0396f0e
Use `check_path_modifications` for detecting local LLVM changes
Kobzol Mar 21, 2025
2228033
Use `check_path_modifications` for detecting local rustc changes
Kobzol Mar 16, 2025
1dabb76
Explicitly model missing upstream
Kobzol Mar 18, 2025
1f5320e
Unify usages of path modifications and log them in verbose mode
Kobzol Mar 21, 2025
8515d7e
Cache result of `check_path_modifications`
Kobzol Mar 18, 2025
64795ec
Remove `setup-upstream-remote.sh` and upstream handling.
Kobzol Mar 17, 2025
432c4a8
Remove the `add_dummy_commit.sh` hack
Kobzol Mar 21, 2025
f414afb
Move freshness test to bootstrap
Kobzol Mar 19, 2025
6ca2a0d
Extend `ci_rustc_if_unchanged` tests
Kobzol Apr 8, 2025
c32c076
Return `PathFreshness::MissingUpstream` from `detect_[gcc|llvm]_fresh…
Kobzol Mar 24, 2025
3a1dd44
Make `git_dir` required in several git functions
Kobzol Mar 24, 2025
e9f3e3a
Clarify comment
Kobzol Mar 24, 2025
4005851
Use `--author-date-order` when looking up upstream commits to support…
Kobzol Apr 10, 2025
fbca453
Fix compiletest and doc comment
Kobzol Apr 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,6 @@ jobs:
# which then uses log commands to actually set them.
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}

- name: setup upstream remote
run: src/ci/scripts/setup-upstream-remote.sh

- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh

Expand Down
93 changes: 84 additions & 9 deletions src/bootstrap/Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ dependencies = [
"sha2",
"sysinfo",
"tar",
"tempfile",
"termcolor",
"toml",
"tracing",
Expand Down Expand Up @@ -225,22 +226,28 @@ dependencies = [

[[package]]
name = "errno"
version = "0.3.9"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]

[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"

[[package]]
name = "fd-lock"
version = "4.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947"
dependencies = [
"cfg-if",
"rustix",
"rustix 0.38.40",
"windows-sys 0.52.0",
]

Expand All @@ -266,6 +273,18 @@ dependencies = [
"version_check",
]

[[package]]
name = "getrandom"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi",
]

[[package]]
name = "globset"
version = "0.4.15"
Expand Down Expand Up @@ -334,9 +353,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"

[[package]]
name = "libc"
version = "0.2.167"
version = "0.2.171"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"

[[package]]
name = "libredox"
Expand All @@ -355,6 +374,12 @@ version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"

[[package]]
name = "linux-raw-sys"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"

[[package]]
name = "log"
version = "0.4.22"
Expand Down Expand Up @@ -486,6 +511,12 @@ dependencies = [
"proc-macro2",
]

[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"

[[package]]
name = "redox_syscall"
version = "0.5.7"
Expand Down Expand Up @@ -548,10 +579,23 @@ dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"linux-raw-sys 0.4.14",
"windows-sys 0.52.0",
]

[[package]]
name = "rustix"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys 0.9.3",
"windows-sys 0.59.0",
]

[[package]]
name = "ryu"
version = "1.0.18"
Expand Down Expand Up @@ -678,6 +722,19 @@ dependencies = [
"xattr",
]

[[package]]
name = "tempfile"
version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600"
dependencies = [
"fastrand",
"getrandom",
"once_cell",
"rustix 1.0.2",
"windows-sys 0.59.0",
]

[[package]]
name = "termcolor"
version = "1.4.1"
Expand Down Expand Up @@ -824,6 +881,15 @@ dependencies = [
"winapi-util",
]

[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
dependencies = [
"wit-bindgen-rt",
]

[[package]]
name = "winapi"
version = "0.3.9"
Expand Down Expand Up @@ -990,15 +1056,24 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]

[[package]]
name = "xattr"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f"
dependencies = [
"libc",
"linux-raw-sys",
"rustix",
"linux-raw-sys 0.4.14",
"rustix 0.38.40",
]

[[package]]
Expand Down
1 change: 1 addition & 0 deletions src/bootstrap/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ features = [

[dev-dependencies]
pretty_assertions = "1.4"
tempfile = "3.15.0"

# We care a lot about bootstrap's compile times, so don't include debuginfo for
# dependencies, only bootstrap itself.
Expand Down
6 changes: 1 addition & 5 deletions src/bootstrap/src/core/build_steps/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,10 @@ impl Step for Std {
// the `rust.download-rustc=true` option.
let force_recompile = builder.rust_info().is_managed_git_subrepository()
&& builder.download_rustc()
&& builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none();
&& builder.config.has_changes_from_upstream(&["library"]);

trace!("is managed git repo: {}", builder.rust_info().is_managed_git_subrepository());
trace!("download_rustc: {}", builder.download_rustc());
trace!(
"last modified commit: {:?}",
builder.config.last_modified_commit(&["library"], "download-rustc", true)
);
trace!(force_recompile);

run.builder.ensure(Std {
Expand Down
77 changes: 43 additions & 34 deletions src/bootstrap/src/core/build_steps/gcc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ pub enum GccBuildStatus {
/// Returns a path to the libgccjit.so file.
#[cfg(not(test))]
fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<PathBuf> {
use build_helper::git::PathFreshness;

// Try to download GCC from CI if configured and available
if !matches!(builder.config.gcc_ci_mode, crate::core::config::GccCiMode::DownloadFromCi) {
return None;
Expand All @@ -104,18 +106,40 @@ fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<Pa
eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target");
return None;
}
let sha =
detect_gcc_sha(&builder.config, builder.config.rust_info.is_managed_git_subrepository());
let root = ci_gcc_root(&builder.config);
let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&sha);
if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
builder.config.download_ci_gcc(&sha, &root);
t!(gcc_stamp.write());
let source = detect_gcc_freshness(
&builder.config,
builder.config.rust_info.is_managed_git_subrepository(),
);
builder.verbose(|| {
eprintln!("GCC freshness: {source:?}");
});
match source {
PathFreshness::LastModifiedUpstream { upstream } => {
// Download from upstream CI
let root = ci_gcc_root(&builder.config);
let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&upstream);
if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
builder.config.download_ci_gcc(&upstream, &root);
t!(gcc_stamp.write());
}

let libgccjit = root.join("lib").join("libgccjit.so");
create_lib_alias(builder, &libgccjit);
Some(libgccjit)
}
PathFreshness::HasLocalModifications { .. } => {
// We have local modifications, rebuild GCC.
eprintln!("Found local GCC modifications, GCC will *not* be downloaded");
None
}
PathFreshness::MissingUpstream => {
eprintln!("error: could not find commit hash for downloading GCC");
eprintln!("HELP: maybe your repository history is too shallow?");
eprintln!("HELP: consider disabling `download-ci-gcc`");
eprintln!("HELP: or fetch enough history to include one upstream commit");
None
}
}

let libgccjit = root.join("lib").join("libgccjit.so");
create_lib_alias(builder, &libgccjit);
Some(libgccjit)
}

#[cfg(test)]
Expand Down Expand Up @@ -264,31 +288,16 @@ fn ci_gcc_root(config: &crate::Config) -> PathBuf {
config.out.join(config.build).join("ci-gcc")
}

/// This retrieves the GCC sha we *want* to use, according to git history.
/// Detect whether GCC sources have been modified locally or not.
#[cfg(not(test))]
fn detect_gcc_sha(config: &crate::Config, is_git: bool) -> String {
use build_helper::git::get_closest_merge_commit;

let gcc_sha = if is_git {
get_closest_merge_commit(
Some(&config.src),
&config.git_config(),
&["src/gcc", "src/bootstrap/download-ci-gcc-stamp"],
)
.unwrap()
fn detect_gcc_freshness(config: &crate::Config, is_git: bool) -> build_helper::git::PathFreshness {
use build_helper::git::PathFreshness;

if is_git {
config.check_path_modifications(&["src/gcc", "src/bootstrap/download-ci-gcc-stamp"])
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like MissingUpstream should be handled here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We handle it one level above, in try_download_gcc. But actually I can remove the ugly exit(1) hack with just returning MissingUpstream.

} else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
info.sha.trim().to_owned()
PathFreshness::LastModifiedUpstream { upstream: info.sha.trim().to_owned() }
} else {
"".to_owned()
};

if gcc_sha.is_empty() {
eprintln!("error: could not find commit hash for downloading GCC");
eprintln!("HELP: maybe your repository history is too shallow?");
eprintln!("HELP: consider disabling `download-ci-gcc`");
eprintln!("HELP: or fetch enough history to include one upstream commit");
panic!();
PathFreshness::MissingUpstream
}

gcc_sha
}
25 changes: 7 additions & 18 deletions src/bootstrap/src/core/build_steps/llvm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use std::path::{Path, PathBuf};
use std::sync::OnceLock;
use std::{env, fs};

use build_helper::git::get_closest_merge_commit;
use build_helper::git::PathFreshness;
#[cfg(feature = "tracing")]
use tracing::instrument;

Expand Down Expand Up @@ -181,26 +181,15 @@ pub const LLVM_INVALIDATION_PATHS: &[&str] = &[
"src/version",
];

/// This retrieves the LLVM sha we *want* to use, according to git history.
pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String {
let llvm_sha = if is_git {
get_closest_merge_commit(Some(&config.src), &config.git_config(), LLVM_INVALIDATION_PATHS)
.unwrap()
/// Detect whether LLVM sources have been modified locally or not.
pub(crate) fn detect_llvm_freshness(config: &Config, is_git: bool) -> PathFreshness {
if is_git {
config.check_path_modifications(LLVM_INVALIDATION_PATHS)
} else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
info.sha.trim().to_owned()
PathFreshness::LastModifiedUpstream { upstream: info.sha.trim().to_owned() }
} else {
"".to_owned()
};

if llvm_sha.is_empty() {
eprintln!("error: could not find commit hash for downloading LLVM");
eprintln!("HELP: maybe your repository history is too shallow?");
eprintln!("HELP: consider disabling `download-ci-llvm`");
eprintln!("HELP: or fetch enough history to include one upstream commit");
panic!();
PathFreshness::MissingUpstream
}

llvm_sha
}

/// Returns whether the CI-found LLVM is currently usable.
Expand Down
3 changes: 1 addition & 2 deletions src/bootstrap/src/core/build_steps/tool.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -696,8 +696,7 @@ impl Step for Rustdoc {
let files_to_track = &["src/librustdoc", "src/tools/rustdoc"];

// Check if unchanged
if builder.config.last_modified_commit(files_to_track, "download-rustc", true).is_some()
{
if !builder.config.has_changes_from_upstream(files_to_track) {
let precompiled_rustdoc = builder
.config
.ci_rustc_dir()
Expand Down
Loading
Loading