Skip to content

Commit 2033bd4

Browse files
committed
Auto merge of rust-lang#138597 - Kobzol:git-ci-beta, r=<try>
[do not merge] beta test for git change detection (rust-lang#138591) Opening to test CI/bootstrap changes. r? `@ghost` try-job: x86_64-gnu-stable try-job: x86_64-gnu try-job: x86_64-gnu-llvm-19-1 try-job: dist-x86_64-linux
2 parents 8c7969a + 3775e26 commit 2033bd4

File tree

18 files changed

+671
-264
lines changed

18 files changed

+671
-264
lines changed

.github/workflows/ci.yml

-3
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,6 @@ jobs:
111111
# which then uses log commands to actually set them.
112112
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
113113

114-
- name: setup upstream remote
115-
run: src/ci/scripts/setup-upstream-remote.sh
116-
117114
- name: ensure the channel matches the target branch
118115
run: src/ci/scripts/verify-channel.sh
119116

Cargo.lock

+27-8
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,7 @@ version = "0.1.0"
317317
dependencies = [
318318
"serde",
319319
"serde_derive",
320+
"tempfile",
320321
]
321322

322323
[[package]]
@@ -2135,6 +2136,12 @@ version = "0.4.15"
21352136
source = "registry+https://github.com/rust-lang/crates.io-index"
21362137
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
21372138

2139+
[[package]]
2140+
name = "linux-raw-sys"
2141+
version = "0.9.3"
2142+
source = "registry+https://github.com/rust-lang/crates.io-index"
2143+
checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
2144+
21382145
[[package]]
21392146
name = "litemap"
21402147
version = "0.7.4"
@@ -4832,7 +4839,20 @@ dependencies = [
48324839
"bitflags",
48334840
"errno",
48344841
"libc",
4835-
"linux-raw-sys",
4842+
"linux-raw-sys 0.4.15",
4843+
"windows-sys 0.59.0",
4844+
]
4845+
4846+
[[package]]
4847+
name = "rustix"
4848+
version = "1.0.2"
4849+
source = "registry+https://github.com/rust-lang/crates.io-index"
4850+
checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
4851+
dependencies = [
4852+
"bitflags",
4853+
"errno",
4854+
"libc",
4855+
"linux-raw-sys 0.9.3",
48364856
"windows-sys 0.59.0",
48374857
]
48384858

@@ -5246,15 +5266,14 @@ dependencies = [
52465266

52475267
[[package]]
52485268
name = "tempfile"
5249-
version = "3.15.0"
5269+
version = "3.19.0"
52505270
source = "registry+https://github.com/rust-lang/crates.io-index"
5251-
checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
5271+
checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600"
52525272
dependencies = [
5253-
"cfg-if",
52545273
"fastrand",
5255-
"getrandom 0.2.15",
5274+
"getrandom 0.3.1",
52565275
"once_cell",
5257-
"rustix",
5276+
"rustix 1.0.2",
52585277
"windows-sys 0.59.0",
52595278
]
52605279

@@ -6599,8 +6618,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
65996618
checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909"
66006619
dependencies = [
66016620
"libc",
6602-
"linux-raw-sys",
6603-
"rustix",
6621+
"linux-raw-sys 0.4.15",
6622+
"rustix 0.38.43",
66046623
]
66056624

66066625
[[package]]

src/bootstrap/Cargo.lock

+5-5
Original file line numberDiff line numberDiff line change
@@ -225,12 +225,12 @@ dependencies = [
225225

226226
[[package]]
227227
name = "errno"
228-
version = "0.3.9"
228+
version = "0.3.10"
229229
source = "registry+https://github.com/rust-lang/crates.io-index"
230-
checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
230+
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
231231
dependencies = [
232232
"libc",
233-
"windows-sys 0.52.0",
233+
"windows-sys 0.59.0",
234234
]
235235

236236
[[package]]
@@ -334,9 +334,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
334334

335335
[[package]]
336336
name = "libc"
337-
version = "0.2.167"
337+
version = "0.2.171"
338338
source = "registry+https://github.com/rust-lang/crates.io-index"
339-
checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
339+
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
340340

341341
[[package]]
342342
name = "libredox"

src/bootstrap/src/core/build_steps/compile.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,11 @@ impl Step for Std {
107107
// the `rust.download-rustc=true` option.
108108
let force_recompile = builder.rust_info().is_managed_git_subrepository()
109109
&& builder.download_rustc()
110-
&& builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none();
110+
&& builder.config.has_changes_from_upstream(&["library"]);
111+
112+
trace!("is managed git repo: {}", builder.rust_info().is_managed_git_subrepository());
113+
trace!("download_rustc: {}", builder.download_rustc());
114+
trace!(force_recompile);
111115

112116
run.builder.ensure(Std {
113117
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),

src/bootstrap/src/core/build_steps/format.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, Str
9494
return Ok(None);
9595
}
9696

97-
get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"])
97+
get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]).map(Some)
9898
}
9999

100100
#[derive(serde_derive::Deserialize)]

src/bootstrap/src/core/build_steps/gcc.rs

+139-5
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,72 @@ use std::sync::OnceLock;
1414

1515
use build_helper::ci::CiEnv;
1616

17-
use crate::Kind;
18-
use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
17+
use crate::core::builder::{Builder, Cargo, Kind, RunConfig, ShouldRun, Step};
1918
use crate::core::config::TargetSelection;
2019
use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
2120
use crate::utils::exec::command;
2221
use crate::utils::helpers::{self, t};
2322

23+
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
24+
pub struct Gcc {
25+
pub target: TargetSelection,
26+
}
27+
28+
#[derive(Clone)]
29+
pub struct GccOutput {
30+
pub libgccjit: PathBuf,
31+
}
32+
33+
impl Step for Gcc {
34+
type Output = GccOutput;
35+
36+
const ONLY_HOSTS: bool = true;
37+
38+
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
39+
run.path("src/gcc").alias("gcc")
40+
}
41+
42+
fn make_run(run: RunConfig<'_>) {
43+
run.builder.ensure(Gcc { target: run.target });
44+
}
45+
46+
/// Compile GCC (specifically `libgccjit`) for `target`.
47+
fn run(self, builder: &Builder<'_>) -> Self::Output {
48+
let target = self.target;
49+
50+
// If GCC has already been built, we avoid building it again.
51+
let metadata = match get_gcc_build_status(builder, target) {
52+
GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
53+
GccBuildStatus::ShouldBuild(m) => m,
54+
};
55+
56+
let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
57+
t!(metadata.stamp.remove());
58+
let _time = helpers::timeit(builder);
59+
60+
let libgccjit_path = libgccjit_built_path(&metadata.install_dir);
61+
if builder.config.dry_run() {
62+
return GccOutput { libgccjit: libgccjit_path };
63+
}
64+
65+
build_gcc(&metadata, builder, target);
66+
create_lib_alias(builder, &libgccjit_path);
67+
68+
t!(metadata.stamp.write());
69+
70+
GccOutput { libgccjit: libgccjit_path }
71+
}
72+
}
73+
74+
/// Creates a libgccjit.so.0 alias next to libgccjit.so if it does not
75+
/// already exist
76+
fn create_lib_alias(builder: &Builder<'_>, libgccjit: &PathBuf) {
77+
let lib_alias = libgccjit.parent().unwrap().join("libgccjit.so.0");
78+
if !lib_alias.exists() {
79+
t!(builder.symlink_file(libgccjit, lib_alias));
80+
}
81+
}
82+
2483
pub struct Meta {
2584
stamp: BuildStamp,
2685
out_dir: PathBuf,
@@ -33,7 +92,54 @@ pub enum GccBuildStatus {
3392
ShouldBuild(Meta),
3493
}
3594

36-
/// This returns whether we've already previously built GCC.
95+
/// Tries to download GCC from CI if it is enabled and GCC artifacts
96+
/// are available for the given target.
97+
/// Returns a path to the libgccjit.so file.
98+
#[cfg(not(test))]
99+
fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<PathBuf> {
100+
use build_helper::git::PathFreshness;
101+
102+
// Try to download GCC from CI if configured and available
103+
if !matches!(builder.config.gcc_ci_mode, crate::core::config::GccCiMode::DownloadFromCi) {
104+
return None;
105+
}
106+
if target != "x86_64-unknown-linux-gnu" {
107+
eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target");
108+
return None;
109+
}
110+
let source = detect_gcc_freshness(
111+
&builder.config,
112+
builder.config.rust_info.is_managed_git_subrepository(),
113+
);
114+
match source {
115+
PathFreshness::LastModifiedUpstream { upstream } => {
116+
// Download from upstream CI
117+
let root = ci_gcc_root(&builder.config);
118+
let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&upstream);
119+
if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
120+
builder.config.download_ci_gcc(&upstream, &root);
121+
t!(gcc_stamp.write());
122+
}
123+
124+
let libgccjit = root.join("lib").join("libgccjit.so");
125+
create_lib_alias(builder, &libgccjit);
126+
Some(libgccjit)
127+
}
128+
PathFreshness::HasLocalModifications { .. } => {
129+
// We have local modifications, rebuild GCC.
130+
eprintln!("Found local GCC modifications, GCC will *not* be downloaded");
131+
None
132+
}
133+
}
134+
}
135+
136+
#[cfg(test)]
137+
fn try_download_gcc(_builder: &Builder<'_>, _target: TargetSelection) -> Option<PathBuf> {
138+
None
139+
}
140+
141+
/// This returns information about whether GCC should be built or if it's already built.
142+
/// It transparently handles downloading GCC from CI if needed.
37143
///
38144
/// It's used to avoid busting caches during x.py check -- if we've already built
39145
/// GCC, it's fine for us to not try to avoid doing so.
@@ -178,6 +284,34 @@ impl Step for Gcc {
178284

179285
t!(stamp.write());
180286

181-
true
182-
}
287+
/// Detect whether GCC sources have been modified locally or not.
288+
#[cfg(not(test))]
289+
fn detect_gcc_freshness(config: &crate::Config, is_git: bool) -> build_helper::git::PathFreshness {
290+
use build_helper::git::{PathFreshness, check_path_modifications};
291+
292+
let freshness = if is_git {
293+
Some(
294+
check_path_modifications(
295+
Some(&config.src),
296+
&config.git_config(),
297+
&["src/gcc", "src/bootstrap/download-ci-gcc-stamp"],
298+
CiEnv::current(),
299+
)
300+
.unwrap(),
301+
)
302+
} else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
303+
Some(PathFreshness::LastModifiedUpstream { upstream: info.sha.trim().to_owned() })
304+
} else {
305+
None
306+
};
307+
308+
let Some(freshness) = freshness else {
309+
eprintln!("error: could not find commit hash for downloading GCC");
310+
eprintln!("HELP: maybe your repository history is too shallow?");
311+
eprintln!("HELP: consider disabling `download-ci-gcc`");
312+
eprintln!("HELP: or fetch enough history to include one upstream commit");
313+
panic!();
314+
};
315+
316+
freshness
183317
}

src/bootstrap/src/core/build_steps/llvm.rs

+25-25
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@ use std::sync::OnceLock;
1515
use std::{env, fs};
1616

1717
use build_helper::ci::CiEnv;
18-
use build_helper::git::get_closest_merge_commit;
18+
use build_helper::git::{PathFreshness, check_path_modifications};
19+
#[cfg(feature = "tracing")]
20+
use tracing::instrument;
1921

2022
use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
2123
use crate::core::config::{Config, TargetSelection};
@@ -172,35 +174,38 @@ pub fn prebuilt_llvm_config(
172174
LlvmBuildStatus::ShouldBuild(Meta { stamp, res, out_dir, root: root.into() })
173175
}
174176

175-
/// This retrieves the LLVM sha we *want* to use, according to git history.
176-
pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String {
177-
let llvm_sha = if is_git {
178-
get_closest_merge_commit(
179-
Some(&config.src),
180-
&config.git_config(),
181-
&[
182-
config.src.join("src/llvm-project"),
183-
config.src.join("src/bootstrap/download-ci-llvm-stamp"),
184-
// the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
185-
config.src.join("src/version"),
186-
],
177+
/// Detect whether LLVM sources have been modified locally or not.
178+
pub(crate) fn detect_llvm_freshness(config: &Config, is_git: bool) -> PathFreshness {
179+
let freshness = if is_git {
180+
Some(
181+
check_path_modifications(
182+
Some(&config.src),
183+
&config.git_config(),
184+
&[
185+
"src/llvm-project",
186+
"src/bootstrap/download-ci-llvm-stamp",
187+
// the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
188+
"src/version",
189+
],
190+
CiEnv::current(),
191+
)
192+
.unwrap(),
187193
)
188-
.unwrap()
189194
} else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
190-
info.sha.trim().to_owned()
195+
Some(PathFreshness::LastModifiedUpstream { upstream: info.sha.trim().to_owned() })
191196
} else {
192-
"".to_owned()
197+
None
193198
};
194199

195-
if llvm_sha.is_empty() {
200+
let Some(freshness) = freshness else {
196201
eprintln!("error: could not find commit hash for downloading LLVM");
197202
eprintln!("HELP: maybe your repository history is too shallow?");
198203
eprintln!("HELP: consider disabling `download-ci-llvm`");
199204
eprintln!("HELP: or fetch enough history to include one upstream commit");
200205
panic!();
201-
}
206+
};
202207

203-
llvm_sha
208+
freshness
204209
}
205210

206211
/// Returns whether the CI-found LLVM is currently usable.
@@ -280,12 +285,7 @@ pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool {
280285
return false;
281286
}
282287

283-
let llvm_sha = detect_llvm_sha(config, true);
284-
let head_sha = crate::output(
285-
helpers::git(Some(&config.src)).arg("rev-parse").arg("HEAD").as_command_mut(),
286-
);
287-
let head_sha = head_sha.trim();
288-
llvm_sha == head_sha
288+
matches!(detect_llvm_freshness(config, true), PathFreshness::HasLocalModifications { .. })
289289
}
290290

291291
#[derive(Debug, Clone, Hash, PartialEq, Eq)]

src/bootstrap/src/core/build_steps/tool.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -604,8 +604,7 @@ impl Step for Rustdoc {
604604
let files_to_track = &["src/librustdoc", "src/tools/rustdoc"];
605605

606606
// Check if unchanged
607-
if builder.config.last_modified_commit(files_to_track, "download-rustc", true).is_some()
608-
{
607+
if !builder.config.has_changes_from_upstream(files_to_track) {
609608
let precompiled_rustdoc = builder
610609
.config
611610
.ci_rustc_dir()

0 commit comments

Comments
 (0)