From dcc3d0fac937e80309e88473b8693a1b07b4cc1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 12:55:29 +0100 Subject: [PATCH 01/13] Implement CI tooling in Rust --- src/ci/citool/Cargo.lock | 345 ++++++++++++++++++++++++++++++++++++++ src/ci/citool/Cargo.toml | 13 ++ src/ci/citool/README.md | 2 + src/ci/citool/src/main.rs | 293 ++++++++++++++++++++++++++++++++ 4 files changed, 653 insertions(+) create mode 100644 src/ci/citool/Cargo.lock create mode 100644 src/ci/citool/Cargo.toml create mode 100644 src/ci/citool/README.md create mode 100644 src/ci/citool/src/main.rs diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock new file mode 100644 index 0000000000000..d7931af464e11 --- /dev/null +++ b/src/ci/citool/Cargo.lock @@ -0,0 +1,345 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + +[[package]] +name = "citool" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", + "serde_yaml", +] + +[[package]] +name = "clap" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "once_cell" +version = "1.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.138" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml new file mode 100644 index 0000000000000..40c6e5b4dc225 --- /dev/null +++ b/src/ci/citool/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "citool" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +clap = { version = "4.5", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1" + +[workspace] diff --git a/src/ci/citool/README.md b/src/ci/citool/README.md new file mode 100644 index 0000000000000..685a255f15251 --- /dev/null +++ b/src/ci/citool/README.md @@ -0,0 +1,2 @@ +# CI tooling +This is a simple Rust script that determines which jobs should be executed on CI based on the situation (pull request, try job, merge attempt). It also provides a simple way of executing (some) CI jobs locally. diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs new file mode 100644 index 0000000000000..5f0202e9c34b8 --- /dev/null +++ b/src/ci/citool/src/main.rs @@ -0,0 +1,293 @@ +use std::collections::HashMap; +use std::path::Path; + +use anyhow::Context; +use clap::Parser; +use serde_yaml::Value; + +const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); +const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml"); + +/// Representation of a job loaded from the jobs.yml file. +#[derive(serde::Deserialize, Debug, Clone)] +struct Job { + /// Name of the job, e.g. mingw-check + name: String, + /// GitHub runner on which the job should be executed + os: String, + env: HashMap, + /// Should the job be only executed on a specific channel? + #[serde(default)] + only_on_channel: Option, + /// Rest of attributes that will be passed through to GitHub actions + #[serde(flatten)] + extra_keys: HashMap, +} + +#[derive(serde::Deserialize, Debug)] +struct JobEnvironments { + #[serde(rename = "pr")] + pr_env: HashMap, + #[serde(rename = "try")] + try_env: HashMap, + #[serde(rename = "auto")] + auto_env: HashMap, +} + +#[derive(serde::Deserialize, Debug)] +struct JobDatabase { + #[serde(rename = "pr")] + pr_jobs: Vec, + #[serde(rename = "try")] + try_jobs: Vec, + #[serde(rename = "auto")] + auto_jobs: Vec, + + /// Shared environments for the individual run types. + envs: JobEnvironments, +} + +impl JobDatabase { + fn find_auto_job_by_name(&self, name: &str) -> Option { + self.auto_jobs.iter().find(|j| j.name == name).cloned() + } +} + +fn load_job_db(path: &Path) -> anyhow::Result { + let db = std::fs::read_to_string(path)?; + let mut db: Value = serde_yaml::from_str(&db)?; + + // We need to expand merge keys (<<), because serde_yaml can't deal with them + // `apply_merge` only applies the merge once, so do it a few times to unwrap nested merges. + db.apply_merge()?; + db.apply_merge()?; + + let db: JobDatabase = serde_yaml::from_value(db)?; + Ok(db) +} + +/// Representation of a job outputted to a GitHub Actions workflow. +#[derive(serde::Serialize, Debug)] +struct GithubActionsJob { + name: String, + full_name: String, + os: String, + env: HashMap, + #[serde(flatten)] + extra_keys: HashMap, +} + +/// Type of workflow that is being executed on CI +#[derive(Debug)] +enum RunType { + /// Workflows that run after a push to a PR branch + PullRequest, + /// Try run started with @bors try + TryJob { custom_jobs: Option> }, + /// Merge attempt workflow + AutoJob, +} + +struct GitHubContext { + event_name: String, + branch_ref: String, + commit_message: Option, +} + +impl GitHubContext { + fn get_run_type(&self) -> Option { + if self.event_name == "pull_request" { + return Some(RunType::PullRequest); + } else if self.event_name == "push" { + let is_try_build = + ["refs/heads/try", "refs/heads/try-perf", "refs/heads/automation/bors/try"] + .iter() + .any(|r| **r == self.branch_ref); + // Unrolled branch from a rollup for testing perf + // This should **not** allow custom try jobs + let is_unrolled_perf_build = self.branch_ref == "refs/heads/try-perf"; + if is_try_build { + let custom_jobs = + if !is_unrolled_perf_build { Some(self.get_custom_jobs()) } else { None }; + return Some(RunType::TryJob { custom_jobs }); + } + + if self.branch_ref == "refs/heads/auto" { + return Some(RunType::AutoJob); + } + } + None + } + + /// Tries to parse names of specific CI jobs that should be executed in the form of + /// try-job: + /// from the commit message of the passed GitHub context. + fn get_custom_jobs(&self) -> Vec { + if let Some(ref msg) = self.commit_message { + msg.lines() + .filter_map(|line| line.trim().strip_prefix("try-job: ")) + .map(|l| l.to_string()) + .collect() + } else { + vec![] + } + } +} + +fn load_env_var(name: &str) -> anyhow::Result { + std::env::var(name).with_context(|| format!("Cannot find variable {name}")) +} + +fn load_github_ctx() -> anyhow::Result { + let event_name = load_env_var("GITHUB_EVENT_NAME")?; + let commit_message = + if event_name == "push" { Some(load_env_var("COMMIT_MESSAGE")?) } else { None }; + + Ok(GitHubContext { event_name, branch_ref: load_env_var("GITHUB_REF")?, commit_message }) +} + +/// Skip CI jobs that are not supposed to be executed on the given `channel`. +fn skip_jobs(jobs: Vec, channel: &str) -> Vec { + jobs.into_iter() + .filter(|job| { + job.only_on_channel.is_none() || job.only_on_channel.as_deref() == Some(channel) + }) + .collect() +} + +fn to_string_map(map: &HashMap) -> HashMap { + map.iter() + .map(|(key, value)| { + ( + key.clone(), + serde_yaml::to_string(value) + .expect("Cannot serialize YAML value to string") + .trim() + .to_string(), + ) + }) + .collect() +} + +fn calculate_jobs( + run_type: &RunType, + db: &JobDatabase, + channel: &str, +) -> anyhow::Result> { + let (jobs, prefix, base_env) = match run_type { + RunType::PullRequest => (db.pr_jobs.clone(), "PR", &db.envs.pr_env), + RunType::TryJob { custom_jobs } => { + let jobs = if let Some(custom_jobs) = custom_jobs { + if custom_jobs.len() > 10 { + return Err(anyhow::anyhow!( + "It is only possible to schedule up to 10 custom jobs, received {} custom jobs", + custom_jobs.len() + )); + } + + let mut jobs = vec![]; + let mut unknown_jobs = vec![]; + for custom_job in custom_jobs { + if let Some(job) = db.find_auto_job_by_name(custom_job) { + jobs.push(job); + } else { + unknown_jobs.push(custom_job.clone()); + } + } + if !unknown_jobs.is_empty() { + return Err(anyhow::anyhow!( + "Custom job(s) `{}` not found in auto jobs", + unknown_jobs.join(", ") + )); + } + jobs + } else { + db.try_jobs.clone() + }; + (jobs, "try", &db.envs.try_env) + } + RunType::AutoJob => (db.auto_jobs.clone(), "auto", &db.envs.auto_env), + }; + let jobs = skip_jobs(jobs, channel); + let jobs = jobs + .into_iter() + .map(|job| { + let mut env: HashMap = to_string_map(base_env); + env.extend(to_string_map(&job.env)); + let full_name = format!("{prefix} - {}", job.name); + + GithubActionsJob { + name: job.name, + full_name, + os: job.os, + env, + extra_keys: job + .extra_keys + .into_iter() + .map(|(key, value)| { + ( + key, + serde_json::to_value(&value) + .expect("Cannot convert extra key value from YAML to JSON"), + ) + }) + .collect(), + } + }) + .collect(); + + Ok(jobs) +} + +fn calculate_job_matrix( + db: JobDatabase, + gh_ctx: GitHubContext, + channel: &str, +) -> anyhow::Result<()> { + let run_type = gh_ctx.get_run_type().ok_or_else(|| { + anyhow::anyhow!("Cannot determine the type of workflow that is being executed") + })?; + eprintln!("Run type: {run_type:?}"); + + let jobs = calculate_jobs(&run_type, &db, channel)?; + if jobs.is_empty() { + return Err(anyhow::anyhow!("Computed job list is empty")); + } + + let run_type = match run_type { + RunType::PullRequest => "pr", + RunType::TryJob { .. } => "try", + RunType::AutoJob => "auto", + }; + + eprintln!("Output:\njobs={jobs:?}\nrun_type={run_type}"); + println!("jobs={}", serde_json::to_string(&jobs)?); + println!("run_type={run_type}"); + + Ok(()) +} + +#[derive(clap::Parser)] +enum Args { + /// Calculate a list of jobs that should be executed on CI. + /// Should only be used on CI inside GitHub actions. + CalculateJobMatrix, +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let db = load_job_db(Path::new(JOBS_YML_PATH)).context("Cannot load jobs.yml")?; + + match args { + Args::CalculateJobMatrix => { + let gh_ctx = load_github_ctx() + .context("Cannot load environment variables from GitHub Actions")?; + let channel = std::fs::read_to_string(Path::new(CI_DIRECTORY).join("channel")) + .context("Cannot read channel file")?; + + calculate_job_matrix(db, gh_ctx, &channel).context("Failed to calculate job matrix")?; + } + } + + Ok(()) +} From 4f0141f6652058f03d76f3ff6195bc4bce13c2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 12:55:34 +0100 Subject: [PATCH 02/13] Add missing base-job directive --- src/ci/github-actions/jobs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 64e64867de26f..bbcc01a0c2998 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -51,9 +51,11 @@ runners: # Free some disk space to avoid running out of space during the build. free_disk: true os: ubuntu-24.04-arm + <<: *base-job - &job-aarch64-linux-8c os: ubuntu-22.04-arm64-8core-32gb + <<: *base-job envs: env-x86_64-apple-tests: &env-x86_64-apple-tests SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact From a789f9e283e26e07a10680dad0d4533b5a33ded9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 13:13:33 +0100 Subject: [PATCH 03/13] Add local job execution to `citool` --- src/ci/citool/src/main.rs | 82 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index 5f0202e9c34b8..d01e32bb575e9 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -1,11 +1,13 @@ use std::collections::HashMap; use std::path::Path; +use std::process::Command; use anyhow::Context; use clap::Parser; use serde_yaml::Value; const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); +const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker"); const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml"); /// Representation of a job loaded from the jobs.yml file. @@ -24,6 +26,21 @@ struct Job { extra_keys: HashMap, } +impl Job { + fn is_linux(&self) -> bool { + self.os.contains("ubuntu") + } + + /// By default, the Docker image of a job is based on its name. + /// However, it can be overridden by its IMAGE environment variable. + fn image(&self) -> String { + self.env + .get("IMAGE") + .map(|v| v.as_str().expect("IMAGE value should be a string").to_string()) + .unwrap_or_else(|| self.name.clone()) + } +} + #[derive(serde::Deserialize, Debug)] struct JobEnvironments { #[serde(rename = "pr")] @@ -267,11 +284,75 @@ fn calculate_job_matrix( Ok(()) } +fn find_linux_job<'a>(jobs: &'a [Job], name: &str) -> anyhow::Result<&'a Job> { + let Some(job) = jobs.iter().find(|j| j.name == name) else { + let available_jobs: Vec<&Job> = jobs.iter().filter(|j| j.is_linux()).collect(); + let mut available_jobs = + available_jobs.iter().map(|j| j.name.to_string()).collect::>(); + available_jobs.sort(); + return Err(anyhow::anyhow!( + "Job {name} not found. The following jobs are available:\n{}", + available_jobs.join(", ") + )); + }; + if !job.is_linux() { + return Err(anyhow::anyhow!("Only Linux jobs can be executed locally")); + } + + Ok(job) +} + +fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> anyhow::Result<()> { + let jobs = match job_type { + JobType::Auto => &db.auto_jobs, + JobType::PR => &db.pr_jobs, + }; + let job = find_linux_job(&jobs, &name).with_context(|| format!("Cannot find job {name}"))?; + + let mut custom_env: HashMap = HashMap::new(); + // Replicate src/ci/scripts/setup-environment.sh + // Adds custom environment variables to the job + if name.starts_with("dist-") { + if name.ends_with("-alt") { + custom_env.insert("DEPLOY_ALT".to_string(), "1".to_string()); + } else { + custom_env.insert("DEPLOY".to_string(), "1".to_string()); + } + } + custom_env.extend(to_string_map(&job.env)); + + let mut cmd = Command::new(Path::new(DOCKER_DIRECTORY).join("run.sh")); + cmd.arg(job.image()); + cmd.envs(custom_env); + + eprintln!("Executing {cmd:?}"); + + let result = cmd.spawn()?.wait()?; + if !result.success() { Err(anyhow::anyhow!("Job failed")) } else { Ok(()) } +} + #[derive(clap::Parser)] enum Args { /// Calculate a list of jobs that should be executed on CI. /// Should only be used on CI inside GitHub actions. CalculateJobMatrix, + /// Execute a given CI job locally. + #[clap(name = "run-local")] + RunJobLocally { + /// Name of the job that should be executed. + name: String, + /// Type of the job that should be executed. + #[clap(long = "type", default_value = "auto")] + job_type: JobType, + }, +} + +#[derive(clap::ValueEnum, Clone)] +enum JobType { + /// Merge attempt ("auto") job + Auto, + /// Pull request job + PR, } fn main() -> anyhow::Result<()> { @@ -287,6 +368,7 @@ fn main() -> anyhow::Result<()> { calculate_job_matrix(db, gh_ctx, &channel).context("Failed to calculate job matrix")?; } + Args::RunJobLocally { job_type, name } => run_workflow_locally(db, job_type, name)?, } Ok(()) From 87c49f025f55c7efac5db30501f7a5255eba895b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 13:14:42 +0100 Subject: [PATCH 04/13] Use citool in CI --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0934e42b277e4..b936028fa2e7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ # This file defines our primary CI workflow that runs on pull requests # and also on pushes to special branches (auto, try). # -# The actual definition of the executed jobs is calculated by a Python -# script located at src/ci/github-actions/ci.py, which +# The actual definition of the executed jobs is calculated by a +# script located at src/ci/citool, which # uses job definition data from src/ci/github-actions/jobs.yml. # You should primarily modify the `jobs.yml` file if you want to modify # what jobs are executed in CI. @@ -56,7 +56,9 @@ jobs: - name: Calculate the CI job matrix env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: python3 src/ci/github-actions/ci.py calculate-job-matrix >> $GITHUB_OUTPUT + run: | + cd src/ci/citool + cargo run calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: name: ${{ matrix.full_name }} From 1a0970d4ce5c10a830cb12581edc73a08cd88b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 13:14:49 +0100 Subject: [PATCH 05/13] Delete the previous Python script --- src/ci/github-actions/ci.py | 318 ------------------------------------ 1 file changed, 318 deletions(-) delete mode 100755 src/ci/github-actions/ci.py diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py deleted file mode 100755 index c93766ef33a0d..0000000000000 --- a/src/ci/github-actions/ci.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script contains CI functionality. -It can be used to generate a matrix of jobs that should -be executed on CI, or run a specific CI job locally. - -It reads job definitions from `src/ci/github-actions/jobs.yml`. -""" - -import argparse -import dataclasses -import json -import logging -import os -import re -import subprocess -import typing -from pathlib import Path -from typing import List, Dict, Any, Optional - -import yaml - -CI_DIR = Path(__file__).absolute().parent.parent -JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" - -Job = Dict[str, Any] - - -def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: - """ - Modify the `name` attribute of each job, based on its base name and the given `prefix`. - Add an `image` attribute to each job, based on its image. - """ - modified_jobs = [] - for job in jobs: - # Create a copy of the `job` dictionary to avoid modifying `jobs` - new_job = dict(job) - new_job["image"] = get_job_image(new_job) - new_job["full_name"] = f"{prefix} - {new_job['name']}" - modified_jobs.append(new_job) - return modified_jobs - - -def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: - """ - Prepends `environment` to the `env` attribute of each job. - The `env` of each job has higher precedence than `environment`. - """ - modified_jobs = [] - for job in jobs: - env = environment.copy() - env.update(job.get("env", {})) - - new_job = dict(job) - new_job["env"] = env - modified_jobs.append(new_job) - return modified_jobs - - -@dataclasses.dataclass -class PRRunType: - pass - - -@dataclasses.dataclass -class TryRunType: - custom_jobs: List[str] - - -@dataclasses.dataclass -class AutoRunType: - pass - - -WorkflowRunType = typing.Union[PRRunType, TryRunType, AutoRunType] - - -@dataclasses.dataclass -class GitHubCtx: - event_name: str - ref: str - repository: str - commit_message: Optional[str] - - -def get_custom_jobs(ctx: GitHubCtx) -> List[str]: - """ - Tries to parse names of specific CI jobs that should be executed in the form of - try-job: - from the commit message of the passed GitHub context. - """ - if ctx.commit_message is None: - return [] - - regex = re.compile(r"^try-job: (.*)", re.MULTILINE) - jobs = [] - for match in regex.finditer(ctx.commit_message): - jobs.append(match.group(1)) - return jobs - - -def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: - if ctx.event_name == "pull_request": - return PRRunType() - elif ctx.event_name == "push": - try_build = ctx.ref in ( - "refs/heads/try", - "refs/heads/try-perf", - "refs/heads/automation/bors/try", - ) - - # Unrolled branch from a rollup for testing perf - # This should **not** allow custom try jobs - is_unrolled_perf_build = ctx.ref == "refs/heads/try-perf" - - if try_build: - custom_jobs = [] - if not is_unrolled_perf_build: - custom_jobs = get_custom_jobs(ctx) - return TryRunType(custom_jobs=custom_jobs) - - if ctx.ref == "refs/heads/auto": - return AutoRunType() - - return None - - -def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: - if isinstance(run_type, PRRunType): - return add_base_env( - add_job_properties(job_data["pr"], "PR"), job_data["envs"]["pr"] - ) - elif isinstance(run_type, TryRunType): - jobs = job_data["try"] - custom_jobs = run_type.custom_jobs - if custom_jobs: - if len(custom_jobs) > 10: - raise Exception( - f"It is only possible to schedule up to 10 custom jobs, " - f"received {len(custom_jobs)} jobs" - ) - - jobs = [] - unknown_jobs = [] - for custom_job in custom_jobs: - job = [j for j in job_data["auto"] if j["name"] == custom_job] - if not job: - unknown_jobs.append(custom_job) - continue - jobs.append(job[0]) - if unknown_jobs: - raise Exception( - f"Custom job(s) `{unknown_jobs}` not found in auto jobs" - ) - - return add_base_env(add_job_properties(jobs, "try"), job_data["envs"]["try"]) - elif isinstance(run_type, AutoRunType): - return add_base_env( - add_job_properties(job_data["auto"], "auto"), job_data["envs"]["auto"] - ) - - return [] - - -def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: - """ - Skip CI jobs that are not supposed to be executed on the given `channel`. - """ - return [j for j in jobs if j.get("only_on_channel", channel) == channel] - - -def get_github_ctx() -> GitHubCtx: - event_name = os.environ["GITHUB_EVENT_NAME"] - - commit_message = None - if event_name == "push": - commit_message = os.environ["COMMIT_MESSAGE"] - return GitHubCtx( - event_name=event_name, - ref=os.environ["GITHUB_REF"], - repository=os.environ["GITHUB_REPOSITORY"], - commit_message=commit_message, - ) - - -def format_run_type(run_type: WorkflowRunType) -> str: - if isinstance(run_type, PRRunType): - return "pr" - elif isinstance(run_type, AutoRunType): - return "auto" - elif isinstance(run_type, TryRunType): - return "try" - else: - raise AssertionError() - - -def get_job_image(job: Job) -> str: - """ - By default, the Docker image of a job is based on its name. - However, it can be overridden by its IMAGE environment variable. - """ - env = job.get("env", {}) - # Return the IMAGE environment variable if it exists, otherwise return the job name - return env.get("IMAGE", job["name"]) - - -def is_linux_job(job: Job) -> bool: - return "ubuntu" in job["os"] - - -def find_linux_job(job_data: Dict[str, Any], job_name: str, pr_jobs: bool) -> Job: - candidates = job_data["pr"] if pr_jobs else job_data["auto"] - jobs = [job for job in candidates if job.get("name") == job_name] - if len(jobs) == 0: - available_jobs = "\n".join( - sorted(job["name"] for job in candidates if is_linux_job(job)) - ) - raise Exception(f"""Job `{job_name}` not found in {'pr' if pr_jobs else 'auto'} jobs. -The following jobs are available: -{available_jobs}""") - assert len(jobs) == 1 - - job = jobs[0] - if not is_linux_job(job): - raise Exception("Only Linux jobs can be executed locally") - return job - - -def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool): - DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" - - job = find_linux_job(job_data, job_name=job_name, pr_jobs=pr_jobs) - - custom_env = {} - # Replicate src/ci/scripts/setup-environment.sh - # Adds custom environment variables to the job - if job_name.startswith("dist-"): - if job_name.endswith("-alt"): - custom_env["DEPLOY_ALT"] = "1" - else: - custom_env["DEPLOY"] = "1" - custom_env.update({k: str(v) for (k, v) in job.get("env", {}).items()}) - - args = [str(DOCKER_DIR / "run.sh"), get_job_image(job)] - env_formatted = [f"{k}={v}" for (k, v) in sorted(custom_env.items())] - print(f"Executing `{' '.join(env_formatted)} {' '.join(args)}`") - - env = os.environ.copy() - env.update(custom_env) - - subprocess.run(args, env=env, check=True) - - -def calculate_job_matrix(job_data: Dict[str, Any]): - github_ctx = get_github_ctx() - - run_type = find_run_type(github_ctx) - logging.info(f"Job type: {run_type}") - - with open(CI_DIR / "channel") as f: - channel = f.read().strip() - - jobs = [] - if run_type is not None: - jobs = calculate_jobs(run_type, job_data) - jobs = skip_jobs(jobs, channel) - - if not jobs: - raise Exception("Scheduled job list is empty, this is an error") - - run_type = format_run_type(run_type) - - logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") - print(f"jobs={json.dumps(jobs)}") - print(f"run_type={run_type}") - - -def create_cli_parser(): - parser = argparse.ArgumentParser( - prog="ci.py", description="Generate or run CI workflows" - ) - subparsers = parser.add_subparsers( - help="Command to execute", dest="command", required=True - ) - subparsers.add_parser( - "calculate-job-matrix", - help="Generate a matrix of jobs that should be executed in CI", - ) - run_parser = subparsers.add_parser( - "run-local", help="Run a CI jobs locally (on Linux)" - ) - run_parser.add_argument( - "job_name", - help="CI job that should be executed. By default, a merge (auto) " - "job with the given name will be executed", - ) - run_parser.add_argument( - "--pr", action="store_true", help="Run a PR job instead of an auto job" - ) - return parser - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - with open(JOBS_YAML_PATH) as f: - data = yaml.safe_load(f) - - parser = create_cli_parser() - args = parser.parse_args() - - if args.command == "calculate-job-matrix": - calculate_job_matrix(data) - elif args.command == "run-local": - run_workflow_locally(data, args.job_name, args.pr) - else: - raise Exception(f"Unknown command {args.command}") From 7f154fa4d1035f9954605dc13d3d257662533e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 13:21:40 +0100 Subject: [PATCH 06/13] Update documentation --- .github/workflows/ci.yml | 4 ++-- src/ci/docker/README.md | 6 +++--- src/doc/rustc-dev-guide/src/building/optimized-build.md | 2 +- src/doc/rustc-dev-guide/src/tests/ci.md | 6 +++--- src/doc/rustc-dev-guide/src/tests/docker.md | 9 +++++++++ 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b936028fa2e7d..b01494630b44f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ # This file defines our primary CI workflow that runs on pull requests # and also on pushes to special branches (auto, try). # -# The actual definition of the executed jobs is calculated by a -# script located at src/ci/citool, which +# The actual definition of the executed jobs is calculated by the +# `src/ci/citool` crate, which # uses job definition data from src/ci/github-actions/jobs.yml. # You should primarily modify the `jobs.yml` file if you want to modify # what jobs are executed in CI. diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md index 2b1de43c2f68c..a1a3a1c37ced7 100644 --- a/src/ci/docker/README.md +++ b/src/ci/docker/README.md @@ -8,15 +8,15 @@ Note that a single Docker image can be used by multiple CI jobs, so the job name is the important thing that you should know. You can examine the existing CI jobs in the [`jobs.yml`](../github-actions/jobs.yml) file. -To run a specific CI job locally, you can use the following script: +To run a specific CI job locally, you can use the `citool` Rust crate: ``` -python3 ./src/ci/github-actions/ci.py run-local +cargo --manifest-path src/ci/citool/Cargo.toml run run-local ``` For example, to run the `x86_64-gnu-llvm-18-1` job: ``` -python3 ./src/ci/github-actions/ci.py run-local x86_64-gnu-llvm-18-1 +cargo --manifest-path src/ci/citool/Cargo.toml run run-local x86_64-gnu-llvm-18-1 ``` The job will output artifacts in an `obj/` dir at the root of a repository. Note diff --git a/src/doc/rustc-dev-guide/src/building/optimized-build.md b/src/doc/rustc-dev-guide/src/building/optimized-build.md index 8feda59829b45..f8ca1d0dc3919 100644 --- a/src/doc/rustc-dev-guide/src/building/optimized-build.md +++ b/src/doc/rustc-dev-guide/src/building/optimized-build.md @@ -126,4 +126,4 @@ Here is an example of how can `opt-dist` be used locally (outside of CI): [`Environment`]: https://github.com/rust-lang/rust/blob/ee451f8faccf3050c76cdcd82543c917b40c7962/src/tools/opt-dist/src/environment.rs#L5 > Note: if you want to run the actual CI pipeline, instead of running `opt-dist` locally, -> you can execute `python3 src/ci/github-actions/ci.py run-local dist-x86_64-linux`. +> you can execute `cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux`. diff --git a/src/doc/rustc-dev-guide/src/tests/ci.md b/src/doc/rustc-dev-guide/src/tests/ci.md index a4b22392f1976..ae6adb678af14 100644 --- a/src/doc/rustc-dev-guide/src/tests/ci.md +++ b/src/doc/rustc-dev-guide/src/tests/ci.md @@ -28,7 +28,7 @@ Our CI is primarily executed on [GitHub Actions], with a single workflow defined in [`.github/workflows/ci.yml`], which contains a bunch of steps that are unified for all CI jobs that we execute. When a commit is pushed to a corresponding branch or a PR, the workflow executes the -[`src/ci/github-actions/ci.py`] script, which dynamically generates the specific CI +[`src/ci/citool`] crate, which dynamically generates the specific CI jobs that should be executed. This script uses the [`jobs.yml`] file as an input, which contains a declarative configuration of all our CI jobs. @@ -299,7 +299,7 @@ platform’s custom [Docker container]. This has a lot of advantages for us: - We can avoid reinstalling tools (like QEMU or the Android emulator) every time thanks to Docker image caching. - Users can run the same tests in the same environment locally by just running - `python3 src/ci/github-actions/ci.py run-local `, which is awesome to debug failures. Note that there are only linux docker images available locally due to licensing and + `cargo run --manifest-path src/ci/citool/Cargo.toml run-local `, which is awesome to debug failures. Note that there are only linux docker images available locally due to licensing and other restrictions. The docker images prefixed with `dist-` are used for building artifacts while @@ -443,7 +443,7 @@ this: [GitHub Actions]: https://github.com/rust-lang/rust/actions [`jobs.yml`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/jobs.yml [`.github/workflows/ci.yml`]: https://github.com/rust-lang/rust/blob/master/.github/workflows/ci.yml -[`src/ci/github-actions/ci.py`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/ci.py +[`src/ci/citool`]: https://github.com/rust-lang/rust/blob/master/src/ci/citool [rust-lang-ci]: https://github.com/rust-lang-ci/rust/actions [bors]: https://github.com/bors [homu]: https://github.com/rust-lang/homu diff --git a/src/doc/rustc-dev-guide/src/tests/docker.md b/src/doc/rustc-dev-guide/src/tests/docker.md index 2ca08d42130a5..a8a388ef90cfb 100644 --- a/src/doc/rustc-dev-guide/src/tests/docker.md +++ b/src/doc/rustc-dev-guide/src/tests/docker.md @@ -53,6 +53,15 @@ Some additional notes about using the interactive mode: containers. With the container name, run `docker exec -it /bin/bash` where `` is the container name like `4ba195e95cef`. +The approach described above is a relatively low-level interface for running the Docker images +directly. If you want to run a full CI Linux job locally with Docker, in a way that is as close to CI as possible, you can use the following command: + +```bash +cargo run --manifest-path src/ci/citool/Cargo.toml run-local +# For example: +cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux-alt +``` + [Docker]: https://www.docker.com/ [`src/ci/docker`]: https://github.com/rust-lang/rust/tree/master/src/ci/docker [`src/ci/docker/run.sh`]: https://github.com/rust-lang/rust/blob/master/src/ci/docker/run.sh From 3742f0b1901462ea1a96c8713c07586ee72d925c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 14:26:03 +0100 Subject: [PATCH 07/13] Apply review comments --- src/ci/citool/src/main.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index d01e32bb575e9..9a22d1264fae5 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -10,7 +10,7 @@ const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker"); const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml"); -/// Representation of a job loaded from the jobs.yml file. +/// Representation of a job loaded from the `src/ci/github-actions/jobs.yml` file. #[derive(serde::Deserialize, Debug, Clone)] struct Job { /// Name of the job, e.g. mingw-check @@ -86,7 +86,10 @@ fn load_job_db(path: &Path) -> anyhow::Result { /// Representation of a job outputted to a GitHub Actions workflow. #[derive(serde::Serialize, Debug)] struct GithubActionsJob { + /// The main identifier of the job, used by CI scripts to determine what should be executed. name: String, + /// Helper label displayed in GitHub Actions interface, containing the job name and a run type + /// prefix (PR/try/auto). full_name: String, os: String, env: HashMap, @@ -277,7 +280,9 @@ fn calculate_job_matrix( RunType::AutoJob => "auto", }; - eprintln!("Output:\njobs={jobs:?}\nrun_type={run_type}"); + eprintln!("Output"); + eprintln!("jobs={jobs:?}"); + eprintln!("run_type={run_type}"); println!("jobs={}", serde_json::to_string(&jobs)?); println!("run_type={run_type}"); From 77bd646023118c10d901cb1d521cf00df6358afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 16:19:14 +0100 Subject: [PATCH 08/13] Add comment to [workspace] --- src/ci/citool/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index 40c6e5b4dc225..d6423d7ae4bae 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -10,4 +10,7 @@ serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +# Tell cargo that citool is its own workspace. +# If this is omitted, cargo will look for a workspace elsewhere. +# We want to avoid this, since citool is independent of the other crates. [workspace] From 8083fd4b495236535ac235c890e1b439b6b1f888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 16:34:16 +0100 Subject: [PATCH 09/13] Add tests --- src/ci/citool/Cargo.lock | 70 +++++++++++++++ src/ci/citool/Cargo.toml | 3 + src/ci/citool/src/main.rs | 81 +++++++++-------- src/ci/citool/tests/jobs.rs | 64 +++++++++++++ src/ci/citool/tests/test-jobs.yml | 145 ++++++++++++++++++++++++++++++ 5 files changed, 324 insertions(+), 39 deletions(-) create mode 100644 src/ci/citool/tests/jobs.rs create mode 100644 src/ci/citool/tests/test-jobs.yml diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock index d7931af464e11..39b6b44da643b 100644 --- a/src/ci/citool/Cargo.lock +++ b/src/ci/citool/Cargo.lock @@ -64,6 +64,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "insta", "serde", "serde_json", "serde_yaml", @@ -115,6 +116,24 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "console" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.1" @@ -143,6 +162,19 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "insta" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "pin-project", + "similar", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -155,6 +187,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "memchr" version = "2.7.4" @@ -167,6 +211,26 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "pin-project" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -236,6 +300,12 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "strsim" version = "0.11.1" diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index d6423d7ae4bae..e77c67c71477a 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -10,6 +10,9 @@ serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +[dev-dependencies] +insta = "1" + # Tell cargo that citool is its own workspace. # If this is omitted, cargo will look for a workspace elsewhere. # We want to avoid this, since citool is independent of the other crates. diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index 9a22d1264fae5..b8d7410042394 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -1,5 +1,5 @@ -use std::collections::HashMap; -use std::path::Path; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::Context; @@ -17,13 +17,13 @@ struct Job { name: String, /// GitHub runner on which the job should be executed os: String, - env: HashMap, + env: BTreeMap, /// Should the job be only executed on a specific channel? #[serde(default)] only_on_channel: Option, /// Rest of attributes that will be passed through to GitHub actions #[serde(flatten)] - extra_keys: HashMap, + extra_keys: BTreeMap, } impl Job { @@ -44,11 +44,11 @@ impl Job { #[derive(serde::Deserialize, Debug)] struct JobEnvironments { #[serde(rename = "pr")] - pr_env: HashMap, + pr_env: BTreeMap, #[serde(rename = "try")] - try_env: HashMap, + try_env: BTreeMap, #[serde(rename = "auto")] - auto_env: HashMap, + auto_env: BTreeMap, } #[derive(serde::Deserialize, Debug)] @@ -71,7 +71,7 @@ impl JobDatabase { } fn load_job_db(path: &Path) -> anyhow::Result { - let db = std::fs::read_to_string(path)?; + let db = read_to_string(path)?; let mut db: Value = serde_yaml::from_str(&db)?; // We need to expand merge keys (<<), because serde_yaml can't deal with them @@ -92,9 +92,9 @@ struct GithubActionsJob { /// prefix (PR/try/auto). full_name: String, os: String, - env: HashMap, + env: BTreeMap, #[serde(flatten)] - extra_keys: HashMap, + extra_keys: BTreeMap, } /// Type of workflow that is being executed on CI @@ -116,27 +116,17 @@ struct GitHubContext { impl GitHubContext { fn get_run_type(&self) -> Option { - if self.event_name == "pull_request" { - return Some(RunType::PullRequest); - } else if self.event_name == "push" { - let is_try_build = - ["refs/heads/try", "refs/heads/try-perf", "refs/heads/automation/bors/try"] - .iter() - .any(|r| **r == self.branch_ref); - // Unrolled branch from a rollup for testing perf - // This should **not** allow custom try jobs - let is_unrolled_perf_build = self.branch_ref == "refs/heads/try-perf"; - if is_try_build { - let custom_jobs = - if !is_unrolled_perf_build { Some(self.get_custom_jobs()) } else { None }; - return Some(RunType::TryJob { custom_jobs }); - } - - if self.branch_ref == "refs/heads/auto" { - return Some(RunType::AutoJob); + match (self.event_name.as_str(), self.branch_ref.as_str()) { + ("pull_request", _) => Some(RunType::PullRequest), + ("push", "refs/heads/try-perf") => Some(RunType::TryJob { custom_jobs: None }), + ("push", "refs/heads/try" | "refs/heads/automation/bors/try") => { + let custom_jobs = self.get_custom_jobs(); + let custom_jobs = if !custom_jobs.is_empty() { Some(custom_jobs) } else { None }; + Some(RunType::TryJob { custom_jobs }) } + ("push", "refs/heads/auto") => Some(RunType::AutoJob), + _ => None, } - None } /// Tries to parse names of specific CI jobs that should be executed in the form of @@ -175,7 +165,7 @@ fn skip_jobs(jobs: Vec, channel: &str) -> Vec { .collect() } -fn to_string_map(map: &HashMap) -> HashMap { +fn to_string_map(map: &BTreeMap) -> BTreeMap { map.iter() .map(|(key, value)| { ( @@ -232,7 +222,7 @@ fn calculate_jobs( let jobs = jobs .into_iter() .map(|job| { - let mut env: HashMap = to_string_map(base_env); + let mut env: BTreeMap = to_string_map(base_env); env.extend(to_string_map(&job.env)); let full_name = format!("{prefix} - {}", job.name); @@ -312,9 +302,9 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any JobType::Auto => &db.auto_jobs, JobType::PR => &db.pr_jobs, }; - let job = find_linux_job(&jobs, &name).with_context(|| format!("Cannot find job {name}"))?; + let job = find_linux_job(jobs, &name).with_context(|| format!("Cannot find job {name}"))?; - let mut custom_env: HashMap = HashMap::new(); + let mut custom_env: BTreeMap = BTreeMap::new(); // Replicate src/ci/scripts/setup-environment.sh // Adds custom environment variables to the job if name.starts_with("dist-") { @@ -340,7 +330,10 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any enum Args { /// Calculate a list of jobs that should be executed on CI. /// Should only be used on CI inside GitHub actions. - CalculateJobMatrix, + CalculateJobMatrix { + #[clap(long)] + jobs_file: Option, + }, /// Execute a given CI job locally. #[clap(name = "run-local")] RunJobLocally { @@ -362,19 +355,29 @@ enum JobType { fn main() -> anyhow::Result<()> { let args = Args::parse(); - let db = load_job_db(Path::new(JOBS_YML_PATH)).context("Cannot load jobs.yml")?; + let default_jobs_file = Path::new(JOBS_YML_PATH); + let load_db = |jobs_path| load_job_db(jobs_path).context("Cannot load jobs.yml"); match args { - Args::CalculateJobMatrix => { + Args::CalculateJobMatrix { jobs_file } => { + let jobs_path = jobs_file.as_deref().unwrap_or(default_jobs_file); let gh_ctx = load_github_ctx() .context("Cannot load environment variables from GitHub Actions")?; - let channel = std::fs::read_to_string(Path::new(CI_DIRECTORY).join("channel")) + let channel = read_to_string(Path::new(CI_DIRECTORY).join("channel")) .context("Cannot read channel file")?; - calculate_job_matrix(db, gh_ctx, &channel).context("Failed to calculate job matrix")?; + calculate_job_matrix(load_db(jobs_path)?, gh_ctx, &channel) + .context("Failed to calculate job matrix")?; + } + Args::RunJobLocally { job_type, name } => { + run_workflow_locally(load_db(default_jobs_file)?, job_type, name)? } - Args::RunJobLocally { job_type, name } => run_workflow_locally(db, job_type, name)?, } Ok(()) } + +fn read_to_string>(path: P) -> anyhow::Result { + let error = format!("Cannot read file {:?}", path.as_ref()); + std::fs::read_to_string(path).context(error) +} diff --git a/src/ci/citool/tests/jobs.rs b/src/ci/citool/tests/jobs.rs new file mode 100644 index 0000000000000..1d81d58f89311 --- /dev/null +++ b/src/ci/citool/tests/jobs.rs @@ -0,0 +1,64 @@ +use std::process::{Command, Stdio}; + +const TEST_JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/test-jobs.yml"); + +#[test] +fn auto_jobs() { + let stdout = get_matrix("push", "commit", "refs/heads/auto"); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"aarch64-gnu","full_name":"auto - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"x86_64-gnu-llvm-18-1","full_name":"auto - x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"0","RUST_BACKTRACE":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"aarch64-apple","full_name":"auto - aarch64-apple","os":"macos-14","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","MACOSX_DEPLOYMENT_TARGET":11.0,"MACOSX_STD_DEPLOYMENT_TARGET":11.0,"NO_DEBUG_ASSERTIONS":1,"NO_LLVM_ASSERTIONS":1,"NO_OVERFLOW_CHECKS":1,"RUSTC_RETRY_LINKER_ON_SEGFAULT":1,"RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set rust.jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","SELECT_XCODE":"/Applications/Xcode_15.4.app","TOOLSTATE_PUBLISH":1,"USE_XCODE_CLANG":1}},{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}] + run_type=auto + "#); +} + +#[test] +fn try_jobs() { + let stdout = get_matrix("push", "commit", "refs/heads/try"); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"dist-x86_64-linux","full_name":"try - dist-x86_64-linux","os":"ubuntu-22.04-16core-64gb","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_TRY_BUILD":1,"TOOLSTATE_PUBLISH":1}}] + run_type=try + "#); +} + +#[test] +fn try_custom_jobs() { + let stdout = get_matrix( + "push", + r#"This is a test PR + +try-job: aarch64-gnu +try-job: dist-i686-msvc"#, + "refs/heads/try", + ); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"aarch64-gnu","full_name":"try - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DIST_TRY_BUILD":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"dist-i686-msvc","full_name":"try - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"DIST_TRY_BUILD":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}] + run_type=try + "#); +} + +#[test] +fn pr_jobs() { + let stdout = get_matrix("pull_request", "commit", "refs/heads/pr/1234"); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"mingw-check","full_name":"PR - mingw-check","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"free_disk":true},{"name":"mingw-check-tidy","full_name":"PR - mingw-check-tidy","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"continue_on_error":true,"free_disk":true}] + run_type=pr + "#); +} + +fn get_matrix(event_name: &str, commit_msg: &str, branch_ref: &str) -> String { + let output = Command::new("cargo") + .args(["run", "-q", "calculate-job-matrix", "--jobs-file", TEST_JOBS_YML_PATH]) + .env("GITHUB_EVENT_NAME", event_name) + .env("COMMIT_MESSAGE", commit_msg) + .env("GITHUB_REF", branch_ref) + .stdout(Stdio::piped()) + .output() + .expect("Failed to execute command"); + + let stdout = String::from_utf8(output.stdout).unwrap(); + let stderr = String::from_utf8(output.stderr).unwrap(); + if !output.status.success() { + panic!("cargo run failed: {}\n{}", stdout, stderr); + } + stdout +} diff --git a/src/ci/citool/tests/test-jobs.yml b/src/ci/citool/tests/test-jobs.yml new file mode 100644 index 0000000000000..56b9ced20719e --- /dev/null +++ b/src/ci/citool/tests/test-jobs.yml @@ -0,0 +1,145 @@ +runners: + - &base-job + env: { } + + - &job-linux-4c + os: ubuntu-24.04 + # Free some disk space to avoid running out of space during the build. + free_disk: true + <<: *base-job + + - &job-linux-16c + os: ubuntu-22.04-16core-64gb + <<: *base-job + + - &job-macos-m1 + os: macos-14 + <<: *base-job + + - &job-windows + os: windows-2022 + <<: *base-job + + - &job-aarch64-linux + # Free some disk space to avoid running out of space during the build. + free_disk: true + os: ubuntu-22.04-arm + <<: *base-job +envs: + env-x86_64-apple-tests: &env-x86_64-apple-tests + SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # Ensure that host tooling is tested on our minimum supported macOS version. + MACOSX_DEPLOYMENT_TARGET: 10.12 + MACOSX_STD_DEPLOYMENT_TARGET: 10.12 + SELECT_XCODE: /Applications/Xcode_15.2.app + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + + production: + &production + DEPLOY_BUCKET: rust-lang-ci2 + # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named + # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to + # rotate them in a single branch while keeping the old key in another + # branch, which wouldn't be possible if the key was named with the kind + # (caches, artifacts...). + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 + AWS_REGION: us-west-1 + TOOLSTATE_PUBLISH: 1 + + try: + <<: *production + # The following env var activates faster `try` builds in `opt-dist` by, e.g. + # - building only the more commonly useful components (we rarely need e.g. rust-docs in try + # builds) + # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain + # + # If you *want* these to happen however, temporarily comment it before triggering a try build. + DIST_TRY_BUILD: 1 + + auto: + <<: *production + + pr: + PR_CI_JOB: 1 + +# Jobs that run on each push to a pull request (PR) +# These jobs automatically inherit envs.pr, to avoid repeating +# it in each job definition. +pr: + - name: mingw-check + <<: *job-linux-4c + - name: mingw-check-tidy + continue_on_error: true + <<: *job-linux-4c + +# Jobs that run when you perform a try build (@bors try) +# These jobs automatically inherit envs.try, to avoid repeating +# it in each job definition. +try: + - name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-linux-16c + +# Main CI jobs that have to be green to merge a commit into master +# These jobs automatically inherit envs.auto, to avoid repeating +# it in each job definition. +auto: + - name: aarch64-gnu + <<: *job-aarch64-linux + + # The x86_64-gnu-llvm-18 job is split into multiple jobs to run tests in parallel. + # x86_64-gnu-llvm-18-1 skips tests that run in x86_64-gnu-llvm-18-{2,3}. + - name: x86_64-gnu-llvm-18-1 + env: + RUST_BACKTRACE: 1 + READ_ONLY_SRC: "0" + IMAGE: x86_64-gnu-llvm-18 + DOCKER_SCRIPT: stage_2_test_set1.sh + <<: *job-linux-4c + + + #################### + # macOS Builders # + #################### + + - name: aarch64-apple + env: + SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin + RUST_CONFIGURE_ARGS: >- + --enable-sanitizers + --enable-profiler + --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + SELECT_XCODE: /Applications/Xcode_15.4.app + USE_XCODE_CLANG: 1 + # Aarch64 tooling only needs to support macOS 11.0 and up as nothing else + # supports the hardware, so only need to test it there. + MACOSX_DEPLOYMENT_TARGET: 11.0 + MACOSX_STD_DEPLOYMENT_TARGET: 11.0 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-m1 + + ###################### + # Windows Builders # + ###################### + + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=i686-pc-windows-msvc + --host=i686-pc-windows-msvc + --target=i686-pc-windows-msvc,i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist bootstrap --include-default-paths + DIST_REQUIRE_ALL_TOOLS: 1 + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-windows From ae6d93ce13406fec8d6649af46f1ae6723172292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Feb 2025 18:08:27 +0100 Subject: [PATCH 10/13] Test citool on CI --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b01494630b44f..fdeb1aa07efdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | cd src/ci/citool + cargo test cargo run calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: From 8ccaf48fcd6e859244aeb476327bdd710da5735b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 17 Feb 2025 12:21:17 +0100 Subject: [PATCH 11/13] Avoid double serialization of environment strings --- src/ci/citool/src/main.rs | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index b8d7410042394..e3eba6b9afc50 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -92,7 +92,7 @@ struct GithubActionsJob { /// prefix (PR/try/auto). full_name: String, os: String, - env: BTreeMap, + env: BTreeMap, #[serde(flatten)] extra_keys: BTreeMap, } @@ -165,15 +165,12 @@ fn skip_jobs(jobs: Vec, channel: &str) -> Vec { .collect() } -fn to_string_map(map: &BTreeMap) -> BTreeMap { - map.iter() +fn yaml_map_to_json(map: &BTreeMap) -> BTreeMap { + map.into_iter() .map(|(key, value)| { ( key.clone(), - serde_yaml::to_string(value) - .expect("Cannot serialize YAML value to string") - .trim() - .to_string(), + serde_json::to_value(&value).expect("Cannot convert map value from YAML to JSON"), ) }) .collect() @@ -222,8 +219,8 @@ fn calculate_jobs( let jobs = jobs .into_iter() .map(|job| { - let mut env: BTreeMap = to_string_map(base_env); - env.extend(to_string_map(&job.env)); + let mut env: BTreeMap = yaml_map_to_json(base_env); + env.extend(yaml_map_to_json(&job.env)); let full_name = format!("{prefix} - {}", job.name); GithubActionsJob { @@ -231,17 +228,7 @@ fn calculate_jobs( full_name, os: job.os, env, - extra_keys: job - .extra_keys - .into_iter() - .map(|(key, value)| { - ( - key, - serde_json::to_value(&value) - .expect("Cannot convert extra key value from YAML to JSON"), - ) - }) - .collect(), + extra_keys: yaml_map_to_json(&job.extra_keys), } }) .collect(); @@ -314,7 +301,15 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any custom_env.insert("DEPLOY".to_string(), "1".to_string()); } } - custom_env.extend(to_string_map(&job.env)); + custom_env.extend(job.env.iter().map(|(key, value)| { + let value = match value { + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + Value::String(value) => value.clone(), + _ => panic!("Unexpected type for environment variable {key} Only bool/number/string is supported.") + }; + (key.clone(), value) + })); let mut cmd = Command::new(Path::new(DOCKER_DIRECTORY).join("run.sh")); cmd.arg(job.image()); From f35c88043b6177710f3c762c886810f28dadbe76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 17 Feb 2025 12:21:24 +0100 Subject: [PATCH 12/13] Trim try-job names --- src/ci/citool/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index e3eba6b9afc50..f0d075d8562b2 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -136,7 +136,7 @@ impl GitHubContext { if let Some(ref msg) = self.commit_message { msg.lines() .filter_map(|line| line.trim().strip_prefix("try-job: ")) - .map(|l| l.to_string()) + .map(|l| l.trim().to_string()) .collect() } else { vec![] From 31acbd3f902da4eba6fe7eaf07cb862ebac9d67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 17 Feb 2025 12:21:41 +0100 Subject: [PATCH 13/13] Trim channel To avoid including a newline at the end, which broke `only_on_channel` comparison. --- src/ci/citool/src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index f0d075d8562b2..ad9cc8b82a6f9 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -359,7 +359,9 @@ fn main() -> anyhow::Result<()> { let gh_ctx = load_github_ctx() .context("Cannot load environment variables from GitHub Actions")?; let channel = read_to_string(Path::new(CI_DIRECTORY).join("channel")) - .context("Cannot read channel file")?; + .context("Cannot read channel file")? + .trim() + .to_string(); calculate_job_matrix(load_db(jobs_path)?, gh_ctx, &channel) .context("Failed to calculate job matrix")?;