diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0934e42b277e4..fdeb1aa07efdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ # This file defines our primary CI workflow that runs on pull requests # and also on pushes to special branches (auto, try). # -# The actual definition of the executed jobs is calculated by a Python -# script located at src/ci/github-actions/ci.py, which +# The actual definition of the executed jobs is calculated by the +# `src/ci/citool` crate, which # uses job definition data from src/ci/github-actions/jobs.yml. # You should primarily modify the `jobs.yml` file if you want to modify # what jobs are executed in CI. @@ -56,7 +56,10 @@ jobs: - name: Calculate the CI job matrix env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: python3 src/ci/github-actions/ci.py calculate-job-matrix >> $GITHUB_OUTPUT + run: | + cd src/ci/citool + cargo test + cargo run calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: name: ${{ matrix.full_name }} diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock new file mode 100644 index 0000000000000..39b6b44da643b --- /dev/null +++ b/src/ci/citool/Cargo.lock @@ -0,0 +1,415 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + +[[package]] +name = "citool" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "insta", + "serde", + "serde_json", + "serde_yaml", +] + +[[package]] +name = "clap" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "console" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "insta" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "pin-project", + "similar", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "once_cell" +version = "1.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" + +[[package]] +name = "pin-project" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.138" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml new file mode 100644 index 0000000000000..e77c67c71477a --- /dev/null +++ b/src/ci/citool/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "citool" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +clap = { version = "4.5", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1" + +[dev-dependencies] +insta = "1" + +# Tell cargo that citool is its own workspace. +# If this is omitted, cargo will look for a workspace elsewhere. +# We want to avoid this, since citool is independent of the other crates. +[workspace] diff --git a/src/ci/citool/README.md b/src/ci/citool/README.md new file mode 100644 index 0000000000000..685a255f15251 --- /dev/null +++ b/src/ci/citool/README.md @@ -0,0 +1,2 @@ +# CI tooling +This is a simple Rust script that determines which jobs should be executed on CI based on the situation (pull request, try job, merge attempt). It also provides a simple way of executing (some) CI jobs locally. diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs new file mode 100644 index 0000000000000..ad9cc8b82a6f9 --- /dev/null +++ b/src/ci/citool/src/main.rs @@ -0,0 +1,380 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::Context; +use clap::Parser; +use serde_yaml::Value; + +const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); +const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker"); +const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml"); + +/// Representation of a job loaded from the `src/ci/github-actions/jobs.yml` file. +#[derive(serde::Deserialize, Debug, Clone)] +struct Job { + /// Name of the job, e.g. mingw-check + name: String, + /// GitHub runner on which the job should be executed + os: String, + env: BTreeMap, + /// Should the job be only executed on a specific channel? + #[serde(default)] + only_on_channel: Option, + /// Rest of attributes that will be passed through to GitHub actions + #[serde(flatten)] + extra_keys: BTreeMap, +} + +impl Job { + fn is_linux(&self) -> bool { + self.os.contains("ubuntu") + } + + /// By default, the Docker image of a job is based on its name. + /// However, it can be overridden by its IMAGE environment variable. + fn image(&self) -> String { + self.env + .get("IMAGE") + .map(|v| v.as_str().expect("IMAGE value should be a string").to_string()) + .unwrap_or_else(|| self.name.clone()) + } +} + +#[derive(serde::Deserialize, Debug)] +struct JobEnvironments { + #[serde(rename = "pr")] + pr_env: BTreeMap, + #[serde(rename = "try")] + try_env: BTreeMap, + #[serde(rename = "auto")] + auto_env: BTreeMap, +} + +#[derive(serde::Deserialize, Debug)] +struct JobDatabase { + #[serde(rename = "pr")] + pr_jobs: Vec, + #[serde(rename = "try")] + try_jobs: Vec, + #[serde(rename = "auto")] + auto_jobs: Vec, + + /// Shared environments for the individual run types. + envs: JobEnvironments, +} + +impl JobDatabase { + fn find_auto_job_by_name(&self, name: &str) -> Option { + self.auto_jobs.iter().find(|j| j.name == name).cloned() + } +} + +fn load_job_db(path: &Path) -> anyhow::Result { + let db = read_to_string(path)?; + let mut db: Value = serde_yaml::from_str(&db)?; + + // We need to expand merge keys (<<), because serde_yaml can't deal with them + // `apply_merge` only applies the merge once, so do it a few times to unwrap nested merges. + db.apply_merge()?; + db.apply_merge()?; + + let db: JobDatabase = serde_yaml::from_value(db)?; + Ok(db) +} + +/// Representation of a job outputted to a GitHub Actions workflow. +#[derive(serde::Serialize, Debug)] +struct GithubActionsJob { + /// The main identifier of the job, used by CI scripts to determine what should be executed. + name: String, + /// Helper label displayed in GitHub Actions interface, containing the job name and a run type + /// prefix (PR/try/auto). + full_name: String, + os: String, + env: BTreeMap, + #[serde(flatten)] + extra_keys: BTreeMap, +} + +/// Type of workflow that is being executed on CI +#[derive(Debug)] +enum RunType { + /// Workflows that run after a push to a PR branch + PullRequest, + /// Try run started with @bors try + TryJob { custom_jobs: Option> }, + /// Merge attempt workflow + AutoJob, +} + +struct GitHubContext { + event_name: String, + branch_ref: String, + commit_message: Option, +} + +impl GitHubContext { + fn get_run_type(&self) -> Option { + match (self.event_name.as_str(), self.branch_ref.as_str()) { + ("pull_request", _) => Some(RunType::PullRequest), + ("push", "refs/heads/try-perf") => Some(RunType::TryJob { custom_jobs: None }), + ("push", "refs/heads/try" | "refs/heads/automation/bors/try") => { + let custom_jobs = self.get_custom_jobs(); + let custom_jobs = if !custom_jobs.is_empty() { Some(custom_jobs) } else { None }; + Some(RunType::TryJob { custom_jobs }) + } + ("push", "refs/heads/auto") => Some(RunType::AutoJob), + _ => None, + } + } + + /// Tries to parse names of specific CI jobs that should be executed in the form of + /// try-job: + /// from the commit message of the passed GitHub context. + fn get_custom_jobs(&self) -> Vec { + if let Some(ref msg) = self.commit_message { + msg.lines() + .filter_map(|line| line.trim().strip_prefix("try-job: ")) + .map(|l| l.trim().to_string()) + .collect() + } else { + vec![] + } + } +} + +fn load_env_var(name: &str) -> anyhow::Result { + std::env::var(name).with_context(|| format!("Cannot find variable {name}")) +} + +fn load_github_ctx() -> anyhow::Result { + let event_name = load_env_var("GITHUB_EVENT_NAME")?; + let commit_message = + if event_name == "push" { Some(load_env_var("COMMIT_MESSAGE")?) } else { None }; + + Ok(GitHubContext { event_name, branch_ref: load_env_var("GITHUB_REF")?, commit_message }) +} + +/// Skip CI jobs that are not supposed to be executed on the given `channel`. +fn skip_jobs(jobs: Vec, channel: &str) -> Vec { + jobs.into_iter() + .filter(|job| { + job.only_on_channel.is_none() || job.only_on_channel.as_deref() == Some(channel) + }) + .collect() +} + +fn yaml_map_to_json(map: &BTreeMap) -> BTreeMap { + map.into_iter() + .map(|(key, value)| { + ( + key.clone(), + serde_json::to_value(&value).expect("Cannot convert map value from YAML to JSON"), + ) + }) + .collect() +} + +fn calculate_jobs( + run_type: &RunType, + db: &JobDatabase, + channel: &str, +) -> anyhow::Result> { + let (jobs, prefix, base_env) = match run_type { + RunType::PullRequest => (db.pr_jobs.clone(), "PR", &db.envs.pr_env), + RunType::TryJob { custom_jobs } => { + let jobs = if let Some(custom_jobs) = custom_jobs { + if custom_jobs.len() > 10 { + return Err(anyhow::anyhow!( + "It is only possible to schedule up to 10 custom jobs, received {} custom jobs", + custom_jobs.len() + )); + } + + let mut jobs = vec![]; + let mut unknown_jobs = vec![]; + for custom_job in custom_jobs { + if let Some(job) = db.find_auto_job_by_name(custom_job) { + jobs.push(job); + } else { + unknown_jobs.push(custom_job.clone()); + } + } + if !unknown_jobs.is_empty() { + return Err(anyhow::anyhow!( + "Custom job(s) `{}` not found in auto jobs", + unknown_jobs.join(", ") + )); + } + jobs + } else { + db.try_jobs.clone() + }; + (jobs, "try", &db.envs.try_env) + } + RunType::AutoJob => (db.auto_jobs.clone(), "auto", &db.envs.auto_env), + }; + let jobs = skip_jobs(jobs, channel); + let jobs = jobs + .into_iter() + .map(|job| { + let mut env: BTreeMap = yaml_map_to_json(base_env); + env.extend(yaml_map_to_json(&job.env)); + let full_name = format!("{prefix} - {}", job.name); + + GithubActionsJob { + name: job.name, + full_name, + os: job.os, + env, + extra_keys: yaml_map_to_json(&job.extra_keys), + } + }) + .collect(); + + Ok(jobs) +} + +fn calculate_job_matrix( + db: JobDatabase, + gh_ctx: GitHubContext, + channel: &str, +) -> anyhow::Result<()> { + let run_type = gh_ctx.get_run_type().ok_or_else(|| { + anyhow::anyhow!("Cannot determine the type of workflow that is being executed") + })?; + eprintln!("Run type: {run_type:?}"); + + let jobs = calculate_jobs(&run_type, &db, channel)?; + if jobs.is_empty() { + return Err(anyhow::anyhow!("Computed job list is empty")); + } + + let run_type = match run_type { + RunType::PullRequest => "pr", + RunType::TryJob { .. } => "try", + RunType::AutoJob => "auto", + }; + + eprintln!("Output"); + eprintln!("jobs={jobs:?}"); + eprintln!("run_type={run_type}"); + println!("jobs={}", serde_json::to_string(&jobs)?); + println!("run_type={run_type}"); + + Ok(()) +} + +fn find_linux_job<'a>(jobs: &'a [Job], name: &str) -> anyhow::Result<&'a Job> { + let Some(job) = jobs.iter().find(|j| j.name == name) else { + let available_jobs: Vec<&Job> = jobs.iter().filter(|j| j.is_linux()).collect(); + let mut available_jobs = + available_jobs.iter().map(|j| j.name.to_string()).collect::>(); + available_jobs.sort(); + return Err(anyhow::anyhow!( + "Job {name} not found. The following jobs are available:\n{}", + available_jobs.join(", ") + )); + }; + if !job.is_linux() { + return Err(anyhow::anyhow!("Only Linux jobs can be executed locally")); + } + + Ok(job) +} + +fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> anyhow::Result<()> { + let jobs = match job_type { + JobType::Auto => &db.auto_jobs, + JobType::PR => &db.pr_jobs, + }; + let job = find_linux_job(jobs, &name).with_context(|| format!("Cannot find job {name}"))?; + + let mut custom_env: BTreeMap = BTreeMap::new(); + // Replicate src/ci/scripts/setup-environment.sh + // Adds custom environment variables to the job + if name.starts_with("dist-") { + if name.ends_with("-alt") { + custom_env.insert("DEPLOY_ALT".to_string(), "1".to_string()); + } else { + custom_env.insert("DEPLOY".to_string(), "1".to_string()); + } + } + custom_env.extend(job.env.iter().map(|(key, value)| { + let value = match value { + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + Value::String(value) => value.clone(), + _ => panic!("Unexpected type for environment variable {key} Only bool/number/string is supported.") + }; + (key.clone(), value) + })); + + let mut cmd = Command::new(Path::new(DOCKER_DIRECTORY).join("run.sh")); + cmd.arg(job.image()); + cmd.envs(custom_env); + + eprintln!("Executing {cmd:?}"); + + let result = cmd.spawn()?.wait()?; + if !result.success() { Err(anyhow::anyhow!("Job failed")) } else { Ok(()) } +} + +#[derive(clap::Parser)] +enum Args { + /// Calculate a list of jobs that should be executed on CI. + /// Should only be used on CI inside GitHub actions. + CalculateJobMatrix { + #[clap(long)] + jobs_file: Option, + }, + /// Execute a given CI job locally. + #[clap(name = "run-local")] + RunJobLocally { + /// Name of the job that should be executed. + name: String, + /// Type of the job that should be executed. + #[clap(long = "type", default_value = "auto")] + job_type: JobType, + }, +} + +#[derive(clap::ValueEnum, Clone)] +enum JobType { + /// Merge attempt ("auto") job + Auto, + /// Pull request job + PR, +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let default_jobs_file = Path::new(JOBS_YML_PATH); + let load_db = |jobs_path| load_job_db(jobs_path).context("Cannot load jobs.yml"); + + match args { + Args::CalculateJobMatrix { jobs_file } => { + let jobs_path = jobs_file.as_deref().unwrap_or(default_jobs_file); + let gh_ctx = load_github_ctx() + .context("Cannot load environment variables from GitHub Actions")?; + let channel = read_to_string(Path::new(CI_DIRECTORY).join("channel")) + .context("Cannot read channel file")? + .trim() + .to_string(); + + calculate_job_matrix(load_db(jobs_path)?, gh_ctx, &channel) + .context("Failed to calculate job matrix")?; + } + Args::RunJobLocally { job_type, name } => { + run_workflow_locally(load_db(default_jobs_file)?, job_type, name)? + } + } + + Ok(()) +} + +fn read_to_string>(path: P) -> anyhow::Result { + let error = format!("Cannot read file {:?}", path.as_ref()); + std::fs::read_to_string(path).context(error) +} diff --git a/src/ci/citool/tests/jobs.rs b/src/ci/citool/tests/jobs.rs new file mode 100644 index 0000000000000..1d81d58f89311 --- /dev/null +++ b/src/ci/citool/tests/jobs.rs @@ -0,0 +1,64 @@ +use std::process::{Command, Stdio}; + +const TEST_JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/test-jobs.yml"); + +#[test] +fn auto_jobs() { + let stdout = get_matrix("push", "commit", "refs/heads/auto"); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"aarch64-gnu","full_name":"auto - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"x86_64-gnu-llvm-18-1","full_name":"auto - x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"0","RUST_BACKTRACE":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"aarch64-apple","full_name":"auto - aarch64-apple","os":"macos-14","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","MACOSX_DEPLOYMENT_TARGET":11.0,"MACOSX_STD_DEPLOYMENT_TARGET":11.0,"NO_DEBUG_ASSERTIONS":1,"NO_LLVM_ASSERTIONS":1,"NO_OVERFLOW_CHECKS":1,"RUSTC_RETRY_LINKER_ON_SEGFAULT":1,"RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set rust.jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","SELECT_XCODE":"/Applications/Xcode_15.4.app","TOOLSTATE_PUBLISH":1,"USE_XCODE_CLANG":1}},{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}] + run_type=auto + "#); +} + +#[test] +fn try_jobs() { + let stdout = get_matrix("push", "commit", "refs/heads/try"); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"dist-x86_64-linux","full_name":"try - dist-x86_64-linux","os":"ubuntu-22.04-16core-64gb","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_TRY_BUILD":1,"TOOLSTATE_PUBLISH":1}}] + run_type=try + "#); +} + +#[test] +fn try_custom_jobs() { + let stdout = get_matrix( + "push", + r#"This is a test PR + +try-job: aarch64-gnu +try-job: dist-i686-msvc"#, + "refs/heads/try", + ); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"aarch64-gnu","full_name":"try - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DIST_TRY_BUILD":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"dist-i686-msvc","full_name":"try - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"DIST_TRY_BUILD":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}] + run_type=try + "#); +} + +#[test] +fn pr_jobs() { + let stdout = get_matrix("pull_request", "commit", "refs/heads/pr/1234"); + insta::assert_snapshot!(stdout, @r#" + jobs=[{"name":"mingw-check","full_name":"PR - mingw-check","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"free_disk":true},{"name":"mingw-check-tidy","full_name":"PR - mingw-check-tidy","os":"ubuntu-24.04","env":{"PR_CI_JOB":1},"continue_on_error":true,"free_disk":true}] + run_type=pr + "#); +} + +fn get_matrix(event_name: &str, commit_msg: &str, branch_ref: &str) -> String { + let output = Command::new("cargo") + .args(["run", "-q", "calculate-job-matrix", "--jobs-file", TEST_JOBS_YML_PATH]) + .env("GITHUB_EVENT_NAME", event_name) + .env("COMMIT_MESSAGE", commit_msg) + .env("GITHUB_REF", branch_ref) + .stdout(Stdio::piped()) + .output() + .expect("Failed to execute command"); + + let stdout = String::from_utf8(output.stdout).unwrap(); + let stderr = String::from_utf8(output.stderr).unwrap(); + if !output.status.success() { + panic!("cargo run failed: {}\n{}", stdout, stderr); + } + stdout +} diff --git a/src/ci/citool/tests/test-jobs.yml b/src/ci/citool/tests/test-jobs.yml new file mode 100644 index 0000000000000..56b9ced20719e --- /dev/null +++ b/src/ci/citool/tests/test-jobs.yml @@ -0,0 +1,145 @@ +runners: + - &base-job + env: { } + + - &job-linux-4c + os: ubuntu-24.04 + # Free some disk space to avoid running out of space during the build. + free_disk: true + <<: *base-job + + - &job-linux-16c + os: ubuntu-22.04-16core-64gb + <<: *base-job + + - &job-macos-m1 + os: macos-14 + <<: *base-job + + - &job-windows + os: windows-2022 + <<: *base-job + + - &job-aarch64-linux + # Free some disk space to avoid running out of space during the build. + free_disk: true + os: ubuntu-22.04-arm + <<: *base-job +envs: + env-x86_64-apple-tests: &env-x86_64-apple-tests + SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # Ensure that host tooling is tested on our minimum supported macOS version. + MACOSX_DEPLOYMENT_TARGET: 10.12 + MACOSX_STD_DEPLOYMENT_TARGET: 10.12 + SELECT_XCODE: /Applications/Xcode_15.2.app + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + + production: + &production + DEPLOY_BUCKET: rust-lang-ci2 + # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named + # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to + # rotate them in a single branch while keeping the old key in another + # branch, which wouldn't be possible if the key was named with the kind + # (caches, artifacts...). + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 + AWS_REGION: us-west-1 + TOOLSTATE_PUBLISH: 1 + + try: + <<: *production + # The following env var activates faster `try` builds in `opt-dist` by, e.g. + # - building only the more commonly useful components (we rarely need e.g. rust-docs in try + # builds) + # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain + # + # If you *want* these to happen however, temporarily comment it before triggering a try build. + DIST_TRY_BUILD: 1 + + auto: + <<: *production + + pr: + PR_CI_JOB: 1 + +# Jobs that run on each push to a pull request (PR) +# These jobs automatically inherit envs.pr, to avoid repeating +# it in each job definition. +pr: + - name: mingw-check + <<: *job-linux-4c + - name: mingw-check-tidy + continue_on_error: true + <<: *job-linux-4c + +# Jobs that run when you perform a try build (@bors try) +# These jobs automatically inherit envs.try, to avoid repeating +# it in each job definition. +try: + - name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-linux-16c + +# Main CI jobs that have to be green to merge a commit into master +# These jobs automatically inherit envs.auto, to avoid repeating +# it in each job definition. +auto: + - name: aarch64-gnu + <<: *job-aarch64-linux + + # The x86_64-gnu-llvm-18 job is split into multiple jobs to run tests in parallel. + # x86_64-gnu-llvm-18-1 skips tests that run in x86_64-gnu-llvm-18-{2,3}. + - name: x86_64-gnu-llvm-18-1 + env: + RUST_BACKTRACE: 1 + READ_ONLY_SRC: "0" + IMAGE: x86_64-gnu-llvm-18 + DOCKER_SCRIPT: stage_2_test_set1.sh + <<: *job-linux-4c + + + #################### + # macOS Builders # + #################### + + - name: aarch64-apple + env: + SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin + RUST_CONFIGURE_ARGS: >- + --enable-sanitizers + --enable-profiler + --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + SELECT_XCODE: /Applications/Xcode_15.4.app + USE_XCODE_CLANG: 1 + # Aarch64 tooling only needs to support macOS 11.0 and up as nothing else + # supports the hardware, so only need to test it there. + MACOSX_DEPLOYMENT_TARGET: 11.0 + MACOSX_STD_DEPLOYMENT_TARGET: 11.0 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-m1 + + ###################### + # Windows Builders # + ###################### + + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=i686-pc-windows-msvc + --host=i686-pc-windows-msvc + --target=i686-pc-windows-msvc,i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist bootstrap --include-default-paths + DIST_REQUIRE_ALL_TOOLS: 1 + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-windows diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md index 2b1de43c2f68c..a1a3a1c37ced7 100644 --- a/src/ci/docker/README.md +++ b/src/ci/docker/README.md @@ -8,15 +8,15 @@ Note that a single Docker image can be used by multiple CI jobs, so the job name is the important thing that you should know. You can examine the existing CI jobs in the [`jobs.yml`](../github-actions/jobs.yml) file. -To run a specific CI job locally, you can use the following script: +To run a specific CI job locally, you can use the `citool` Rust crate: ``` -python3 ./src/ci/github-actions/ci.py run-local +cargo --manifest-path src/ci/citool/Cargo.toml run run-local ``` For example, to run the `x86_64-gnu-llvm-18-1` job: ``` -python3 ./src/ci/github-actions/ci.py run-local x86_64-gnu-llvm-18-1 +cargo --manifest-path src/ci/citool/Cargo.toml run run-local x86_64-gnu-llvm-18-1 ``` The job will output artifacts in an `obj/` dir at the root of a repository. Note diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py deleted file mode 100755 index c93766ef33a0d..0000000000000 --- a/src/ci/github-actions/ci.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script contains CI functionality. -It can be used to generate a matrix of jobs that should -be executed on CI, or run a specific CI job locally. - -It reads job definitions from `src/ci/github-actions/jobs.yml`. -""" - -import argparse -import dataclasses -import json -import logging -import os -import re -import subprocess -import typing -from pathlib import Path -from typing import List, Dict, Any, Optional - -import yaml - -CI_DIR = Path(__file__).absolute().parent.parent -JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" - -Job = Dict[str, Any] - - -def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: - """ - Modify the `name` attribute of each job, based on its base name and the given `prefix`. - Add an `image` attribute to each job, based on its image. - """ - modified_jobs = [] - for job in jobs: - # Create a copy of the `job` dictionary to avoid modifying `jobs` - new_job = dict(job) - new_job["image"] = get_job_image(new_job) - new_job["full_name"] = f"{prefix} - {new_job['name']}" - modified_jobs.append(new_job) - return modified_jobs - - -def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: - """ - Prepends `environment` to the `env` attribute of each job. - The `env` of each job has higher precedence than `environment`. - """ - modified_jobs = [] - for job in jobs: - env = environment.copy() - env.update(job.get("env", {})) - - new_job = dict(job) - new_job["env"] = env - modified_jobs.append(new_job) - return modified_jobs - - -@dataclasses.dataclass -class PRRunType: - pass - - -@dataclasses.dataclass -class TryRunType: - custom_jobs: List[str] - - -@dataclasses.dataclass -class AutoRunType: - pass - - -WorkflowRunType = typing.Union[PRRunType, TryRunType, AutoRunType] - - -@dataclasses.dataclass -class GitHubCtx: - event_name: str - ref: str - repository: str - commit_message: Optional[str] - - -def get_custom_jobs(ctx: GitHubCtx) -> List[str]: - """ - Tries to parse names of specific CI jobs that should be executed in the form of - try-job: - from the commit message of the passed GitHub context. - """ - if ctx.commit_message is None: - return [] - - regex = re.compile(r"^try-job: (.*)", re.MULTILINE) - jobs = [] - for match in regex.finditer(ctx.commit_message): - jobs.append(match.group(1)) - return jobs - - -def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: - if ctx.event_name == "pull_request": - return PRRunType() - elif ctx.event_name == "push": - try_build = ctx.ref in ( - "refs/heads/try", - "refs/heads/try-perf", - "refs/heads/automation/bors/try", - ) - - # Unrolled branch from a rollup for testing perf - # This should **not** allow custom try jobs - is_unrolled_perf_build = ctx.ref == "refs/heads/try-perf" - - if try_build: - custom_jobs = [] - if not is_unrolled_perf_build: - custom_jobs = get_custom_jobs(ctx) - return TryRunType(custom_jobs=custom_jobs) - - if ctx.ref == "refs/heads/auto": - return AutoRunType() - - return None - - -def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: - if isinstance(run_type, PRRunType): - return add_base_env( - add_job_properties(job_data["pr"], "PR"), job_data["envs"]["pr"] - ) - elif isinstance(run_type, TryRunType): - jobs = job_data["try"] - custom_jobs = run_type.custom_jobs - if custom_jobs: - if len(custom_jobs) > 10: - raise Exception( - f"It is only possible to schedule up to 10 custom jobs, " - f"received {len(custom_jobs)} jobs" - ) - - jobs = [] - unknown_jobs = [] - for custom_job in custom_jobs: - job = [j for j in job_data["auto"] if j["name"] == custom_job] - if not job: - unknown_jobs.append(custom_job) - continue - jobs.append(job[0]) - if unknown_jobs: - raise Exception( - f"Custom job(s) `{unknown_jobs}` not found in auto jobs" - ) - - return add_base_env(add_job_properties(jobs, "try"), job_data["envs"]["try"]) - elif isinstance(run_type, AutoRunType): - return add_base_env( - add_job_properties(job_data["auto"], "auto"), job_data["envs"]["auto"] - ) - - return [] - - -def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: - """ - Skip CI jobs that are not supposed to be executed on the given `channel`. - """ - return [j for j in jobs if j.get("only_on_channel", channel) == channel] - - -def get_github_ctx() -> GitHubCtx: - event_name = os.environ["GITHUB_EVENT_NAME"] - - commit_message = None - if event_name == "push": - commit_message = os.environ["COMMIT_MESSAGE"] - return GitHubCtx( - event_name=event_name, - ref=os.environ["GITHUB_REF"], - repository=os.environ["GITHUB_REPOSITORY"], - commit_message=commit_message, - ) - - -def format_run_type(run_type: WorkflowRunType) -> str: - if isinstance(run_type, PRRunType): - return "pr" - elif isinstance(run_type, AutoRunType): - return "auto" - elif isinstance(run_type, TryRunType): - return "try" - else: - raise AssertionError() - - -def get_job_image(job: Job) -> str: - """ - By default, the Docker image of a job is based on its name. - However, it can be overridden by its IMAGE environment variable. - """ - env = job.get("env", {}) - # Return the IMAGE environment variable if it exists, otherwise return the job name - return env.get("IMAGE", job["name"]) - - -def is_linux_job(job: Job) -> bool: - return "ubuntu" in job["os"] - - -def find_linux_job(job_data: Dict[str, Any], job_name: str, pr_jobs: bool) -> Job: - candidates = job_data["pr"] if pr_jobs else job_data["auto"] - jobs = [job for job in candidates if job.get("name") == job_name] - if len(jobs) == 0: - available_jobs = "\n".join( - sorted(job["name"] for job in candidates if is_linux_job(job)) - ) - raise Exception(f"""Job `{job_name}` not found in {'pr' if pr_jobs else 'auto'} jobs. -The following jobs are available: -{available_jobs}""") - assert len(jobs) == 1 - - job = jobs[0] - if not is_linux_job(job): - raise Exception("Only Linux jobs can be executed locally") - return job - - -def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool): - DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" - - job = find_linux_job(job_data, job_name=job_name, pr_jobs=pr_jobs) - - custom_env = {} - # Replicate src/ci/scripts/setup-environment.sh - # Adds custom environment variables to the job - if job_name.startswith("dist-"): - if job_name.endswith("-alt"): - custom_env["DEPLOY_ALT"] = "1" - else: - custom_env["DEPLOY"] = "1" - custom_env.update({k: str(v) for (k, v) in job.get("env", {}).items()}) - - args = [str(DOCKER_DIR / "run.sh"), get_job_image(job)] - env_formatted = [f"{k}={v}" for (k, v) in sorted(custom_env.items())] - print(f"Executing `{' '.join(env_formatted)} {' '.join(args)}`") - - env = os.environ.copy() - env.update(custom_env) - - subprocess.run(args, env=env, check=True) - - -def calculate_job_matrix(job_data: Dict[str, Any]): - github_ctx = get_github_ctx() - - run_type = find_run_type(github_ctx) - logging.info(f"Job type: {run_type}") - - with open(CI_DIR / "channel") as f: - channel = f.read().strip() - - jobs = [] - if run_type is not None: - jobs = calculate_jobs(run_type, job_data) - jobs = skip_jobs(jobs, channel) - - if not jobs: - raise Exception("Scheduled job list is empty, this is an error") - - run_type = format_run_type(run_type) - - logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") - print(f"jobs={json.dumps(jobs)}") - print(f"run_type={run_type}") - - -def create_cli_parser(): - parser = argparse.ArgumentParser( - prog="ci.py", description="Generate or run CI workflows" - ) - subparsers = parser.add_subparsers( - help="Command to execute", dest="command", required=True - ) - subparsers.add_parser( - "calculate-job-matrix", - help="Generate a matrix of jobs that should be executed in CI", - ) - run_parser = subparsers.add_parser( - "run-local", help="Run a CI jobs locally (on Linux)" - ) - run_parser.add_argument( - "job_name", - help="CI job that should be executed. By default, a merge (auto) " - "job with the given name will be executed", - ) - run_parser.add_argument( - "--pr", action="store_true", help="Run a PR job instead of an auto job" - ) - return parser - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - with open(JOBS_YAML_PATH) as f: - data = yaml.safe_load(f) - - parser = create_cli_parser() - args = parser.parse_args() - - if args.command == "calculate-job-matrix": - calculate_job_matrix(data) - elif args.command == "run-local": - run_workflow_locally(data, args.job_name, args.pr) - else: - raise Exception(f"Unknown command {args.command}") diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 64e64867de26f..bbcc01a0c2998 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -51,9 +51,11 @@ runners: # Free some disk space to avoid running out of space during the build. free_disk: true os: ubuntu-24.04-arm + <<: *base-job - &job-aarch64-linux-8c os: ubuntu-22.04-arm64-8core-32gb + <<: *base-job envs: env-x86_64-apple-tests: &env-x86_64-apple-tests SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact diff --git a/src/doc/rustc-dev-guide/src/building/optimized-build.md b/src/doc/rustc-dev-guide/src/building/optimized-build.md index 8feda59829b45..f8ca1d0dc3919 100644 --- a/src/doc/rustc-dev-guide/src/building/optimized-build.md +++ b/src/doc/rustc-dev-guide/src/building/optimized-build.md @@ -126,4 +126,4 @@ Here is an example of how can `opt-dist` be used locally (outside of CI): [`Environment`]: https://github.com/rust-lang/rust/blob/ee451f8faccf3050c76cdcd82543c917b40c7962/src/tools/opt-dist/src/environment.rs#L5 > Note: if you want to run the actual CI pipeline, instead of running `opt-dist` locally, -> you can execute `python3 src/ci/github-actions/ci.py run-local dist-x86_64-linux`. +> you can execute `cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux`. diff --git a/src/doc/rustc-dev-guide/src/tests/ci.md b/src/doc/rustc-dev-guide/src/tests/ci.md index a4b22392f1976..ae6adb678af14 100644 --- a/src/doc/rustc-dev-guide/src/tests/ci.md +++ b/src/doc/rustc-dev-guide/src/tests/ci.md @@ -28,7 +28,7 @@ Our CI is primarily executed on [GitHub Actions], with a single workflow defined in [`.github/workflows/ci.yml`], which contains a bunch of steps that are unified for all CI jobs that we execute. When a commit is pushed to a corresponding branch or a PR, the workflow executes the -[`src/ci/github-actions/ci.py`] script, which dynamically generates the specific CI +[`src/ci/citool`] crate, which dynamically generates the specific CI jobs that should be executed. This script uses the [`jobs.yml`] file as an input, which contains a declarative configuration of all our CI jobs. @@ -299,7 +299,7 @@ platform’s custom [Docker container]. This has a lot of advantages for us: - We can avoid reinstalling tools (like QEMU or the Android emulator) every time thanks to Docker image caching. - Users can run the same tests in the same environment locally by just running - `python3 src/ci/github-actions/ci.py run-local `, which is awesome to debug failures. Note that there are only linux docker images available locally due to licensing and + `cargo run --manifest-path src/ci/citool/Cargo.toml run-local `, which is awesome to debug failures. Note that there are only linux docker images available locally due to licensing and other restrictions. The docker images prefixed with `dist-` are used for building artifacts while @@ -443,7 +443,7 @@ this: [GitHub Actions]: https://github.com/rust-lang/rust/actions [`jobs.yml`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/jobs.yml [`.github/workflows/ci.yml`]: https://github.com/rust-lang/rust/blob/master/.github/workflows/ci.yml -[`src/ci/github-actions/ci.py`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/ci.py +[`src/ci/citool`]: https://github.com/rust-lang/rust/blob/master/src/ci/citool [rust-lang-ci]: https://github.com/rust-lang-ci/rust/actions [bors]: https://github.com/bors [homu]: https://github.com/rust-lang/homu diff --git a/src/doc/rustc-dev-guide/src/tests/docker.md b/src/doc/rustc-dev-guide/src/tests/docker.md index 2ca08d42130a5..a8a388ef90cfb 100644 --- a/src/doc/rustc-dev-guide/src/tests/docker.md +++ b/src/doc/rustc-dev-guide/src/tests/docker.md @@ -53,6 +53,15 @@ Some additional notes about using the interactive mode: containers. With the container name, run `docker exec -it /bin/bash` where `` is the container name like `4ba195e95cef`. +The approach described above is a relatively low-level interface for running the Docker images +directly. If you want to run a full CI Linux job locally with Docker, in a way that is as close to CI as possible, you can use the following command: + +```bash +cargo run --manifest-path src/ci/citool/Cargo.toml run-local +# For example: +cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux-alt +``` + [Docker]: https://www.docker.com/ [`src/ci/docker`]: https://github.com/rust-lang/rust/tree/master/src/ci/docker [`src/ci/docker/run.sh`]: https://github.com/rust-lang/rust/blob/master/src/ci/docker/run.sh