diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e83bbf643fee..7c46871569646 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,17 @@ jobs: # Check the `calculate_matrix` job to see how is the matrix defined. include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} steps: + - name: Install cargo in AWS CodeBuild + if: matrix.codebuild + run: | + # Check if cargo is installed + if ! command -v cargo &> /dev/null; then + echo "Cargo not found, installing Rust..." + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal + # Make cargo available in PATH + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: disable git crlf conversion run: git config --global core.autocrlf false @@ -168,6 +179,8 @@ jobs: run: src/ci/scripts/install-ninja.sh - name: enable ipv6 on Docker + # Don't run on codebuild because systemctl is not available + if: ${{ !matrix.codebuild }} run: src/ci/scripts/enable-docker-ipv6.sh # Disable automatic line ending conversion (again). On Windows, when we're diff --git a/src/ci/citool/src/jobs.rs b/src/ci/citool/src/jobs.rs index 13880ad466a6b..5600d7b4db59b 100644 --- a/src/ci/citool/src/jobs.rs +++ b/src/ci/citool/src/jobs.rs @@ -3,12 +3,15 @@ mod tests; use std::collections::BTreeMap; +use anyhow::Context as _; use serde_yaml::Value; use crate::GitHubContext; +use crate::utils::load_env_var; /// Representation of a job loaded from the `src/ci/github-actions/jobs.yml` file. #[derive(serde::Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub struct Job { /// Name of the job, e.g. mingw-check pub name: String, @@ -26,6 +29,8 @@ pub struct Job { pub free_disk: Option, /// Documentation link to a resource that could help people debug this CI job. pub doc_url: Option, + /// Whether the job is executed on AWS CodeBuild. + pub codebuild: Option, } impl Job { @@ -80,7 +85,7 @@ impl JobDatabase { } pub fn load_job_db(db: &str) -> anyhow::Result { - let mut db: Value = serde_yaml::from_str(&db)?; + let mut db: Value = serde_yaml::from_str(db)?; // We need to expand merge keys (<<), because serde_yaml can't deal with them // `apply_merge` only applies the merge once, so do it a few times to unwrap nested merges. @@ -107,6 +112,29 @@ struct GithubActionsJob { free_disk: Option, #[serde(skip_serializing_if = "Option::is_none")] doc_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + codebuild: Option, +} + +/// Replace GitHub context variables with environment variables in job configs. +/// Used for codebuild jobs like +/// `codebuild-ubuntu-22-8c-$github.run_id-$github.run_attempt` +fn substitute_github_vars(jobs: Vec) -> anyhow::Result> { + let run_id = load_env_var("GITHUB_RUN_ID")?; + let run_attempt = load_env_var("GITHUB_RUN_ATTEMPT")?; + + let jobs = jobs + .into_iter() + .map(|mut job| { + job.os = job + .os + .replace("$github.run_id", &run_id) + .replace("$github.run_attempt", &run_attempt); + job + }) + .collect(); + + Ok(jobs) } /// Skip CI jobs that are not supposed to be executed on the given `channel`. @@ -177,6 +205,8 @@ fn calculate_jobs( } RunType::AutoJob => (db.auto_jobs.clone(), "auto", &db.envs.auto_env), }; + let jobs = substitute_github_vars(jobs.clone()) + .context("Failed to substitute GitHub context variables in jobs")?; let jobs = skip_jobs(jobs, channel); let jobs = jobs .into_iter() @@ -207,6 +237,7 @@ fn calculate_jobs( continue_on_error: job.continue_on_error, free_disk: job.free_disk, doc_url: job.doc_url, + codebuild: job.codebuild, } }) .collect(); diff --git a/src/ci/docker/host-x86_64/dist-arm-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-arm-linux/Dockerfile index 420c42bc9d807..3795859f308e6 100644 --- a/src/ci/docker/host-x86_64/dist-arm-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-arm-linux/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ghcr.io/rust-lang/ubuntu:22.04 COPY scripts/cross-apt-packages.sh /scripts/ RUN sh /scripts/cross-apt-packages.sh diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 00d791eeb6b38..36f7df2b06907 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -288,7 +288,7 @@ args="$args --privileged" # `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all # read/written as the same user as the bare-metal user. if [ -f /.dockerenv ]; then - docker create -v /checkout --name checkout alpine:3.4 /bin/true + docker create -v /checkout --name checkout ghcr.io/rust-lang/alpine:3.4 /bin/true docker cp . checkout:/checkout args="$args --volumes-from checkout" else diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index cb2bec5a9dfa6..367e45ebe2006 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -56,6 +56,15 @@ runners: - &job-aarch64-linux-8c os: ubuntu-24.04-arm64-8core-32gb <<: *base-job + + # Codebuild runners are provisioned in + # https://github.com/rust-lang/simpleinfra/blob/b7ddd5e6bec8a93ec30510cdddec02c5666fefe9/terragrunt/accounts/ci-prod/ci-runners/terragrunt.hcl#L2 + - &job-linux-36c-codebuild + free_disk: true + codebuild: true + os: codebuild-ubuntu-22-36c-$github.run_id-$github.run_attempt + <<: *base-job + envs: env-x86_64-apple-tests: &env-x86_64-apple-tests SCRIPT: ./x.py check compiletest --set build.compiletest-use-stage0-libtest=true && ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact @@ -153,7 +162,7 @@ auto: <<: *job-linux-4c - name: dist-arm-linux - <<: *job-linux-8c + <<: *job-linux-36c-codebuild - name: dist-armhf-linux <<: *job-linux-4c diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 055a6ac2211e3..ad7ee136e9c27 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -14,6 +14,17 @@ isX86() { fi } +# Check if we're on a GitHub hosted runner. +# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted". +isGitHubRunner() { + # `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string". + if [[ "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then + return 0 + else + return 1 + fi +} + # print a line of the specified character printSeparationLine() { for ((i = 0; i < 80; i++)); do @@ -32,7 +43,7 @@ getAvailableSpace() { # make Kb human readable (assume the input is Kb) # REF: https://unix.stackexchange.com/a/44087/60849 formatByteCount() { - numfmt --to=iec-i --suffix=B --padding=7 "$1"'000' + numfmt --to=iec-i --suffix=B --padding=7 "${1}000" } # macro to output saved space @@ -45,6 +56,11 @@ printSavedSpace() { after=$(getAvailableSpace) local saved=$((after - before)) + if [ "$saved" -lt 0 ]; then + echo "::warning::Saved space is negative: $saved. Using '0' as saved space." + saved=0 + fi + echo "" printSeparationLine "*" if [ -n "${title}" ]; then @@ -118,10 +134,14 @@ removeUnusedFilesAndDirs() { # Azure "/opt/az" "/usr/share/az_"* + ) + if [ -n "${AGENT_TOOLSDIRECTORY:-}" ]; then # Environment variable set by GitHub Actions - "$AGENT_TOOLSDIRECTORY" - ) + to_remove+=( + "${AGENT_TOOLSDIRECTORY}" + ) + fi for element in "${to_remove[@]}"; do if [ ! -e "$element" ]; then @@ -155,20 +175,25 @@ cleanPackages() { '^dotnet-.*' '^llvm-.*' '^mongodb-.*' - 'azure-cli' 'firefox' 'libgl1-mesa-dri' 'mono-devel' 'php.*' ) - if isX86; then + if isGitHubRunner; then packages+=( - 'google-chrome-stable' - 'google-cloud-cli' - 'google-cloud-sdk' - 'powershell' + azure-cli ) + + if isX86; then + packages+=( + 'google-chrome-stable' + 'google-cloud-cli' + 'google-cloud-sdk' + 'powershell' + ) + fi fi sudo apt-get -qq remove -y --fix-missing "${packages[@]}"