[CI] Moved Oracle GraalVM checks to a new nightly checks workflow. (#…

…9580) This PR introduces [a new workflow — nightly checks](https://github.com/enso-org/enso/actions/workflows/nightly-tests.yml). It consists of the whole array of Backend checks: * build check, Scala tests and Standard Library tests; * covers both Community and Oracle (Enterprise) GraalVM editions (Linux-only); * includes checks for Aarch64 macOS runner. We do not want to run these checks on each PR due to limited runners capacity. By running them nightly, we can still catch any issues that might arise on `develop` branch. # Important Notes * [ ] Before merging, this requires updating the GH required checks list.
enso-org · Apr 3, 2024 · 5038582 · 5038582
1 parent 4971907
commit 5038582
Show file tree

Hide file tree

Showing 9 changed files with 1,913 additions and 244 deletions.
diff --git a/.github/workflows/engine-nightly.yml b/.github/workflows/engine-nightly.yml
diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml
diff --git a/.github/workflows/scala-new.yml b/.github/workflows/scala-new.yml
@@ -179,8 +179,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     env:
       GRAAL_EDITION: GraalVM CE
-  enso-build-ci-gen-job-ci-check-backend-oracle-graal-vm-linux-x86_64:
-    name: Engine (Oracle GraalVM) (linux, x86_64)
+  enso-build-ci-gen-job-jvm-tests-graal-vm-ce-linux-x86_64:
+    name: JVM Tests (GraalVM CE) (linux, x86_64)
     runs-on:
       - self-hosted
       - Linux
@@ -214,58 +214,7 @@ jobs:
         run: ./run git-clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: ./run backend ci-check
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - if: failure() && runner.os == 'Windows'
-        name: List files if failed (Windows)
-        run: Get-ChildItem -Force -Recurse
-      - if: failure() && runner.os != 'Windows'
-        name: List files if failed (non-Windows)
-        run: ls -lAR
-      - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
-        name: Clean after
-        run: ./run git-clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    env:
-      GRAAL_EDITION: Oracle GraalVM
-  enso-build-ci-gen-job-scala-tests-graal-vm-ce-linux-x86_64:
-    name: Scala Tests (GraalVM CE) (linux, x86_64)
-    runs-on:
-      - self-hosted
-      - Linux
-    steps:
-      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
-        name: Setup conda (GH runners only)
-        uses: s-weigand/[email protected]
-        with:
-          update-conda: false
-          conda-channels: anaconda, conda-forge
-      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
-        name: Installing wasm-pack
-        uses: jetli/[email protected]
-        with:
-          version: v0.10.2
-      - name: Expose Artifact API and context information.
-        uses: actions/github-script@v7
-        with:
-          script: "\n    core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n    core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n    core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n    console.log(context)\n    "
-      - name: Checking out the repository
-        uses: actions/checkout@v4
-        with:
-          clean: false
-          submodules: recursive
-      - name: Build Script Setup
-        run: ./run --help || (git clean -ffdx && ./run --help)
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
-        name: Clean before
-        run: ./run git-clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: ./run backend test scala
+      - run: ./run backend test jvm
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - if: success() || failure()
@@ -292,8 +241,8 @@ jobs:
       GRAAL_EDITION: GraalVM CE
     permissions:
       checks: write
-  enso-build-ci-gen-job-scala-tests-graal-vm-ce-macos-x86_64:
-    name: Scala Tests (GraalVM CE) (macos, x86_64)
+  enso-build-ci-gen-job-jvm-tests-graal-vm-ce-macos-x86_64:
+    name: JVM Tests (GraalVM CE) (macos, x86_64)
     runs-on:
       - macos-latest
     steps:
@@ -326,7 +275,7 @@ jobs:
         run: ./run git-clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: ./run backend test scala
+      - run: ./run backend test jvm
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - if: success() || failure()
@@ -353,8 +302,8 @@ jobs:
       GRAAL_EDITION: GraalVM CE
     permissions:
       checks: write
-  enso-build-ci-gen-job-scala-tests-graal-vm-ce-windows-x86_64:
-    name: Scala Tests (GraalVM CE) (windows, x86_64)
+  enso-build-ci-gen-job-jvm-tests-graal-vm-ce-windows-x86_64:
+    name: JVM Tests (GraalVM CE) (windows, x86_64)
     runs-on:
       - self-hosted
       - Windows
@@ -388,7 +337,7 @@ jobs:
         run: ./run git-clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: ./run backend test scala
+      - run: ./run backend test jvm
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - if: success() || failure()
@@ -415,68 +364,6 @@ jobs:
       GRAAL_EDITION: GraalVM CE
     permissions:
       checks: write
-  enso-build-ci-gen-job-scala-tests-oracle-graal-vm-linux-x86_64:
-    name: Scala Tests (Oracle GraalVM) (linux, x86_64)
-    runs-on:
-      - self-hosted
-      - Linux
-    steps:
-      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
-        name: Setup conda (GH runners only)
-        uses: s-weigand/[email protected]
-        with:
-          update-conda: false
-          conda-channels: anaconda, conda-forge
-      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
-        name: Installing wasm-pack
-        uses: jetli/[email protected]
-        with:
-          version: v0.10.2
-      - name: Expose Artifact API and context information.
-        uses: actions/github-script@v7
-        with:
-          script: "\n    core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n    core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n    core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n    console.log(context)\n    "
-      - name: Checking out the repository
-        uses: actions/checkout@v4
-        with:
-          clean: false
-          submodules: recursive
-      - name: Build Script Setup
-        run: ./run --help || (git clean -ffdx && ./run --help)
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
-        name: Clean before
-        run: ./run git-clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: ./run backend test scala
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - if: success() || failure()
-        name: Engine Test Reporter
-        uses: dorny/test-reporter@v1
-        with:
-          max-annotations: 50
-          name: Engine Tests Report (Oracle GraalVM, linux, x86_64)
-          path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*.xml
-          path-replace-backslashes: true
-          reporter: java-junit
-      - if: failure() && runner.os == 'Windows'
-        name: List files if failed (Windows)
-        run: Get-ChildItem -Force -Recurse
-      - if: failure() && runner.os != 'Windows'
-        name: List files if failed (non-Windows)
-        run: ls -lAR
-      - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
-        name: Clean after
-        run: ./run git-clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    env:
-      GRAAL_EDITION: Oracle GraalVM
-    permissions:
-      checks: write
   enso-build-ci-gen-job-standard-library-tests-graal-vm-ce-linux-x86_64:
     name: Standard Library Tests (GraalVM CE) (linux, x86_64)
     runs-on:
@@ -671,71 +558,6 @@ jobs:
       GRAAL_EDITION: GraalVM CE
     permissions:
       checks: write
-  enso-build-ci-gen-job-standard-library-tests-oracle-graal-vm-linux-x86_64:
-    name: Standard Library Tests (Oracle GraalVM) (linux, x86_64)
-    runs-on:
-      - self-hosted
-      - Linux
-    steps:
-      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
-        name: Setup conda (GH runners only)
-        uses: s-weigand/[email protected]
-        with:
-          update-conda: false
-          conda-channels: anaconda, conda-forge
-      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
-        name: Installing wasm-pack
-        uses: jetli/[email protected]
-        with:
-          version: v0.10.2
-      - name: Expose Artifact API and context information.
-        uses: actions/github-script@v7
-        with:
-          script: "\n    core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n    core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n    core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n    console.log(context)\n    "
-      - name: Checking out the repository
-        uses: actions/checkout@v4
-        with:
-          clean: false
-          submodules: recursive
-      - name: Build Script Setup
-        run: ./run --help || (git clean -ffdx && ./run --help)
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
-        name: Clean before
-        run: ./run git-clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: ./run backend test standard-library
-        env:
-          ENSO_LIB_S3_AWS_ACCESS_KEY_ID: ${{ secrets.ENSO_LIB_S3_AWS_ACCESS_KEY_ID }}
-          ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
-          ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - if: success() || failure()
-        name: Standard Library Test Reporter
-        uses: dorny/test-reporter@v1
-        with:
-          max-annotations: 50
-          name: Standard Library Tests Report (Oracle GraalVM, linux, x86_64)
-          path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml
-          path-replace-backslashes: true
-          reporter: java-junit
-      - if: failure() && runner.os == 'Windows'
-        name: List files if failed (Windows)
-        run: Get-ChildItem -Force -Recurse
-      - if: failure() && runner.os != 'Windows'
-        name: List files if failed (non-Windows)
-        run: ls -lAR
-      - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
-        name: Clean after
-        run: ./run git-clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    env:
-      GRAAL_EDITION: Oracle GraalVM
-    permissions:
-      checks: write
   enso-build-ci-gen-job-verify-license-packages-linux-x86_64:
     name: Verify License Packages (linux, x86_64)
     runs-on:

diff --git a/build/build/paths.yaml b/build/build/paths.yaml
@@ -5,16 +5,17 @@
 <repo_root>/:
   .github/:
     workflows/:
-      engine-benchmark.yml:
-      std-libs-benchmark.yml:
       changelog.yml:
-      gui.yml:
+      engine-benchmark.yml:
       gui-tests.yml:
+      gui.yml:
+      engine-nightly.yml:
       nightly.yml:
       promote.yml:
       release.yml:
       scala-new.yml:
       shader-tools.yml:
+      std-libs-benchmark.yml:
   app/:
     gui/:
     gui2/: # The new, Vue-based GUI.

diff --git a/build/build/src/ci_gen.rs b/build/build/src/ci_gen.rs
@@ -73,7 +73,10 @@ pub const RELEASE_TARGETS: [(OS, Arch); 4] = [
     (OS::MacOS, Arch::AArch64),
 ];
 
-pub const CHECKED_TARGETS: [(OS, Arch); 3] =
+/// Targets for which we run PR checks.
+///
+/// The macOS AArch64 is intentionally omitted, as the runner availability is limited.
+pub const PR_CHECKED_TARGETS: [(OS, Arch); 3] =
     [(OS::Windows, Arch::X86_64), (OS::Linux, Arch::X86_64), (OS::MacOS, Arch::X86_64)];
 
 pub const DEFAULT_BRANCH_NAME: &str = "develop";
@@ -515,6 +518,17 @@ fn add_release_steps(workflow: &mut Workflow) -> Result {
     Ok(())
 }
 
+/// Add jobs that perform backend checks ,including Scala and Standard Library tests.
+pub fn add_backend_checks(
+    workflow: &mut Workflow,
+    target: Target,
+    graal_edition: graalvm::Edition,
+) {
+    workflow.add(target, job::CiCheckBackend { graal_edition });
+    workflow.add(target, job::JvmTests { graal_edition });
+    workflow.add(target, job::StandardLibraryTests { graal_edition });
+}
+
 pub fn workflow_call_job(name: impl Into<String>, path: impl Into<String>) -> Job {
     Job {
         name: name.into(),
@@ -583,14 +597,24 @@ pub fn promote() -> Result<Workflow> {
     Ok(workflow)
 }
 
-pub fn typical_check_triggers() -> Event {
+/// Trigger for a workflow that allows running it manually, on user request.
+///
+/// The workflow can be run either through the web interface or through the API.
+///
+/// The generated trigger will include an additional input, corresponding to the PR labels.
+pub fn manual_workflow_dispatch() -> WorkflowDispatch {
     let clean_build_input =
         WorkflowDispatchInput::new_boolean("Clean before and after the run.", false, false);
     let workflow_dispatch = WorkflowDispatch::default()
         .with_input(crate::ci::inputs::CLEAN_BUILD_REQUIRED, clean_build_input);
+    workflow_dispatch
+}
+
+/// The typical set of triggers for a CI workflow - it will be run on PRs and default branch pushes.
+pub fn typical_check_triggers() -> Event {
     Event {
         pull_request: Some(default()),
-        workflow_dispatch: Some(workflow_dispatch),
+        workflow_dispatch: Some(manual_workflow_dispatch()),
         push: Some(on_default_branch_push()),
         ..default()
     }
@@ -601,7 +625,7 @@ pub fn gui() -> Result<Workflow> {
     let mut workflow = Workflow { name: "GUI Packaging".into(), on, ..default() };
     workflow.add(PRIMARY_TARGET, job::CancelWorkflow);
 
-    for target in CHECKED_TARGETS {
+    for target in PR_CHECKED_TARGETS {
         let project_manager_job = workflow.add(target, job::BuildBackend);
         workflow.add_customized(target, job::PackageIde, |job| {
             job.needs.insert(project_manager_job.clone());
@@ -627,22 +651,32 @@ pub fn backend() -> Result<Workflow> {
     let mut workflow = Workflow { name: "Engine CI".into(), on, ..default() };
     workflow.add(PRIMARY_TARGET, job::CancelWorkflow);
     workflow.add(PRIMARY_TARGET, job::VerifyLicensePackages);
-    for target in CHECKED_TARGETS {
-        workflow.add(target, job::CiCheckBackend { graal_edition: graalvm::Edition::Community });
-        workflow.add(target, job::ScalaTests { graal_edition: graalvm::Edition::Community });
-        workflow
-            .add(target, job::StandardLibraryTests { graal_edition: graalvm::Edition::Community });
+    for target in PR_CHECKED_TARGETS {
+        add_backend_checks(&mut workflow, target, graalvm::Edition::Community);
     }
+    Ok(workflow)
+}
+
+pub fn engine_nightly() -> Result<Workflow> {
+    let on = Event {
+        schedule: vec![Schedule::new("0 3 * * *")?],
+        workflow_dispatch: Some(manual_workflow_dispatch()),
+        ..default()
+    };
+    let mut workflow = Workflow { name: "Engine Nightly Checks".into(), on, ..default() };
+
     // Oracle GraalVM jobs run only on Linux
-    workflow
-        .add(PRIMARY_TARGET, job::CiCheckBackend { graal_edition: graalvm::Edition::Enterprise });
-    workflow.add(PRIMARY_TARGET, job::ScalaTests { graal_edition: graalvm::Edition::Enterprise });
-    workflow.add(PRIMARY_TARGET, job::StandardLibraryTests {
-        graal_edition: graalvm::Edition::Enterprise,
-    });
+    add_backend_checks(&mut workflow, PRIMARY_TARGET, graalvm::Edition::Enterprise);
+
+    // Run macOS AArch64 tests only once a day, as we have only one self-hosted runner for this.
+    for target in PR_CHECKED_TARGETS {
+        add_backend_checks(&mut workflow, target, graalvm::Edition::Community);
+    }
+    add_backend_checks(&mut workflow, (OS::MacOS, Arch::AArch64), graalvm::Edition::Community);
     Ok(workflow)
 }
 
+
 pub fn engine_benchmark() -> Result<Workflow> {
     benchmark_workflow("Benchmark Engine", "backend benchmark runtime", Some(4 * 60))
 }
@@ -710,6 +744,7 @@ pub fn generate(
         (repo_root.changelog_yml.to_path_buf(), changelog()?),
         (repo_root.nightly_yml.to_path_buf(), nightly()?),
         (repo_root.scala_new_yml.to_path_buf(), backend()?),
+        (repo_root.engine_nightly_yml.to_path_buf(), engine_nightly()?),
         (repo_root.gui_yml.to_path_buf(), gui()?),
         (repo_root.gui_tests_yml.to_path_buf(), gui_tests()?),
         (repo_root.engine_benchmark_yml.to_path_buf(), engine_benchmark()?),