Initial template for the Extra Tests workflow (#10636)

- Closes #10618
- adjusts some edge case tests in Snowflake

.github/workflows/extra-nightly-tests.yml

@@ -0,0 +1,79 @@
+# This file is auto-generated. Do not edit it manually!
+# Edit the enso_build::ci_gen module instead and run `cargo run --package enso-build-ci-gen`.
+
+name: Extra Nightly Tests
+on:
+  schedule:
+    - cron: 0 3 * * *
+  workflow_dispatch:
+    inputs:
+      clean_build_required:
+        description: Clean before and after the run.
+        required: false
+        type: boolean
+        default: false
+jobs:
+  enso-build-ci-gen-job-snowflake-tests-linux-x86_64:
+    name: Snowflake Tests (linux, x86_64)
+    runs-on:
+      - self-hosted
+      - Linux
+    steps:
+      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
+        name: Installing wasm-pack
+        uses: jetli/[email protected]
+        with:
+          version: v0.10.2
+      - name: Expose Artifact API and context information.
+        uses: actions/github-script@v7
+        with:
+          script: "\n    core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n    core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n    core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n    console.log(context)\n    "
+      - name: Checking out the repository
+        uses: actions/checkout@v4
+        with:
+          clean: false
+          submodules: recursive
+      - name: Build Script Setup
+        run: ./run --help || (git clean -ffdx && ./run --help)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
+        name: Clean before
+        run: ./run git-clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: ./run backend test std-snowflake
+        env:
+          ENSO_SNOWFLAKE_ACCOUNT: ${{ secrets.ENSO_SNOWFLAKE_ACCOUNT }}
+          ENSO_SNOWFLAKE_DATABASE: ${{ secrets.ENSO_SNOWFLAKE_DATABASE }}
+          ENSO_SNOWFLAKE_PASSWORD: ${{ secrets.ENSO_SNOWFLAKE_PASSWORD }}
+          ENSO_SNOWFLAKE_SCHEMA: ${{ secrets.ENSO_SNOWFLAKE_SCHEMA }}
+          ENSO_SNOWFLAKE_USER: ${{ secrets.ENSO_SNOWFLAKE_USER }}
+          ENSO_SNOWFLAKE_WAREHOUSE: ${{ secrets.ENSO_SNOWFLAKE_WAREHOUSE }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
+        name: Extra Library Test Reporter
+        uses: dorny/test-reporter@v1
+        with:
+          max-annotations: 50
+          name: Extra Library Tests Report (GraalVM CE, linux, x86_64)
+          path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml
+          path-replace-backslashes: true
+          reporter: java-junit
+      - if: failure() && runner.os == 'Windows'
+        name: List files if failed (Windows)
+        run: Get-ChildItem -Force -Recurse
+      - if: failure() && runner.os != 'Windows'
+        name: List files if failed (non-Windows)
+        run: ls -lAR
+      - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
+        name: Clean after
+        run: ./run git-clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    env:
+      GRAAL_EDITION: GraalVM CE
+    permissions:
+      checks: write
+env:
+  ENSO_BUILD_SKIP_VERSION_CHECK: "true"

build/build/paths.yaml

@@ -10,6 +10,7 @@
       gui-tests.yml:
       gui.yml:
       engine-nightly.yml:
+      extra-nightly-tests.yml:
       nightly.yml:
       promote.yml:
       release.yml:

build/build/src/ci_gen.rs

@@ -119,6 +119,14 @@ pub mod secret {
     pub const APPLE_NOTARIZATION_PASSWORD: &str = "APPLE_NOTARIZATION_PASSWORD";
     pub const APPLE_NOTARIZATION_TEAM_ID: &str = "APPLE_NOTARIZATION_TEAM_ID";
 
+    // === Snowflake Test Account ===
+    pub const ENSO_SNOWFLAKE_ACCOUNT: &str = "ENSO_SNOWFLAKE_ACCOUNT";
+    pub const ENSO_SNOWFLAKE_USER: &str = "ENSO_SNOWFLAKE_USER";
+    pub const ENSO_SNOWFLAKE_PASSWORD: &str = "ENSO_SNOWFLAKE_PASSWORD";
+    pub const ENSO_SNOWFLAKE_DATABASE: &str = "ENSO_SNOWFLAKE_DATABASE";
+    pub const ENSO_SNOWFLAKE_SCHEMA: &str = "ENSO_SNOWFLAKE_SCHEMA";
+    pub const ENSO_SNOWFLAKE_WAREHOUSE: &str = "ENSO_SNOWFLAKE_WAREHOUSE";
+
     // === Windows Code Signing ===
     /// Name of the GitHub Actions secret that stores path to the Windows code signing certificate
     /// within the runner.
@@ -682,6 +690,22 @@ pub fn engine_nightly() -> Result<Workflow> {
     Ok(workflow)
 }
 
+pub fn extra_nightly_tests() -> Result<Workflow> {
+    let on = Event {
+        // We start at running the tests daily at 3 am, but we may adjust to run it every few days
+        // or only once a week.
+        schedule: vec![Schedule::new("0 3 * * *")?],
+        workflow_dispatch: Some(manual_workflow_dispatch()),
+        ..default()
+    };
+    let mut workflow = Workflow { name: "Extra Nightly Tests".into(), on, ..default() };
+
+    // We run the extra tests only on Linux, as they should not contain any platform-specific
+    // behavior.
+    let target = (OS::Linux, Arch::X86_64);
+    workflow.add(target, job::SnowflakeTests {});
+    Ok(workflow)
+}
 
 pub fn engine_benchmark() -> Result<Workflow> {
     benchmark_workflow("Benchmark Engine", "backend benchmark runtime", Some(4 * 60))
@@ -751,6 +775,7 @@ pub fn generate(
         (repo_root.nightly_yml.to_path_buf(), nightly()?),
         (repo_root.scala_new_yml.to_path_buf(), backend()?),
         (repo_root.engine_nightly_yml.to_path_buf(), engine_nightly()?),
+        (repo_root.extra_nightly_tests_yml.to_path_buf(), extra_nightly_tests()?),
         (repo_root.gui_yml.to_path_buf(), gui()?),
         (repo_root.gui_tests_yml.to_path_buf(), gui_tests()?),
         (repo_root.engine_benchmark_yml.to_path_buf(), engine_benchmark()?),

build/build/src/ci_gen/job.rs

@@ -256,6 +256,57 @@ impl JobArchetype for StandardLibraryTests {
     }
 }
 
+#[derive(Clone, Copy, Debug)]
+pub struct SnowflakeTests {}
+
+const GRAAL_EDITION_FOR_EXTRA_TESTS: graalvm::Edition = graalvm::Edition::Community;
+
+impl JobArchetype for SnowflakeTests {
+    fn job(&self, target: Target) -> Job {
+        let job_name = "Snowflake Tests";
+        let mut job = RunStepsBuilder::new("backend test std-snowflake")
+            .customize(move |step| {
+                let main_step = step
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_ACCOUNT,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_ACCOUNT,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_USER,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_USER,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_PASSWORD,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_PASSWORD,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_DATABASE,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_DATABASE,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_SCHEMA,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_SCHEMA,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_WAREHOUSE,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_WAREHOUSE,
+                    );
+                vec![
+                    main_step,
+                    step::extra_stdlib_test_reporter(target, GRAAL_EDITION_FOR_EXTRA_TESTS),
+                ]
+            })
+            .build_job(job_name, target)
+            .with_permission(Permission::Checks, Access::Write);
+        job.env(env::GRAAL_EDITION, GRAAL_EDITION_FOR_EXTRA_TESTS);
+        job
+    }
+
+    fn key(&self, (os, arch): Target) -> String {
+        format!("{}-{os}-{arch}", self.id_key_base())
+    }
+}
+
 #[derive(Clone, Copy, Debug)]
 pub struct Lint;
 

build/build/src/ci_gen/step.rs

@@ -42,3 +42,10 @@ pub fn engine_test_reporter((os, arch): Target, graal_edition: graalvm::Edition)
     let path = format!("{}/*.xml", env_expression(&paths::ENSO_TEST_JUNIT_DIR));
     test_reporter(step_name, report_name, path)
 }
+
+pub fn extra_stdlib_test_reporter((os, arch): Target, graal_edition: graalvm::Edition) -> Step {
+    let step_name = "Extra Library Test Reporter";
+    let report_name = format!("Extra Library Tests Report ({graal_edition}, {os}, {arch})");
+    let path = format!("{}/*/*.xml", env_expression(&paths::ENSO_TEST_JUNIT_DIR));
+    test_reporter(step_name, report_name, path)
+}

build/build/src/engine.rs

@@ -97,6 +97,7 @@ pub enum Tests {
     Jvm,
     #[clap(alias = "stdlib")]
     StandardLibrary,
+    StdSnowflake,
 }
 
 impl Benchmarks {
@@ -118,7 +119,7 @@ pub struct BuildConfigurationFlags {
     /// Run JVM tests.
     pub test_jvm: bool,
     /// Whether the Enso standard library should be tested.
-    pub test_standard_library: bool,
+    pub test_standard_library: Option<StandardLibraryTestsSelection>,
     /// Whether benchmarks are compiled.
     ///
     /// Note that this does not run the benchmarks, only ensures that they are buildable.
@@ -147,6 +148,12 @@ pub struct BuildConfigurationFlags {
     pub verify_packages: bool,
 }
 
+#[derive(Clone, Debug)]
+pub enum StandardLibraryTestsSelection {
+    All,
+    Selected(Vec<String>),
+}
+
 impl From<BuildConfigurationFlags> for BuildConfigurationResolved {
     fn from(value: BuildConfigurationFlags) -> Self {
         Self::new(value)
@@ -170,7 +177,7 @@ impl BuildConfigurationResolved {
 
         // Check for components that require Enso Engine runner. Basically everything that needs to
         // run pure Enso code.
-        if config.test_standard_library
+        if config.test_standard_library.is_some()
             || config.execute_benchmarks.contains(&Benchmarks::Enso)
             || config.check_enso_benchmarks
         {
@@ -195,7 +202,7 @@ impl BuildConfigurationFlags {
         self.build_engine_package
             || self.build_launcher_bundle
             || self.build_project_manager_bundle
-            || self.test_standard_library
+            || self.test_standard_library.is_some()
             || self.build_native_runner
     }
 
@@ -206,13 +213,29 @@ impl BuildConfigurationFlags {
     pub fn build_launcher_package(&self) -> bool {
         self.build_launcher_package || self.build_launcher_bundle
     }
+
+    pub fn add_standard_library_test_selection(
+        &mut self,
+        selection: StandardLibraryTestsSelection,
+    ) {
+        use StandardLibraryTestsSelection::*;
+        let combined_selection = match (self.test_standard_library.take(), selection) {
+            (None, selection) => selection,
+            (Some(All), _) | (_, All) => All,
+            (Some(Selected(mut selection)), Selected(new_selection)) => {
+                selection.extend(new_selection);
+                Selected(selection)
+            }
+        };
+        self.test_standard_library = Some(combined_selection);
+    }
 }
 
 impl Default for BuildConfigurationFlags {
     fn default() -> Self {
         Self {
             test_jvm: false,
-            test_standard_library: false,
+            test_standard_library: None,
             build_benchmarks: false,
             check_enso_benchmarks: false,
             execute_benchmarks: default(),

build/build/src/engine/context.rs

@@ -243,7 +243,7 @@ impl RunContext {
 
 
         let _test_results_upload_guard =
-            if self.config.test_jvm || self.config.test_standard_library {
+            if self.config.test_jvm || self.config.test_standard_library.is_some() {
                 // If we run tests, make sure that old and new results won't end up mixed together.
                 let test_results_dir = ENSO_TEST_JUNIT_DIR
                     .get()
@@ -393,14 +393,16 @@ impl RunContext {
             Ok(())
         };
 
-        if self.config.test_standard_library {
-            enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS).await?;
+        match &self.config.test_standard_library {
+            Some(selection) => {
+                enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS, selection.clone()).await?;
+            }
+            None => {}
         }
 
         perhaps_test_java_generated_from_rust_job.await.transpose()?;
 
         // === Run benchmarks ===
-        debug!("Running benchmarks.");
         let build_benchmark_task = if self.config.build_benchmarks {
             let build_benchmark_task_names = [
                 "runtime-benchmarks/compile",
@@ -422,6 +424,7 @@ impl RunContext {
             build_benchmark_task.as_deref().into_iter().chain(execute_benchmark_tasks);
         let benchmark_command = Sbt::sequential_tasks(build_and_execute_benchmark_task);
         if !benchmark_command.is_empty() {
+            debug!("Running benchmarks.");
             sbt.call_arg(benchmark_command).await?;
         } else {
             debug!("No SBT tasks to run.");

build/build/src/enso.rs

@@ -1,3 +1,4 @@
+use crate::engine::StandardLibraryTestsSelection;
 use crate::prelude::*;
 
 use crate::paths::Paths;
@@ -113,6 +114,7 @@ impl BuiltEnso {
         ir_caches: IrCaches,
         sbt: &crate::engine::sbt::Context,
         async_policy: AsyncPolicy,
+        test_selection: StandardLibraryTestsSelection,
     ) -> Result {
         let paths = &self.paths;
         // Environment for meta-tests. See:
@@ -131,9 +133,22 @@ impl BuiltEnso {
             ide_ci::fs::write(google_api_test_data_dir.join("secret.json"), gdoc_key)?;
         }
 
+        let std_tests = match &test_selection {
+            StandardLibraryTestsSelection::All =>
+                crate::paths::discover_standard_library_tests(&paths.repo_root)?,
+            StandardLibraryTestsSelection::Selected(only) =>
+                only.iter().map(|test| paths.repo_root.test.join(test)).collect(),
+        };
+        let may_need_postgres = match &test_selection {
+            StandardLibraryTestsSelection::All => true,
+            StandardLibraryTestsSelection::Selected(only) =>
+                only.iter().any(|test| test.contains("Postgres_Tests")),
+        };
+
         let _httpbin = crate::httpbin::get_and_spawn_httpbin_on_free_port(sbt).await?;
+
         let _postgres = match TARGET_OS {
-            OS::Linux => {
+            OS::Linux if may_need_postgres => {
                 let runner_context_string = crate::env::ENSO_RUNNER_CONTAINER_NAME
                     .get_raw()
                     .or_else(|_| ide_ci::actions::env::RUNNER_NAME.get())
@@ -156,7 +171,6 @@ impl BuiltEnso {
             _ => None,
         };
 
-        let std_tests = crate::paths::discover_standard_library_tests(&paths.repo_root)?;
         let futures = std_tests.into_iter().map(|test_path| {
             let command = self.run_test(test_path, ir_caches);
             async move { command?.run_ok().await }

build/build/src/libraries_tests.rs

@@ -8,3 +8,17 @@ pub mod s3 {
         }
     }
 }
+
+pub mod snowflake {
+    /// Environment variables used inside of the Snowflake tests.
+    pub mod env {
+        ide_ci::define_env_var! {
+            ENSO_SNOWFLAKE_ACCOUNT, String;
+            ENSO_SNOWFLAKE_USER, String;
+            ENSO_SNOWFLAKE_PASSWORD, String;
+            ENSO_SNOWFLAKE_DATABASE, String;
+            ENSO_SNOWFLAKE_SCHEMA, String;
+            ENSO_SNOWFLAKE_WAREHOUSE, String;
+        }
+    }
+}

build/build/src/rust/parser.rs

@@ -69,7 +69,6 @@ pub async fn run_self_tests(repo_root: &RepoRoot) -> Result {
         .output_ok()
         .await?
         .into_stdout_string()?;
-    trace!("Generated test code:\n{tests_code}");
     ide_ci::fs::tokio::write(&test, tests_code).await?;
 
     Javac