Run integration in CI. (#11198)

Added a step for packaging IDE which runs integration tests.

This is an additional step, not a job, because here I'm sure the package will exist and won't be built unnecessarily twice. Technically I could make a job downloading it from GH action, but didn't want to invest time to rust scripts. Once Bazel will be fully introduced, the integration test will be improved.

# Important Notes
Fixes #8487

Author: farmaazon

.github/workflows/gui.yml

@@ -365,6 +365,13 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VITE_ENSO_AG_GRID_LICENSE_KEY: ${{ vars.ENSO_AG_GRID_LICENSE_KEY }}
           VITE_ENSO_MAPBOX_API_TOKEN: ${{ vars.ENSO_MAPBOX_API_TOKEN }}
+      - run: xvfb-run corepack pnpm -r --filter enso exec playwright test
+        env:
+          DEBUG: "pw:browser log:"
+          ENSO_TEST_APP_ARGS: --no-sandbox
+          ENSO_TEST_USER: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
+          ENSO_TEST_USER_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - if: failure() && runner.os == 'Windows'
         name: List files if failed (Windows)
         run: Get-ChildItem -Force -Recurse
@@ -428,6 +435,12 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VITE_ENSO_AG_GRID_LICENSE_KEY: ${{ vars.ENSO_AG_GRID_LICENSE_KEY }}
           VITE_ENSO_MAPBOX_API_TOKEN: ${{ vars.ENSO_MAPBOX_API_TOKEN }}
+      - run: corepack pnpm -r --filter enso exec playwright test
+        env:
+          DEBUG: "pw:browser log:"
+          ENSO_TEST_USER: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
+          ENSO_TEST_USER_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - if: failure() && runner.os == 'Windows'
         name: List files if failed (Windows)
         run: Get-ChildItem -Force -Recurse
@@ -487,6 +500,12 @@ jobs:
           VITE_ENSO_MAPBOX_API_TOKEN: ${{ vars.ENSO_MAPBOX_API_TOKEN }}
           WIN_CSC_KEY_PASSWORD: ${{ secrets.MICROSOFT_CODE_SIGNING_CERT_PASSWORD }}
           WIN_CSC_LINK: ${{ secrets.MICROSOFT_CODE_SIGNING_CERT }}
+      - run: corepack pnpm -r --filter enso exec playwright test
+        env:
+          DEBUG: "pw:browser log:"
+          ENSO_TEST_USER: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
+          ENSO_TEST_USER_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - if: failure() && runner.os == 'Windows'
         name: List files if failed (Windows)
         run: Get-ChildItem -Force -Recurse

app/ide-desktop/client/src/globals.d.ts

@@ -191,6 +191,7 @@ declare global {
       // === Integration test variables ===
 
       readonly ENSO_TEST?: string
+      readonly ENSO_TEST_APP_ARGS?: string
       readonly ENSO_TEST_USER?: string
       readonly ENSO_TEST_USER_PASSWORD?: string
       ENSO_TEST_EXEC_PATH?: string

app/ide-desktop/client/tests/createNewProject.spec.ts

@@ -9,5 +9,26 @@ electronTest('Create new project', async page => {
   await loginAsTestUser(page)
   await expect(page.getByRole('button', { name: 'New Project', exact: true })).toBeVisible()
   await page.getByRole('button', { name: 'New Project', exact: true }).click()
-  await expect(page.locator('.GraphNode'), {}).toBeVisible({ timeout: 30000 })
+  await expect(page.locator('.GraphNode')).toHaveCount(1, { timeout: 30000 })
+
+  // We see the node type and visualization, so the engine is running the program
+  await expect(page.locator('.node-type')).toHaveText('Table')
+  await expect(page.locator('.TableVisualization')).toBeVisible()
+  await expect(page.locator('.TableVisualization')).toContainText('Welcome To Enso!')
+
+  // We can add new node and see suggestions.
+  await page.locator('.GraphNode').click()
+  await page.keyboard.press('Enter')
+  await expect(page.locator('.ComponentBrowser')).toBeVisible()
+  const entry = page.locator('.ComponentList .list-variant.selected .component', {
+    hasText: 'column_count',
+  })
+  await expect(entry).toBeVisible()
+  await entry.click()
+  await expect(page.locator('.GraphNode'), {}).toHaveCount(2)
+  await page.locator('.GraphNode', { hasText: 'column_count' }).click()
+  await page
+    .locator('.GraphNode', { hasText: 'column_count' })
+    .getByRole('button', { name: 'Visualization' })
+    .click()
 })

app/ide-desktop/client/tests/electronTest.ts

@@ -2,6 +2,8 @@
 
 import { _electron, expect, type Page, test } from '@playwright/test'
 
+const LOADING_TIMEOUT = 10000
+
 /**
  * Tests run on electron executable.
  *
@@ -11,9 +13,13 @@ export function electronTest(name: string, body: (page: Page) => Promise<void> |
   test(name, async () => {
     const app = await _electron.launch({
       executablePath: process.env.ENSO_TEST_EXEC_PATH ?? '',
+      args: process.env.ENSO_TEST_APP_ARGS != null ? process.env.ENSO_TEST_APP_ARGS.split(',') : [],
       env: { ...process.env, ['ENSO_TEST']: name },
     })
     const page = await app.firstWindow()
+    // Wait until page will be finally loaded: we expect login screen.
+    // There's bigger timeout, because the page may load longer on CI machines.
+    await expect(page.getByText('Login to your account')).toBeVisible({ timeout: LOADING_TIMEOUT })
     await body(page)
     await app.close()
   })
@@ -32,6 +38,7 @@ export async function loginAsTestUser(page: Page) {
       'Cannot log in; `ENSO_TEST_USER` and `ENSO_TEST_USER_PASSWORD` env variables are not provided',
     )
   }
+  await page.getByRole('textbox', { name: 'email' }).click()
   await page.keyboard.insertText(process.env.ENSO_TEST_USER)
   await page.keyboard.press('Tab')
   await page.keyboard.insertText(process.env.ENSO_TEST_USER_PASSWORD)

build/build/src/ci_gen/job.rs

@@ -14,6 +14,7 @@ use crate::ide::web::env::VITE_ENSO_AG_GRID_LICENSE_KEY;
 use crate::ide::web::env::VITE_ENSO_MAPBOX_API_TOKEN;
 
 use ide_ci::actions::workflow::definition::cancel_workflow_action;
+use ide_ci::actions::workflow::definition::shell;
 use ide_ci::actions::workflow::definition::Access;
 use ide_ci::actions::workflow::definition::Job;
 use ide_ci::actions::workflow::definition::JobArchetype;
@@ -528,7 +529,26 @@ impl JobArchetype for PackageIde {
         RunStepsBuilder::new(
             "ide build --backend-source current-ci-run --gui-upload-artifact false",
         )
-        .customize(with_packaging_steps(target.0))
+        .customize(move |step| {
+            let mut steps = prepare_packaging_steps(target.0, step);
+            const TEST_COMMAND: &str = "corepack pnpm -r --filter enso exec playwright test";
+            let test_step = if target.0 == OS::Linux {
+                shell(format!("xvfb-run {TEST_COMMAND}"))
+                    // See https://askubuntu.com/questions/1512287/obsidian-appimage-the-suid-sandbox-helper-binary-was-found-but-is-not-configu
+                    .with_env("ENSO_TEST_APP_ARGS", "--no-sandbox")
+            } else {
+                shell(TEST_COMMAND)
+            };
+            let test_step = test_step
+                .with_env("DEBUG", "pw:browser log:")
+                .with_secret_exposed_as(secret::ENSO_CLOUD_TEST_ACCOUNT_USERNAME, "ENSO_TEST_USER")
+                .with_secret_exposed_as(
+                    secret::ENSO_CLOUD_TEST_ACCOUNT_PASSWORD,
+                    "ENSO_TEST_USER_PASSWORD",
+                );
+            steps.push(test_step);
+            steps
+        })
         .build_job("Package New IDE", target)
     }
 }