Skip to content

Commit cf1b966

Browse files
YgnasYgnas
committed
UI visual regression testing to cover UI widgets visibility
1 parent fb59ba6 commit cf1b966

File tree

9 files changed

+3149
-0
lines changed

9 files changed

+3149
-0
lines changed

‎.github/workflows/ui_notebooks_test.yaml

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: UI notebooks tests
2+
3+
on: [pull_request]
4+
5+
concurrency:
6+
group: ${{ github.head_ref }}-${{ github.workflow }}
7+
cancel-in-progress: true
8+
9+
env:
10+
CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
11+
12+
jobs:
13+
verify-0_basic_ray:
14+
runs-on: ubuntu-20.04-4core
15+
16+
steps:
17+
- name: Checkout code
18+
uses: actions/checkout@v4
19+
with:
20+
submodules: recursive
21+
22+
- name: Checkout common repo code
23+
uses: actions/checkout@v4
24+
with:
25+
repository: "project-codeflare/codeflare-common"
26+
ref: "main"
27+
path: "common"
28+
29+
- name: Checkout CodeFlare operator repository
30+
uses: actions/checkout@v4
31+
with:
32+
repository: project-codeflare/codeflare-operator
33+
path: codeflare-operator
34+
35+
- name: Set Go
36+
uses: actions/setup-go@v5
37+
with:
38+
go-version-file: "./codeflare-operator/go.mod"
39+
cache-dependency-path: "./codeflare-operator/go.sum"
40+
41+
- name: Set up gotestfmt
42+
uses: gotesttools/gotestfmt-action@v2
43+
with:
44+
token: ${{ secrets.GITHUB_TOKEN }}
45+
46+
- name: Set up specific Python version
47+
uses: actions/setup-python@v5
48+
with:
49+
python-version: "3.9"
50+
cache: "pip" # caching pip dependencies
51+
52+
- name: Setup and start KinD cluster
53+
uses: ./common/github-actions/kind
54+
55+
- name: Deploy CodeFlare stack
56+
id: deploy
57+
run: |
58+
cd codeflare-operator
59+
echo Setting up CodeFlare stack
60+
make setup-e2e
61+
echo Deploying CodeFlare operator
62+
make deploy -e IMG="${CODEFLARE_OPERATOR_IMG}" -e ENV="e2e"
63+
kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
64+
cd ..
65+
66+
- name: Setup Guided notebooks execution
67+
run: |
68+
echo "Installing papermill and dependencies..."
69+
pip install poetry ipython ipykernel jupyterlab
70+
poetry config virtualenvs.create false
71+
echo "Installing SDK..."
72+
poetry install --with test,docs
73+
74+
- name: Install Yarn dependencies
75+
run: |
76+
poetry run yarn install
77+
poetry run yarn playwright install chromium
78+
working-directory: ui-tests
79+
80+
- name: Run UI notebook tests
81+
run: |
82+
set -euo pipefail
83+
84+
# Remove login/logout cells, as KinD doesn't support authentication using token
85+
jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' 0_basic_ray.ipynb > 0_basic_ray.ipynb.tmp && mv 0_basic_ray.ipynb.tmp 0_basic_ray.ipynb
86+
jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' 0_basic_ray.ipynb > 0_basic_ray.ipynb.tmp && mv 0_basic_ray.ipynb.tmp 0_basic_ray.ipynb
87+
# Set explicit namespace as SDK need it (currently) to resolve local queues
88+
sed -i "s/head_memory=2,/head_memory=2, namespace='default',/" 0_basic_ray.ipynb
89+
90+
poetry run yarn test
91+
working-directory: ui-tests
92+
93+
- name: Print CodeFlare operator logs
94+
if: always() && steps.deploy.outcome == 'success'
95+
run: |
96+
echo "Printing CodeFlare operator logs"
97+
kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
98+
99+
- name: Print KubeRay operator logs
100+
if: always() && steps.deploy.outcome == 'success'
101+
run: |
102+
echo "Printing KubeRay operator logs"
103+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
104+
105+
- name: Upload Playwright Test assets
106+
if: always()
107+
uses: actions/upload-artifact@v4
108+
with:
109+
name: ipywidgets-test-assets
110+
path: |
111+
ui-tests/test-results
112+
113+
- name: Upload Playwright Test report
114+
if: always()
115+
uses: actions/upload-artifact@v4
116+
with:
117+
name: ipywidgets-test-report
118+
path: |
119+
ui-tests/playwright-report

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,7 @@ Pipfile.lock
88
build/
99
tls-cluster-namespace
1010
quicktest.yaml
11+
node_modules
12+
.DS_Store
13+
ui-tests/playwright-report
14+
ui-tests/test-results

‎ui-tests/.yarnrc

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
disable-self-update-check true
2+
ignore-optional true
3+
network-timeout "300000"
4+
registry "https://registry.npmjs.org/"

‎ui-tests/jupyter_server_config.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from jupyterlab.galata import configure_jupyter_server
2+
3+
configure_jupyter_server(c)
4+
5+
# Uncomment to set server log level to debug level
6+
# c.ServerApp.log_level = "DEBUG"

‎ui-tests/package.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"name": "@jupyter-widgets/ui-tests",
3+
"private": true,
4+
"version": "0.1.0",
5+
"description": "ipywidgets UI Tests",
6+
"scripts": {
7+
"start": "jupyter lab --config ./jupyter_server_config.py",
8+
"start:detached": "jlpm start&",
9+
"test": "npx playwright test",
10+
"test:debug": "PWDEBUG=1 npx playwright test",
11+
"test:report": "http-server ./playwright-report -a localhost -o",
12+
"test:update": "npx playwright test --update-snapshots",
13+
"deduplicate": "jlpm && yarn-deduplicate -s fewer --fail"
14+
},
15+
"author": "Project Jupyter",
16+
"license": "BSD-3-Clause",
17+
"devDependencies": {
18+
"@jupyterlab/galata": "^5.0.1",
19+
"@playwright/test": "^1.32.0",
20+
"yarn-deduplicate": "^6.0.1"
21+
}
22+
}

‎ui-tests/playwright.config.js

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
const baseConfig = require('@jupyterlab/galata/lib/playwright-config');
2+
3+
module.exports = {
4+
...baseConfig,
5+
timeout: 240000,
6+
webServer: {
7+
command: 'yarn start',
8+
url: 'http://localhost:8888/lab',
9+
timeout: 120 * 1000,
10+
reuseExistingServer: !process.env.CI,
11+
},
12+
retries: 0,
13+
};

‎ui-tests/tests/0_basic_ray.test.ts

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Copyright (c) Jupyter Development Team.
2+
// Distributed under the terms of the Modified BSD License.
3+
4+
import { test } from "@jupyterlab/galata";
5+
import { expect } from "@playwright/test";
6+
import * as path from "path";
7+
8+
test.setTimeout(460000);
9+
10+
test.describe("Visual Regression", () => {
11+
test.beforeEach(async ({ page, tmpPath }) => {
12+
await page.contents.uploadDirectory(
13+
path.resolve(__dirname, "../../demo-notebooks/guided-demos"),
14+
tmpPath
15+
);
16+
await page.filebrowser.openDirectory(tmpPath);
17+
});
18+
19+
test("Run notebook 0_basic_ray.ipynb and capture cell outputs", async ({
20+
page,
21+
tmpPath,
22+
}) => {
23+
const notebook = "0_basic_ray.ipynb";
24+
await page.notebook.openByPath(`${tmpPath}/${notebook}`);
25+
await page.notebook.activate(notebook);
26+
27+
const captures: (Buffer | null)[] = []; // Array to store cell screenshots
28+
const cellCount = await page.notebook.getCellCount();
29+
30+
// Run all cells and capture their screenshots
31+
await page.notebook.runCellByCell({
32+
onAfterCellRun: async (cellIndex: number) => {
33+
const cell = await page.notebook.getCellOutput(cellIndex);
34+
if (cell && (await cell.isVisible())) {
35+
captures[cellIndex] = await cell.screenshot(); // Save the screenshot by cell index
36+
}
37+
},
38+
});
39+
40+
await page.notebook.save();
41+
42+
// Ensure that each cell's screenshot is captured
43+
for (let i = 0; i < cellCount; i++) {
44+
const image = `widgets-cell-${i}.png`;
45+
46+
if (captures[i]) {
47+
expect.soft(captures[i]).toMatchSnapshot(image); // Compare pre-existing capture
48+
continue;
49+
}
50+
}
51+
});
52+
});

‎ui-tests/tests/notebooks/0_basic_ray.ipynb

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"# Import pieces from codeflare-sdk\n",
11+
"from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication"
12+
]
13+
},
14+
{
15+
"cell_type": "code",
16+
"execution_count": null,
17+
"id": "0f4bc870-091f-4e11-9642-cba145710159",
18+
"metadata": {},
19+
"outputs": [],
20+
"source": [
21+
"# Create and configure our cluster object\n",
22+
"# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n",
23+
"cluster = Cluster(ClusterConfiguration(\n",
24+
" name='raytest',\n",
25+
" namespace='default',\n",
26+
" head_cpus='500m',\n",
27+
" head_memory=2,\n",
28+
" head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n",
29+
" num_gpus=0,\n",
30+
" num_workers=2,\n",
31+
" min_cpus='250m',\n",
32+
" max_cpus=1,\n",
33+
" min_memory=1,\n",
34+
" max_memory=2,\n",
35+
" # image=\"\", # Optional Field \n",
36+
" write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n",
37+
" # local_queue=\"local-queue-name\" # Specify the local queue manually\n",
38+
"))"
39+
]
40+
},
41+
{
42+
"cell_type": "code",
43+
"execution_count": null,
44+
"id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200",
45+
"metadata": {},
46+
"outputs": [],
47+
"source": [
48+
"# Bring up the cluster\n",
49+
"cluster.up()"
50+
]
51+
},
52+
{
53+
"cell_type": "code",
54+
"execution_count": null,
55+
"id": "0d513912",
56+
"metadata": {},
57+
"outputs": [],
58+
"source": [
59+
"cluster.wait_ready()"
60+
]
61+
},
62+
{
63+
"cell_type": "code",
64+
"execution_count": null,
65+
"id": "0ce99d84",
66+
"metadata": {},
67+
"outputs": [],
68+
"source": [
69+
"cluster.status()"
70+
]
71+
},
72+
{
73+
"cell_type": "code",
74+
"execution_count": null,
75+
"id": "1767a342",
76+
"metadata": {},
77+
"outputs": [],
78+
"source": [
79+
"cluster.down()"
80+
]
81+
},
82+
{
83+
"cell_type": "code",
84+
"execution_count": null,
85+
"id": "7e9152ce",
86+
"metadata": {},
87+
"outputs": [],
88+
"source": []
89+
}
90+
],
91+
"metadata": {
92+
"kernelspec": {
93+
"display_name": "Python 3 (ipykernel)",
94+
"language": "python",
95+
"name": "python3"
96+
},
97+
"language_info": {
98+
"codemirror_mode": {
99+
"name": "ipython",
100+
"version": 3
101+
},
102+
"file_extension": ".py",
103+
"mimetype": "text/x-python",
104+
"name": "python",
105+
"nbconvert_exporter": "python",
106+
"pygments_lexer": "ipython3",
107+
"version": "3.9.19"
108+
},
109+
"vscode": {
110+
"interpreter": {
111+
"hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac"
112+
}
113+
}
114+
},
115+
"nbformat": 4,
116+
"nbformat_minor": 5
117+
}

0 commit comments

Comments
 (0)