Skip to content

Commit deb5680

Browse files
yatarkanyatarkan
authored
Enable GPU tests runner for notebooks nightly (#2039)
CVS-140398
1 parent 6494bbb commit deb5680

File tree

15 files changed

+442
-454
lines changed

15 files changed

+442
-454
lines changed

‎.ci/aggregate_notebooks_reports.py

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,29 @@
1010
class ValidationMatrix:
1111
os = ("ubuntu-20.04", "ubuntu-22.04", "windows-2019", "macos-12")
1212
python = ("3.8", "3.9", "3.10")
13+
device = ("cpu", "gpu")
1314

1415
@classmethod
1516
def values(cls):
16-
return product(cls.os, cls.python)
17+
return product(cls.device, cls.os, cls.python)
1718

1819

19-
def get_report_file_path(os: str, python: str) -> Path:
20-
return Path(REPORTS_DIR) / f"{os}-{python}" / "test_report.csv"
20+
def get_report_file_path(device: str, os: str, python: str) -> Path:
21+
return Path(REPORTS_DIR) / f"{device}-{os}-{python}" / "test_report.csv"
2122

2223

2324
def get_default_status_dict(notebook_name: str) -> Dict:
2425
default_status = None
2526

26-
def _get_python_status_dict():
27+
def _get_python_dict():
2728
return dict((python, default_status) for python in ValidationMatrix.python)
2829

30+
def _get_device_dict():
31+
return dict((device, _get_python_dict()) for device in ValidationMatrix.device)
32+
2933
return {
3034
"name": notebook_name,
31-
"status": dict((os, _get_python_status_dict()) for os in ValidationMatrix.os),
35+
"status": dict((os, _get_device_dict()) for os in ValidationMatrix.os),
3236
}
3337

3438

@@ -39,8 +43,11 @@ def write_json_file(filename: str, data: Dict):
3943

4044
def main():
4145
NOTEBOOKS_STATUS_MAP = {}
42-
for os, python in ValidationMatrix.values():
43-
report_file_path = get_report_file_path(os, python)
46+
for device, os, python in ValidationMatrix.values():
47+
if device == "gpu" and not os.startswith("ubuntu"):
48+
print(f'Tests are not available for "{device}" device and "{os}".')
49+
continue
50+
report_file_path = get_report_file_path(device, os, python)
4451
if not report_file_path.exists():
4552
print(f'Report file "{report_file_path}" does not exists.')
4653
continue
@@ -51,7 +58,7 @@ def main():
5158
status = row["status"]
5259
if name not in NOTEBOOKS_STATUS_MAP:
5360
NOTEBOOKS_STATUS_MAP[name] = get_default_status_dict(name)
54-
NOTEBOOKS_STATUS_MAP[name]["status"][os][python] = status
61+
NOTEBOOKS_STATUS_MAP[name]["status"][os][device][python] = status
5562
write_json_file(Path(REPORTS_DIR) / "notebooks-status-map.json", NOTEBOOKS_STATUS_MAP)
5663

5764

‎.ci/validate_notebooks.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ def prepare_test_plan(test_list: Optional[List[str]], ignore_list: List[str], nb
9393
raise ValueError(
9494
f"Ignore list items should be relative to repo root (e.g. 'notebooks/subdir/notebook.ipynb').\nInvalid ignored notebooks: {ignored_notebooks}"
9595
)
96+
ignored_notebooks = sorted(ignored_notebooks)
9697
print(f"Ignored notebooks: {ignored_notebooks}")
9798

9899
testing_notebooks: List[Path] = []
@@ -121,7 +122,7 @@ def prepare_test_plan(test_list: Optional[List[str]], ignore_list: List[str], nb
121122
"Testing notebooks should be provided to '--test_list' argument as a txt file or should be empty to test all notebooks.\n"
122123
f"Received test list: {test_list}"
123124
)
124-
testing_notebooks = list(set(testing_notebooks))
125+
testing_notebooks = sorted(list(set(testing_notebooks)))
125126
print(f"Testing notebooks: {testing_notebooks}")
126127

127128
for notebook in test_plan:

‎.github/workflows/build_treon_reusable.yml

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
name: Build Treon Reusable Workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runs_on:
7+
required: true
8+
type: string
9+
python:
10+
required: true
11+
type: string
12+
container:
13+
required: false
14+
type: string
15+
default: null
16+
test_only_changed:
17+
required: false
18+
type: boolean
19+
default: false
20+
21+
jobs:
22+
build_treon:
23+
runs-on: ${{ inputs.runs_on }}
24+
container:
25+
image: ${{ inputs.container }}
26+
options: --device /dev/dri:/dev/dri --group-add 109 --group-add 44
27+
volumes:
28+
- /dev/dri:/dev/dri
29+
steps:
30+
- name: Set env variables
31+
uses: actions/github-script@v7
32+
with:
33+
script: |
34+
const container = "${{ inputs.container }}";
35+
const runsOn = "${{ inputs.runs_on }}";
36+
const osName = !container ? "${{ inputs.runs_on }}" : container.replace(':', '-');
37+
const testDevice = runsOn === 'gpu' ? 'gpu' : 'cpu';
38+
const testReportDir = `${testDevice}-${osName}-${{ inputs.python }}`;
39+
core.exportVariable('OS_NAME', osName);
40+
core.exportVariable('TEST_DEVICE', testDevice);
41+
core.exportVariable('TEST_REPORT_DIR', testReportDir);
42+
core.exportVariable('GIT_CLONE_PROTECTION_ACTIVE', 'false');
43+
44+
#### Installation/preparation ####
45+
#
46+
# These steps are also copied to convert_notebooks.yml
47+
- name: Checkout repository
48+
uses: actions/checkout@v4
49+
50+
- name: Get changed files
51+
if: ${{ inputs.test_only_changed }}
52+
id: changed-files
53+
uses: tj-actions/changed-files@v44
54+
with:
55+
files: |
56+
notebooks/*/**
57+
requirements.txt
58+
59+
- name: List all changed files
60+
if: ${{ inputs.test_only_changed }}
61+
shell: bash
62+
run: |
63+
touch test_notebooks.txt
64+
changed_files="${{ steps.changed-files.outputs.all_changed_files }}"
65+
changed_files=$(echo $changed_files | tr '\\' '/')
66+
for file in $changed_files; do
67+
echo "$file was changed"
68+
echo $file >> test_notebooks.txt
69+
done
70+
71+
- name: Dotenv Action
72+
id: dotenv
73+
uses: xom9ikk/[email protected]
74+
with:
75+
path: ./.github/workflows
76+
77+
- name: Install required packages
78+
if: ${{ !inputs.container }}
79+
shell: bash
80+
run: |
81+
if [ "$RUNNER_OS" == "Linux" ]; then
82+
sudo apt-get update -y
83+
sudo apt-get install libsndfile1 -y
84+
fi
85+
86+
- name: Install required packages (container)
87+
if: ${{ inputs.container }}
88+
shell: bash
89+
env:
90+
DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
91+
TZ: 'Europe/London' # to prevent tzdata from waiting user input
92+
run: |
93+
if [ "$RUNNER_OS" == "Linux" ]; then
94+
apt-get update -y
95+
apt-get install git curl wget libsndfile1 libssl-dev unzip libsqlite3-dev libedit-dev libgl1 libgl1-mesa-glx libglib2.0-0 -y
96+
wget https://raw.githubusercontent.com/openvinotoolkit/openvino/master/scripts/install_dependencies/install_openvino_dependencies.sh
97+
chmod +x ./install_openvino_dependencies.sh
98+
./install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y
99+
fi
100+
101+
- name: Install GPU Drivers
102+
if: ${{ inputs.container }}
103+
shell: bash
104+
run: |
105+
wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.15985.7/intel-igc-core_1.0.15985.7_amd64.deb
106+
wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.15985.7/intel-igc-opencl_1.0.15985.7_amd64.deb
107+
wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-level-zero-gpu-dbgsym_1.3.28454.6_amd64.ddeb
108+
wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-level-zero-gpu_1.3.28454.6_amd64.deb
109+
wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-opencl-icd-dbgsym_24.05.28454.6_amd64.ddeb
110+
wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-opencl-icd_24.05.28454.6_amd64.deb
111+
wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/libigdgmm12_22.3.11_amd64.deb
112+
dpkg -i *.deb
113+
114+
- name: Set up Python
115+
uses: actions/setup-python@v5
116+
with:
117+
python-version: '${{ inputs.python }}'
118+
env:
119+
AGENT_TOOLSDIRECTORY: ${{ inputs.container && '/opt/hostedtoolcache' || '' }}
120+
121+
- name: Cache Pip Packages
122+
id: cachepip
123+
uses: actions/cache@v3
124+
with:
125+
path: |
126+
pipcache
127+
key: ${{ env.PIP_CACHE_KEY }}-${{ env.OS_NAME }}-${{ inputs.python }}
128+
129+
# Cache specific files to reduce downloads or prevent network issues
130+
- name: Cache Files
131+
id: cachefiles
132+
uses: actions/cache@v3 # TODO Consider updating cache action to v4
133+
with:
134+
path: |
135+
# NOTE: when modifying cache paths, update FILES_CACHE_KEY in .env
136+
# and change cache paths in both treon.yml and convert_notebooks.yml
137+
case_00030.zip
138+
notebooks/ct-segmentation-quantize/kits19_frames_1
139+
notebooks/pytorch-post-training-quantization-nncf/output/tiny-imagenet-200.zip
140+
# omz cache location is set to this with test_replace
141+
notebooks/optical-character-recognition/open_model_zoo_cache
142+
notebooks/ct-scan-live-inference/kits19_frames_1
143+
notebooks/pytorch-quantization-aware-training/data/tiny-imagenet-200.zip
144+
key: ${{ env.FILES_CACHE_KEY }}
145+
146+
# PaddleGAN stores cache in ppgan directory in CACHE_DIR
147+
- name: Set CACHE_DIR
148+
shell: bash
149+
run: |
150+
python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))'
151+
# replace backslashes with forward slashes for Windows paths
152+
python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))' | sed -e 's/\\/\//g' >> $GITHUB_ENV
153+
154+
# PaddleHub stores cache in directory pointed to by HUB_HOME environment variable
155+
- name: Set HUB_HOME
156+
shell: bash
157+
run: |
158+
echo HUB_HOME=${{ env.CACHE_DIR }}/.paddlehub >> $GITHUB_ENV
159+
160+
# Cache PaddlePaddle cache directories to prevent CI failing due to network/download issues
161+
- name: Cache PaddlePaddle cache directories
162+
id: cacheusercache
163+
uses: actions/cache@v3
164+
with:
165+
path: |
166+
${{ env.HUB_HOME }}
167+
${{ env.CACHE_DIR }}/paddle
168+
${{ env.CACHE_DIR }}/ppgan
169+
key: ${{ env.USER_CACHE_KEY }}-${{ env.OS_NAME }}
170+
171+
- name: Cache openvino packages
172+
if: steps.cachepip.outputs.cache-hit != 'true'
173+
run: |
174+
python -m pip install --upgrade pip
175+
mkdir pipcache
176+
python -m pip install --cache-dir pipcache --no-deps openvino openvino-dev nncf
177+
cp -r pipcache pipcache_openvino
178+
# python -m pip uninstall -y openvino openvino-dev nncf
179+
180+
# Download a small dataset to use for testing purposes in monai-kidney training notebook
181+
- name: Download CT files
182+
if: steps.cachefiles.outputs.cache-hit != 'true'
183+
run: |
184+
curl -O https://storage.openvinotoolkit.org/data/test_data/openvino_notebooks/kits19/case_00030.zip
185+
- name: Copy CT files
186+
run: |
187+
mkdir notebooks/ct-segmentation-quantize/kits19
188+
mkdir notebooks/ct-segmentation-quantize/kits19/kits19_frames
189+
unzip case_00030.zip
190+
cp -r case_00030 case_00001
191+
mv case_00030 notebooks/ct-segmentation-quantize/kits19/kits19_frames
192+
mv case_00001 notebooks/ct-segmentation-quantize/kits19/kits19_frames
193+
194+
# Prevent test aborting by timeout for 'meter-reader' and 'paddle-ocr-webcam' notebooks
195+
- name: Download long loading models for 'meter-reader' and 'paddle-ocr-webcam' notebooks
196+
run: |
197+
mkdir notebooks/meter-reader/model
198+
curl -o notebooks/meter-reader/model/meter_det_model.tar.gz 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/meter-reader/meter_det_model.tar.gz'
199+
curl -o notebooks/meter-reader/model/meter_seg_model.tar.gz 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/meter-reader/meter_seg_model.tar.gz'
200+
mkdir notebooks/paddle-ocr-webcam/model
201+
curl -o notebooks/paddle-ocr-webcam/model/ch_PP-OCRv3_det_infer.tar 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/paddle-ocr/ch_PP-OCRv3_det_infer.tar'
202+
curl -o notebooks/paddle-ocr-webcam/model/ch_PP-OCRv3_rec_infer.tar 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/paddle-ocr/ch_PP-OCRv3_rec_infer.tar'
203+
204+
- name: Install python dependencies
205+
run: |
206+
python -m pip install --upgrade pip
207+
python -m pip install -r .ci/dev-requirements.txt --cache-dir pipcache
208+
python -m ipykernel install --user --name openvino_env
209+
210+
# Cache OpenVINO packages (`mv` works cross-platform)
211+
- name: Make pipcache directory with OpenVINO packages
212+
if: steps.cachepip.outputs.cache-hit != 'true'
213+
run: |
214+
mv pipcache pipcache_full
215+
mv pipcache_openvino pipcache
216+
217+
# Create list of installed pip packages that can be downloaded as artifacts
218+
# to verify the exact environment of a specific test run
219+
- name: pip freeze
220+
run: |
221+
python -m pip freeze
222+
python -m pip freeze > pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
223+
- name: Upload pip freeze artifact
224+
uses: actions/upload-artifact@v4
225+
with:
226+
name: pip-freeze-${{ env.TEST_DEVICE }}-${{ env.OS_NAME }}-${{ inputs.python }}
227+
path: pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
228+
#### End installation/preparation
229+
230+
- name: Check install
231+
run: |
232+
python check_install.py
233+
234+
# Patch long running cells to run faster
235+
- name: Patch notebooks
236+
run: |
237+
python .ci/patch_notebooks.py . -td ${{ env.TEST_DEVICE }}
238+
239+
# Test that JupyterLab runs without errors
240+
- name: Test Jupyterlab
241+
run: |
242+
jupyter lab notebooks --help
243+
244+
# Main notebooks test. Verifies that all notebooks run without errors
245+
- name: Analysing with treon (Windows) python > 3.8
246+
if: runner.os == 'Windows' && inputs.python != '3.8'
247+
shell: bash
248+
run: |
249+
python .ci/validate_notebooks.py \
250+
${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
251+
--ignore_list .ci/ignore_treon_win.txt \
252+
--report_dir test_report/${{ env.TEST_REPORT_DIR }} \
253+
--move_notebooks_dir c:/notebooks \
254+
--timeout 1200
255+
- name: Analysing with treon (Windows) python 3.8
256+
if: runner.os == 'Windows' && inputs.python == '3.8'
257+
shell: bash
258+
run: |
259+
python .ci/validate_notebooks.py \
260+
${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
261+
--ignore_list .ci/ignore_treon_win.txt .ci/ignore_treon_py38.txt \
262+
--report_dir test_report/${{ env.TEST_REPORT_DIR }} \
263+
--move_notebooks_dir c:/notebooks \
264+
--timeout 1200
265+
- name: Analysing with treon (Linux) python > 3.8
266+
if: runner.os == 'Linux' && inputs.python != '3.8'
267+
shell: bash
268+
run: |
269+
python .ci/validate_notebooks.py \
270+
${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
271+
--ignore_list .ci/ignore_treon_linux.txt ${{ env.TEST_DEVICE == 'gpu' && '.ci/heavy_ubuntu_gpu.txt' || '' }} \
272+
--report_dir test_report/${{ env.TEST_REPORT_DIR }} \
273+
--timeout 1200
274+
- name: Analysing with treon (Linux) python 3.8
275+
if: runner.os == 'Linux' && inputs.python == '3.8'
276+
shell: bash
277+
run: |
278+
python .ci/validate_notebooks.py \
279+
${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
280+
--ignore_list .ci/ignore_treon_linux.txt .ci/ignore_treon_py38.txt ${{ env.TEST_DEVICE == 'gpu' && '.ci/heavy_ubuntu_gpu.txt' || '' }} \
281+
--report_dir test_report/${{ env.TEST_REPORT_DIR }} \
282+
--timeout 1200
283+
- name: Analysing with treon (MacOS) python > 3.8
284+
if: runner.os == 'MacOS' && inputs.python != '3.8'
285+
shell: bash
286+
run: |
287+
python .ci/validate_notebooks.py \
288+
${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
289+
--ignore_list .ci/ignore_treon_mac.txt \
290+
--report_dir test_report/${{ env.TEST_REPORT_DIR }} \
291+
--timeout 1200
292+
- name: Analysing with treon (MacOS) python 3.8
293+
if: runner.os == 'MacOS' && inputs.python == '3.8'
294+
shell: bash
295+
run: |
296+
python .ci/validate_notebooks.py \
297+
${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
298+
--ignore_list .ci/ignore_treon_mac.txt .ci/ignore_treon_py38.txt \
299+
--report_dir test_report/${{ env.TEST_REPORT_DIR }} \
300+
--timeout 1200
301+
302+
- name: Archive notebook test report
303+
if: always()
304+
uses: actions/upload-artifact@v4
305+
with:
306+
name: test_report-${{ env.TEST_REPORT_DIR }}
307+
path: test_report/
308+
309+
# Show the cache after running the notebooks
310+
- name: Show cache
311+
if: runner.os != 'Windows'
312+
run: |
313+
ls -laR ${{ env.CACHE_DIR }}
314+
du -sh ${{ env.CACHE_DIR }}

‎.github/workflows/codecheck.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
uses: actions/checkout@v4
3636
- name: Dotenv Action
3737
id: dotenv
38-
uses: xom9ikk/dotenv@v1.0.2
38+
uses: xom9ikk/dotenv@v2.3.0
3939
with:
4040
path: ./.github/workflows
4141
- name: Set up Python

0 commit comments

Comments
 (0)