[CI] [GHA] Use self-hosted runners

.ci/precommit_list.txt

@@ -1 +1,2 @@
-notebooks/hello-world/hello-world.ipynb
+notebooks/hello-world/hello-world.ipynb
+notebooks/async-api/async-api.ipynb

.ci/validate_notebooks.py

@@ -14,6 +14,8 @@
 from clonevirtualenv import clone_virtualenv
 import traceback
 import tempfile
+import nbformat
+from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
 
 from argparse import ArgumentParser
 from pathlib import Path
@@ -65,8 +67,8 @@ class NotebookReport(TypedDict):
 def parse_arguments():
     parser = ArgumentParser()
     parser.add_argument("--ignore_config", required=False, default=SKIPPED_NOTEBOOKS_CONFIG_FILENAME)
-    parser.add_argument("--ignore_list", required=False, nargs="+")
-    parser.add_argument("--test_list", required=False, nargs="+")
+    parser.add_argument("--ignore_list", required=False, nargs="+", default=None)
+    parser.add_argument("--test_list", required=False, nargs="+", default=None)
     parser.add_argument("--os", type=validation_config_arg("os"))
     parser.add_argument("--python", type=validation_config_arg("python"))
     parser.add_argument("--device", type=validation_config_arg("device"))
@@ -427,6 +429,57 @@ def kill_process_tree(pid):
         print(f"Error killing process tree PID {pid}: {e}", flush=True)
 
 
+def execute_notebook_with_nbconvert(notebook_path: Path, 
+                                    timeout: int) -> tuple[int, float]:
+    """
+    Execute a Jupyter notebook using nbconvert's ExecutePreprocessor.
+
+    Args:
+        notebook_path: Path to the notebook file to execute
+        timeout: Timeout in seconds for cell execution
+
+    Returns:
+        tuple: (return_code, duration) where return_code is 0 for success,
+               -42 for timeout, or 1 for other errors
+    """
+    start_time = time.perf_counter()
+    retcode = 0
+    notebook_original_name = notebook_path.name
+    test_notebook_path = notebook_path.parent / f"test_{notebook_original_name}"
+    print(f"Executing notebook with nbconvert: {test_notebook_path}", flush=True)
+    with open(test_notebook_path, 'r', encoding='utf-8') as f:
+        nb = nbformat.read(f, as_version=4)
+    try:
+        # Ensure environment variables (including PATH with MSVC DLLs) are passed to kernel
+        ep = ExecutePreprocessor(
+            timeout=timeout,
+        )
+
+        # Pass current environment to ensure DLL paths and other configs are available
+        resources = {
+            'metadata': {
+                'path': str(notebook_path.parent)
+            }
+        }
+        ep.preprocess(nb, resources)           
+        print(f"Notebook executed successfully", flush=True)
+
+    except CellExecutionError as e:
+        print(f"Notebook execution failed with cell error:\n{e}", flush=True)
+        retcode = 1
+    except TimeoutError:
+        print(f"Notebook execution timed out after {timeout} seconds", flush=True)
+        retcode = -42  # Special timeout exit code
+
+    except Exception as e:
+        print(f"Notebook execution failed with error:\n{e}", flush=True)
+        print(traceback.format_exc(), flush=True)
+        retcode = 1
+
+    duration = time.perf_counter() - start_time
+    return retcode, duration
+
+
 def run_subprocess_with_timeout(cmd, timeout, shell=False, description="Process"):
     """
     Run a subprocess with real-time output and timeout protection.
@@ -583,14 +636,11 @@ def run_test(
                     return result
 
                 collect_python_packages(python_executable, report_dir / (patched_notebook.stem + "_env_before.txt"))
+                print(f'Python executable for notebook test: {python_executable}', flush=True)
 
-                main_command = [python_executable, "-m", "treon", "--verbose", str(patched_notebook)]
-
-                retcode, duration = run_subprocess_with_timeout(
-                    main_command,
-                    timeout,
-                    shell=(platform.system() == "Windows"),
-                    description=f"Notebook test [{patched_notebook.name}]",
+                retcode, duration = execute_notebook_with_nbconvert(
+                    notebook_path,
+                    timeout
                 )
 
                 ov_version_after = get_pip_package_version(python_executable, "openvino", "OpenVINO after notebook execution", "OpenVINO is missing")
@@ -697,6 +747,7 @@ def write_single_notebook_report(
     job_name: str,
     device: str,
     saving_dir: Path,
+    notebook_content: str,
 ) -> Path:
     report_file = saving_dir / notebook_name.replace(".ipynb", ".json")
     report = {
@@ -708,6 +759,7 @@ def write_single_notebook_report(
         "ov_version_after": ov_version_after,
         "job_name": job_name,
         "device_used": device,
+        "notebook_content": notebook_content
     }
     with report_file.open("w") as f:
         json.dump(report, f)
@@ -784,8 +836,10 @@ def main():
             if args.collect_reports:
                 job_name = args.job_name or "Unknown"
                 device = args.device or "Unknown"
+                print(f'Notebook directory content: {list(report["path"].parent.iterdir())}', flush=True)
+                notebook_content = json.loads(report["path"].parent.joinpath(f'test_{report["path"].name}').read_text(encoding="utf-8"))
                 report_path = write_single_notebook_report(
-                    base_version, patched_notebook, status_code, duration, ov_version_before, ov_version_after, job_name, device, reports_dir
+                    base_version, patched_notebook, status_code, duration, ov_version_before, ov_version_after, job_name, device, reports_dir, notebook_content=notebook_content
                 )
                 if args.upload_to_db:
                     cmd = [sys.executable, args.upload_to_db, report_path]

.github/workflows/build_treon_reusable.yml

@@ -29,28 +29,33 @@ on:
 permissions:
   contents: read
 
+env:
+  PIP_CACHE_PATH_LINUX: /mount/caches/pip
+  PIP_CACHE_PATH_WIN: "C:/mount/caches/pip/win"
+
 jobs:
   build_treon:
     runs-on: ${{ inputs.runs_on }}
     container:
       image: ${{ inputs.container }}
-      options: --device /dev/dri:/dev/dri --group-add 109 --group-add 44
-      volumes:
-        - /dev/dri:/dev/dri
+      # options: --device /dev/dri:/dev/dri --group-add 109 --group-add 44
+      # volumes:
+      #   - /dev/dri:/dev/dri
     steps:
       - name: Set env variables
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           script: |
             const container = "${{ inputs.container }}";
             const runsOn = "${{ inputs.runs_on }}";
-            const osName = !container ? "${{ inputs.runs_on }}" : container.replace(':', '-');
+            const osName = runsOn.includes('linux') ? 'ubuntu-22.04' : 'windows-2022';
             const testDevice = runsOn === 'gpu' ? 'gpu' : 'cpu';
             const testReportDir = `${testDevice}-${osName}-${{ inputs.python }}`;
             core.exportVariable('OS_NAME', osName);
             core.exportVariable('TEST_DEVICE', testDevice);
             core.exportVariable('TEST_REPORT_DIR', testReportDir);
             core.exportVariable('GIT_CLONE_PROTECTION_ACTIVE', 'false');
+            core.exportVariable('PIP_CACHE_PATH', runsOn.includes('linux') ? "${{ env.PIP_CACHE_PATH_LINUX }}" : "${{ env.PIP_CACHE_PATH_WIN }}");
 
       #### Installation/preparation ####
       #
@@ -101,19 +106,15 @@ jobs:
         uses: xom9ikk/dotenv@ac290ca23a42155a0cba1031d23afa46240116a9 # v2.3.0
         with:
           path: ./.github/workflows
+
       # Packages that notebooks need to run in plain os
-      - name: Install required packages
-        if: ${{ !inputs.container }}
+      - name: Install required packages (Linux)
+        if: ${{ !inputs.container && runner.os == 'Linux' }}
         shell: bash
         run: |
-          if [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt-get update -y
-            sudo apt-get install libsndfile1 ffmpeg espeak-ng -y
-          elif [ "$RUNNER_OS" == "macOS" ]; then
-            brew install ffmpeg espeak-ng
-          elif [ "$RUNNER_OS" == "Windows" ]; then
-            choco install ffmpeg-full --version 7.1.1 -y
-          fi
+          sudo apt-get update -y
+          sudo apt-get install libsndfile1 ffmpeg espeak-ng -y
+
       # Packages that notebooks need to run in docker container
       - name: Install required packages (container)
         if: ${{ inputs.container }}
@@ -131,7 +132,7 @@ jobs:
           fi
 
       - name: Install GPU Drivers
-        if: ${{ inputs.container }}
+        if: ${{ env.TEST_DEVICE == 'gpu' && runner.os == 'Linux' }}
         shell: bash
         run: |
           wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.15985.7/intel-igc-core_1.0.15985.7_amd64.deb
@@ -143,20 +144,13 @@ jobs:
           wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/libigdgmm12_22.3.11_amd64.deb
           dpkg -i *.deb
 
-      - name: Set up Python
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+      - name: Setup Python ${{ inputs.python }}
+        uses: openvinotoolkit/openvino/.github/actions/setup_python@e4b65a0ab6c946c72d56240223620837e6d34bf8
         with:
-          python-version: '${{ inputs.python }}'
-        env:
-          AGENT_TOOLSDIRECTORY: ${{ inputs.container && '/opt/hostedtoolcache' || '' }}
-
-      - name: Cache Pip Packages
-        id: cachepip
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
-        with:
-          path: |
-            pipcache
-          key: ${{ env.PIP_CACHE_KEY }}-${{ env.OS_NAME }}-${{ inputs.python }}
+          version: ${{ inputs.python }}
+          pip-cache-path: ${{ env.PIP_CACHE_PATH }}
+          should-setup-pip-paths: 'true'
+          self-hosted-runner: 'true'
 
       # Cache specific files to reduce downloads or prevent network issues
       - name: Cache Files
@@ -177,17 +171,12 @@ jobs:
 
       # PaddleGAN stores cache in ppgan directory in CACHE_DIR
       - name: Set CACHE_DIR
-        shell: bash
-        run: |
-          python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))'
-          # replace backslashes with forward slashes for Windows paths
-          python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))' | sed -e 's/\\/\//g' >> $GITHUB_ENV
+        run: python${{ inputs.python }} -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))' | sed -e 's/\\/\//g' >> $GITHUB_ENV
 
       # PaddleHub stores cache in directory pointed to by HUB_HOME environment variable
       - name: Set HUB_HOME
         shell: bash
-        run: |
-          echo HUB_HOME=${{ env.CACHE_DIR }}/.paddlehub >> $GITHUB_ENV
+        run: echo HUB_HOME=${{ env.CACHE_DIR }}/.paddlehub >> $GITHUB_ENV
 
       # Cache PaddlePaddle cache directories to prevent CI failing due to network/download issues
       - name: Cache PaddlePaddle cache directories
@@ -203,17 +192,17 @@ jobs:
       - name: Cache openvino packages
         if: steps.cachepip.outputs.cache-hit != 'true'
         run: |
-          python -m pip install --upgrade pip
+          python${{ inputs.python }} -m pip install --upgrade pip
           mkdir pipcache
-          python -m pip install --cache-dir pipcache --no-deps openvino nncf
+          python${{ inputs.python }} -m pip install --cache-dir pipcache --no-deps openvino nncf
           cp -r pipcache pipcache_openvino
-          # python -m pip uninstall -y openvino nncf
+          # python${{ inputs.python }} -m pip uninstall -y openvino nncf
 
       # Download a small dataset to use for testing purposes in monai-kidney training notebook
       - name: Download CT files
         if: steps.cachefiles.outputs.cache-hit != 'true'
-        run: |
-          curl -O https://storage.openvinotoolkit.org/data/test_data/openvino_notebooks/kits19/case_00030.zip
+        run: curl -O https://storage.openvinotoolkit.org/data/test_data/openvino_notebooks/kits19/case_00030.zip
+
       - name: Copy CT files
         run: |
           mkdir notebooks/ct-segmentation-quantize/kits19
@@ -235,71 +224,35 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install -r .ci/dev-requirements.txt --cache-dir pipcache
-          python -m ipykernel install --user --name openvino_env
-
-      # Cache OpenVINO packages (`mv` works cross-platform)
-      - name: Make pipcache directory with OpenVINO packages
-        if: steps.cachepip.outputs.cache-hit != 'true'
-        run: |
-          mv pipcache pipcache_full
-          mv pipcache_openvino pipcache
+          python${{ inputs.python }} -m pip install --upgrade pip wheel setuptools
+          python${{ inputs.python }} -m pip install -r .ci/dev-requirements.txt
+          python${{ inputs.python }} -m ipykernel install --user
 
       # Create list of installed pip packages that can be downloaded as artifacts
       # to verify the exact environment of a specific test run
-      - name: pip freeze
-        run: |
-          python -m pip freeze
-          python -m pip freeze > pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
-      - name: Upload pip freeze artifact
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
-        with:
-          name: pip-freeze-${{ env.TEST_DEVICE }}-${{ env.OS_NAME }}-${{ inputs.python }}
-          path: pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
+      # - name: pip freeze
+      #   run: |
+      #     python${{ inputs.python }} -m pip freeze
+      #     python${{ inputs.python }} -m pip freeze > pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
+      # - name: Upload pip freeze artifact
+      #   uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      #   with:
+      #     name: pip-freeze-${{ env.TEST_DEVICE }}-${{ env.OS_NAME }}-${{ inputs.python }}
+      #     path: pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
       #### End installation/preparation
 
       - name: Check install
-        run: |
-          python check_install.py
+        run: python${{ inputs.python }} check_install.py
 
       # Patch long running cells to run faster
       - name: Patch notebooks
-        run: |
-          python .ci/patch_notebooks.py . -td ${{ env.TEST_DEVICE }}
+        run: python${{ inputs.python }} .ci/patch_notebooks.py . -td ${{ env.TEST_DEVICE }}
 
       # Test that JupyterLab runs without errors
       - name: Test Jupyterlab
-        run: |
-          jupyter lab notebooks --help
+        run: jupyter lab notebooks --help
 
       # Main notebooks test. Verifies that all notebooks run without errors
-      - name: Analysing with treon (Windows)
-        if: runner.os == 'Windows'
-        shell: bash
-        run: |
-          IGNORE_ARGS=""
-          if [[ -n "${{ inputs.ignore_list }}" ]]; then
-            IGNORE_ARGS="$IGNORE_ARGS ${{ inputs.ignore_list }}"
-          fi
-          if [[ "${{ env.TEST_DEVICE }}" == "gpu" ]]; then
-             IGNORE_ARGS="$IGNORE_ARGS .ci/heavy_win_gpu.txt"
-          fi
-
-          if [[ -n "$IGNORE_ARGS" ]]; then
-            IGNORE_ARGS="--ignore_list $IGNORE_ARGS"
-          fi
-
-          python .ci/validate_notebooks.py \
-            --os ${{ env.OS_NAME }} \
-            --python ${{ inputs.python }} \
-            --device ${{ env.TEST_DEVICE }} \
-            ${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
-            $IGNORE_ARGS \
-            --report_dir test_report/${{ env.TEST_REPORT_DIR }} \
-            --move_notebooks_dir c:/notebooks \
-            ${{ inputs.separate_venv && '--separate_venv' || '' }} \
-            --timeout 1200
       - name: Analysing with treon (Linux)
         if: runner.os == 'Linux'
         shell: bash
@@ -316,7 +269,7 @@ jobs:
             IGNORE_ARGS="--ignore_list $IGNORE_ARGS"
           fi
 
-          python .ci/validate_notebooks.py \
+          python${{ inputs.python }} .ci/validate_notebooks.py \
             --os ${{ env.OS_NAME }} \
             --python ${{ inputs.python }} \
             --device ${{ env.TEST_DEVICE }} \
@@ -325,11 +278,12 @@ jobs:
             --report_dir test_report/${{ env.TEST_REPORT_DIR }} \
             ${{ inputs.separate_venv && '--separate_venv' || '' }} \
             --timeout 1200
+
       - name: Analysing with treon (MacOS)
         if: runner.os == 'MacOS'
         shell: bash
         run: |
-          python .ci/validate_notebooks.py \
+          python${{ inputs.python }} .ci/validate_notebooks.py \
             --os ${{ env.OS_NAME }} \
             --python ${{ inputs.python }} \
             --device ${{ env.TEST_DEVICE }} \
@@ -347,7 +301,7 @@ jobs:
 
       # Show the cache after running the notebooks
       - name: Show cache
-        if: runner.os != 'Windows'
+        if: 'false'
         run: |
           ls -laR ${{ env.CACHE_DIR }}
           du -sh ${{ env.CACHE_DIR }}