Cell execution telemetry (#2610)

Ticket: CVS-149317 CVS-160620

.ci/check_notebooks.py

@@ -4,6 +4,7 @@
 from patch_notebooks import DEVICE_WIDGET, DEVICE_WIDGET_NEW
 from install_instructions import check_install_instructions
 from scarf_pixel import check_scarf_tag
+from telemetry_snippet import check_telemetry_snippet
 from pathlib import Path
 
 NOTEBOOKS_ROOT = Path(__file__).resolve().parents[1]
@@ -34,6 +35,7 @@ def main():
     no_tocs = []
     no_device = []
     no_scarf_tag = []
+    no_telemetry_snippet = []
     no_install_instructions = []
 
     def complain(message):
@@ -73,6 +75,9 @@ def complain(message):
             if not check_scarf_tag(nb_path):
                 no_scarf_tag.append(str(nb_path.relative_to(NOTEBOOKS_ROOT)))
                 complain(f"FAILED: {nb_path.relative_to(NOTEBOOKS_ROOT)}: Scarf Pixel tag is not found")
+            if not check_telemetry_snippet(nb_path):
+                no_telemetry_snippet.append(str(nb_path.relative_to(NOTEBOOKS_ROOT)))
+                complain(f"FAILED: {nb_path.relative_to(NOTEBOOKS_ROOT)}: telemetry snippet is not found")
             if not check_install_instructions(nb_path):
                 no_install_instructions.append(str(nb_path.relative_to(NOTEBOOKS_ROOT)))
                 complain(f"FAILED: {nb_path.relative_to(NOTEBOOKS_ROOT)}: Install Instructions section is not found")
@@ -94,6 +99,11 @@ def complain(message):
             print("\n".join(no_scarf_tag))
             print("\nYou can generate Scarf Pixel tag with the following command:\n    python .ci/scarf_pixel.py -s <PATH>")
             print("==================================")
+        if no_telemetry_snippet:
+            print("NO TELEMETRY SNIPPET:")
+            print("\n".join(no_telemetry_snippet))
+            print("\nYou can generate telemetry snippet with the following command:\n    python .ci/telemetry_snippet.py -s <PATH>")
+            print("==================================")
         if no_install_instructions:
             print("NO INSTALL INSTRUCTIONS SECTION:")
             print("\n".join(no_install_instructions))

.ci/spellcheck/.pyspelling.wordlist.txt

@@ -687,6 +687,7 @@ PixelShuffleUpsampleNetwork
 pixelwise
 Pixtral
 pixtral
+PII
 PIL
 PNDM
 png

.ci/telemetry_snippet.py

@@ -0,0 +1,72 @@
+import argparse
+from pathlib import Path
+import nbformat
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _get_telemetry_snippet(notebook_path: Path) -> str:
+    if notebook_path.is_absolute():
+        notebook_path = notebook_path.relative_to(REPO_ROOT)
+    return "".join(
+        [
+            "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+            "from notebook_utils import collect_telemetry\n\n",
+            f'collect_telemetry("{notebook_path.name}")',
+        ]
+    )
+
+
+def check_telemetry_snippet(notebook_path: Path) -> bool:
+    if notebook_path.suffix != ".ipynb":
+        print(f'Invalid file extension at path "{str(notebook_path)}". Only .ipynb files are supported.')
+        return False
+    telemetry_snippet = _get_telemetry_snippet(notebook_path)
+    with open(notebook_path, "r") as notebook_file:
+        nb_node: nbformat.NotebookNode = nbformat.read(notebook_file, as_version=4)
+    for cell in nb_node["cells"]:
+        if cell["cell_type"] != "code":
+            continue
+        cell_content: str = cell["source"]
+        if telemetry_snippet in cell_content:
+            return True
+    return False
+
+
+def _add_telemetry_snippet(notebook_path: Path):
+    if notebook_path.suffix != ".ipynb":
+        raise Exception(f'Invalid file extension at path "{str(notebook_path)}". Only .ipynb files are supported.')
+    with open(notebook_path, "r") as fr:
+        nb_node: nbformat.NotebookNode = nbformat.read(fr, as_version=4)
+        # Find cell with notebook_utils
+        notebook_utils_url = "https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py"
+        for i, cell in enumerate(nb_node["cells"]):
+            if cell["cell_type"] != "code":
+                continue
+            cell_content: str = cell["source"]
+            if notebook_utils_url in cell_content:
+                nb_node["cells"][i]["source"] = cell_content + "\n\n" + _get_telemetry_snippet(notebook_path)
+                break
+    with open(notebook_path, "w") as fw:
+        nbformat.write(nb_node, fw)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-s",
+        "--source",
+        help="Specify the path to the notebook file, where telemetry snippet should be added",
+        required=True,
+    )
+
+    args = parser.parse_args()
+    file_path = Path(args.source)
+    if not file_path.exists():
+        print(f'File does not exist at path "{file_path}"')
+        exit(1)
+    if not file_path.is_file():
+        print(f"Provided path is not a file")
+        exit(1)
+    _add_telemetry_snippet(file_path)

CONTRIBUTING.md

@@ -132,7 +132,13 @@ To do this, there are a few requirements that all notebooks need to pass.
 5. Add **Table of content** to top of the Notebook, it helps to get quick fist understanding of content and ease of navigation in the dev environment. There is no need to think about it during development, it can be built or updated after changes with `.ci\table_of_content.py`. Just run the script with the parameter `-s/--source`, specifying a Notebook or a folder with several notebooks as value, the changes will be applied to all of them.
 6. Add **Installation Instructions** section to the top of the notebook (after "Table of content") and to the corresponding `README.md` file in the notebook directory. See existing notebooks for the reference.
 7. Add Scarf Pixel tag for analytics to the notebook (at the end of the first cell) and to the corresponding `README.md` file (to the end of the file). Add relative path to the notebook or `README.md` file as `path` URL query parameter. Example: `<img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=5b5a4db0-7875-4bfb-bdbd-01698b5b1a77&file=<RELATIVE_FILE_PATH>" />`. You can use the following command to generate the tag and add it to the file: `python .ci/scarf_pixel.py -s <PATH>`.
-8. In case if notebook has specific requirements on python version or OS, it should be noted on top of notebook (before any code blocks) using
+8. Add telemetry snippet to the notebook file (e.g. to the cell with fetching notebook_utils file) for tracking notebook execution. Add notebook file name as a parameter to `collect_telemetry()` function. You can use the following command to generate the snippet and add it to the notebook file: `python .ci/telemetry_snippet.py -s <PATH>`. Snippet example:
+   ```python
+   # Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry
+   from notebook_utils import collect_telemetry
+   collect_telemetry("<NOTEBOOK_NAME>.ipynb")
+   ```
+1.  In case if notebook has specific requirements on python version or OS, it should be noted on top of notebook (before any code blocks) using
    following colored block:
    ```
    <div class="alert alert-block alert-danger"> <b>Important note:</b> This notebook requires python >= 3.9. Please make sure that your environment fulfill to this requirement  before running it </div>

README.md

@@ -28,6 +28,7 @@ List of all notebooks is available in [index file](./notebooks/README.md).
 	- [To Launch all Notebooks](#to-launch-all-notebooks)
 - [🧹 Cleaning Up](#-cleaning-up)
 - [⚠️ Troubleshooting](#️-troubleshooting)
+- [📊 Telemetry](#-telemetry)
 - [📚 Additional Resources](#-additional-resources)
 - [🧑‍💻 Contributors](#-contributors)
 - [❓ FAQ](#-faq)
@@ -171,6 +172,30 @@ or create an [issue](https://github.com/openvinotoolkit/openvino_notebooks/issue
 - If OpenVINO is installed globally, do not run installation commands in a terminal where `setupvars.bat` or `setupvars.sh` are sourced.
 - For Windows installation, it is recommended to use _Command Prompt (`cmd.exe`)_, not _PowerShell_.
 
+[![-----------------------------------------------------](https://user-images.githubusercontent.com/10940214/155750931-fc094349-b6ec-4e1f-9f9a-113e67941119.jpg)]()
+<div id='-telemetry'></div>
+
+## 📊 Telemetry
+
+When you execute a notebook cell that contains `collect_telemetry()` function, telemetry data is collected to help us improve your experience.
+This data only indicates that the cell was executed and does **not** include any personally identifiable information (PII).
+
+By default, anonymous telemetry data is collected, limited solely to the execution of the notebook.
+This telemetry does **not** extend to any Intel software, hardware, websites, or products.
+
+If you prefer to disable telemetry, you can do so at any time by commenting out the specific line responsible for data collection in the notebook:
+```python
+# collect_telemetry(...)
+```
+Also you can disable telemetry collection by setting `SCARF_NO_ANALYTICS` or `DO_NOT_TRACK` environment variable to `1`:
+```bash
+export SCARF_NO_ANALYTICS=1
+# or
+export DO_NOT_TRACK=1
+```
+
+Scarf is used for telemetry purposes. Refer to [Scarf documentation](https://docs.scarf.sh/) to understand how the data is collected and processed.
+
 [![-----------------------------------------------------](https://user-images.githubusercontent.com/10940214/155750931-fc094349-b6ec-4e1f-9f9a-113e67941119.jpg)]()
 <div id='-additional-resource'></div>
 

notebooks/3D-pose-estimation-webcam/3D-pose-estimation.ipynb

@@ -135,7 +135,12 @@
     "        f.write(r.text)\n",
     "\n",
     "import notebook_utils as utils\n",
-    "import engine3js as engine"
+    "import engine3js as engine\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"3D-pose-estimation.ipynb\")"
    ]
   },
   {

notebooks/3D-segmentation-point-clouds/3D-segmentation-point-clouds.ipynb

@@ -73,7 +73,12 @@
     "    )\n",
     "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
     "\n",
-    "from notebook_utils import download_file, device_widget"
+    "from notebook_utils import download_file, device_widget\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"3D-segmentation-point-clouds.ipynb\")"
    ]
   },
   {

notebooks/action-recognition-webcam/action-recognition-webcam.ipynb

@@ -106,7 +106,12 @@
     "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
     "    )\n",
     "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
-    "import notebook_utils as utils"
+    "import notebook_utils as utils\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"action-recognition-webcam.ipynb\")"
    ]
   },
   {

notebooks/animate-anyone/animate-anyone.ipynb

@@ -92,7 +92,12 @@
     "\n",
     "clone_repo(\"https://github.com/itrushkin/Moore-AnimateAnyone.git\")\n",
     "\n",
-    "%load_ext skip_kernel_extension"
+    "%load_ext skip_kernel_extension\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"animate-anyone.ipynb\")"
    ]
   },
   {

notebooks/async-api/async-api.ipynb

@@ -82,7 +82,12 @@
     "    )\n",
     "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
     "\n",
-    "import notebook_utils as utils"
+    "import notebook_utils as utils\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"async-api.ipynb\")"
    ]
   },
   {

notebooks/auto-device/auto-device.ipynb

@@ -311,7 +311,12 @@
     "    r = requests.get(url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\")\n",
     "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
     "\n",
-    "from notebook_utils import download_file"
+    "from notebook_utils import download_file\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"auto-device.ipynb\")"
    ]
   },
   {

notebooks/bark-text-to-audio/bark-text-to-audio.ipynb

@@ -89,7 +89,12 @@
     "    r = requests.get(\n",
     "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
     "    )\n",
-    "    open(\"notebook_utils.py\", \"w\").write(r.text)"
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"bark-text-to-audio.ipynb\")"
    ]
   },
   {

notebooks/big-transfer-quantization/tensorflow-bit-image-classification-nncf-quantization.ipynb

@@ -92,7 +92,12 @@
     "    r = requests.get(\n",
     "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
     "    )\n",
-    "    open(\"notebook_utils.py\", \"w\").write(r.text)"
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"tensorflow-bit-image-classification-nncf-quantization.ipynb\")"
    ]
   },
   {

notebooks/blip-visual-language-processing/blip-visual-language-processing.ipynb

@@ -219,7 +219,12 @@
     "end = time.perf_counter() - start\n",
     "\n",
     "# postprocess result\n",
-    "answer = processor.decode(out[0], skip_special_tokens=True)"
+    "answer = processor.decode(out[0], skip_special_tokens=True)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"blip-visual-language-processing.ipynb\")"
    ]
   },
   {

notebooks/catvton/catvton.ipynb

@@ -91,7 +91,12 @@
     "    r = requests.get(\n",
     "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py\",\n",
     "    )\n",
-    "    open(\"cmd_helper.py\", \"w\").write(r.text)"
+    "    open(\"cmd_helper.py\", \"w\").write(r.text)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"catvton.ipynb\")"
    ]
   },
   {

notebooks/clip-language-saliency-map/clip-language-saliency-map.ipynb

@@ -118,7 +118,12 @@
     "    r = requests.get(\n",
     "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
     "    )\n",
-    "    open(\"notebook_utils.py\", \"w\").write(r.text)"
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"clip-language-saliency-map.ipynb\")"
    ]
   },
   {

notebooks/clip-zero-shot-image-classification/clip-zero-shot-classification.ipynb

@@ -97,7 +97,12 @@
     "    r = requests.get(\n",
     "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
     "    )\n",
-    "    open(\"notebook_utils.py\", \"w\").write(r.text)"
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"clip-zero-shot-classification.ipynb\")"
    ]
   },
   {

notebooks/controlnet-stable-diffusion/controlnet-stable-diffusion.ipynb

@@ -149,7 +149,12 @@
     "    \"https://download.pytorch.org/whl/cpu\",\n",
     ")\n",
     "pip_install(\"--no-deps\", \"controlnet-aux>=0.0.6\")\n",
-    "pip_install(\"openvino>=2023.1.0\")"
+    "pip_install(\"openvino>=2023.1.0\")\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"controlnet-stable-diffusion.ipynb\")"
    ]
   },
   {

notebooks/convert-to-openvino/convert-to-openvino.ipynb

@@ -51,6 +51,27 @@
     "\"openvino>=2024.4.0\" \"requests\" \"tqdm\" \"transformers>=4.31\" \"onnx!=1.16.2\" \"torch>=2.1\" \"torchvision\" \"tf_keras\""
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if not Path(\"notebook_utils.py\").exists():\n",
+    "    r = requests.get(\n",
+    "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
+    "    )\n",
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"convert-to-openvino.ipynb\")"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",

notebooks/ct-segmentation-quantize/ct-segmentation-quantize-nncf.ipynb

@@ -150,7 +150,12 @@
     "\n",
     "if not Path(\"./async_pipeline.py\").exists():\n",
     "    download_file(url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/ct-segmentation-quantize/async_pipeline.py\")\n",
-    "from async_pipeline import show_live_inference"
+    "from async_pipeline import show_live_inference\n",
+    "\n",
+    "# Read more about telemetry collection at https://github.com/openvinotoolkit/openvino_notebooks?tab=readme-ov-file#-telemetry\n",
+    "from notebook_utils import collect_telemetry\n",
+    "\n",
+    "collect_telemetry(\"ct-segmentation-quantize-nncf.ipynb\")"
    ]
   },
   {