Add CI coverage for examples that use remote resources (#439)

by adding support for prefetching with no-sandbox, so that each test
doesn't have to re-download the resources.

Author: RussTedrake

.github/workflows/main.yml

@@ -63,7 +63,9 @@ jobs:
         run:  ./setup/submodule_checkout
         shell: bash
       - name: post-checkout setup
-        run: ./book/htmlbook/workflows/ci/jammy/setup
+        run: |
+          ./book/htmlbook/workflows/ci/jammy/setup
+          bazel run //manipulation:prefetch_remotes
         shell: bash
 #      - name: Check disk space
 #        run: |
@@ -85,7 +87,9 @@ jobs:
         run:  ./setup/submodule_checkout
         shell: zsh -efuo pipefail {0}
       - name: setup
-        run: ./book/htmlbook/workflows/ci/sonoma/setup
+        run: |
+          ./book/htmlbook/workflows/ci/sonoma/setup
+          bazel run //manipulation:prefetch_remotes
         shell: zsh -efuo pipefail {0}
       - name: test
         run: ./book/htmlbook/workflows/ci/sonoma/test
@@ -148,6 +152,7 @@ jobs:
           pip install -U manipulation[all] --extra-index-url https://drake-packages.csail.mit.edu/whl/nightly/
           truncate -s0 requirements.txt 
           sed -i '/PYTHONNOUSERSITE/d' .bazelrc
+          bazel run //manipulation:prefetch_remotes
         shell: bash
       - name: test
         run: EXCLUDE_TESTS="//book/htmlbook/..." ./book/htmlbook/workflows/ci/jammy/test
@@ -179,6 +184,7 @@ jobs:
           pip install manipulation --extra-index-url https://drake-packages.csail.mit.edu/whl/nightly/
           truncate -s0 requirements.txt
           sed -i '/PYTHONNOUSERSITE/d' .bazelrc
+          bazel run //manipulation:prefetch_remotes
         shell: bash
       - name: test
         run: |
@@ -216,6 +222,7 @@ jobs:
           pip install manipulation[all] --extra-index-url https://drake-packages.csail.mit.edu/whl/nightly/
           truncate -s0 requirements.txt
           sed -i '/PYTHONNOUSERSITE/d' .bazelrc
+          bazel run //manipulation:prefetch_remotes
         shell: bash
       - name: test
         run: |

book/htmlbook

@@ -26,15 +26,16 @@ rt_ipynb_test(
     ],
 )
 
-
-# TODO(russt): Pre-download the model
-#rt_ipynb_test(
-#    name = "spot",
-#    srcs = ["spot.ipynb"],
-#    deps = [
-#      "//manipulation",
-#    ],
-#)
+rt_ipynb_test(
+    name = "spot",
+    srcs = ["spot.ipynb"],
+    tags = ["requires-network", "no-sandbox"],
+    deps = [
+      "//manipulation",
+      "//manipulation:remotes",
+      "//manipulation:station",
+    ],
+)
 
 rt_ipynb_test(
     name = "pr2",

book/mobile/BUILD.bazel

@@ -26,6 +26,7 @@
     "from pydrake.all import ModelVisualizer, PackageMap, Simulator, StartMeshcat\n",
     "\n",
     "from manipulation import ConfigureParser, FindResource, running_as_notebook\n",
+    "from manipulation.remotes import AddSpotRemote\n",
     "from manipulation.station import LoadScenario, MakeHardwareStation"
    ]
   },
@@ -40,25 +41,6 @@
     "meshcat = StartMeshcat()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def AddSpotRemote(parser):\n",
-    "    parser.package_map().AddRemote(\n",
-    "        package_name=\"spot_description\",\n",
-    "        params=PackageMap.RemoteParams(\n",
-    "            urls=[\n",
-    "                f\"https://github.com/wrangel-bdai/spot_ros2/archive/20965ef7bba98598ee10878c7b54e6ef28a300c6.tar.gz\"\n",
-    "            ],\n",
-    "            sha256=(\"20a4f12896b04cc73e186cf876bf2c7e905ee88f8add8ea51bf52dfc888674b4\"),\n",
-    "            strip_prefix=\"spot_ros2-20965ef7bba98598ee10878c7b54e6ef28a300c6/spot_description/\",\n",
-    "        ),\n",
-    "    )"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "7112a8be",
@@ -82,7 +64,7 @@
    "source": [
     "visualizer = ModelVisualizer(meshcat=meshcat)\n",
     "ConfigureParser(visualizer.parser())\n",
-    "AddSpotRemote(visualizer.parser())\n",
+    "AddSpotRemote(visualizer.parser().package_map())\n",
     "visualizer.AddModels(\n",
     "    url=\"package://manipulation/spot/spot_with_arm_and_floating_base_actuators.urdf\"\n",
     ")\n",
@@ -109,7 +91,11 @@
     "        \"models/spot/spot_with_arm_and_floating_base_actuators.scenario.yaml\"\n",
     "    )\n",
     ")\n",
-    "station = MakeHardwareStation(scenario, meshcat, parser_preload_callback=AddSpotRemote)\n",
+    "station = MakeHardwareStation(\n",
+    "    scenario,\n",
+    "    meshcat,\n",
+    "    parser_preload_callback=lambda parser: AddSpotRemote(parser.package_map()),\n",
+    ")\n",
     "simulator = Simulator(station)\n",
     "context = simulator.get_mutable_context()\n",
     "x0 = station.GetOutputPort(\"spot.state_estimated\").Eval(context)\n",

book/mobile/spot.ipynb

@@ -36,6 +36,7 @@ rt_ipynb_test(
 
 rt_ipynb_test(
     name = "qdot_vs_v",
+    data = ["@drake_models"],
     srcs = ["qdot_vs_v.ipynb"],
     deps = [
       requirement("drake"),

book/pick/BUILD.bazel

@@ -18,18 +18,22 @@ rt_ipynb_test(
 rt_ipynb_test(
     name = "menagerie",
     srcs = ["menagerie.ipynb"],
+    tags = ["requires-network", "no-sandbox"],
     deps = [
       "//manipulation",
       "//manipulation:make_drake_compatible_model",
+      "//manipulation:remotes",
     ],
 )
 
 rt_ipynb_test(
     name = "gymnasium_robotics",
     srcs = ["gymnasium_robotics.ipynb"],
+    tags = ["requires-network", "no-sandbox"],
     deps = [
       "//manipulation",
       "//manipulation:make_drake_compatible_model",
+      "//manipulation:remotes",
     ],
 )
 

book/robot/BUILD.bazel

@@ -28,6 +28,7 @@
     "\n",
     "from manipulation import running_as_notebook\n",
     "from manipulation.make_drake_compatible_model import MakeDrakeCompatibleModel\n",
+    "from manipulation.remotes import AddGymnasiumRobotics\n",
     "from manipulation.utils import ApplyDefaultVisualization"
    ]
   },
@@ -68,16 +69,7 @@
     "    plant = builder.plant()\n",
     "    parser = Parser(plant)\n",
     "    package_map = parser.package_map()\n",
-    "    package_map.AddRemote(\n",
-    "        package_name=\"gymnasium_robotics\",\n",
-    "        params=PackageMap.RemoteParams(\n",
-    "            urls=[\n",
-    "                f\"https://github.com/Farama-Foundation/Gymnasium-Robotics/archive/refs/tags/v1.3.1.tar.gz\"\n",
-    "            ],\n",
-    "            sha256=(\"d274b3ee1d34337aa35d4686447fda6a6d20dfcb4d375eab91b4e49b1108afde\"),\n",
-    "            strip_prefix=\"Gymnasium-Robotics-1.3.1/gymnasium_robotics/envs/assets/\",\n",
-    "        ),\n",
-    "    )\n",
+    "    AddGymnasiumRobotics(package_map)\n",
     "    original_file = package_map.ResolveUrl(\n",
     "        \"package://gymnasium_robotics/kitchen_franka/kitchen_assets/kitchen_env_model.xml\"\n",
     "    )\n",

book/robot/gymnasium_robotics.ipynb

@@ -38,7 +38,8 @@
     "\n",
     "from manipulation import running_as_notebook\n",
     "from manipulation.make_drake_compatible_model import MakeDrakeCompatibleModel\n",
-    "from manipulation.utils import AddMujocoMenagerie, ApplyDefaultVisualization"
+    "from manipulation.remotes import AddMujocoMenagerie\n",
+    "from manipulation.utils import ApplyDefaultVisualization"
    ]
   },
   {
@@ -150,8 +151,24 @@
     "    \"universal_robots_ur10e\": (\"scene.xml\", \"ur10e.png\"),\n",
     "    \"universal_robots_ur5e\": (\"scene.xml\", \"ur5e.png\"),\n",
     "    \"wonik_allegro\": (\"scene_left.xml\", \"allegro_hand.png\"),\n",
-    "}\n",
-    "\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ee27292",
+   "metadata": {},
+   "source": [
+    "Show a particular model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e80d70ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "if running_as_notebook:\n",
     "    show_model(\"unitree_g1\")"
    ]
@@ -197,17 +214,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "if running_as_notebook:\n",
-    "    vis = ModelVisualizer(meshcat=meshcat)\n",
-    "    parser = vis.parser()\n",
-    "    AddMujocoMenagerie(parser.package_map())\n",
-    "    model_path = parser.package_map().ResolveUrl(\n",
-    "        \"package://mujoco_menagerie/agility_cassie/scene.xml\"\n",
-    "    )\n",
-    "    drake_model_path = model_path.replace(\".xml\", \".drake.xml\")\n",
-    "    MakeDrakeCompatibleModel(model_path, drake_model_path, overwrite=True)\n",
-    "    vis.AddModels(drake_model_path)\n",
-    "    vis.Run(loop_once=False)"
+    "vis = ModelVisualizer(meshcat=meshcat)\n",
+    "parser = vis.parser()\n",
+    "AddMujocoMenagerie(parser.package_map())\n",
+    "model_path = parser.package_map().ResolveUrl(\n",
+    "    \"package://mujoco_menagerie/agility_cassie/scene.xml\"\n",
+    ")\n",
+    "drake_model_path = model_path.replace(\".xml\", \".drake.xml\")\n",
+    "MakeDrakeCompatibleModel(model_path, drake_model_path, overwrite=True)\n",
+    "vis.AddModels(drake_model_path)\n",
+    "vis.Run(loop_once=not running_as_notebook)"
    ]
   },
   {
@@ -221,7 +237,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "venv",
    "language": "python",
    "name": "python3"
   },

book/robot/menagerie.ipynb

@@ -5,14 +5,17 @@
 # Licensed under the BSD 3-Clause License. See LICENSE.TXT for details.
 
 load("@pip_deps//:requirements.bzl", "requirement")
-load("//book/htmlbook/tools/python:defs.bzl", "rt_py_library", "rt_py_test")
+load("//book/htmlbook/tools/python:defs.bzl", "rt_py_library", "rt_py_test", "rt_py_binary")
 
 rt_py_library(
     name = "manipulation",
     srcs = [
         "__init__.py",
         "utils.py",
     ],
+    data = [
+      "@drake_models",
+    ],
     imports = [".."],
     visibility = ["//visibility:public"],
     deps = [
@@ -65,6 +68,23 @@ rt_py_library(
     visibility = ["//visibility:public"],
 )
 
+rt_py_library(
+    name = "remotes",
+    srcs = ["remotes.py"],
+    imports = [".."],
+    deps = [
+      requirement("drake"),
+    ],
+    visibility = ["//visibility:public"],
+)
+
+rt_py_binary(
+    name = "prefetch_remotes",
+    tags = ["requires-network", "no-sandbox"], 
+    srcs = ["prefetch_remotes.py"],
+    deps = ["remotes"],
+)
+
 rt_py_library(
     name = "directives_tree",
     deps = [
@@ -152,12 +172,15 @@ rt_py_test(
     name = "test_make_drake_compatible_model",
     srcs = ["test/test_make_drake_compatible_model.py"],
     imports = [".."],
+    tags = ["requires-network", "no-sandbox"],
+    timeout = "moderate",  # TODO(russt): consider moving the menagerie work to the prefetch_remotes
     data = [
       "//manipulation/test/models:models",
     ],
     deps = [
       "manipulation",
       "//manipulation:make_drake_compatible_model",
+      "//manipulation:remotes",
     ],
 )
 

manipulation/BUILD.bazel

@@ -0,0 +1,4 @@
+from manipulation.remotes import PrefetchAllRemotePackages
+
+if __name__ == "__main__":
+    PrefetchAllRemotePackages()

manipulation/prefetch_remotes.py

@@ -0,0 +1,5 @@
+# `manipulation.remotes`
+
+```{eval-rst}
+.. automodule:: manipulation.remotes
+   :members: