Track camera origins in Newton visualizers

Previously, Newton visualizers interpreted eye and lookat as absolute world coordinates. The Kit visualizer already supported world, environment, and asset origins, so the same visualizer config produced different recordings when used with Newton.

Add the Kit origin selectors to the Newton visualizer config and apply the resolved origin before pushing the camera pose to GL or RTX. Asset origins are refreshed during normal steps and on RGB capture so recorded frames follow moving assets.

Author: diegoferigo-rai

source/isaaclab_visualizers/changelog.d/newton-visualizer-camera-origin.rst

@@ -0,0 +1,4 @@
+Fixed
+^^^^^
+
+* Fixed Newton visualizer cameras ignoring environment and asset origins when interpreting ``eye`` and ``lookat``.

source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py

@@ -971,6 +971,8 @@ def __init__(self, cfg: NewtonVisualizerCfg):
         self._scene_cameras: dict = {}
         self._scene_camera_names: list[str] = []
         self._active_camera_idx: int = 0
+        self._interactive_scene = None
+        self._viewer_origin: torch.Tensor | None = None
 
     # ------------------------------------------------------------------
     # Shared lifecycle
@@ -1040,6 +1042,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None:
             self._viewer.set_visible_worlds(self._resolved_visible_env_ids)
             self._viewer.set_world_offsets(self.cfg.world_spacing)
             self._apply_camera_focal_length()
+            self._setup_initial_camera_origin(apply=False)
             initial_pose = self._resolve_initial_camera_pose()
             self._apply_camera_pose(initial_pose)
             self._viewer._paused = False
@@ -1113,6 +1116,9 @@ def step(self, dt: float) -> None:
         self._sim_time += dt
         self._step_counter += 1
 
+        if self.cfg.origin_type == "asset":
+            self._update_asset_tracking_camera()
+
         # Headless mode renders on demand via render_rgb_array(). Keep the latest
         # physics state available without paying the per-step render cost.
         if self._runtime_headless:
@@ -1268,6 +1274,15 @@ def set_camera_view(
         self.cfg.lookat = target_t
         self._apply_camera_pose((eye_t, target_t))
 
+    def reapply_origin(self) -> None:
+        """Recompute the camera position from the current origin config."""
+        self._setup_initial_camera_origin()
+
+    @property
+    def viewer_origin(self) -> torch.Tensor | None:
+        """World-space origin offset applied to the configured eye and look-at target."""
+        return self._viewer_origin
+
     # ------------------------------------------------------------------
     # Hook methods — override in subclasses
     # ------------------------------------------------------------------
@@ -1323,7 +1338,86 @@ def _uses_streaming_view(self) -> bool:
 
     def _resolve_initial_camera_pose(self) -> tuple[tuple[float, float, float], tuple[float, float, float]]:
         """Resolve initial camera pose from config or USD camera path."""
-        return self._resolve_cfg_camera_pose(type(self).__name__)
+        return self._camera_pose_with_origin(self._resolve_cfg_camera_pose(type(self).__name__))
+
+    def _setup_initial_camera_origin(self, *, apply: bool = True) -> None:
+        """Position the viewer camera according to :attr:`NewtonVisualizerCfg.origin_type`."""
+        from isaaclab.sim import SimulationContext  # noqa: PLC0415
+
+        self._interactive_scene = getattr(SimulationContext.instance(), "_interactive_scene", None)
+        self._viewer_origin = self._resolve_configured_origin()
+
+        if apply:
+            self._apply_viewer_origin_to_camera()
+
+    def _resolve_configured_origin(self) -> torch.Tensor:
+        """Resolve the configured camera origin."""
+        if self.cfg.origin_type == "world":
+            return torch.zeros(3)
+        if self.cfg.origin_type == "env":
+            return self._resolve_env_origin()
+        if self.cfg.origin_type == "asset":
+            return self._resolve_asset_origin()
+        logger.warning("[NewtonVisualizer] Unknown origin_type '%s'; defaulting to world.", self.cfg.origin_type)
+        return torch.zeros(3)
+
+    def _resolve_env_origin(self) -> torch.Tensor:
+        """Resolve the configured environment origin."""
+        scene = self._interactive_scene
+        if scene is None:
+            logger.warning("[NewtonVisualizer] origin_type='env' requested but no scene is registered yet.")
+            return torch.zeros(3)
+
+        num_envs = scene.num_envs
+        if not (0 <= self.cfg.origin_env_index < num_envs):
+            raise ValueError(
+                f"[NewtonVisualizer] origin_env_index {self.cfg.origin_env_index} is out of range "
+                f"[0, {num_envs - 1}] for origin_type='env'."
+            )
+        return scene.env_origins[self.cfg.origin_env_index]
+
+    def _resolve_asset_origin(self) -> torch.Tensor:
+        """Resolve the initial asset origin."""
+        if self.cfg.origin_track_path is None:
+            raise ValueError("[NewtonVisualizer] origin_type='asset' requires origin_track_path to be set.")
+        return torch.zeros(3)
+
+    def _update_asset_tracking_camera(self) -> None:
+        """Update the viewer camera to track an asset root or body."""
+        scene = self._interactive_scene
+        if scene is None or self.cfg.origin_track_path is None:
+            return
+        asset_name, _, body_name = self.cfg.origin_track_path.partition("/")
+        try:
+            asset = scene[asset_name]
+        except KeyError:
+            return
+        if body_name:
+            body_ids, _ = asset.find_bodies(body_name)
+            origin = asset.data.body_pos_w.torch[self.cfg.origin_env_index, body_ids[0]]
+        else:
+            origin = asset.data.root_pos_w.torch[self.cfg.origin_env_index]
+        self._viewer_origin = origin
+        self._apply_viewer_origin_to_camera()
+
+    def _apply_viewer_origin_to_camera(self) -> None:
+        """Compute absolute eye/target from :attr:`_viewer_origin` and push to the viewer."""
+        self._apply_camera_pose(self._resolve_initial_camera_pose())
+
+    def _camera_pose_with_origin(
+        self,
+        pose: tuple[tuple[float, float, float], tuple[float, float, float]],
+    ) -> tuple[tuple[float, float, float], tuple[float, float, float]]:
+        if self._viewer_origin is None:
+            return pose
+        origin = self._viewer_origin.detach().cpu().numpy()
+        eye = np.array(pose[0], dtype=float) + origin
+        target = np.array(pose[1], dtype=float) + origin
+        return (float(eye[0]), float(eye[1]), float(eye[2])), (
+            float(target[0]),
+            float(target[1]),
+            float(target[2]),
+        )
 
     def _resolve_streaming_renderer_cfg(self):
         """Return the renderer cfg for the auto-created streaming camera.
@@ -1919,6 +2013,8 @@ def render_rgb_array(self) -> np.ndarray:
         """
         if self._viewer is None:
             raise RuntimeError("NewtonGLVisualizer must be initialized before capturing an RGB frame.")
+        if self.cfg.origin_type == "asset":
+            self._update_asset_tracking_camera()
         if self._runtime_headless and self._state is not None and not self._viewer.is_paused():
             self._pre_step()
             self._viewer.begin_frame(self._sim_time)
@@ -2178,6 +2274,8 @@ def render_rgb_array(self) -> np.ndarray | None:
         """
         if self._viewer is None:
             return None
+        if self.cfg.origin_type == "asset":
+            self._update_asset_tracking_camera()
         if self._runtime_headless and self._state is not None and not self._viewer.is_paused():
             self._pre_step()
             self._viewer.begin_frame(self._sim_time)

source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py

@@ -40,6 +40,35 @@ def __post_init__(self):
     window_height: int = 1080
     """Window height in pixels."""
 
+    origin_type: str = "world"
+    """Frame in which :attr:`~isaaclab.visualizers.VisualizerCfg.eye` and
+    :attr:`~isaaclab.visualizers.VisualizerCfg.lookat` are interpreted.
+
+    Options:
+
+    * ``"world"``: global origin.
+    * ``"env"``: origin of the environment at :attr:`origin_env_index`.
+    * ``"asset"``: a scene asset (or body) specified by :attr:`origin_track_path`.
+    """
+
+    origin_env_index: int = 0
+    """Index of the environment used as the viewer camera origin.
+
+    Only meaningful when :attr:`origin_type` is ``"env"`` or ``"asset"``.
+    """
+
+    origin_track_path: str | None = None
+    """Asset tracking path for the viewer camera origin.
+
+    Format: ``"<asset_name>"`` to track the asset root, or ``"<asset_name>/<body_name>"``
+    to track a specific body on the asset. Required when :attr:`origin_type` is ``"asset"``.
+
+    Examples::
+
+        origin_track_path = "robot"             # track robot root
+        origin_track_path = "robot/panda_hand"  # track panda_hand body on robot
+    """
+
     headless: bool = False
     """Run the Newton viewer without requiring a display server."""
 

source/isaaclab_visualizers/test/test_newton_adapter.py

@@ -22,7 +22,7 @@
 )
 from isaaclab_visualizers.newton import newton_visualization_markers as newton_markers
 from isaaclab_visualizers.newton import newton_visualizer as newton_visualizer_module
-from isaaclab_visualizers.newton.newton_visualizer import NewtonViewerGL, _eye_lookat_to_pitch_yaw
+from isaaclab_visualizers.newton.newton_visualizer import NewtonViewerGL, NewtonVisualizer, _eye_lookat_to_pitch_yaw
 from isaaclab_visualizers.newton_adapter import (
     VISUALIZER_INFINITE_PLANE_SIZE,
     apply_viewer_visible_worlds,
@@ -125,6 +125,57 @@ def test_newton_visualizer_cfg_exposes_viewer_options():
     assert cfg.particle_color == (0.1, 0.2, 0.3)
 
 
+def test_newton_visualizer_offsets_cfg_camera_by_viewer_origin():
+    visualizer = NewtonVisualizer.__new__(NewtonVisualizer)
+    visualizer._viewer_origin = torch.tensor([10.0, 20.0, 30.0])
+
+    pose = visualizer._camera_pose_with_origin(((1.0, 2.0, 3.0), (4.0, 5.0, 6.0)))
+
+    assert pose == ((11.0, 22.0, 33.0), (14.0, 25.0, 36.0))
+
+
+def test_newton_visualizer_set_camera_view_uses_absolute_coordinates():
+    visualizer = NewtonVisualizer.__new__(NewtonVisualizer)
+    visualizer.cfg = SimpleNamespace(eye=(0.0, 0.0, 0.0), lookat=(1.0, 0.0, 0.0))
+    visualizer._viewer_origin = torch.tensor([10.0, 20.0, 30.0])
+    visualizer._apply_camera_pose = Mock()
+
+    NewtonVisualizer.set_camera_view(visualizer, (1.0, 2.0, 3.0), (4.0, 5.0, 6.0))
+
+    assert visualizer.cfg.eye == (1.0, 2.0, 3.0)
+    assert visualizer.cfg.lookat == (4.0, 5.0, 6.0)
+    visualizer._apply_camera_pose.assert_called_once_with(((1.0, 2.0, 3.0), (4.0, 5.0, 6.0)))
+
+
+def test_newton_visualizer_reapply_origin_updates_env_camera(monkeypatch: pytest.MonkeyPatch):
+    import isaaclab.sim as sim_utils
+
+    scene = SimpleNamespace(num_envs=2, env_origins=torch.tensor([[0.0, 0.0, 0.0], [10.0, 20.0, 30.0]]))
+    context = SimpleNamespace(_interactive_scene=scene)
+
+    class _SimulationContext:
+        @staticmethod
+        def instance():
+            return context
+
+    monkeypatch.setattr(sim_utils, "SimulationContext", _SimulationContext)
+
+    visualizer = NewtonVisualizer.__new__(NewtonVisualizer)
+    visualizer.cfg = SimpleNamespace(
+        eye=(1.0, 2.0, 3.0),
+        lookat=(4.0, 5.0, 6.0),
+        origin_type="env",
+        origin_env_index=1,
+        origin_track_path=None,
+    )
+    visualizer._resolve_cfg_camera_pose = Mock(return_value=(visualizer.cfg.eye, visualizer.cfg.lookat))
+    visualizer._apply_camera_pose = Mock()
+
+    NewtonVisualizer.reapply_origin(visualizer)
+
+    visualizer._apply_camera_pose.assert_called_once_with(((11.0, 22.0, 33.0), (14.0, 25.0, 36.0)))
+
+
 def test_newton_marker_registry_lifecycle(monkeypatch: pytest.MonkeyPatch):
     """Construction caches the registry; close survives context teardown and is idempotent."""