Merge pull request #3523 from sibocw:fix-python-ortho-cam-depth

PiperOrigin-RevId: 982434876
Change-Id: I932619e0128119d01d6b3c374b34f056cbac6e00

Author: copybara-github

python/mujoco/renderer_test.py

@@ -167,6 +167,62 @@ def test_renderer_output_with_out(self):
       with self.assertRaises(ValueError):
         renderer.render(out=np.zeros((*failing_render_size, 3), np.uint8))
 
+  def test_renderer_depth_rendering_camera_projection(self):
+    """Tests orthographic vs perspective camera depth rendering projections.
+
+    Orthographic depth uses a linear NDC mapping (glOrtho); perspective depth
+    uses a hyperbolic mapping (glFrustum).
+
+    Render the same tilted plane using identically positioned orthographic and
+    perspective cameras. The two should agree on the depth at the center pixel
+    (actual distance), but they should have different depth values at non-center
+    pixels because the rays of the perspective camera diverge while those of the
+    orthographic camera remain parallel.
+    """
+    distance = 3.0
+    ortho_fovy = 2.0 * distance * np.tan(np.deg2rad(45.0 / 2.0))
+    xml = f"""
+<mujoco>
+  <statistic extent="1"/>
+  <visual>
+    <quality offsamples="0"/>
+  </visual>
+  <worldbody>
+    <camera name="ortho" pos="0 0 {distance}" xyaxes="1 0 0 0 1 0" projection="orthographic" fovy="{ortho_fovy}"/>
+    <camera name="persp" pos="0 0 {distance}" xyaxes="1 0 0 0 1 0" projection="perspective" fovy="45"/>
+    <geom name="floor" type="plane" size="5 5 0.1" zaxis="0 0.3 1"/>
+  </worldbody>
+</mujoco>
+"""
+    model = mujoco.MjModel.from_xml_string(xml)
+    data = mujoco.MjData(model)
+    mujoco.mj_forward(model, data)
+
+    # Odd resolution so a single pixel sits exactly on the optical axis.
+    def render_depth(camera_name):
+      with mujoco.Renderer(model, 51, 51) as renderer:
+        renderer.enable_depth_rendering()
+        renderer.update_scene(data, camera_name)
+        return renderer.scene.camera[0].orthographic, renderer.render().copy()
+
+    is_ortho, ortho_depth = render_depth('ortho')
+    is_persp, persp_depth = render_depth('persp')
+
+    # Check that the scenes are actually rendered with the expected cameras.
+    self.assertTrue(is_ortho)
+    self.assertFalse(is_persp)
+
+    # Center pixels of both cameras should show the actual distance
+    center = ortho_depth.shape[0] // 2, ortho_depth.shape[1] // 2
+    self.assertAlmostEqual(ortho_depth[center], distance, delta=1e-2)
+    self.assertAlmostEqual(persp_depth[center], distance, delta=1e-2)
+
+    # Depth values at the corner should differ
+    corner = (0, 0)
+    self.assertNotAlmostEqual(
+        ortho_depth[corner], persp_depth[corner], delta=1e-2
+    )
+
   def test_renderer_del_safe_when_init_fails_early(self):
     """Regression test for #3213.
 

python/mujoco/rendering/classic/renderer.py

@@ -195,22 +195,28 @@ def render(self, *, out: Optional[np.ndarray] = None) -> np.ndarray:
       # https://registry.khronos.org/OpenGL-Refpages/gl2.1/xhtml/glFrustum.xml
       zfar = np.float32(far)
       znear = np.float32(near)
-      c_coef = -(zfar + znear) / (zfar - znear)
-      d_coef = -(np.float32(2) * zfar * znear) / (zfar - znear)
-
-      # In reverse Z mode the perspective matrix is transformed by the following
-      c_coef = np.float32(-0.5) * c_coef - np.float32(0.5)
-      d_coef = np.float32(-0.5) * d_coef
 
       # We need 64 bits to convert Z from ndc to metric depth without noticeable
       # losses in precision
       out_64 = out.astype(np.float64)
 
-      # Undo OpenGL projection
-      # Note: We do not need to take action to convert from window coordinates
-      # to normalized device coordinates because in reversed Z mode the mapping
-      # is identity
-      out_64 = d_coef / (out_64 + c_coef)
+      if self._scene.camera[0].orthographic:
+        # Orthographic cameras: mapping is linear
+        out_64 = zfar - out_64 * (zfar - znear)
+      else:
+        # Perspective cameras: mapping is hyperbolic
+        c_coef = -(zfar + znear) / (zfar - znear)
+        d_coef = -(np.float32(2) * zfar * znear) / (zfar - znear)
+
+        # In reverse Z mode the perspective matrix is transformed by
+        c_coef = np.float32(-0.5) * c_coef - np.float32(0.5)
+        d_coef = np.float32(-0.5) * d_coef
+
+        # Undo OpenGL projection
+        # Note: We do not need to take action to convert from window coordinates
+        # to normalized device coordinates because in reversed Z mode
+        # the mapping is identity
+        out_64 = d_coef / (out_64 + c_coef)
 
       # Cast result back to float32 for backwards compatibility
       # This has a small accuracy cost