Pointmap: triangulate the depth grid into a mesh (no more sparse-on-zoom); drop point_cloud display_mode

app.py

@@ -211,40 +211,79 @@ def _floor_ring(radius: float = 1.5, n_points: int = 360, y: float = 0.0,
     return verts, cols
 
 
+def _triangulate_grid(pointmap_hwc: np.ndarray, image_rgb: np.ndarray,
+                      mask_hw: np.ndarray, max_edge: float = 0.05):
+    """Build a triangulated mesh from the (H, W) pointmap grid.
+
+    Each valid pixel becomes a vertex; adjacent valid pixels form quads
+    (two triangles). Triangles whose edges exceed `max_edge` (meters) are
+    dropped so we don't stretch skin between the subject and the background.
+    """
+    H, W = pointmap_hwc.shape[:2]
+    z = pointmap_hwc[:, :, 2]
+    valid = mask_hw & np.isfinite(pointmap_hwc).all(axis=2) & (z > 0.05) & (z < 25.0)
+
+    # Vertex index per pixel; -1 if invalid.
+    idx_map = np.full((H, W), -1, dtype=np.int64)
+    yy, xx = np.where(valid)
+    idx_map[yy, xx] = np.arange(len(yy))
+
+    verts = pointmap_hwc[yy, xx]                         # (N, 3) float32
+    cols = image_rgb[yy, xx]                             # (N, 3) uint8
+
+    # Quad corners
+    a = idx_map[:-1, :-1]   # top-left
+    b = idx_map[:-1, 1:]    # top-right
+    c = idx_map[1:, :-1]    # bottom-left
+    d = idx_map[1:, 1:]     # bottom-right
+    quad_valid = (a != -1) & (b != -1) & (c != -1) & (d != -1)
+
+    a_v, b_v, c_v, d_v = a[quad_valid], b[quad_valid], c[quad_valid], d[quad_valid]
+    tri1 = np.stack([a_v, c_v, b_v], axis=1)             # (M, 3)
+    tri2 = np.stack([b_v, c_v, d_v], axis=1)             # (M, 3)
+    faces = np.concatenate([tri1, tri2], axis=0)          # (2M, 3)
+
+    # Drop triangles with any edge longer than max_edge — kills stretched skins.
+    p0 = verts[faces[:, 0]]
+    p1 = verts[faces[:, 1]]
+    p2 = verts[faces[:, 2]]
+    e01 = np.linalg.norm(p1 - p0, axis=1)
+    e12 = np.linalg.norm(p2 - p1, axis=1)
+    e20 = np.linalg.norm(p0 - p2, axis=1)
+    keep = (e01 < max_edge) & (e12 < max_edge) & (e20 < max_edge)
+    faces = faces[keep]
+
+    return verts.astype(np.float32), cols.astype(np.uint8), faces.astype(np.int64)
+
+
 def _make_glb(image_pil_native: Image.Image, pointmap_hwc: np.ndarray,
-              mask_hw: np.ndarray, max_points: int = 200_000) -> str:
+              mask_hw: np.ndarray) -> str:
     h, w = pointmap_hwc.shape[:2]
     image_rgb = np.asarray(image_pil_native.resize((w, h), Image.LANCZOS))
 
-    pts = pointmap_hwc.reshape(-1, 3).astype(np.float32)
-    cols_rgb = image_rgb.reshape(-1, 3).astype(np.uint8)
-
-    z = pts[:, 2]
-    finite = np.isfinite(pts).all(axis=1) & (z > 0.05) & (z < 25.0) & mask_hw.reshape(-1)
-    pts, cols_rgb = pts[finite], cols_rgb[finite]
+    verts, cols_rgb, faces = _triangulate_grid(pointmap_hwc, image_rgb, mask_hw)
 
-    if len(pts) > max_points:
-        idx = np.random.default_rng(0).choice(len(pts), size=max_points, replace=False)
-        pts, cols_rgb = pts[idx], cols_rgb[idx]
+    # Y-up flip so the viewer's default orientation matches photographic intuition.
+    flip = np.array([1.0, -1.0, -1.0], dtype=np.float32)
+    verts = verts * flip
 
-    # Add scene aids: camera marker + XYZ axes + floor ring.
-    cam_verts, cam_cols = _camera_marker()
-    axis_verts, axis_cols = _xyz_axes()
-    ring_verts, ring_cols = _floor_ring()
-    verts = np.concatenate([pts, cam_verts, axis_verts, ring_verts], axis=0)
     cols_rgba = np.concatenate(
-        [np.concatenate([cols_rgb, np.full((len(cols_rgb), 1), 255, dtype=np.uint8)], axis=1),
-         cam_cols, axis_cols, ring_cols], axis=0,
+        [cols_rgb, np.full((len(cols_rgb), 1), 255, dtype=np.uint8)], axis=1
     )
-
-    # Three.js viewers (and gr.Model3D) typically use Y-up. Sapiens2 pointmaps
-    # come in camera frame with Y down, Z forward — flip Y so the viewer's
-    # default orientation matches photographic intuition.
-    verts = verts * np.array([1.0, -1.0, -1.0], dtype=np.float32)
-
-    pc = trimesh.PointCloud(vertices=verts, colors=cols_rgba)
+    mesh = trimesh.Trimesh(vertices=verts, faces=faces, vertex_colors=cols_rgba, process=False)
+
+    # Scene aids as separate point clouds.
+    aids_v, aids_c = [], []
+    for fn in (_camera_marker, _xyz_axes, _floor_ring):
+        v, c = fn()
+        aids_v.append(v * flip)
+        aids_c.append(c)
+    aids = trimesh.PointCloud(vertices=np.concatenate(aids_v, axis=0),
+                              colors=np.concatenate(aids_c, axis=0))
+
+    scene = trimesh.Scene([mesh, aids])
     out_path = tempfile.NamedTemporaryFile(delete=False, suffix=".glb").name
-    pc.export(out_path)
+    scene.export(out_path)
     return out_path
 
 
@@ -354,10 +393,9 @@ with gr.Blocks(title="Sapiens2 Pointmap", theme=gr.themes.Soft(), css=CUSTOM_CSS
         inp = gr.Image(label="Input", type="pil", height=640, scale=2)
         out_depth = gr.Image(label="Depth (Z)", type="pil", height=640, scale=2)
         out_glb = gr.Model3D(
-            label="Point cloud  ·  drag to orbit  ·  scroll to zoom  ·  shift+drag to pan",
+            label="3D mesh  ·  drag to orbit  ·  scroll to zoom  ·  shift+drag to pan",
             height=640,
             clear_color=[0.10, 0.11, 0.14, 1.0],
-            display_mode="point_cloud",
             camera_position=(35, 70, 4.0),  # azimuth, elevation, radius — flattering 3/4 view
             zoom_speed=0.7,
             pan_speed=0.5,