Daankular commited on
Commit
51cdda6
Β·
1 Parent(s): 9800a58

Add CPU texture-bake fallback: xatlas UV-unwrap + numpy projection

Browse files

nvdiffrast fails on ZeroGPU A10G with error 209 (cudaErrorInvalidDeviceFunction)
for both GL and CUDA contexts. Add _bake_texture_cpu() that uses:
- xatlas for UV parametrization (no GPU required)
- Per-face orthographic projection from the 6 MV-Adapter views
- trimesh for GLB export with embedded texture

The primary nvdiffrast path is tried first; on any exception the CPU
fallback runs automatically. Also add xatlas to requirements.txt.

Files changed (2) hide show
  1. app.py +160 -43
  2. requirements.txt +1 -0
app.py CHANGED
@@ -905,6 +905,123 @@ def generate_shape(input_image, remove_background, num_steps, guidance_scale,
905
  return None, f"Error:\n{traceback.format_exc()}"
906
 
907
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
908
  # ── Stage 2: Texture ──────────────────────────────────────────────────────────
909
 
910
  @spaces.GPU(duration=300)
@@ -1025,54 +1142,54 @@ def apply_texture(glb_path, input_image, remove_background, variant, tex_seed,
1025
  print(f"[apply_texture] face enhance failed: {_fe}")
1026
 
1027
  # ── Bake textures onto mesh ─────────────────────────────────────
1028
- # Use CameraProjection + replace_mesh_texture_and_save directly.
1029
- # TexturePipeline imports mesh_process which requires open3d+pymeshlab
1030
- # (not available); the UV projection itself has no such dependency.
1031
  progress(0.85, desc="Baking UV texture onto mesh...")
1032
- from mvadapter.utils.mesh_utils import (
1033
- load_mesh, replace_mesh_texture_and_save,
1034
- )
1035
- from mvadapter.utils.mesh_utils.projection import CameraProjection
1036
- from mvadapter.utils import image_to_tensor, tensor_to_image
1037
 
1038
  # Split the saved horizontal 6-view grid back into individual images
1039
- mv_img = Image.open(mv_path)
1040
- mv_np = np.array(mv_img)
1041
- mv_views = [Image.fromarray(v) for v in np.array_split(mv_np, 6, axis=1)]
1042
-
1043
- # Cameras must match those used during MV-Adapter generation
1044
- tex_cameras = get_orthogonal_camera(
1045
- elevation_deg=[0, 0, 0, 0, 0, 0],
1046
- distance=[1.8] * 6,
1047
- left=-0.55, right=0.55, bottom=-0.55, top=0.55,
1048
- azimuth_deg=[x - 90 for x in [0, 45, 90, 180, 270, 315]],
1049
- device=DEVICE,
1050
- )
1051
- mesh_obj = load_mesh(glb_path, rescale=True, device=DEVICE, default_uv_size=1024)
1052
- cam_proj = CameraProjection(pb_backend="torch-cuda", bg_remover=None, device=DEVICE, context_type="cuda")
1053
- mod_tensor = image_to_tensor(mv_views, device=DEVICE)
1054
-
1055
- cam_out = cam_proj(
1056
- mod_tensor, mesh_obj, tex_cameras,
1057
- from_scratch=True,
1058
- poisson_blending=False,
1059
- depth_grad_dilation=5,
1060
- depth_grad_threshold=0.1,
1061
- uv_exp_blend_alpha=3,
1062
- uv_exp_blend_view_weight=torch.as_tensor([1, 1, 1, 1, 1, 1]),
1063
- aoi_cos_valid_threshold=0.2,
1064
- uv_size=1024,
1065
- uv_padding=True,
1066
- return_dict=True,
1067
- )
1068
 
1069
  out_glb = os.path.join(out_dir, "textured_shaded.glb")
1070
- replace_mesh_texture_and_save(
1071
- glb_path, out_glb,
1072
- texture=tensor_to_image(cam_out.uv_proj),
1073
- backend="gltflib",
1074
- task_id="textured",
1075
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1076
 
1077
  final_path = "/tmp/triposg_textured.glb"
1078
  shutil.copy(out_glb, final_path)
 
905
  return None, f"Error:\n{traceback.format_exc()}"
906
 
907
 
908
+ # ── CPU texture-bake fallback (no nvdiffrast) ─────────────────────────────────
909
+
910
+ def _bake_texture_cpu(glb_path: str, mv_views: list, out_path: str,
911
+ uv_size: int = 1024) -> str:
912
+ """
913
+ CPU texture baking via xatlas UV-unwrap + per-face numpy projection.
914
+ Used when CameraProjection / nvdiffrast is unavailable (error 209 on ZeroGPU).
915
+ """
916
+ import xatlas
917
+ import trimesh as _trimesh
918
+
919
+ print("[_bake_texture_cpu] Loading mesh...")
920
+ scene = _trimesh.load(glb_path)
921
+ if isinstance(scene, _trimesh.Scene):
922
+ parts = [g for g in scene.geometry.values() if isinstance(g, _trimesh.Trimesh)]
923
+ mesh = _trimesh.util.concatenate(parts) if len(parts) > 1 else parts[0]
924
+ else:
925
+ mesh = scene
926
+
927
+ verts = np.array(mesh.vertices, dtype=np.float32)
928
+ faces = np.array(mesh.faces, dtype=np.uint32)
929
+
930
+ # Normalize to camera projection space (coords β‰ˆ Β±0.55)
931
+ center = (verts.max(0) + verts.min(0)) * 0.5
932
+ scale = (verts.max(0) - verts.min(0)).max() / 1.1
933
+ verts_n = (verts - center) / scale
934
+
935
+ print("[_bake_texture_cpu] Running xatlas UV parametrize...")
936
+ vmapping, new_faces, uvs = xatlas.parametrize(verts_n, faces)
937
+ verts_new = verts_n[vmapping] # (N_new, 3)
938
+
939
+ v0 = verts_new[new_faces[:, 0]]; uv0 = uvs[new_faces[:, 0]]
940
+ v1 = verts_new[new_faces[:, 1]]; uv1 = uvs[new_faces[:, 1]]
941
+ v2 = verts_new[new_faces[:, 2]]; uv2 = uvs[new_faces[:, 2]]
942
+
943
+ # Face normals
944
+ normals = np.cross(v1 - v0, v2 - v0)
945
+ norms_len = np.linalg.norm(normals, axis=1, keepdims=True)
946
+ valid = norms_len[:, 0] > 1e-8
947
+ normals[valid] /= norms_len[valid]
948
+
949
+ # 6 cameras: azimuth = [-90,-45,0,90,180,225] deg (matches MV-Adapter setup)
950
+ azims = np.radians(np.array([-90., -45., 0., 90., 180., 225.]))
951
+ cam_dirs = np.stack([np.sin(azims), np.zeros(6), np.cos(azims)], axis=1)
952
+ dots = normals @ cam_dirs.T # (F, 6)
953
+ best_view = dots.argmax(1)
954
+ max_dot = dots.max(1)
955
+
956
+ view_imgs = [np.array(v.resize((768, 768)))[..., :3] for v in mv_views]
957
+
958
+ print(f"[_bake_texture_cpu] Baking {len(new_faces)} faces into {uv_size}x{uv_size} texture...")
959
+ tex = np.full((uv_size, uv_size, 3), 200, dtype=np.uint8)
960
+
961
+ for fi in range(len(new_faces)):
962
+ if not valid[fi] or max_dot[fi] < 0.05:
963
+ continue
964
+
965
+ bv = int(best_view[fi])
966
+ az = float(azims[bv])
967
+
968
+ uv_tri = np.stack([uv0[fi], uv1[fi], uv2[fi]])
969
+ px = uv_tri * (uv_size - 1)
970
+ u_min = max(0, int(np.floor(px[:, 0].min())))
971
+ u_max = min(uv_size-1, int(np.ceil (px[:, 0].max())))
972
+ v_min = max(0, int(np.floor(px[:, 1].min())))
973
+ v_max = min(uv_size-1, int(np.ceil (px[:, 1].max())))
974
+ if u_max < u_min or v_max < v_min:
975
+ continue
976
+
977
+ pu = np.arange(u_min, u_max + 1, dtype=np.float32) / (uv_size - 1)
978
+ pv = np.arange(v_min, v_max + 1, dtype=np.float32) / (uv_size - 1)
979
+ PU, PV = np.meshgrid(pu, pv)
980
+ P = np.stack([PU.ravel(), PV.ravel()], axis=1)
981
+
982
+ d1 = uv1[fi] - uv0[fi]
983
+ d2 = uv2[fi] - uv0[fi]
984
+ dp = P - uv0[fi]
985
+ denom = d1[0] * d2[1] - d1[1] * d2[0]
986
+ if abs(denom) < 1e-10:
987
+ continue
988
+ b1 = (dp[:, 0] * d2[1] - dp[:, 1] * d2[0]) / denom
989
+ b2 = (d1[0] * dp[:, 1] - d1[1] * dp[:, 0]) / denom
990
+ b0 = 1.0 - b1 - b2
991
+ inside = (b0 >= -0.01) & (b1 >= -0.01) & (b2 >= -0.01)
992
+ if not inside.any():
993
+ continue
994
+
995
+ b0i = b0[inside, None]; b1i = b1[inside, None]; b2i = b2[inside, None]
996
+ p3d = b0i * v0[fi] + b1i * v1[fi] + b2i * v2[fi]
997
+
998
+ right = np.array([ np.cos(az), 0.0, -np.sin(az)])
999
+ up = np.array([ 0.0, 1.0, 0.0 ])
1000
+ u_cam = np.clip(p3d @ right / 1.1 * 0.5 + 0.5, 0.0, 1.0)
1001
+ v_cam = np.clip(1.0 - (p3d @ up / 1.1 * 0.5 + 0.5), 0.0, 1.0)
1002
+ u_img = (u_cam * 767).astype(np.int32)
1003
+ v_img = (v_cam * 767).astype(np.int32)
1004
+
1005
+ colors = view_imgs[bv][v_img, u_img]
1006
+
1007
+ pu_in = np.round(PU.ravel()[inside] * (uv_size - 1)).astype(np.int32)
1008
+ pv_in = np.round(PV.ravel()[inside] * (uv_size - 1)).astype(np.int32)
1009
+ tex[pv_in, pu_in] = colors
1010
+
1011
+ print("[_bake_texture_cpu] Saving textured GLB...")
1012
+ new_mesh = _trimesh.Trimesh(
1013
+ vertices = verts_new,
1014
+ faces = new_faces.astype(np.int64),
1015
+ visual = _trimesh.visual.TextureVisuals(
1016
+ uv = uvs,
1017
+ image = Image.fromarray(tex),
1018
+ ),
1019
+ process=False,
1020
+ )
1021
+ new_mesh.export(out_path)
1022
+ return out_path
1023
+
1024
+
1025
  # ── Stage 2: Texture ──────────────────────────────────────────────────────────
1026
 
1027
  @spaces.GPU(duration=300)
 
1142
  print(f"[apply_texture] face enhance failed: {_fe}")
1143
 
1144
  # ── Bake textures onto mesh ─────────────────────────────────────
 
 
 
1145
  progress(0.85, desc="Baking UV texture onto mesh...")
 
 
 
 
 
1146
 
1147
  # Split the saved horizontal 6-view grid back into individual images
1148
+ mv_img = Image.open(mv_path)
1149
+ mv_np = np.array(mv_img)
1150
+ mv_views = [Image.fromarray(v) for v in np.array_split(mv_np, 6, axis=1)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1151
 
1152
  out_glb = os.path.join(out_dir, "textured_shaded.glb")
1153
+ try:
1154
+ # Primary path: CameraProjection via nvdiffrast
1155
+ from mvadapter.utils.mesh_utils import (
1156
+ load_mesh, replace_mesh_texture_and_save,
1157
+ )
1158
+ from mvadapter.utils.mesh_utils.projection import CameraProjection
1159
+ from mvadapter.utils import image_to_tensor, tensor_to_image
1160
+
1161
+ tex_cameras = get_orthogonal_camera(
1162
+ elevation_deg=[0, 0, 0, 0, 0, 0],
1163
+ distance=[1.8] * 6,
1164
+ left=-0.55, right=0.55, bottom=-0.55, top=0.55,
1165
+ azimuth_deg=[x - 90 for x in [0, 45, 90, 180, 270, 315]],
1166
+ device=DEVICE,
1167
+ )
1168
+ mesh_obj = load_mesh(glb_path, rescale=True, device=DEVICE, default_uv_size=1024)
1169
+ cam_proj = CameraProjection(pb_backend="torch-cuda", bg_remover=None,
1170
+ device=DEVICE, context_type="cuda")
1171
+ mod_tensor = image_to_tensor(mv_views, device=DEVICE)
1172
+
1173
+ cam_out = cam_proj(
1174
+ mod_tensor, mesh_obj, tex_cameras,
1175
+ from_scratch=True, poisson_blending=False,
1176
+ depth_grad_dilation=5, depth_grad_threshold=0.1,
1177
+ uv_exp_blend_alpha=3,
1178
+ uv_exp_blend_view_weight=torch.as_tensor([1, 1, 1, 1, 1, 1]),
1179
+ aoi_cos_valid_threshold=0.2,
1180
+ uv_size=1024, uv_padding=True, return_dict=True,
1181
+ )
1182
+ replace_mesh_texture_and_save(
1183
+ glb_path, out_glb,
1184
+ texture=tensor_to_image(cam_out.uv_proj),
1185
+ backend="gltflib", task_id="textured",
1186
+ )
1187
+ print("[apply_texture] nvdiffrast texture baking succeeded.")
1188
+ except Exception as _nv_err:
1189
+ # Fallback: CPU xatlas UV-unwrap + per-face numpy projection
1190
+ print(f"[apply_texture] nvdiffrast baking failed ({type(_nv_err).__name__}): {_nv_err}")
1191
+ print("[apply_texture] Falling back to CPU xatlas texture bake...")
1192
+ _bake_texture_cpu(glb_path, mv_views, out_glb)
1193
 
1194
  final_path = "/tmp/triposg_textured.glb"
1195
  shutil.copy(out_glb, final_path)
requirements.txt CHANGED
@@ -31,6 +31,7 @@ trimesh
31
  fast-simplification
32
  open3d # needed by mvadapter pipeline_texture.py (Space runs Python 3.10 which has wheels)
33
  # pymeshlab: no Python 3.13 wheels β€” not used in this Space
 
34
  pygltflib
35
  # pyrender: no Python 3.13 wheels β€” not used in this Space
36
  moderngl
 
31
  fast-simplification
32
  open3d # needed by mvadapter pipeline_texture.py (Space runs Python 3.10 which has wheels)
33
  # pymeshlab: no Python 3.13 wheels β€” not used in this Space
34
+ xatlas # CPU UV unwrap fallback for texture baking when nvdiffrast unavailable
35
  pygltflib
36
  # pyrender: no Python 3.13 wheels β€” not used in this Space
37
  moderngl