pipeline/face_project.py

"""
face_project.py — Project reference face image onto TripoSG mesh UV texture.

Keeps geometry 100% intact. Paints the face-region UV triangles using
barycentric rasterization — never interpolates across UV island boundaries.

Usage:
    python face_project.py --body /tmp/triposg_textured.glb \
                           --face /tmp/triposg_face_ref.png \
                           --out  /tmp/face_projected.glb \
                           [--blend 0.9] [--neck_frac 0.84] [--debug_tex /tmp/tex.png]
"""

import os, argparse, warnings
warnings.filterwarnings('ignore')

import numpy as np
import cv2
from PIL import Image
import trimesh
from trimesh.visual.texture import TextureVisuals
from trimesh.visual.material import PBRMaterial


# ── Face alignment ─────────────────────────────────────────────────────────────

def _aligned_face_bgr(face_img_bgr, target_size=512):
    """Detect + align face via InsightFace 5-pt warp; falls back to square crop."""
    try:
        from insightface.app import FaceAnalysis
        from insightface.utils import face_align
        app = FaceAnalysis(providers=['CPUExecutionProvider'])
        app.prepare(ctx_id=0, det_size=(640, 640))
        faces = app.get(face_img_bgr)
        if faces:
            faces.sort(
                key=lambda f: (f.bbox[2]-f.bbox[0]) * (f.bbox[3]-f.bbox[1]),
                reverse=True)
            aligned = face_align.norm_crop(face_img_bgr, faces[0].kps,
                                           image_size=target_size)
            print(f'  InsightFace aligned: {aligned.shape}')
            return aligned
    except Exception as e:
        print(f'  InsightFace unavailable ({e}), using centre-crop')
    h, w = face_img_bgr.shape[:2]
    side = min(h, w)
    y0, x0 = (h - side) // 2, (w - side) // 2
    return cv2.resize(face_img_bgr[y0:y0+side, x0:x0+side], (target_size, target_size))


# ── Triangle rasterizer ────────────────────────────────────────────────────────

def _rasterize_triangles(face_tri_uvs_px, face_tri_img_xy,
                          face_img_rgb, tex, blend,
                          max_uv_span=300):
    """
    Paint face_img_rgb colour into tex at UV locations, triangle by triangle.

    face_tri_uvs_px  : (M, 3, 2)  UV pixel coords of M triangles
    face_tri_img_xy  : (M, 3, 2)  projected image coords of M triangles
    face_img_rgb     : (H, W, 3)  reference face image
    tex              : (texH, texW, 3) float32 texture (modified in-place)
    blend            : float 0–1
    max_uv_span      : skip triangles whose UV bounding box exceeds this (UV seams)
    """
    H_f, W_f   = face_img_rgb.shape[:2]
    tex_H, tex_W = tex.shape[:2]
    painted = 0

    for fi in range(len(face_tri_uvs_px)):
        uv  = face_tri_uvs_px[fi]    # (3, 2) in texture pixel coords
        img = face_tri_img_xy[fi]    # (3, 2) in face-image pixel coords

        # Skip UV-seam triangles (vertices far apart in UV space)
        if (uv[:, 0].max() - uv[:, 0].min() > max_uv_span or
                uv[:, 1].max() - uv[:, 1].min() > max_uv_span):
            continue

        # Bounding box in texture space
        u_lo = max(0,      int(uv[:, 0].min()))
        u_hi = min(tex_W,  int(uv[:, 0].max()) + 2)
        v_lo = max(0,      int(uv[:, 1].min()))
        v_hi = min(tex_H,  int(uv[:, 1].max()) + 2)
        if u_hi <= u_lo or v_hi <= v_lo:
            continue

        # Grid of texel centres in this bounding box
        gu, gv = np.meshgrid(np.arange(u_lo, u_hi), np.arange(v_lo, v_hi))
        pts = np.column_stack([gu.ravel().astype(np.float32),
                               gv.ravel().astype(np.float32)])  # (K, 2)

        # Barycentric coordinates (in UV pixel space)
        A   = uv[0].astype(np.float64)
        AB  = (uv[1] - uv[0]).astype(np.float64)
        AC  = (uv[2] - uv[0]).astype(np.float64)
        denom = AB[0] * AC[1] - AB[1] * AC[0]
        if abs(denom) < 0.5:
            continue
        P  = pts.astype(np.float64) - A
        b1 = (P[:, 0] * AC[1] - P[:, 1] * AC[0]) / denom
        b2 = (P[:, 1] * AB[0] - P[:, 0] * AB[1]) / denom
        b0 = 1.0 - b1 - b2

        inside = (b0 >= 0) & (b1 >= 0) & (b2 >= 0)
        if not inside.any():
            continue

        # Interpolate reference-face image coordinates
        ix_f = (b0[inside] * img[0, 0] +
                b1[inside] * img[1, 0] +
                b2[inside] * img[2, 0])
        iy_f = (b0[inside] * img[0, 1] +
                b1[inside] * img[1, 1] +
                b2[inside] * img[2, 1])

        valid = ((ix_f >= 0) & (ix_f < W_f) & (iy_f >= 0) & (iy_f < H_f))
        if not valid.any():
            continue

        ix = np.clip(ix_f[valid].astype(int), 0, W_f - 1)
        iy = np.clip(iy_f[valid].astype(int), 0, H_f - 1)
        colours = face_img_rgb[iy, ix].astype(np.float32)   # (P, 3)

        tu = pts[inside][valid, 0].astype(int)
        tv = pts[inside][valid, 1].astype(int)
        in_tex = (tu >= 0) & (tu < tex_W) & (tv >= 0) & (tv < tex_H)

        tex[tv[in_tex], tu[in_tex]] = (
            blend * colours[in_tex] +
            (1.0 - blend) * tex[tv[in_tex], tu[in_tex]]
        )
        painted += int(in_tex.sum())

    return painted


# ── Main ───────────────────────────────────────────────────────────────────────

def project_face(body_glb, face_img_path, out_glb,
                 blend=0.90, neck_frac=0.84, debug_tex=None):
    """
    Project reference face onto TripoSG UV texture via per-triangle rasterization.
    """

    # ── Load mesh ─────────────────────────────────────────────────────────────
    print(f'[face_project] Loading {body_glb}')
    scene = trimesh.load(body_glb)
    if isinstance(scene, trimesh.Scene):
        geom_name = list(scene.geometry.keys())[0]
        mesh = scene.geometry[geom_name]
    else:
        mesh = scene
        geom_name = None

    verts    = np.array(mesh.vertices,  dtype=np.float64)   # (N, 3)
    faces    = np.array(mesh.faces,     dtype=np.int32)     # (F, 3)
    uvs      = np.array(mesh.visual.uv, dtype=np.float64)   # (N, 2)
    mat      = mesh.visual.material
    orig_tex = np.array(mat.baseColorTexture, dtype=np.float32)  # (H, W, 3) RGB
    tex_H, tex_W = orig_tex.shape[:2]
    print(f'  {len(verts)} verts | {len(faces)} faces | texture {orig_tex.shape}')

    # ── Identify face-region vertices ─────────────────────────────────────────
    y_min, y_max = verts[:, 1].min(), verts[:, 1].max()
    neck_y = float(y_min + (y_max - y_min) * neck_frac)

    head_mask = verts[:, 1] > neck_y
    head_idx  = np.where(head_mask)[0]
    hv = verts[head_idx]

    # Front half only (z >= median — face faces +Z)
    z_med  = float(np.median(hv[:, 2]))
    front  = hv[:, 2] >= z_med
    if front.sum() < 30:
        front = np.ones(len(hv), bool)

    face_vert_idx = head_idx[front]         # indices into the full vertex array

    # Build boolean mask for fast triangle selection
    face_vert_mask = np.zeros(len(verts), bool)
    face_vert_mask[face_vert_idx] = True

    # Select triangles where ALL 3 vertices are in the face region
    face_tri_mask = face_vert_mask[faces].all(axis=1)
    face_tris     = faces[face_tri_mask]    # (M, 3)
    print(f'  neck_y={neck_y:.4f} | head={len(head_idx)} '
          f'| face-front={front.sum()} | face triangles={len(face_tris)}')

    # ── Load and align reference face ─────────────────────────────────────────
    print(f'[face_project] Reference face: {face_img_path}')
    raw_bgr     = cv2.imread(face_img_path)
    aligned_bgr = _aligned_face_bgr(raw_bgr, target_size=512)
    aligned_rgb = cv2.cvtColor(aligned_bgr, cv2.COLOR_BGR2RGB).astype(np.float32)
    H_f, W_f    = aligned_rgb.shape[:2]

    # ── Compute face projection axes from actual face normal ─────────────────
    fv  = verts[face_vert_idx]

    # Average normal of the front-facing face triangles defines projection dir
    face_tri_normals = np.array(mesh.face_normals)[face_tri_mask]
    face_fwd = face_tri_normals.mean(axis=0)
    face_fwd /= np.linalg.norm(face_fwd)

    # Build orthonormal right/up axes in the face plane
    world_up   = np.array([0., 1., 0.])
    face_right = np.cross(face_fwd, world_up)
    face_right /= np.linalg.norm(face_right)
    face_up    = np.cross(face_right, face_fwd)
    face_up    /= np.linalg.norm(face_up)
    print(f'  Face normal: {face_fwd.round(3)}')

    # Project face vertices onto local (right, up) plane
    fv_centroid = fv.mean(axis=0)
    fv_c  = fv - fv_centroid
    lx    = fv_c @ face_right
    ly    = fv_c @ face_up
    x_span = float(lx.max() - lx.min())
    y_span = float(ly.max() - ly.min())

    # InsightFace norm_crop places eyes at ~37% from top of the 512px image.
    # In 3D the eyes are ~78% up from neck → 28% above centroid.
    # Shift the vertical origin up by 0.112*y_span so eye level → 37% in image.
    cy_shift = 0.112 * y_span
    pad = 0.10   # tighter crop so face features fill more of the image

    def vert_to_img(v):
        """Project 3D vertex to reference-face image using the face normal."""
        c  = v - fv_centroid                          # (N, 3)
        lx = c @ face_right
        ly = c @ face_up
        pu = lx / (x_span * (1 + 2*pad)) + 0.5
        pv = -(ly - cy_shift) / (y_span * (1 + 2*pad)) + 0.5
        return np.column_stack([pu * W_f, pv * H_f])  # (N, 2)

    def vert_to_uv_px(v_idx):
        """Convert vertex UV coords to texture pixel coordinates."""
        uv = uvs[v_idx]
        # trimesh loads GLB UV with (0,0)=bottom-left; flip V for image row
        col = uv[:, 0] * tex_W
        row = (1.0 - uv[:, 1]) * tex_H
        return np.column_stack([col, row])   # (N, 2)

    # Pre-compute image + UV pixel coords for every vertex
    all_img_px = vert_to_img(verts)           # (N, 2)
    all_uv_px  = vert_to_uv_px(np.arange(len(verts)))  # (N, 2)

    # Gather per-triangle arrays
    face_tri_uvs_px  = all_uv_px[face_tris]   # (M, 3, 2)
    face_tri_img_xy  = all_img_px[face_tris]  # (M, 3, 2)

    print(f'  UV pixel range: u={face_tri_uvs_px[:,:,0].min():.0f}→'
          f'{face_tri_uvs_px[:,:,0].max():.0f} '
          f'v={face_tri_uvs_px[:,:,1].min():.0f}→'
          f'{face_tri_uvs_px[:,:,1].max():.0f}')
    print(f'  Image coord range: x={face_tri_img_xy[:,:,0].min():.1f}→'
          f'{face_tri_img_xy[:,:,0].max():.1f} '
          f'y={face_tri_img_xy[:,:,1].min():.1f}→'
          f'{face_tri_img_xy[:,:,1].max():.1f}')

    # ── Rasterize face triangles into UV texture ──────────────────────────────
    print(f'[face_project] Rasterizing {len(face_tris)} triangles into texture...')
    new_tex = orig_tex.copy()
    painted = _rasterize_triangles(
        face_tri_uvs_px, face_tri_img_xy,
        aligned_rgb, new_tex, blend,
        max_uv_span=300
    )
    print(f'  Painted {painted} texels across {len(face_tris)} triangles')

    # ── Save debug texture if requested ──────────────────────────────────────
    if debug_tex:
        dbg = np.clip(new_tex, 0, 255).astype(np.uint8)
        Image.fromarray(dbg).save(debug_tex)
        print(f'  Debug texture: {debug_tex}')

    # ── Write modified texture back to mesh ───────────────────────────────────
    new_pil     = Image.fromarray(np.clip(new_tex, 0, 255).astype(np.uint8))
    new_mat     = PBRMaterial(baseColorTexture=new_pil)
    mesh.visual = TextureVisuals(uv=uvs, material=new_mat)

    os.makedirs(os.path.dirname(os.path.abspath(out_glb)), exist_ok=True)
    if geom_name and isinstance(scene, trimesh.Scene):
        scene.geometry[geom_name] = mesh
        scene.export(out_glb)
    else:
        mesh.export(out_glb)

    print(f'[face_project] Saved: {out_glb}  ({os.path.getsize(out_glb)//1024} KB)')
    return out_glb


# ── CLI ────────────────────────────────────────────────────────────────────────

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--body',      required=True)
    ap.add_argument('--face',      required=True)
    ap.add_argument('--out',       required=True)
    ap.add_argument('--blend',     type=float, default=0.90)
    ap.add_argument('--neck_frac', type=float, default=0.84)
    ap.add_argument('--debug_tex', default=None)
    args = ap.parse_args()
    project_face(args.body, args.face, args.out,
                 blend=args.blend, neck_frac=args.neck_frac,
                 debug_tex=args.debug_tex)