[diffusion] perf: implement paint_use_remesh (decimate to 40k faces before UV unwrap) for Hunyuan3D (#42770)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -66,6 +66,7 @@ class Hunyuan3D2PipelineConfig(PipelineConfig):
|
||||
paint_resolution: int = 512
|
||||
paint_render_size: int = 2048
|
||||
paint_texture_size: int = 2048
|
||||
# Decimate the mesh to 40k faces before UV unwrapping, as Hunyuan3D-2.1 does
|
||||
paint_use_remesh: bool = True
|
||||
paint_save_glb: bool = True
|
||||
paint_turbo_mode: bool = False
|
||||
|
||||
+9
-3
@@ -181,10 +181,16 @@ class Hunyuan3DPaintPreprocessStage(PipelineStage):
|
||||
ComponentUse(stage_name, "delight_vae", phase="decode"),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _unwrap_mesh(mesh: Any) -> Any:
|
||||
from sglang.multimodal_gen.runtime.utils.mesh3d_utils import mesh_uv_wrap
|
||||
def _unwrap_mesh(self, mesh: Any) -> Any:
|
||||
from sglang.multimodal_gen.runtime.utils.mesh3d_utils import (
|
||||
mesh_simplify,
|
||||
mesh_uv_wrap,
|
||||
)
|
||||
|
||||
# xatlas time grows superlinearly with face count; the raw shape mesh
|
||||
# has ~650k faces at the default octree resolution.
|
||||
if self.config.paint_use_remesh:
|
||||
mesh = mesh_simplify(mesh)
|
||||
return mesh_uv_wrap(mesh)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -142,6 +142,136 @@ def export_to_trimesh(mesh_output: Any) -> Any:
|
||||
return trimesh.Trimesh(mesh_output.mesh_v, mesh_output.mesh_f)
|
||||
|
||||
|
||||
# Hunyuan3D-2.1's use_remesh decimates to this many faces before UV unwrapping.
|
||||
REMESH_TARGET_FACES = 40000
|
||||
|
||||
|
||||
def _qem_vertex_quadrics(v: np.ndarray, f: np.ndarray) -> np.ndarray:
|
||||
"""Sum of the plane quadrics of the faces around each vertex, (V, 4, 4)."""
|
||||
p0 = v[f[:, 0]]
|
||||
n = np.cross(v[f[:, 1]] - p0, v[f[:, 2]] - p0)
|
||||
n /= np.maximum(np.linalg.norm(n, axis=1), 1e-20)[:, None]
|
||||
plane = np.concatenate([n, -(n * p0).sum(1, keepdims=True)], axis=1)
|
||||
face_q = (plane[:, :, None] * plane[:, None, :]).reshape(len(f), 16)
|
||||
idx = f.T.ravel()
|
||||
q = np.stack(
|
||||
[
|
||||
np.bincount(idx, weights=np.tile(face_q[:, c], 3), minlength=len(v))
|
||||
for c in range(16)
|
||||
],
|
||||
axis=1,
|
||||
)
|
||||
return q.reshape(-1, 4, 4)
|
||||
|
||||
|
||||
def _qem_collapse_targets(
|
||||
q: np.ndarray, va: np.ndarray, vb: np.ndarray
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""Optimal position and quadric error for collapsing each edge (va, vb)."""
|
||||
mid = (va + vb) / 2
|
||||
x = mid.copy()
|
||||
a = q[:, :3, :3]
|
||||
solvable = np.abs(np.linalg.det(a)) > 1e-12
|
||||
if solvable.any():
|
||||
x[solvable] = np.linalg.solve(a[solvable], -q[solvable, :3, 3:])[:, :, 0]
|
||||
# an optimum far from the edge is unstable; use the midpoint instead
|
||||
far = np.linalg.norm(x - mid, axis=1) > 2 * np.linalg.norm(vb - va, axis=1)
|
||||
x[far] = mid[far]
|
||||
xh = np.concatenate([x, np.ones((len(x), 1))], axis=1)
|
||||
return x, np.maximum(np.einsum("ni,nij,nj->n", xh, q, xh), 0)
|
||||
|
||||
|
||||
def _qem_decimate(
|
||||
vertices: np.ndarray, faces: np.ndarray, target_faces: int, max_rounds: int = 100
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""Quadric-error edge-collapse decimation, vectorized in numpy.
|
||||
|
||||
Each round collapses a set of edges at once: an edge is taken if it is the
|
||||
cheapest edge of both its endpoints, so no two collapses share a vertex. The
|
||||
cheapest such edges are taken up to the remaining face budget, skipping any
|
||||
collapse that would flip a triangle. Boundary edges and vertices are kept.
|
||||
"""
|
||||
v = np.asarray(vertices, dtype=np.float64).copy()
|
||||
f = np.asarray(faces, dtype=np.int64).copy()
|
||||
for _ in range(max_rounds):
|
||||
if len(f) <= target_faces:
|
||||
break
|
||||
nv = len(v)
|
||||
e = np.sort(np.concatenate([f[:, [0, 1]], f[:, [1, 2]], f[:, [2, 0]]]), axis=1)
|
||||
key, count = np.unique(e[:, 0] * nv + e[:, 1], return_counts=True)
|
||||
e = np.stack([key // nv, key % nv], axis=1)
|
||||
on_boundary = np.zeros(nv, dtype=bool)
|
||||
on_boundary[e[count == 1].ravel()] = True
|
||||
e = e[(count == 2) & ~on_boundary[e[:, 0]] & ~on_boundary[e[:, 1]]]
|
||||
q = _qem_vertex_quadrics(v, f)
|
||||
x, cost = _qem_collapse_targets(q[e[:, 0]] + q[e[:, 1]], v[e[:, 0]], v[e[:, 1]])
|
||||
normals = np.cross(v[f[:, 1]] - v[f[:, 0]], v[f[:, 2]] - v[f[:, 0]])
|
||||
budget = max(1, (len(f) - target_faces) // 2) # a collapse removes ~2 faces
|
||||
|
||||
def collapse(idx):
|
||||
new_v, remap = v.copy(), np.arange(nv)
|
||||
new_v[e[idx, 0]] = x[idx]
|
||||
remap[e[idx, 1]] = e[idx, 0]
|
||||
new_f = remap[f]
|
||||
live = (
|
||||
(new_f[:, 0] != new_f[:, 1])
|
||||
& (new_f[:, 1] != new_f[:, 2])
|
||||
& (new_f[:, 0] != new_f[:, 2])
|
||||
)
|
||||
new_normals = np.cross(
|
||||
new_v[new_f[:, 1]] - new_v[new_f[:, 0]],
|
||||
new_v[new_f[:, 2]] - new_v[new_f[:, 0]],
|
||||
)
|
||||
flipped = live & ((normals * new_normals).sum(1) <= 0)
|
||||
owner = np.full(nv, -1)
|
||||
owner[e[idx, 0]] = owner[e[idx, 1]] = np.arange(len(idx))
|
||||
bad = np.unique(owner[f[flipped]].ravel())
|
||||
return new_v, new_f[live], bad[bad >= 0]
|
||||
|
||||
blocked = np.zeros(len(e), dtype=bool)
|
||||
idx, bad = np.empty(0, dtype=np.int64), np.empty(0, dtype=np.int64)
|
||||
for _ in range(8):
|
||||
c = np.where(blocked, np.inf, cost)
|
||||
best = np.full(nv, np.inf)
|
||||
np.minimum.at(best, e[:, 0], c)
|
||||
np.minimum.at(best, e[:, 1], c)
|
||||
idx = np.flatnonzero(
|
||||
np.isfinite(c) & (c == best[e[:, 0]]) & (c == best[e[:, 1]])
|
||||
)
|
||||
idx = idx[np.argsort(cost[idx])[:budget]]
|
||||
if len(idx) == 0:
|
||||
break
|
||||
new_v, new_f, bad = collapse(idx)
|
||||
if len(bad) == 0:
|
||||
break
|
||||
blocked[idx[bad]] = True
|
||||
for _ in range(8):
|
||||
if len(idx) == 0 or len(bad) == 0:
|
||||
break
|
||||
idx = np.delete(idx, bad)
|
||||
if len(idx):
|
||||
new_v, new_f, bad = collapse(idx)
|
||||
if len(idx) == 0 or len(bad):
|
||||
break
|
||||
# collapses can leave two faces on the same vertices; keep one
|
||||
_, first = np.unique(np.sort(new_f, axis=1), axis=0, return_index=True)
|
||||
v, f = new_v, new_f[np.sort(first)]
|
||||
used = np.unique(f)
|
||||
remap = np.full(len(v), -1)
|
||||
remap[used] = np.arange(len(used))
|
||||
return v[used], remap[f]
|
||||
|
||||
|
||||
def mesh_simplify(mesh: Any, target_faces: int = REMESH_TARGET_FACES) -> Any:
|
||||
"""Decimate mesh to at most target_faces faces (Hunyuan3D-2.1 remesh)."""
|
||||
if isinstance(mesh, trimesh.Scene):
|
||||
mesh = mesh.dump(concatenate=True)
|
||||
if len(mesh.faces) <= target_faces:
|
||||
return mesh
|
||||
vertices, faces = _qem_decimate(mesh.vertices, mesh.faces, target_faces)
|
||||
return trimesh.Trimesh(vertices=vertices, faces=faces)
|
||||
|
||||
|
||||
def mesh_uv_wrap(mesh: Any) -> Any:
|
||||
"""Apply UV unwrapping to mesh. In-place like native Hunyuan3D-2 for same layout."""
|
||||
try:
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import trimesh
|
||||
from diffusers import AutoencoderKL as DiffusersAutoencoderKL
|
||||
from diffusers import LCMScheduler, UNet2DConditionModel
|
||||
|
||||
@@ -29,11 +30,13 @@ from sglang.multimodal_gen.runtime.pipelines.hunyuan3d_pipeline import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan3d.paint import (
|
||||
Hunyuan3DPaintPostprocessStage,
|
||||
Hunyuan3DPaintPreprocessStage,
|
||||
Hunyuan3DPaintTexGenStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan3d.shape import (
|
||||
Hunyuan3DShapeSaveStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.mesh3d_utils import mesh_simplify
|
||||
|
||||
|
||||
def _unet_config() -> dict:
|
||||
@@ -236,5 +239,44 @@ class TestHunyuan3DPaintTurboSchedule(unittest.TestCase):
|
||||
self.assertFalse(stage.scheduler.custom_timesteps)
|
||||
|
||||
|
||||
class TestHunyuan3DPaintRemesh(unittest.TestCase):
|
||||
def _unwrap(self, use_remesh: bool) -> trimesh.Trimesh:
|
||||
stage = Hunyuan3DPaintPreprocessStage.__new__(Hunyuan3DPaintPreprocessStage)
|
||||
stage.config = Hunyuan3D2PipelineConfig(paint_use_remesh=use_remesh)
|
||||
mesh = trimesh.creation.icosphere(subdivisions=6) # 81,920 faces
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.utils.mesh3d_utils.mesh_uv_wrap",
|
||||
side_effect=lambda m: m,
|
||||
):
|
||||
return stage._unwrap_mesh(mesh)
|
||||
|
||||
def test_remesh_decimates_to_40k_faces_before_uv_unwrap(self):
|
||||
mesh = self._unwrap(use_remesh=True)
|
||||
self.assertLessEqual(len(mesh.faces), 40000)
|
||||
self.assertGreater(len(mesh.faces), 39000)
|
||||
|
||||
def test_remesh_disabled_keeps_all_faces(self):
|
||||
self.assertEqual(len(self._unwrap(use_remesh=False).faces), 81920)
|
||||
|
||||
def test_decimation_keeps_shape_and_topology(self):
|
||||
sphere = trimesh.creation.icosphere(subdivisions=6, radius=1.0)
|
||||
mesh = mesh_simplify(sphere, target_faces=5000)
|
||||
self.assertLessEqual(len(mesh.faces), 5000)
|
||||
self.assertGreater(len(mesh.faces), 4500)
|
||||
self.assertTrue(mesh.is_watertight)
|
||||
radius = np.linalg.norm(mesh.vertices, axis=1)
|
||||
self.assertLess(np.abs(radius - 1.0).max(), 0.02)
|
||||
|
||||
def test_decimation_keeps_open_boundary(self):
|
||||
sphere = trimesh.creation.icosphere(subdivisions=5)
|
||||
half = trimesh.Trimesh(
|
||||
sphere.vertices, sphere.faces[sphere.triangles_center[:, 2] > 0]
|
||||
)
|
||||
mesh = mesh_simplify(half, target_faces=2000)
|
||||
self.assertLessEqual(len(mesh.faces), 2000)
|
||||
boundary = lambda m: (np.bincount(m.edges_unique_inverse) == 1).sum()
|
||||
self.assertEqual(boundary(mesh), boundary(half))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user