[diffusion] perf: implement paint_use_remesh (decimate to 40k faces before UV unwrap) for Hunyuan3D (#42770)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
WenhaoZhang
2026-10-13 12:28:32 +08:00
committed by GitHub
co-authored by Mick Qian
parent 6a89181614
commit ea13359da9
4 changed files with 182 additions and 3 deletions
@@ -66,6 +66,7 @@ class Hunyuan3D2PipelineConfig(PipelineConfig):
paint_resolution: int = 512
paint_render_size: int = 2048
paint_texture_size: int = 2048
# Decimate the mesh to 40k faces before UV unwrapping, as Hunyuan3D-2.1 does
paint_use_remesh: bool = True
paint_save_glb: bool = True
paint_turbo_mode: bool = False
@@ -181,10 +181,16 @@ class Hunyuan3DPaintPreprocessStage(PipelineStage):
ComponentUse(stage_name, "delight_vae", phase="decode"),
]
@staticmethod
def _unwrap_mesh(mesh: Any) -> Any:
from sglang.multimodal_gen.runtime.utils.mesh3d_utils import mesh_uv_wrap
def _unwrap_mesh(self, mesh: Any) -> Any:
from sglang.multimodal_gen.runtime.utils.mesh3d_utils import (
mesh_simplify,
mesh_uv_wrap,
)
# xatlas time grows superlinearly with face count; the raw shape mesh
# has ~650k faces at the default octree resolution.
if self.config.paint_use_remesh:
mesh = mesh_simplify(mesh)
return mesh_uv_wrap(mesh)
@staticmethod
@@ -142,6 +142,136 @@ def export_to_trimesh(mesh_output: Any) -> Any:
return trimesh.Trimesh(mesh_output.mesh_v, mesh_output.mesh_f)
# Hunyuan3D-2.1's use_remesh decimates to this many faces before UV unwrapping.
REMESH_TARGET_FACES = 40000
def _qem_vertex_quadrics(v: np.ndarray, f: np.ndarray) -> np.ndarray:
"""Sum of the plane quadrics of the faces around each vertex, (V, 4, 4)."""
p0 = v[f[:, 0]]
n = np.cross(v[f[:, 1]] - p0, v[f[:, 2]] - p0)
n /= np.maximum(np.linalg.norm(n, axis=1), 1e-20)[:, None]
plane = np.concatenate([n, -(n * p0).sum(1, keepdims=True)], axis=1)
face_q = (plane[:, :, None] * plane[:, None, :]).reshape(len(f), 16)
idx = f.T.ravel()
q = np.stack(
[
np.bincount(idx, weights=np.tile(face_q[:, c], 3), minlength=len(v))
for c in range(16)
],
axis=1,
)
return q.reshape(-1, 4, 4)
def _qem_collapse_targets(
q: np.ndarray, va: np.ndarray, vb: np.ndarray
) -> Tuple[np.ndarray, np.ndarray]:
"""Optimal position and quadric error for collapsing each edge (va, vb)."""
mid = (va + vb) / 2
x = mid.copy()
a = q[:, :3, :3]
solvable = np.abs(np.linalg.det(a)) > 1e-12
if solvable.any():
x[solvable] = np.linalg.solve(a[solvable], -q[solvable, :3, 3:])[:, :, 0]
# an optimum far from the edge is unstable; use the midpoint instead
far = np.linalg.norm(x - mid, axis=1) > 2 * np.linalg.norm(vb - va, axis=1)
x[far] = mid[far]
xh = np.concatenate([x, np.ones((len(x), 1))], axis=1)
return x, np.maximum(np.einsum("ni,nij,nj->n", xh, q, xh), 0)
def _qem_decimate(
vertices: np.ndarray, faces: np.ndarray, target_faces: int, max_rounds: int = 100
) -> Tuple[np.ndarray, np.ndarray]:
"""Quadric-error edge-collapse decimation, vectorized in numpy.
Each round collapses a set of edges at once: an edge is taken if it is the
cheapest edge of both its endpoints, so no two collapses share a vertex. The
cheapest such edges are taken up to the remaining face budget, skipping any
collapse that would flip a triangle. Boundary edges and vertices are kept.
"""
v = np.asarray(vertices, dtype=np.float64).copy()
f = np.asarray(faces, dtype=np.int64).copy()
for _ in range(max_rounds):
if len(f) <= target_faces:
break
nv = len(v)
e = np.sort(np.concatenate([f[:, [0, 1]], f[:, [1, 2]], f[:, [2, 0]]]), axis=1)
key, count = np.unique(e[:, 0] * nv + e[:, 1], return_counts=True)
e = np.stack([key // nv, key % nv], axis=1)
on_boundary = np.zeros(nv, dtype=bool)
on_boundary[e[count == 1].ravel()] = True
e = e[(count == 2) & ~on_boundary[e[:, 0]] & ~on_boundary[e[:, 1]]]
q = _qem_vertex_quadrics(v, f)
x, cost = _qem_collapse_targets(q[e[:, 0]] + q[e[:, 1]], v[e[:, 0]], v[e[:, 1]])
normals = np.cross(v[f[:, 1]] - v[f[:, 0]], v[f[:, 2]] - v[f[:, 0]])
budget = max(1, (len(f) - target_faces) // 2) # a collapse removes ~2 faces
def collapse(idx):
new_v, remap = v.copy(), np.arange(nv)
new_v[e[idx, 0]] = x[idx]
remap[e[idx, 1]] = e[idx, 0]
new_f = remap[f]
live = (
(new_f[:, 0] != new_f[:, 1])
& (new_f[:, 1] != new_f[:, 2])
& (new_f[:, 0] != new_f[:, 2])
)
new_normals = np.cross(
new_v[new_f[:, 1]] - new_v[new_f[:, 0]],
new_v[new_f[:, 2]] - new_v[new_f[:, 0]],
)
flipped = live & ((normals * new_normals).sum(1) <= 0)
owner = np.full(nv, -1)
owner[e[idx, 0]] = owner[e[idx, 1]] = np.arange(len(idx))
bad = np.unique(owner[f[flipped]].ravel())
return new_v, new_f[live], bad[bad >= 0]
blocked = np.zeros(len(e), dtype=bool)
idx, bad = np.empty(0, dtype=np.int64), np.empty(0, dtype=np.int64)
for _ in range(8):
c = np.where(blocked, np.inf, cost)
best = np.full(nv, np.inf)
np.minimum.at(best, e[:, 0], c)
np.minimum.at(best, e[:, 1], c)
idx = np.flatnonzero(
np.isfinite(c) & (c == best[e[:, 0]]) & (c == best[e[:, 1]])
)
idx = idx[np.argsort(cost[idx])[:budget]]
if len(idx) == 0:
break
new_v, new_f, bad = collapse(idx)
if len(bad) == 0:
break
blocked[idx[bad]] = True
for _ in range(8):
if len(idx) == 0 or len(bad) == 0:
break
idx = np.delete(idx, bad)
if len(idx):
new_v, new_f, bad = collapse(idx)
if len(idx) == 0 or len(bad):
break
# collapses can leave two faces on the same vertices; keep one
_, first = np.unique(np.sort(new_f, axis=1), axis=0, return_index=True)
v, f = new_v, new_f[np.sort(first)]
used = np.unique(f)
remap = np.full(len(v), -1)
remap[used] = np.arange(len(used))
return v[used], remap[f]
def mesh_simplify(mesh: Any, target_faces: int = REMESH_TARGET_FACES) -> Any:
"""Decimate mesh to at most target_faces faces (Hunyuan3D-2.1 remesh)."""
if isinstance(mesh, trimesh.Scene):
mesh = mesh.dump(concatenate=True)
if len(mesh.faces) <= target_faces:
return mesh
vertices, faces = _qem_decimate(mesh.vertices, mesh.faces, target_faces)
return trimesh.Trimesh(vertices=vertices, faces=faces)
def mesh_uv_wrap(mesh: Any) -> Any:
"""Apply UV unwrapping to mesh. In-place like native Hunyuan3D-2 for same layout."""
try:
@@ -6,6 +6,7 @@ from unittest.mock import Mock, patch
import numpy as np
import torch
import trimesh
from diffusers import AutoencoderKL as DiffusersAutoencoderKL
from diffusers import LCMScheduler, UNet2DConditionModel
@@ -29,11 +30,13 @@ from sglang.multimodal_gen.runtime.pipelines.hunyuan3d_pipeline import (
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan3d.paint import (
Hunyuan3DPaintPostprocessStage,
Hunyuan3DPaintPreprocessStage,
Hunyuan3DPaintTexGenStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan3d.shape import (
Hunyuan3DShapeSaveStage,
)
from sglang.multimodal_gen.runtime.utils.mesh3d_utils import mesh_simplify
def _unet_config() -> dict:
@@ -236,5 +239,44 @@ class TestHunyuan3DPaintTurboSchedule(unittest.TestCase):
self.assertFalse(stage.scheduler.custom_timesteps)
class TestHunyuan3DPaintRemesh(unittest.TestCase):
def _unwrap(self, use_remesh: bool) -> trimesh.Trimesh:
stage = Hunyuan3DPaintPreprocessStage.__new__(Hunyuan3DPaintPreprocessStage)
stage.config = Hunyuan3D2PipelineConfig(paint_use_remesh=use_remesh)
mesh = trimesh.creation.icosphere(subdivisions=6) # 81,920 faces
with patch(
"sglang.multimodal_gen.runtime.utils.mesh3d_utils.mesh_uv_wrap",
side_effect=lambda m: m,
):
return stage._unwrap_mesh(mesh)
def test_remesh_decimates_to_40k_faces_before_uv_unwrap(self):
mesh = self._unwrap(use_remesh=True)
self.assertLessEqual(len(mesh.faces), 40000)
self.assertGreater(len(mesh.faces), 39000)
def test_remesh_disabled_keeps_all_faces(self):
self.assertEqual(len(self._unwrap(use_remesh=False).faces), 81920)
def test_decimation_keeps_shape_and_topology(self):
sphere = trimesh.creation.icosphere(subdivisions=6, radius=1.0)
mesh = mesh_simplify(sphere, target_faces=5000)
self.assertLessEqual(len(mesh.faces), 5000)
self.assertGreater(len(mesh.faces), 4500)
self.assertTrue(mesh.is_watertight)
radius = np.linalg.norm(mesh.vertices, axis=1)
self.assertLess(np.abs(radius - 1.0).max(), 0.02)
def test_decimation_keeps_open_boundary(self):
sphere = trimesh.creation.icosphere(subdivisions=5)
half = trimesh.Trimesh(
sphere.vertices, sphere.faces[sphere.triangles_center[:, 2] > 0]
)
mesh = mesh_simplify(half, target_faces=2000)
self.assertLessEqual(len(mesh.faces), 2000)
boundary = lambda m: (np.bincount(m.edges_unique_inverse) == 1).sum()
self.assertEqual(boundary(mesh), boundary(half))
if __name__ == "__main__":
unittest.main()