diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py index 028ff65e5a..69cbdf7c9f 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py @@ -66,6 +66,7 @@ class Hunyuan3D2PipelineConfig(PipelineConfig): paint_resolution: int = 512 paint_render_size: int = 2048 paint_texture_size: int = 2048 + # Decimate the mesh to 40k faces before UV unwrapping, as Hunyuan3D-2.1 does paint_use_remesh: bool = True paint_save_glb: bool = True paint_turbo_mode: bool = False diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/paint.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/paint.py index 9cd08e3c86..c6582d8463 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/paint.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/paint.py @@ -181,10 +181,16 @@ class Hunyuan3DPaintPreprocessStage(PipelineStage): ComponentUse(stage_name, "delight_vae", phase="decode"), ] - @staticmethod - def _unwrap_mesh(mesh: Any) -> Any: - from sglang.multimodal_gen.runtime.utils.mesh3d_utils import mesh_uv_wrap + def _unwrap_mesh(self, mesh: Any) -> Any: + from sglang.multimodal_gen.runtime.utils.mesh3d_utils import ( + mesh_simplify, + mesh_uv_wrap, + ) + # xatlas time grows superlinearly with face count; the raw shape mesh + # has ~650k faces at the default octree resolution. + if self.config.paint_use_remesh: + mesh = mesh_simplify(mesh) return mesh_uv_wrap(mesh) @staticmethod diff --git a/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py b/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py index 55919b2585..22f909c0ad 100644 --- a/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/mesh3d_utils.py @@ -142,6 +142,136 @@ def export_to_trimesh(mesh_output: Any) -> Any: return trimesh.Trimesh(mesh_output.mesh_v, mesh_output.mesh_f) +# Hunyuan3D-2.1's use_remesh decimates to this many faces before UV unwrapping. +REMESH_TARGET_FACES = 40000 + + +def _qem_vertex_quadrics(v: np.ndarray, f: np.ndarray) -> np.ndarray: + """Sum of the plane quadrics of the faces around each vertex, (V, 4, 4).""" + p0 = v[f[:, 0]] + n = np.cross(v[f[:, 1]] - p0, v[f[:, 2]] - p0) + n /= np.maximum(np.linalg.norm(n, axis=1), 1e-20)[:, None] + plane = np.concatenate([n, -(n * p0).sum(1, keepdims=True)], axis=1) + face_q = (plane[:, :, None] * plane[:, None, :]).reshape(len(f), 16) + idx = f.T.ravel() + q = np.stack( + [ + np.bincount(idx, weights=np.tile(face_q[:, c], 3), minlength=len(v)) + for c in range(16) + ], + axis=1, + ) + return q.reshape(-1, 4, 4) + + +def _qem_collapse_targets( + q: np.ndarray, va: np.ndarray, vb: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: + """Optimal position and quadric error for collapsing each edge (va, vb).""" + mid = (va + vb) / 2 + x = mid.copy() + a = q[:, :3, :3] + solvable = np.abs(np.linalg.det(a)) > 1e-12 + if solvable.any(): + x[solvable] = np.linalg.solve(a[solvable], -q[solvable, :3, 3:])[:, :, 0] + # an optimum far from the edge is unstable; use the midpoint instead + far = np.linalg.norm(x - mid, axis=1) > 2 * np.linalg.norm(vb - va, axis=1) + x[far] = mid[far] + xh = np.concatenate([x, np.ones((len(x), 1))], axis=1) + return x, np.maximum(np.einsum("ni,nij,nj->n", xh, q, xh), 0) + + +def _qem_decimate( + vertices: np.ndarray, faces: np.ndarray, target_faces: int, max_rounds: int = 100 +) -> Tuple[np.ndarray, np.ndarray]: + """Quadric-error edge-collapse decimation, vectorized in numpy. + + Each round collapses a set of edges at once: an edge is taken if it is the + cheapest edge of both its endpoints, so no two collapses share a vertex. The + cheapest such edges are taken up to the remaining face budget, skipping any + collapse that would flip a triangle. Boundary edges and vertices are kept. + """ + v = np.asarray(vertices, dtype=np.float64).copy() + f = np.asarray(faces, dtype=np.int64).copy() + for _ in range(max_rounds): + if len(f) <= target_faces: + break + nv = len(v) + e = np.sort(np.concatenate([f[:, [0, 1]], f[:, [1, 2]], f[:, [2, 0]]]), axis=1) + key, count = np.unique(e[:, 0] * nv + e[:, 1], return_counts=True) + e = np.stack([key // nv, key % nv], axis=1) + on_boundary = np.zeros(nv, dtype=bool) + on_boundary[e[count == 1].ravel()] = True + e = e[(count == 2) & ~on_boundary[e[:, 0]] & ~on_boundary[e[:, 1]]] + q = _qem_vertex_quadrics(v, f) + x, cost = _qem_collapse_targets(q[e[:, 0]] + q[e[:, 1]], v[e[:, 0]], v[e[:, 1]]) + normals = np.cross(v[f[:, 1]] - v[f[:, 0]], v[f[:, 2]] - v[f[:, 0]]) + budget = max(1, (len(f) - target_faces) // 2) # a collapse removes ~2 faces + + def collapse(idx): + new_v, remap = v.copy(), np.arange(nv) + new_v[e[idx, 0]] = x[idx] + remap[e[idx, 1]] = e[idx, 0] + new_f = remap[f] + live = ( + (new_f[:, 0] != new_f[:, 1]) + & (new_f[:, 1] != new_f[:, 2]) + & (new_f[:, 0] != new_f[:, 2]) + ) + new_normals = np.cross( + new_v[new_f[:, 1]] - new_v[new_f[:, 0]], + new_v[new_f[:, 2]] - new_v[new_f[:, 0]], + ) + flipped = live & ((normals * new_normals).sum(1) <= 0) + owner = np.full(nv, -1) + owner[e[idx, 0]] = owner[e[idx, 1]] = np.arange(len(idx)) + bad = np.unique(owner[f[flipped]].ravel()) + return new_v, new_f[live], bad[bad >= 0] + + blocked = np.zeros(len(e), dtype=bool) + idx, bad = np.empty(0, dtype=np.int64), np.empty(0, dtype=np.int64) + for _ in range(8): + c = np.where(blocked, np.inf, cost) + best = np.full(nv, np.inf) + np.minimum.at(best, e[:, 0], c) + np.minimum.at(best, e[:, 1], c) + idx = np.flatnonzero( + np.isfinite(c) & (c == best[e[:, 0]]) & (c == best[e[:, 1]]) + ) + idx = idx[np.argsort(cost[idx])[:budget]] + if len(idx) == 0: + break + new_v, new_f, bad = collapse(idx) + if len(bad) == 0: + break + blocked[idx[bad]] = True + for _ in range(8): + if len(idx) == 0 or len(bad) == 0: + break + idx = np.delete(idx, bad) + if len(idx): + new_v, new_f, bad = collapse(idx) + if len(idx) == 0 or len(bad): + break + # collapses can leave two faces on the same vertices; keep one + _, first = np.unique(np.sort(new_f, axis=1), axis=0, return_index=True) + v, f = new_v, new_f[np.sort(first)] + used = np.unique(f) + remap = np.full(len(v), -1) + remap[used] = np.arange(len(used)) + return v[used], remap[f] + + +def mesh_simplify(mesh: Any, target_faces: int = REMESH_TARGET_FACES) -> Any: + """Decimate mesh to at most target_faces faces (Hunyuan3D-2.1 remesh).""" + if isinstance(mesh, trimesh.Scene): + mesh = mesh.dump(concatenate=True) + if len(mesh.faces) <= target_faces: + return mesh + vertices, faces = _qem_decimate(mesh.vertices, mesh.faces, target_faces) + return trimesh.Trimesh(vertices=vertices, faces=faces) + + def mesh_uv_wrap(mesh: Any) -> Any: """Apply UV unwrapping to mesh. In-place like native Hunyuan3D-2 for same layout.""" try: diff --git a/python/sglang/multimodal_gen/test/unit/test_hunyuan3d_native_texture_models.py b/python/sglang/multimodal_gen/test/unit/test_hunyuan3d_native_texture_models.py index 264ff798db..cdf3d6a81c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_hunyuan3d_native_texture_models.py +++ b/python/sglang/multimodal_gen/test/unit/test_hunyuan3d_native_texture_models.py @@ -6,6 +6,7 @@ from unittest.mock import Mock, patch import numpy as np import torch +import trimesh from diffusers import AutoencoderKL as DiffusersAutoencoderKL from diffusers import LCMScheduler, UNet2DConditionModel @@ -29,11 +30,13 @@ from sglang.multimodal_gen.runtime.pipelines.hunyuan3d_pipeline import ( ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan3d.paint import ( Hunyuan3DPaintPostprocessStage, + Hunyuan3DPaintPreprocessStage, Hunyuan3DPaintTexGenStage, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan3d.shape import ( Hunyuan3DShapeSaveStage, ) +from sglang.multimodal_gen.runtime.utils.mesh3d_utils import mesh_simplify def _unet_config() -> dict: @@ -236,5 +239,44 @@ class TestHunyuan3DPaintTurboSchedule(unittest.TestCase): self.assertFalse(stage.scheduler.custom_timesteps) +class TestHunyuan3DPaintRemesh(unittest.TestCase): + def _unwrap(self, use_remesh: bool) -> trimesh.Trimesh: + stage = Hunyuan3DPaintPreprocessStage.__new__(Hunyuan3DPaintPreprocessStage) + stage.config = Hunyuan3D2PipelineConfig(paint_use_remesh=use_remesh) + mesh = trimesh.creation.icosphere(subdivisions=6) # 81,920 faces + with patch( + "sglang.multimodal_gen.runtime.utils.mesh3d_utils.mesh_uv_wrap", + side_effect=lambda m: m, + ): + return stage._unwrap_mesh(mesh) + + def test_remesh_decimates_to_40k_faces_before_uv_unwrap(self): + mesh = self._unwrap(use_remesh=True) + self.assertLessEqual(len(mesh.faces), 40000) + self.assertGreater(len(mesh.faces), 39000) + + def test_remesh_disabled_keeps_all_faces(self): + self.assertEqual(len(self._unwrap(use_remesh=False).faces), 81920) + + def test_decimation_keeps_shape_and_topology(self): + sphere = trimesh.creation.icosphere(subdivisions=6, radius=1.0) + mesh = mesh_simplify(sphere, target_faces=5000) + self.assertLessEqual(len(mesh.faces), 5000) + self.assertGreater(len(mesh.faces), 4500) + self.assertTrue(mesh.is_watertight) + radius = np.linalg.norm(mesh.vertices, axis=1) + self.assertLess(np.abs(radius - 1.0).max(), 0.02) + + def test_decimation_keeps_open_boundary(self): + sphere = trimesh.creation.icosphere(subdivisions=5) + half = trimesh.Trimesh( + sphere.vertices, sphere.faces[sphere.triangles_center[:, 2] > 0] + ) + mesh = mesh_simplify(half, target_faces=2000) + self.assertLessEqual(len(mesh.faces), 2000) + boundary = lambda m: (np.bincount(m.edges_unique_inverse) == 1).sum() + self.assertEqual(boundary(mesh), boundary(half)) + + if __name__ == "__main__": unittest.main()