diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/minimax_h3_qwen3vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/minimax_h3_qwen3vl.py index f6d7940957..a025509482 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/minimax_h3_qwen3vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/minimax_h3_qwen3vl.py @@ -410,6 +410,14 @@ class MiniMaxH3Qwen3VLEncoder(TextEncoder): ) weight_loader = getattr(param, "weight_loader", default_weight_loader) try: + if ( + name == "model.visual.patch_embed.proj.weight" + and param.ndim == 5 + and tuple(loaded_weight.shape) + == (param.shape[0] * param.shape[1], *param.shape[2:]) + ): + # H3 GGUF folds the Conv3D output and input channel axes. + loaded_weight = loaded_weight.reshape(param.shape) can_keep_checkpoint_tensor = bool( getattr(self, "_keep_checkpoint_mapping", False) and weight_loader is default_weight_loader diff --git a/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py b/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py index e1ed2c9ed8..c20748fc6a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py +++ b/python/sglang/multimodal_gen/test/unit/test_gguf_diffusion.py @@ -35,6 +35,9 @@ from sglang.multimodal_gen.runtime.loader.gguf_weights import ( read_gguf_tensor_meta, remap_gguf_tensor_meta, ) +from sglang.multimodal_gen.runtime.models.encoders.minimax_h3_qwen3vl import ( + MiniMaxH3Qwen3VLEncoder, +) from sglang.srt.layers.quantization.gguf import UNQUANTIZED_TYPES from sglang.srt.utils.hf_transformers import check_gguf_file @@ -102,6 +105,97 @@ def _write_gguf( path.write_bytes(body) +class TestMiniMaxH3GGUFPatchEmbedding(unittest.TestCase): + @staticmethod + def _encoder(dtype=torch.float32): + encoder = MiniMaxH3Qwen3VLEncoder.__new__(MiniMaxH3Qwen3VLEncoder) + torch.nn.Module.__init__(encoder) + encoder.selected_lm_layer = 50 + encoder.model = torch.nn.Module() + encoder.model.visual = torch.nn.Module() + encoder.model.visual.patch_embed = torch.nn.Module() + encoder.model.visual.patch_embed.proj = torch.nn.Conv3d( + 3, 4, (2, 2, 2), bias=False, dtype=dtype + ) + return encoder + + def test_folded_patch_weight_from_gguf_matches_conv3d(self): + # H3 GGUF stores (out * in, temporal, height, width), without an + # original-shape field. Distinct channel values catch axis mixups. + source = torch.arange(96, dtype=torch.float32).reshape(4, 3, 2, 2, 2) + source = source.sub(48).div(32).to(torch.bfloat16) + name = "visual.patch_embed.proj.weight" + target_name = f"model.{name}" + inputs = torch.arange(3 * 4 * 5 * 6).reshape(1, 3, 4, 5, 6).float() / 100 + expected = torch.nn.functional.conv3d(inputs, source.float()) + + for original_shape in (False, True): + with self.subTest(original_shape=original_shape): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "patch.gguf" + metadata = ( + ( + _kv_u64_array( + f"comfy.gguf.orig_shape.{name}", list(source.shape) + ), + ) + if original_shape + else () + ) + _write_gguf( + path, + [ + ( + name, + [2, 2, 2, 12], + _BF16, + source.view(torch.int16).numpy().tobytes(), + ) + ], + metadata=metadata, + ) + meta = read_gguf_tensor_meta(str(path)) + self.assertEqual( + meta[name].logical_shape, + tuple(source.shape) if original_shape else (12, 2, 2, 2), + ) + encoder = self._encoder() + loaded = encoder.load_weights( + gguf_weights_iterator(str(path), meta) + ) + + self.assertEqual(loaded, {target_name}) + projection = encoder.model.visual.patch_embed.proj + torch.testing.assert_close( + projection.weight, source.float(), rtol=0, atol=0 + ) + torch.testing.assert_close(projection(inputs), expected, rtol=0, atol=0) + + def test_folded_patch_weight_keeps_checkpoint_mapping(self): + encoder = self._encoder(torch.bfloat16) + encoder._keep_checkpoint_mapping = True + source = torch.arange(96).to(torch.bfloat16).reshape(12, 2, 2, 2) + encoder.load_weights([("model.visual.patch_embed.proj.weight", source)]) + weight = encoder.model.visual.patch_embed.proj.weight + self.assertEqual(weight.data_ptr(), source.data_ptr()) + torch.testing.assert_close(weight, source.reshape(4, 3, 2, 2, 2)) + + def test_other_equal_size_shapes_are_rejected(self): + for shape in ((4, 6, 2, 2), (12, 2, 4), (96,)): + with self.subTest(shape=shape): + encoder = self._encoder() + with self.assertRaisesRegex(RuntimeError, "checkpoint=.*parameter="): + encoder.load_weights( + [("visual.patch_embed.proj.weight", torch.ones(shape))] + ) + + def test_other_conv3d_weights_are_not_reshaped(self): + encoder = self._encoder() + encoder.model.visual.other = torch.nn.Conv3d(3, 4, (2, 2, 2), bias=False) + with self.assertRaisesRegex(RuntimeError, "checkpoint=.*parameter="): + encoder.load_weights([("visual.other.weight", torch.ones(12, 2, 2, 2))]) + + class TestGGUFTensorMeta(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory()