[diffusion] model: restore folded H3 GGUF patch embedding (#39523)

Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
liuke
2026-10-05 21:10:06 +08:00
committed by GitHub
co-authored by Mick
parent 83a6e1d39b
commit f20f8e120d
2 changed files with 102 additions and 0 deletions
@@ -410,6 +410,14 @@ class MiniMaxH3Qwen3VLEncoder(TextEncoder):
)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
try:
if (
name == "model.visual.patch_embed.proj.weight"
and param.ndim == 5
and tuple(loaded_weight.shape)
== (param.shape[0] * param.shape[1], *param.shape[2:])
):
# H3 GGUF folds the Conv3D output and input channel axes.
loaded_weight = loaded_weight.reshape(param.shape)
can_keep_checkpoint_tensor = bool(
getattr(self, "_keep_checkpoint_mapping", False)
and weight_loader is default_weight_loader
@@ -35,6 +35,9 @@ from sglang.multimodal_gen.runtime.loader.gguf_weights import (
read_gguf_tensor_meta,
remap_gguf_tensor_meta,
)
from sglang.multimodal_gen.runtime.models.encoders.minimax_h3_qwen3vl import (
MiniMaxH3Qwen3VLEncoder,
)
from sglang.srt.layers.quantization.gguf import UNQUANTIZED_TYPES
from sglang.srt.utils.hf_transformers import check_gguf_file
@@ -102,6 +105,97 @@ def _write_gguf(
path.write_bytes(body)
class TestMiniMaxH3GGUFPatchEmbedding(unittest.TestCase):
@staticmethod
def _encoder(dtype=torch.float32):
encoder = MiniMaxH3Qwen3VLEncoder.__new__(MiniMaxH3Qwen3VLEncoder)
torch.nn.Module.__init__(encoder)
encoder.selected_lm_layer = 50
encoder.model = torch.nn.Module()
encoder.model.visual = torch.nn.Module()
encoder.model.visual.patch_embed = torch.nn.Module()
encoder.model.visual.patch_embed.proj = torch.nn.Conv3d(
3, 4, (2, 2, 2), bias=False, dtype=dtype
)
return encoder
def test_folded_patch_weight_from_gguf_matches_conv3d(self):
# H3 GGUF stores (out * in, temporal, height, width), without an
# original-shape field. Distinct channel values catch axis mixups.
source = torch.arange(96, dtype=torch.float32).reshape(4, 3, 2, 2, 2)
source = source.sub(48).div(32).to(torch.bfloat16)
name = "visual.patch_embed.proj.weight"
target_name = f"model.{name}"
inputs = torch.arange(3 * 4 * 5 * 6).reshape(1, 3, 4, 5, 6).float() / 100
expected = torch.nn.functional.conv3d(inputs, source.float())
for original_shape in (False, True):
with self.subTest(original_shape=original_shape):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "patch.gguf"
metadata = (
(
_kv_u64_array(
f"comfy.gguf.orig_shape.{name}", list(source.shape)
),
)
if original_shape
else ()
)
_write_gguf(
path,
[
(
name,
[2, 2, 2, 12],
_BF16,
source.view(torch.int16).numpy().tobytes(),
)
],
metadata=metadata,
)
meta = read_gguf_tensor_meta(str(path))
self.assertEqual(
meta[name].logical_shape,
tuple(source.shape) if original_shape else (12, 2, 2, 2),
)
encoder = self._encoder()
loaded = encoder.load_weights(
gguf_weights_iterator(str(path), meta)
)
self.assertEqual(loaded, {target_name})
projection = encoder.model.visual.patch_embed.proj
torch.testing.assert_close(
projection.weight, source.float(), rtol=0, atol=0
)
torch.testing.assert_close(projection(inputs), expected, rtol=0, atol=0)
def test_folded_patch_weight_keeps_checkpoint_mapping(self):
encoder = self._encoder(torch.bfloat16)
encoder._keep_checkpoint_mapping = True
source = torch.arange(96).to(torch.bfloat16).reshape(12, 2, 2, 2)
encoder.load_weights([("model.visual.patch_embed.proj.weight", source)])
weight = encoder.model.visual.patch_embed.proj.weight
self.assertEqual(weight.data_ptr(), source.data_ptr())
torch.testing.assert_close(weight, source.reshape(4, 3, 2, 2, 2))
def test_other_equal_size_shapes_are_rejected(self):
for shape in ((4, 6, 2, 2), (12, 2, 4), (96,)):
with self.subTest(shape=shape):
encoder = self._encoder()
with self.assertRaisesRegex(RuntimeError, "checkpoint=.*parameter="):
encoder.load_weights(
[("visual.patch_embed.proj.weight", torch.ones(shape))]
)
def test_other_conv3d_weights_are_not_reshaped(self):
encoder = self._encoder()
encoder.model.visual.other = torch.nn.Conv3d(3, 4, (2, 2, 2), bias=False)
with self.assertRaisesRegex(RuntimeError, "checkpoint=.*parameter="):
encoder.load_weights([("visual.other.weight", torch.ones(12, 2, 2, 2))])
class TestGGUFTensorMeta(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()