[diffusion] model: restore folded H3 GGUF patch embedding (#39523)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -410,6 +410,14 @@ class MiniMaxH3Qwen3VLEncoder(TextEncoder):
|
||||
)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
try:
|
||||
if (
|
||||
name == "model.visual.patch_embed.proj.weight"
|
||||
and param.ndim == 5
|
||||
and tuple(loaded_weight.shape)
|
||||
== (param.shape[0] * param.shape[1], *param.shape[2:])
|
||||
):
|
||||
# H3 GGUF folds the Conv3D output and input channel axes.
|
||||
loaded_weight = loaded_weight.reshape(param.shape)
|
||||
can_keep_checkpoint_tensor = bool(
|
||||
getattr(self, "_keep_checkpoint_mapping", False)
|
||||
and weight_loader is default_weight_loader
|
||||
|
||||
@@ -35,6 +35,9 @@ from sglang.multimodal_gen.runtime.loader.gguf_weights import (
|
||||
read_gguf_tensor_meta,
|
||||
remap_gguf_tensor_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.minimax_h3_qwen3vl import (
|
||||
MiniMaxH3Qwen3VLEncoder,
|
||||
)
|
||||
from sglang.srt.layers.quantization.gguf import UNQUANTIZED_TYPES
|
||||
from sglang.srt.utils.hf_transformers import check_gguf_file
|
||||
|
||||
@@ -102,6 +105,97 @@ def _write_gguf(
|
||||
path.write_bytes(body)
|
||||
|
||||
|
||||
class TestMiniMaxH3GGUFPatchEmbedding(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _encoder(dtype=torch.float32):
|
||||
encoder = MiniMaxH3Qwen3VLEncoder.__new__(MiniMaxH3Qwen3VLEncoder)
|
||||
torch.nn.Module.__init__(encoder)
|
||||
encoder.selected_lm_layer = 50
|
||||
encoder.model = torch.nn.Module()
|
||||
encoder.model.visual = torch.nn.Module()
|
||||
encoder.model.visual.patch_embed = torch.nn.Module()
|
||||
encoder.model.visual.patch_embed.proj = torch.nn.Conv3d(
|
||||
3, 4, (2, 2, 2), bias=False, dtype=dtype
|
||||
)
|
||||
return encoder
|
||||
|
||||
def test_folded_patch_weight_from_gguf_matches_conv3d(self):
|
||||
# H3 GGUF stores (out * in, temporal, height, width), without an
|
||||
# original-shape field. Distinct channel values catch axis mixups.
|
||||
source = torch.arange(96, dtype=torch.float32).reshape(4, 3, 2, 2, 2)
|
||||
source = source.sub(48).div(32).to(torch.bfloat16)
|
||||
name = "visual.patch_embed.proj.weight"
|
||||
target_name = f"model.{name}"
|
||||
inputs = torch.arange(3 * 4 * 5 * 6).reshape(1, 3, 4, 5, 6).float() / 100
|
||||
expected = torch.nn.functional.conv3d(inputs, source.float())
|
||||
|
||||
for original_shape in (False, True):
|
||||
with self.subTest(original_shape=original_shape):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "patch.gguf"
|
||||
metadata = (
|
||||
(
|
||||
_kv_u64_array(
|
||||
f"comfy.gguf.orig_shape.{name}", list(source.shape)
|
||||
),
|
||||
)
|
||||
if original_shape
|
||||
else ()
|
||||
)
|
||||
_write_gguf(
|
||||
path,
|
||||
[
|
||||
(
|
||||
name,
|
||||
[2, 2, 2, 12],
|
||||
_BF16,
|
||||
source.view(torch.int16).numpy().tobytes(),
|
||||
)
|
||||
],
|
||||
metadata=metadata,
|
||||
)
|
||||
meta = read_gguf_tensor_meta(str(path))
|
||||
self.assertEqual(
|
||||
meta[name].logical_shape,
|
||||
tuple(source.shape) if original_shape else (12, 2, 2, 2),
|
||||
)
|
||||
encoder = self._encoder()
|
||||
loaded = encoder.load_weights(
|
||||
gguf_weights_iterator(str(path), meta)
|
||||
)
|
||||
|
||||
self.assertEqual(loaded, {target_name})
|
||||
projection = encoder.model.visual.patch_embed.proj
|
||||
torch.testing.assert_close(
|
||||
projection.weight, source.float(), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(projection(inputs), expected, rtol=0, atol=0)
|
||||
|
||||
def test_folded_patch_weight_keeps_checkpoint_mapping(self):
|
||||
encoder = self._encoder(torch.bfloat16)
|
||||
encoder._keep_checkpoint_mapping = True
|
||||
source = torch.arange(96).to(torch.bfloat16).reshape(12, 2, 2, 2)
|
||||
encoder.load_weights([("model.visual.patch_embed.proj.weight", source)])
|
||||
weight = encoder.model.visual.patch_embed.proj.weight
|
||||
self.assertEqual(weight.data_ptr(), source.data_ptr())
|
||||
torch.testing.assert_close(weight, source.reshape(4, 3, 2, 2, 2))
|
||||
|
||||
def test_other_equal_size_shapes_are_rejected(self):
|
||||
for shape in ((4, 6, 2, 2), (12, 2, 4), (96,)):
|
||||
with self.subTest(shape=shape):
|
||||
encoder = self._encoder()
|
||||
with self.assertRaisesRegex(RuntimeError, "checkpoint=.*parameter="):
|
||||
encoder.load_weights(
|
||||
[("visual.patch_embed.proj.weight", torch.ones(shape))]
|
||||
)
|
||||
|
||||
def test_other_conv3d_weights_are_not_reshaped(self):
|
||||
encoder = self._encoder()
|
||||
encoder.model.visual.other = torch.nn.Conv3d(3, 4, (2, 2, 2), bias=False)
|
||||
with self.assertRaisesRegex(RuntimeError, "checkpoint=.*parameter="):
|
||||
encoder.load_weights([("visual.other.weight", torch.ones(12, 2, 2, 2))])
|
||||
|
||||
|
||||
class TestGGUFTensorMeta(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
Reference in New Issue
Block a user