[diffusion] refactor: share stacked encoder weight loading (#42988)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 09:16:03 +08:00
committed by GitHub
co-authored by Mick Qian
parent 0427ea8032
commit 1e2f524726
5 changed files with 146 additions and 279 deletions
@@ -395,6 +395,34 @@ def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> N
raise
def load_stacked_weight(
name: str,
loaded_weight: torch.Tensor,
params_dict: dict[str, torch.nn.Parameter],
stacked_params_mapping: Iterable[tuple[str, str, str | int]],
) -> str | None:
"""Load a known parameter or its fused shard; return None for unknown names."""
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
param.weight_loader(param, loaded_weight, shard_id)
return name
if name not in params_dict:
return None
param = params_dict[name]
try:
weight_loader = param.weight_loader
except AttributeError:
weight_loader = default_weight_loader
weight_loader(param, loaded_weight)
return name
def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
"""Remap the name of FP8 k/v_scale parameters.
@@ -51,7 +51,7 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
default_weight_loader,
load_stacked_weight,
maybe_remap_kv_scale_name,
)
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
@@ -400,17 +400,6 @@ class LlamaModel(TextEncoder):
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
# if (self.quant_config is not None and
# (scale_name := self.quant_config.get_cache_scale(name))):
# # Loading kv cache quantization scales
# param = params_dict[scale_name]
# weight_loader = getattr(param, "weight_loader",
# default_weight_loader)
# loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
# loaded_weight[0])
# weight_loader(param, loaded_weight)
# loaded_params.add(scale_name)
# continue
if "scale" in name:
# Remapping the name of FP8 kv-scale.
kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict)
@@ -418,37 +407,14 @@ class LlamaModel(TextEncoder):
continue
else:
name = kv_scale_name
for (
param_name,
weight_name,
shard_id,
) in self.config.arch_config.stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
name = load_stacked_weight(
name,
loaded_weight,
params_dict,
self.config.arch_config.stacked_params_mapping,
)
if name is not None:
loaded_params.add(name)
return loaded_params
@@ -20,7 +20,7 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
default_weight_loader,
load_stacked_weight,
maybe_remap_kv_scale_name,
)
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
@@ -454,40 +454,14 @@ class Qwen3ForCausalLM(TextEncoder):
else:
name = kv_scale_name
# Handle stacked params mapping (qkv_proj, gate_up_proj)
for (
param_name,
weight_name,
shard_id,
) in self.config.arch_config.stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
name = load_stacked_weight(
name,
loaded_weight,
params_dict,
self.config.arch_config.stacked_params_mapping,
)
if name is not None:
loaded_params.add(name)
return loaded_params
@@ -42,7 +42,7 @@ from sglang.multimodal_gen.runtime.layers.utils import get_group_rank, get_group
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader
from sglang.multimodal_gen.runtime.loader.weight_utils import load_stacked_weight
from sglang.multimodal_gen.runtime.models.encoders.base import (
TextEncoder,
get_folding_tp_group,
@@ -618,37 +618,13 @@ class T5EncoderModel(TextEncoder):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
loaded = False
if "decoder" in name or "lm_head" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded = True
break
if not loaded:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
name = load_stacked_weight(
name, loaded_weight, params_dict, stacked_params_mapping
)
if name is not None:
loaded_params.add(name)
return loaded_params
@@ -709,41 +685,16 @@ class UMT5EncoderModel(TextEncoder):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
loaded = False
if "decoder" in name or "lm_head" in name:
continue
for (
param_name,
weight_name,
shard_id,
) in self.config.arch_config.stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded = True
break
if not loaded:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
name = load_stacked_weight(
name,
loaded_weight,
params_dict,
self.config.arch_config.stacked_params_mapping,
)
if name is not None:
loaded_params.add(name)
return loaded_params
@@ -600,6 +600,26 @@ class TestTextEncoderQuantization(unittest.TestCase):
self.addCleanup(self.quant_config_patcher.stop)
self.serialized = serialized
def _configure_h3_quantization(self, tensors, *, metadata=None):
self.get_quant_config.return_value = None
with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint:
save_file(tensors, checkpoint.name, metadata=metadata)
model_config = SimpleNamespace(quant_config=None)
with mock.patch(
"sglang.multimodal_gen.runtime.loader.component_loaders."
"text_encoder_loader.get_quant_config_from_safetensors_metadata",
return_value=None,
):
_configure_encoder_quantization(
model_config,
MiniMaxH3Qwen3VLEncoder,
{},
"/model/text_encoder",
checkpoint.name,
"text_encoder",
)
return model_config.quant_config
def test_serialized_checkpoint_configures_native_encoder(self):
model_config = SimpleNamespace(quant_config=None)
_configure_encoder_quantization(
@@ -652,7 +672,6 @@ class TestTextEncoderQuantization(unittest.TestCase):
get_file_quant_config.assert_called_once_with("/weights/encoder.safetensors")
def test_comfy_int8_weight_file_configures_native_encoder(self):
self.get_quant_config.return_value = None
marker = json.dumps(
{
"format": "int8_tensorwise",
@@ -660,81 +679,47 @@ class TestTextEncoderQuantization(unittest.TestCase):
"convrot_groupsize": 256,
}
).encode()
with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint:
save_file(
{
"visual.blocks.0.attn.qkv.weight": torch.ones(
(2, 256), dtype=torch.int8
),
"visual.blocks.0.attn.qkv.weight_scale": torch.ones((2, 1)),
"visual.blocks.0.attn.qkv.comfy_quant": torch.tensor(
list(marker), dtype=torch.uint8
),
},
checkpoint.name,
)
model_config = SimpleNamespace(quant_config=None)
with mock.patch(
"sglang.multimodal_gen.runtime.loader.component_loaders."
"text_encoder_loader.get_quant_config_from_safetensors_metadata",
return_value=None,
):
_configure_encoder_quantization(
model_config,
MiniMaxH3Qwen3VLEncoder,
{},
"/model/text_encoder",
checkpoint.name,
"text_encoder",
)
quant_config = self._configure_h3_quantization(
{
"visual.blocks.0.attn.qkv.weight": torch.ones(
(2, 256), dtype=torch.int8
),
"visual.blocks.0.attn.qkv.weight_scale": torch.ones((2, 1)),
"visual.blocks.0.attn.qkv.comfy_quant": torch.tensor(
list(marker), dtype=torch.uint8
),
},
)
self.assertIsInstance(model_config.quant_config, ConvRotInt8Config)
self.assertIsInstance(quant_config, ConvRotInt8Config)
self.assertEqual(
set(model_config.quant_config.layer_markers),
set(quant_config.layer_markers),
{"model.visual.blocks.0.attn.qkv_proj"},
)
def test_comfy_w4a4_weight_file_configures_native_encoder(self):
self.get_quant_config.return_value = None
marker = json.dumps(
{"format": "convrot_w4a4", "convrot_groupsize": 256}
).encode()
with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint:
save_file(
{
"model.layers.0.self_attn.q_proj.weight": torch.ones(
(2, 128), dtype=torch.int8
),
"model.layers.0.self_attn.q_proj.weight_scale": torch.ones(2),
"model.layers.0.self_attn.q_proj.comfy_quant": torch.tensor(
list(marker), dtype=torch.uint8
),
},
checkpoint.name,
)
model_config = SimpleNamespace(quant_config=None)
with mock.patch(
"sglang.multimodal_gen.runtime.loader.component_loaders."
"text_encoder_loader.get_quant_config_from_safetensors_metadata",
return_value=None,
):
_configure_encoder_quantization(
model_config,
MiniMaxH3Qwen3VLEncoder,
{},
"/model/text_encoder",
checkpoint.name,
"text_encoder",
)
quant_config = self._configure_h3_quantization(
{
"model.layers.0.self_attn.q_proj.weight": torch.ones(
(2, 128), dtype=torch.int8
),
"model.layers.0.self_attn.q_proj.weight_scale": torch.ones(2),
"model.layers.0.self_attn.q_proj.comfy_quant": torch.tensor(
list(marker), dtype=torch.uint8
),
},
)
self.assertIsInstance(model_config.quant_config, KitchenW4A4Config)
self.assertIsInstance(quant_config, KitchenW4A4Config)
self.assertEqual(
set(model_config.quant_config.layer_markers),
set(quant_config.layer_markers),
{"model.language_model.layers.0.self_attn.q_proj"},
)
def test_mixed_w4a8_weight_file_maps_embedding_and_linear_markers(self):
self.get_quant_config.return_value = None
layers = {
"model.embed_tokens": {"format": "int8_tensorwise"},
"model.layers.0.mlp.down_proj": {
@@ -744,51 +729,32 @@ class TestTextEncoderQuantization(unittest.TestCase):
"convrot_groupsize": 256,
},
}
with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint:
save_file(
{
"model.embed_tokens.weight": torch.ones((4, 256), dtype=torch.int8),
"model.embed_tokens.weight_scale": torch.tensor(0.25),
"model.layers.0.mlp.down_proj.weight": torch.ones(
(2, 128), dtype=torch.int8
),
"model.layers.0.mlp.down_proj.weight_s_rel": torch.ones(
(2, 16), dtype=torch.float8_e4m3fn
),
"model.layers.0.mlp.down_proj.weight_s_channel": torch.ones(2),
"model.layers.0.mlp.down_proj.weight_codebook": torch.ones(16),
},
checkpoint.name,
metadata={"_quantization_metadata": json.dumps({"layers": layers})},
)
model_config = SimpleNamespace(quant_config=None)
with mock.patch(
"sglang.multimodal_gen.runtime.loader.component_loaders."
"text_encoder_loader.get_quant_config_from_safetensors_metadata",
return_value=None,
):
_configure_encoder_quantization(
model_config,
MiniMaxH3Qwen3VLEncoder,
{},
"/model/text_encoder",
checkpoint.name,
"text_encoder",
)
quant_config = self._configure_h3_quantization(
{
"model.embed_tokens.weight": torch.ones((4, 256), dtype=torch.int8),
"model.embed_tokens.weight_scale": torch.tensor(0.25),
"model.layers.0.mlp.down_proj.weight": torch.ones(
(2, 128), dtype=torch.int8
),
"model.layers.0.mlp.down_proj.weight_s_rel": torch.ones(
(2, 16), dtype=torch.float8_e4m3fn
),
"model.layers.0.mlp.down_proj.weight_s_channel": torch.ones(2),
"model.layers.0.mlp.down_proj.weight_codebook": torch.ones(16),
},
metadata={"_quantization_metadata": json.dumps({"layers": layers})},
)
self.assertIsInstance(model_config.quant_config, KitchenW4A8Config)
self.assertIsInstance(quant_config, KitchenW4A8Config)
self.assertTrue(
model_config.quant_config.quantizes_embedding(
"model.language_model.embed_tokens"
)
quant_config.quantizes_embedding("model.language_model.embed_tokens")
)
self.assertIn(
"model.language_model.layers.0.mlp.down_proj",
model_config.quant_config.layer_markers,
quant_config.layer_markers,
)
def test_nvfp4_awq_weight_file_maps_embedding_and_linear_markers(self):
self.get_quant_config.return_value = None
layers = {
"model.embed_tokens": {"format": "int8_tensorwise"},
"model.layers.0.self_attn.o_proj": {
@@ -796,47 +762,29 @@ class TestTextEncoderQuantization(unittest.TestCase):
"full_precision_matrix_mult": True,
},
}
with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint:
save_file(
{
"model.embed_tokens.weight": torch.ones((4, 64), dtype=torch.int8),
"model.embed_tokens.weight_scale": torch.ones(4, 1),
"model.layers.0.self_attn.o_proj.weight": torch.full(
(128, 32), 0x21, dtype=torch.uint8
),
"model.layers.0.self_attn.o_proj.weight_scale": torch.ones(
(128, 4), dtype=torch.float8_e4m3fn
),
"model.layers.0.self_attn.o_proj.weight_scale_2": torch.tensor(0.5),
"model.layers.0.self_attn.o_proj.pre_quant_scale": torch.ones(
64, dtype=torch.bfloat16
),
},
checkpoint.name,
metadata={"_quantization_metadata": json.dumps({"layers": layers})},
)
model_config = SimpleNamespace(quant_config=None)
with mock.patch(
"sglang.multimodal_gen.runtime.loader.component_loaders."
"text_encoder_loader.get_quant_config_from_safetensors_metadata",
return_value=None,
):
_configure_encoder_quantization(
model_config,
MiniMaxH3Qwen3VLEncoder,
{},
"/model/text_encoder",
checkpoint.name,
"text_encoder",
)
self.assertIsInstance(model_config.quant_config, ComfyNvfp4Config)
self.assertTrue(
model_config.quant_config.quantizes_embedding(
"model.language_model.embed_tokens"
)
quant_config = self._configure_h3_quantization(
{
"model.embed_tokens.weight": torch.ones((4, 64), dtype=torch.int8),
"model.embed_tokens.weight_scale": torch.ones(4, 1),
"model.layers.0.self_attn.o_proj.weight": torch.full(
(128, 32), 0x21, dtype=torch.uint8
),
"model.layers.0.self_attn.o_proj.weight_scale": torch.ones(
(128, 4), dtype=torch.float8_e4m3fn
),
"model.layers.0.self_attn.o_proj.weight_scale_2": torch.tensor(0.5),
"model.layers.0.self_attn.o_proj.pre_quant_scale": torch.ones(
64, dtype=torch.bfloat16
),
},
metadata={"_quantization_metadata": json.dumps({"layers": layers})},
)
marker = model_config.quant_config.layer_markers[
self.assertIsInstance(quant_config, ComfyNvfp4Config)
self.assertTrue(
quant_config.quantizes_embedding("model.language_model.embed_tokens")
)
marker = quant_config.layer_markers[
"model.language_model.layers.0.self_attn.o_proj"
]
self.assertTrue(marker["_has_pre_quant_scale"])