From 1e2f524726ff6d7d8c7d3d062dbe922f3dff7ff0 Mon Sep 17 00:00:00 2001 From: Mick Date: Thu, 8 Oct 2026 09:16:03 +0800 Subject: [PATCH] [diffusion] refactor: share stacked encoder weight loading (#42988) Co-authored-by: Mick Qian --- .../runtime/loader/weight_utils.py | 28 +++ .../runtime/models/encoders/llama.py | 52 +--- .../runtime/models/encoders/qwen3.py | 44 +--- .../runtime/models/encoders/t5.py | 77 ++---- .../test/unit/test_text_encoder_loader.py | 224 +++++++----------- 5 files changed, 146 insertions(+), 279 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py index 5cfa320169..831d6e0347 100644 --- a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py @@ -395,6 +395,34 @@ def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> N raise +def load_stacked_weight( + name: str, + loaded_weight: torch.Tensor, + params_dict: dict[str, torch.nn.Parameter], + stacked_params_mapping: Iterable[tuple[str, str, str | int]], +) -> str | None: + """Load a known parameter or its fused shard; return None for unknown names.""" + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if name not in params_dict: + continue + param = params_dict[name] + param.weight_loader(param, loaded_weight, shard_id) + return name + + if name not in params_dict: + return None + param = params_dict[name] + try: + weight_loader = param.weight_loader + except AttributeError: + weight_loader = default_weight_loader + weight_loader(param, loaded_weight) + return name + + def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None: """Remap the name of FP8 k/v_scale parameters. diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py index 5debd97354..2b8190cb25 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py @@ -51,7 +51,7 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from sglang.multimodal_gen.runtime.loader.weight_utils import ( - default_weight_loader, + load_stacked_weight, maybe_remap_kv_scale_name, ) from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder @@ -400,17 +400,6 @@ class LlamaModel(TextEncoder): # Models trained using ColossalAI may include these tensors in # the checkpoint. Skip them. continue - # if (self.quant_config is not None and - # (scale_name := self.quant_config.get_cache_scale(name))): - # # Loading kv cache quantization scales - # param = params_dict[scale_name] - # weight_loader = getattr(param, "weight_loader", - # default_weight_loader) - # loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else - # loaded_weight[0]) - # weight_loader(param, loaded_weight) - # loaded_params.add(scale_name) - # continue if "scale" in name: # Remapping the name of FP8 kv-scale. kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict) @@ -418,37 +407,14 @@ class LlamaModel(TextEncoder): continue else: name = kv_scale_name - for ( - param_name, - weight_name, - shard_id, - ) in self.config.arch_config.stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) + name = load_stacked_weight( + name, + loaded_weight, + params_dict, + self.config.arch_config.stacked_params_mapping, + ) + if name is not None: + loaded_params.add(name) return loaded_params diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py index 6b5e80ef56..b74d93c7fb 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3.py @@ -20,7 +20,7 @@ from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from sglang.multimodal_gen.runtime.loader.weight_utils import ( - default_weight_loader, + load_stacked_weight, maybe_remap_kv_scale_name, ) from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder @@ -454,40 +454,14 @@ class Qwen3ForCausalLM(TextEncoder): else: name = kv_scale_name - # Handle stacked params mapping (qkv_proj, gate_up_proj) - for ( - param_name, - weight_name, - shard_id, - ) in self.config.arch_config.stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - # Skip loading extra bias for GPTQ models - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - loaded_params.add(name) + name = load_stacked_weight( + name, + loaded_weight, + params_dict, + self.config.arch_config.stacked_params_mapping, + ) + if name is not None: + loaded_params.add(name) return loaded_params diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py index 7eb86621ee..566a4ddbd3 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py @@ -42,7 +42,7 @@ from sglang.multimodal_gen.runtime.layers.utils import get_group_rank, get_group from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) -from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader +from sglang.multimodal_gen.runtime.loader.weight_utils import load_stacked_weight from sglang.multimodal_gen.runtime.models.encoders.base import ( TextEncoder, get_folding_tp_group, @@ -618,37 +618,13 @@ class T5EncoderModel(TextEncoder): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() for name, loaded_weight in weights: - loaded = False if "decoder" in name or "lm_head" in name: continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded = True - break - if not loaded: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) + name = load_stacked_weight( + name, loaded_weight, params_dict, stacked_params_mapping + ) + if name is not None: + loaded_params.add(name) return loaded_params @@ -709,41 +685,16 @@ class UMT5EncoderModel(TextEncoder): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() for name, loaded_weight in weights: - loaded = False if "decoder" in name or "lm_head" in name: continue - for ( - param_name, - weight_name, - shard_id, - ) in self.config.arch_config.stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded = True - break - if not loaded: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) + name = load_stacked_weight( + name, + loaded_weight, + params_dict, + self.config.arch_config.stacked_params_mapping, + ) + if name is not None: + loaded_params.add(name) return loaded_params diff --git a/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py b/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py index ca39ab0583..b40305fba3 100644 --- a/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py +++ b/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py @@ -600,6 +600,26 @@ class TestTextEncoderQuantization(unittest.TestCase): self.addCleanup(self.quant_config_patcher.stop) self.serialized = serialized + def _configure_h3_quantization(self, tensors, *, metadata=None): + self.get_quant_config.return_value = None + with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint: + save_file(tensors, checkpoint.name, metadata=metadata) + model_config = SimpleNamespace(quant_config=None) + with mock.patch( + "sglang.multimodal_gen.runtime.loader.component_loaders." + "text_encoder_loader.get_quant_config_from_safetensors_metadata", + return_value=None, + ): + _configure_encoder_quantization( + model_config, + MiniMaxH3Qwen3VLEncoder, + {}, + "/model/text_encoder", + checkpoint.name, + "text_encoder", + ) + return model_config.quant_config + def test_serialized_checkpoint_configures_native_encoder(self): model_config = SimpleNamespace(quant_config=None) _configure_encoder_quantization( @@ -652,7 +672,6 @@ class TestTextEncoderQuantization(unittest.TestCase): get_file_quant_config.assert_called_once_with("/weights/encoder.safetensors") def test_comfy_int8_weight_file_configures_native_encoder(self): - self.get_quant_config.return_value = None marker = json.dumps( { "format": "int8_tensorwise", @@ -660,81 +679,47 @@ class TestTextEncoderQuantization(unittest.TestCase): "convrot_groupsize": 256, } ).encode() - with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint: - save_file( - { - "visual.blocks.0.attn.qkv.weight": torch.ones( - (2, 256), dtype=torch.int8 - ), - "visual.blocks.0.attn.qkv.weight_scale": torch.ones((2, 1)), - "visual.blocks.0.attn.qkv.comfy_quant": torch.tensor( - list(marker), dtype=torch.uint8 - ), - }, - checkpoint.name, - ) - model_config = SimpleNamespace(quant_config=None) - with mock.patch( - "sglang.multimodal_gen.runtime.loader.component_loaders." - "text_encoder_loader.get_quant_config_from_safetensors_metadata", - return_value=None, - ): - _configure_encoder_quantization( - model_config, - MiniMaxH3Qwen3VLEncoder, - {}, - "/model/text_encoder", - checkpoint.name, - "text_encoder", - ) + quant_config = self._configure_h3_quantization( + { + "visual.blocks.0.attn.qkv.weight": torch.ones( + (2, 256), dtype=torch.int8 + ), + "visual.blocks.0.attn.qkv.weight_scale": torch.ones((2, 1)), + "visual.blocks.0.attn.qkv.comfy_quant": torch.tensor( + list(marker), dtype=torch.uint8 + ), + }, + ) - self.assertIsInstance(model_config.quant_config, ConvRotInt8Config) + self.assertIsInstance(quant_config, ConvRotInt8Config) self.assertEqual( - set(model_config.quant_config.layer_markers), + set(quant_config.layer_markers), {"model.visual.blocks.0.attn.qkv_proj"}, ) def test_comfy_w4a4_weight_file_configures_native_encoder(self): - self.get_quant_config.return_value = None marker = json.dumps( {"format": "convrot_w4a4", "convrot_groupsize": 256} ).encode() - with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint: - save_file( - { - "model.layers.0.self_attn.q_proj.weight": torch.ones( - (2, 128), dtype=torch.int8 - ), - "model.layers.0.self_attn.q_proj.weight_scale": torch.ones(2), - "model.layers.0.self_attn.q_proj.comfy_quant": torch.tensor( - list(marker), dtype=torch.uint8 - ), - }, - checkpoint.name, - ) - model_config = SimpleNamespace(quant_config=None) - with mock.patch( - "sglang.multimodal_gen.runtime.loader.component_loaders." - "text_encoder_loader.get_quant_config_from_safetensors_metadata", - return_value=None, - ): - _configure_encoder_quantization( - model_config, - MiniMaxH3Qwen3VLEncoder, - {}, - "/model/text_encoder", - checkpoint.name, - "text_encoder", - ) + quant_config = self._configure_h3_quantization( + { + "model.layers.0.self_attn.q_proj.weight": torch.ones( + (2, 128), dtype=torch.int8 + ), + "model.layers.0.self_attn.q_proj.weight_scale": torch.ones(2), + "model.layers.0.self_attn.q_proj.comfy_quant": torch.tensor( + list(marker), dtype=torch.uint8 + ), + }, + ) - self.assertIsInstance(model_config.quant_config, KitchenW4A4Config) + self.assertIsInstance(quant_config, KitchenW4A4Config) self.assertEqual( - set(model_config.quant_config.layer_markers), + set(quant_config.layer_markers), {"model.language_model.layers.0.self_attn.q_proj"}, ) def test_mixed_w4a8_weight_file_maps_embedding_and_linear_markers(self): - self.get_quant_config.return_value = None layers = { "model.embed_tokens": {"format": "int8_tensorwise"}, "model.layers.0.mlp.down_proj": { @@ -744,51 +729,32 @@ class TestTextEncoderQuantization(unittest.TestCase): "convrot_groupsize": 256, }, } - with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint: - save_file( - { - "model.embed_tokens.weight": torch.ones((4, 256), dtype=torch.int8), - "model.embed_tokens.weight_scale": torch.tensor(0.25), - "model.layers.0.mlp.down_proj.weight": torch.ones( - (2, 128), dtype=torch.int8 - ), - "model.layers.0.mlp.down_proj.weight_s_rel": torch.ones( - (2, 16), dtype=torch.float8_e4m3fn - ), - "model.layers.0.mlp.down_proj.weight_s_channel": torch.ones(2), - "model.layers.0.mlp.down_proj.weight_codebook": torch.ones(16), - }, - checkpoint.name, - metadata={"_quantization_metadata": json.dumps({"layers": layers})}, - ) - model_config = SimpleNamespace(quant_config=None) - with mock.patch( - "sglang.multimodal_gen.runtime.loader.component_loaders." - "text_encoder_loader.get_quant_config_from_safetensors_metadata", - return_value=None, - ): - _configure_encoder_quantization( - model_config, - MiniMaxH3Qwen3VLEncoder, - {}, - "/model/text_encoder", - checkpoint.name, - "text_encoder", - ) + quant_config = self._configure_h3_quantization( + { + "model.embed_tokens.weight": torch.ones((4, 256), dtype=torch.int8), + "model.embed_tokens.weight_scale": torch.tensor(0.25), + "model.layers.0.mlp.down_proj.weight": torch.ones( + (2, 128), dtype=torch.int8 + ), + "model.layers.0.mlp.down_proj.weight_s_rel": torch.ones( + (2, 16), dtype=torch.float8_e4m3fn + ), + "model.layers.0.mlp.down_proj.weight_s_channel": torch.ones(2), + "model.layers.0.mlp.down_proj.weight_codebook": torch.ones(16), + }, + metadata={"_quantization_metadata": json.dumps({"layers": layers})}, + ) - self.assertIsInstance(model_config.quant_config, KitchenW4A8Config) + self.assertIsInstance(quant_config, KitchenW4A8Config) self.assertTrue( - model_config.quant_config.quantizes_embedding( - "model.language_model.embed_tokens" - ) + quant_config.quantizes_embedding("model.language_model.embed_tokens") ) self.assertIn( "model.language_model.layers.0.mlp.down_proj", - model_config.quant_config.layer_markers, + quant_config.layer_markers, ) def test_nvfp4_awq_weight_file_maps_embedding_and_linear_markers(self): - self.get_quant_config.return_value = None layers = { "model.embed_tokens": {"format": "int8_tensorwise"}, "model.layers.0.self_attn.o_proj": { @@ -796,47 +762,29 @@ class TestTextEncoderQuantization(unittest.TestCase): "full_precision_matrix_mult": True, }, } - with tempfile.NamedTemporaryFile(suffix=".safetensors") as checkpoint: - save_file( - { - "model.embed_tokens.weight": torch.ones((4, 64), dtype=torch.int8), - "model.embed_tokens.weight_scale": torch.ones(4, 1), - "model.layers.0.self_attn.o_proj.weight": torch.full( - (128, 32), 0x21, dtype=torch.uint8 - ), - "model.layers.0.self_attn.o_proj.weight_scale": torch.ones( - (128, 4), dtype=torch.float8_e4m3fn - ), - "model.layers.0.self_attn.o_proj.weight_scale_2": torch.tensor(0.5), - "model.layers.0.self_attn.o_proj.pre_quant_scale": torch.ones( - 64, dtype=torch.bfloat16 - ), - }, - checkpoint.name, - metadata={"_quantization_metadata": json.dumps({"layers": layers})}, - ) - model_config = SimpleNamespace(quant_config=None) - with mock.patch( - "sglang.multimodal_gen.runtime.loader.component_loaders." - "text_encoder_loader.get_quant_config_from_safetensors_metadata", - return_value=None, - ): - _configure_encoder_quantization( - model_config, - MiniMaxH3Qwen3VLEncoder, - {}, - "/model/text_encoder", - checkpoint.name, - "text_encoder", - ) - - self.assertIsInstance(model_config.quant_config, ComfyNvfp4Config) - self.assertTrue( - model_config.quant_config.quantizes_embedding( - "model.language_model.embed_tokens" - ) + quant_config = self._configure_h3_quantization( + { + "model.embed_tokens.weight": torch.ones((4, 64), dtype=torch.int8), + "model.embed_tokens.weight_scale": torch.ones(4, 1), + "model.layers.0.self_attn.o_proj.weight": torch.full( + (128, 32), 0x21, dtype=torch.uint8 + ), + "model.layers.0.self_attn.o_proj.weight_scale": torch.ones( + (128, 4), dtype=torch.float8_e4m3fn + ), + "model.layers.0.self_attn.o_proj.weight_scale_2": torch.tensor(0.5), + "model.layers.0.self_attn.o_proj.pre_quant_scale": torch.ones( + 64, dtype=torch.bfloat16 + ), + }, + metadata={"_quantization_metadata": json.dumps({"layers": layers})}, ) - marker = model_config.quant_config.layer_markers[ + + self.assertIsInstance(quant_config, ComfyNvfp4Config) + self.assertTrue( + quant_config.quantizes_embedding("model.language_model.embed_tokens") + ) + marker = quant_config.layer_markers[ "model.language_model.layers.0.self_attn.o_proj" ] self.assertTrue(marker["_has_pre_quant_scale"])