From 96d91ef9266d2bebd8e8c09ef1f28b2d521631ff Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" <53919306+nvpohanh@users.noreply.github.com> Date: Thu, 10 Sep 2026 04:51:08 +0800 Subject: [PATCH] [Model] Support GLM-5.3 Flash NVFP4 loading (#38621) Co-authored-by: YAMY <74099316+YAMY1234@users.noreply.github.com> --- python/sglang/srt/models/glm5_next.py | 19 ++++ .../unit/models/test_glm5_next_modelopt.py | 104 ++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 test/registered/unit/models/test_glm5_next_modelopt.py diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index f5253cfdf..c6ec66530 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -94,6 +94,7 @@ from sglang.srt.models.glm_ocr import ( GlmOcrVisionPatchEmbed, GlmOcrVisionPatchMerger, ) +from sglang.srt.models.utils import WeightsMapper from sglang.srt.multimodal.mm_utils import ( run_dp_presharded_mrope_vision_model, run_dp_sharded_mrope_vision_model, @@ -1077,6 +1078,13 @@ class Glm5NextModel(nn.Module): class Glm5NextForConditionalGeneration(nn.Module): + hf_to_sglang_mapper = WeightsMapper( + orig_to_new_substr={ + "model.language_model.": "model.", + "model.visual": "visual", + } + ) + packed_modules_mapping = { "fused_qkv_a_proj_with_mqa": ["q_a_proj", "kv_a_proj_with_mqa"], "fused_qkvbfg_a_proj": [ @@ -1200,6 +1208,17 @@ class Glm5NextForConditionalGeneration(nn.Module): text_config = getattr(hf_config, "text_config", hf_config) if not getattr(text_config, "n_shared_experts", None): return "No shared experts are defined in the config." + if quant_config is not None and quant_config.get_name() == "modelopt_fp4": + first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0) + for layer_id in range(first_sparse_layer, text_config.num_hidden_layers): + moe_prefix = f"model.layers.{layer_id}.mlp" + if quant_config.is_layer_excluded( + f"{moe_prefix}.shared_experts" + ) and not quant_config.is_layer_excluded(f"{moe_prefix}.experts"): + return ( + "ModelOpt FP4 keeps shared experts unquantized while routed " + "experts are quantized." + ) if not _is_cuda: return "Shared experts fusion currently requires CUDA devices." if _device_sm is not None and _device_sm < 80: diff --git a/test/registered/unit/models/test_glm5_next_modelopt.py b/test/registered/unit/models/test_glm5_next_modelopt.py new file mode 100644 index 000000000..459805680 --- /dev/null +++ b/test/registered/unit/models/test_glm5_next_modelopt.py @@ -0,0 +1,104 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config +from sglang.srt.models import glm5_next +from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +NVIDIA_EXCLUDE_MODULES = [ + "model.language_model.embed_tokens", + "model.language_model.layers.11.self_attn*", + "model.language_model.layers.11.mlp.gate", + "model.language_model.layers.11.mlp.shared_experts*", + "model.visual*", +] + +RADIXARK_EXCLUDE_MODULES = [ + "model.language_model.embed_tokens", + "model.language_model.layers.11.self_attn*", + "model.language_model.layers.11.mlp.gate", + "model.visual*", + "model.embed_tokens", + "model.layers.11.self_attn*", + "model.layers.11.mlp.gate", + "model.language_model.layers.45*", + "model.layers.45*", + "visual*", +] + + +class TestGlm5NextModelOpt(CustomTestCase): + def _config(self, exclude_modules): + config = ModelOptFp4Config( + is_checkpoint_nvfp4_serialized=True, + group_size=16, + exclude_modules=exclude_modules, + ) + config.apply_weight_name_mapper( + Glm5NextForConditionalGeneration.hf_to_sglang_mapper + ) + return config + + def _hf_config(self): + return SimpleNamespace( + text_config=SimpleNamespace( + first_k_dense_replace=3, + num_hidden_layers=45, + n_shared_experts=1, + ) + ) + + def test_checkpoint_exclusions_match_sglang_module_names(self): + checkpoints = { + "nvidia/GLM-5.3-Flash-NVFP4": NVIDIA_EXCLUDE_MODULES, + "RadixArk/GLM-5.3-Flash-NVFP4": RADIXARK_EXCLUDE_MODULES, + } + + for checkpoint, exclude_modules in checkpoints.items(): + with self.subTest(checkpoint=checkpoint): + config = self._config(exclude_modules) + self.assertTrue(config.is_layer_excluded("model.embed_tokens")) + self.assertTrue( + config.is_layer_excluded("model.layers.11.self_attn.kv_b_proj") + ) + self.assertTrue(config.is_layer_excluded("model.layers.11.mlp.gate")) + self.assertTrue( + config.is_layer_excluded("visual.blocks.0.attn.qkv_proj") + ) + + def test_nvidia_mixed_precision_shared_experts_disable_fusion(self): + config = self._config(NVIDIA_EXCLUDE_MODULES) + + reason = Glm5NextForConditionalGeneration.shared_experts_fusion_disable_reason( + self._hf_config(), config + ) + + self.assertIn("shared experts unquantized", reason) + + def test_radixark_uniform_fp4_shared_experts_keep_fusion(self): + config = self._config(RADIXARK_EXCLUDE_MODULES) + a2a_backend = SimpleNamespace(is_deepep=lambda: False) + + with ( + patch.object(glm5_next, "_is_cuda", True), + patch.object(glm5_next, "_device_sm", 100), + patch.object(glm5_next, "get_moe_a2a_backend", return_value=a2a_backend), + get_parallel().override(moe_ep_size=1), + ): + reason = ( + Glm5NextForConditionalGeneration.shared_experts_fusion_disable_reason( + self._hf_config(), config + ) + ) + + self.assertIsNone(reason) + + +if __name__ == "__main__": + unittest.main()