[bugfix]GLM-4V model (#17122)

python/sglang/srt/models/glm4v.py

@@ -414,6 +414,7 @@ class Glm4vVisionModel(nn.Module):
                     num_heads=self.num_heads,
                     quant_config=quant_config,
                     prefix=add_prefix(f"blocks.{layer_idx}", prefix),
+                    num_dummy_heads=vision_config.num_dummy_heads,
                     rms_norm_eps=vision_config.rms_norm_eps,
                     attn_qkv_bias=vision_config.attention_bias,
                     use_data_parallel=use_data_parallel,
@@ -553,6 +554,7 @@ class Glm4vForConditionalGeneration(nn.Module):
         self.pp_group = get_pp_group()
         self.config = config
         self.use_data_parallel = get_global_server_args().mm_enable_dp_encoder
+        vision_utils.update_vit_attn_dummy_heads_config(self.config)
         self.visual = Glm4vVisionModel(
             config.vision_config,
             quant_config=quant_config,
@@ -560,8 +562,6 @@ class Glm4vForConditionalGeneration(nn.Module):
             use_data_parallel=self.use_data_parallel,
         )
 
-        vision_utils.update_vit_attn_dummy_heads_config(self.config)
-
         self.model = Glm4Model(
             config,
             quant_config=quant_config,

python/sglang/srt/multimodal/processors/base_processor.py

@@ -409,7 +409,9 @@ class BaseMultimodalProcessor(ABC):
                 kwargs["device"] = "xpu"
             elif not _is_npu:
                 kwargs["device"] = "cuda"
-            else:
+            elif processor.__class__.__name__ not in {
+                "Glm4vProcessor",
+            }:
                 # Note: for qwen-vl, processor has some reshape issue because of dims restriction on Ascend.
                 from sglang.srt.hardware_backend.npu.modules.qwen_vl_processor import (
                     npu_apply_qwen_image_preprocess_patch,

test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py

@@ -0,0 +1,33 @@
+import unittest
+
+from sglang.test.ascend.vlm_utils import TestVLMModels
+from sglang.test.ci.ci_register import register_npu_ci
+
+register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
+
+
+class TestGLM4Models(TestVLMModels):
+    model = "/root/.cache/modelscope/hub/models/ZhipuAI/GLM-4.5V"
+    mmmu_accuracy = 0.2
+    other_args = [
+        "--trust-remote-code",
+        "--cuda-graph-max-bs",
+        "32",
+        "--enable-multimodal",
+        "--mem-fraction-static",
+        0.7,
+        "--log-level",
+        "info",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+        "--tp-size",
+        8,
+    ]
+
+    def test_vlm_mmmu_benchmark(self):
+        self._run_vlm_mmmu_test()
+
+
+if __name__ == "__main__":
+    unittest.main()