[bugfix]GLM-4V model (#17122)

python/sglang/srt/models/glm4v.py

@@ -414,6 +414,7 @@ class Glm4vVisionModel(nn.Module):
                     num_heads=self.num_heads,
                     quant_config=quant_config,
                     prefix=add_prefix(f"blocks.{layer_idx}", prefix),
+                    num_dummy_heads=vision_config.num_dummy_heads,
                     rms_norm_eps=vision_config.rms_norm_eps,
                     attn_qkv_bias=vision_config.attention_bias,
                     use_data_parallel=use_data_parallel,
@@ -553,6 +554,7 @@ class Glm4vForConditionalGeneration(nn.Module):
         self.pp_group = get_pp_group()
         self.config = config
         self.use_data_parallel = get_global_server_args().mm_enable_dp_encoder
+        vision_utils.update_vit_attn_dummy_heads_config(self.config)
         self.visual = Glm4vVisionModel(
             config.vision_config,
             quant_config=quant_config,
@@ -560,8 +562,6 @@ class Glm4vForConditionalGeneration(nn.Module):
             use_data_parallel=self.use_data_parallel,
         )
 
-        vision_utils.update_vit_attn_dummy_heads_config(self.config)
-
         self.model = Glm4Model(
             config,
             quant_config=quant_config,

python/sglang/srt/multimodal/processors/base_processor.py

@@ -409,7 +409,9 @@ class BaseMultimodalProcessor(ABC):
                 kwargs["device"] = "xpu"
             elif not _is_npu:
                 kwargs["device"] = "cuda"
-            else:
+            elif processor.__class__.__name__ not in {
+                "Glm4vProcessor",
+            }:
                 # Note: for qwen-vl, processor has some reshape issue because of dims restriction on Ascend.
                 from sglang.srt.hardware_backend.npu.modules.qwen_vl_processor import (
                     npu_apply_qwen_image_preprocess_patch,