From 44d10eab5e7d31e9deab73b142d1288e1bfc74a4 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:17:06 +0000 Subject: [PATCH] [GLM-V] Shard the vision tower over the attention-TP group under DP attention (#42931) Co-authored-by: Mohammad Angkad --- python/sglang/srt/models/glm4v.py | 30 +++++++++++++++++++++++---- python/sglang/srt/models/glm5_next.py | 4 ++++ python/sglang/srt/models/glm_ocr.py | 4 ++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/models/glm4v.py b/python/sglang/srt/models/glm4v.py index 66bb19de1e..714065edd4 100644 --- a/python/sglang/srt/models/glm4v.py +++ b/python/sglang/srt/models/glm4v.py @@ -36,6 +36,7 @@ from sglang.srt.layers.attention.vision import ( prepare_vision_attention_metadata, ) from sglang.srt.layers.conv import Conv3dLayer +from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import LayerNorm, RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -75,6 +76,20 @@ class Glm4vRMSNorm(RMSNorm): return x +# TODO: fold into parallel_group selection ("attn_tp" under DP attention). +def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool: + return is_dp_attention_enabled() and not use_data_parallel + + +def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]: + if use_data_parallel: + return 1, 0 + parallel = get_parallel() + if glm4v_vision_reduces_over_attn_tp(use_data_parallel): + return parallel.attn_tp_size, parallel.attn_tp_rank + return parallel.tp_size, parallel.tp_rank + + class Glm4vVisionMLP(nn.Module): def __init__( self, @@ -86,8 +101,7 @@ class Glm4vVisionMLP(nn.Module): use_data_parallel: bool = False, ): super().__init__() - self.tp_size = 1 if use_data_parallel else get_parallel().tp_size - self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank + self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel) self.gate_up_proj = MergedColumnParallelLinear( input_size=in_features, output_sizes=[hidden_features] * 2, # [gate_proj, up_proj] @@ -105,6 +119,9 @@ class Glm4vVisionMLP(nn.Module): prefix=add_prefix("down_proj", prefix), tp_size=self.tp_size, tp_rank=self.tp_rank, + use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( + use_data_parallel + ), ) self.act_fn = SiluAndMul() @@ -144,6 +161,9 @@ class Glm4vVisionBlock(nn.Module): prefix=add_prefix("attn", prefix), num_dummy_heads=num_dummy_heads, use_data_parallel=use_data_parallel, + use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( + use_data_parallel + ), ) self.mlp = Glm4vVisionMLP( dim, @@ -237,8 +257,7 @@ class Glm4vPatchMerger(nn.Module): ) -> None: super().__init__() self.hidden_size = d_model - tp_size = 1 if use_data_parallel else get_parallel().tp_size - tp_rank = 0 if use_data_parallel else get_parallel().tp_rank + tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel) self.proj = ReplicatedLinear( self.hidden_size, self.hidden_size, @@ -264,6 +283,9 @@ class Glm4vPatchMerger(nn.Module): prefix=add_prefix("down_proj", prefix), tp_size=tp_size, tp_rank=tp_rank, + use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( + use_data_parallel + ), ) self.extra_activation_func = nn.GELU() diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 8bdf599b69..1cfdfefd28 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -98,6 +98,7 @@ from sglang.srt.models.deepseek_common.utils import ( from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA from sglang.srt.models.deepseek_v2 import DeepseekV2MLP as Glm5NextMLP from sglang.srt.models.deepseek_v2 import DeepseekV2MoE as Glm5NextMoE +from sglang.srt.models.glm4v import glm4v_vision_reduces_over_attn_tp from sglang.srt.models.glm_ocr import ( GlmOcrRMSNorm, GlmOcrVisionBlock, @@ -240,6 +241,9 @@ class Glm5NextVisionBlock(GlmOcrVisionBlock): prefix=add_prefix("attn", prefix), num_dummy_heads=num_dummy_heads, use_data_parallel=use_data_parallel, + use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( + use_data_parallel + ), ) self.mlp = Glm5NextVisionMLP( dim, diff --git a/python/sglang/srt/models/glm_ocr.py b/python/sglang/srt/models/glm_ocr.py index e26385d6e9..5d9edffc7f 100644 --- a/python/sglang/srt/models/glm_ocr.py +++ b/python/sglang/srt/models/glm_ocr.py @@ -51,6 +51,7 @@ from sglang.srt.models.glm4v import ( Glm4vVisionMLP, Glm4vVisionModel, Glm4vVisionPatchEmbed, + glm4v_vision_reduces_over_attn_tp, ) from sglang.srt.runtime_context import get_mm, get_parallel from sglang.srt.utils import add_prefix @@ -98,6 +99,9 @@ class GlmOcrVisionBlock(nn.Module): prefix=add_prefix("attn", prefix), num_dummy_heads=num_dummy_heads, use_data_parallel=use_data_parallel, + use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( + use_data_parallel + ), ) self.mlp = GlmOcrVisionMLP( dim,