[GLM-V] Shard the vision tower over the attention-TP group under DP attention (#42931)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-07 23:17:06 +08:00
committed by GitHub
co-authored by Mohammad Angkad
parent d9a0f1cb34
commit 44d10eab5e
3 changed files with 34 additions and 4 deletions
+26 -4
View File
@@ -36,6 +36,7 @@ from sglang.srt.layers.attention.vision import (
prepare_vision_attention_metadata,
)
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import LayerNorm, RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -75,6 +76,20 @@ class Glm4vRMSNorm(RMSNorm):
return x
# TODO: fold into parallel_group selection ("attn_tp" under DP attention).
def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool:
return is_dp_attention_enabled() and not use_data_parallel
def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]:
if use_data_parallel:
return 1, 0
parallel = get_parallel()
if glm4v_vision_reduces_over_attn_tp(use_data_parallel):
return parallel.attn_tp_size, parallel.attn_tp_rank
return parallel.tp_size, parallel.tp_rank
class Glm4vVisionMLP(nn.Module):
def __init__(
self,
@@ -86,8 +101,7 @@ class Glm4vVisionMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
self.tp_size = 1 if use_data_parallel else get_parallel().tp_size
self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel)
self.gate_up_proj = MergedColumnParallelLinear(
input_size=in_features,
output_sizes=[hidden_features] * 2, # [gate_proj, up_proj]
@@ -105,6 +119,9 @@ class Glm4vVisionMLP(nn.Module):
prefix=add_prefix("down_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
)
self.act_fn = SiluAndMul()
@@ -144,6 +161,9 @@ class Glm4vVisionBlock(nn.Module):
prefix=add_prefix("attn", prefix),
num_dummy_heads=num_dummy_heads,
use_data_parallel=use_data_parallel,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
)
self.mlp = Glm4vVisionMLP(
dim,
@@ -237,8 +257,7 @@ class Glm4vPatchMerger(nn.Module):
) -> None:
super().__init__()
self.hidden_size = d_model
tp_size = 1 if use_data_parallel else get_parallel().tp_size
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel)
self.proj = ReplicatedLinear(
self.hidden_size,
self.hidden_size,
@@ -264,6 +283,9 @@ class Glm4vPatchMerger(nn.Module):
prefix=add_prefix("down_proj", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
)
self.extra_activation_func = nn.GELU()
+4
View File
@@ -98,6 +98,7 @@ from sglang.srt.models.deepseek_common.utils import (
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
from sglang.srt.models.deepseek_v2 import DeepseekV2MLP as Glm5NextMLP
from sglang.srt.models.deepseek_v2 import DeepseekV2MoE as Glm5NextMoE
from sglang.srt.models.glm4v import glm4v_vision_reduces_over_attn_tp
from sglang.srt.models.glm_ocr import (
GlmOcrRMSNorm,
GlmOcrVisionBlock,
@@ -240,6 +241,9 @@ class Glm5NextVisionBlock(GlmOcrVisionBlock):
prefix=add_prefix("attn", prefix),
num_dummy_heads=num_dummy_heads,
use_data_parallel=use_data_parallel,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
)
self.mlp = Glm5NextVisionMLP(
dim,
+4
View File
@@ -51,6 +51,7 @@ from sglang.srt.models.glm4v import (
Glm4vVisionMLP,
Glm4vVisionModel,
Glm4vVisionPatchEmbed,
glm4v_vision_reduces_over_attn_tp,
)
from sglang.srt.runtime_context import get_mm, get_parallel
from sglang.srt.utils import add_prefix
@@ -98,6 +99,9 @@ class GlmOcrVisionBlock(nn.Module):
prefix=add_prefix("attn", prefix),
num_dummy_heads=num_dummy_heads,
use_data_parallel=use_data_parallel,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
)
self.mlp = GlmOcrVisionMLP(
dim,