[GLM-V] Shard the vision tower over the attention-TP group under DP attention (#42931)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
d9a0f1cb34
commit
44d10eab5e
@@ -36,6 +36,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import LayerNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -75,6 +76,20 @@ class Glm4vRMSNorm(RMSNorm):
|
||||
return x
|
||||
|
||||
|
||||
# TODO: fold into parallel_group selection ("attn_tp" under DP attention).
|
||||
def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool:
|
||||
return is_dp_attention_enabled() and not use_data_parallel
|
||||
|
||||
|
||||
def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]:
|
||||
if use_data_parallel:
|
||||
return 1, 0
|
||||
parallel = get_parallel()
|
||||
if glm4v_vision_reduces_over_attn_tp(use_data_parallel):
|
||||
return parallel.attn_tp_size, parallel.attn_tp_rank
|
||||
return parallel.tp_size, parallel.tp_rank
|
||||
|
||||
|
||||
class Glm4vVisionMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -86,8 +101,7 @@ class Glm4vVisionMLP(nn.Module):
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = 1 if use_data_parallel else get_parallel().tp_size
|
||||
self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
|
||||
self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel)
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=in_features,
|
||||
output_sizes=[hidden_features] * 2, # [gate_proj, up_proj]
|
||||
@@ -105,6 +119,9 @@ class Glm4vVisionMLP(nn.Module):
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
@@ -144,6 +161,9 @@ class Glm4vVisionBlock(nn.Module):
|
||||
prefix=add_prefix("attn", prefix),
|
||||
num_dummy_heads=num_dummy_heads,
|
||||
use_data_parallel=use_data_parallel,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
)
|
||||
self.mlp = Glm4vVisionMLP(
|
||||
dim,
|
||||
@@ -237,8 +257,7 @@ class Glm4vPatchMerger(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = d_model
|
||||
tp_size = 1 if use_data_parallel else get_parallel().tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
|
||||
tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel)
|
||||
self.proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
@@ -264,6 +283,9 @@ class Glm4vPatchMerger(nn.Module):
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
)
|
||||
self.extra_activation_func = nn.GELU()
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ from sglang.srt.models.deepseek_common.utils import (
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2MLP as Glm5NextMLP
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2MoE as Glm5NextMoE
|
||||
from sglang.srt.models.glm4v import glm4v_vision_reduces_over_attn_tp
|
||||
from sglang.srt.models.glm_ocr import (
|
||||
GlmOcrRMSNorm,
|
||||
GlmOcrVisionBlock,
|
||||
@@ -240,6 +241,9 @@ class Glm5NextVisionBlock(GlmOcrVisionBlock):
|
||||
prefix=add_prefix("attn", prefix),
|
||||
num_dummy_heads=num_dummy_heads,
|
||||
use_data_parallel=use_data_parallel,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
)
|
||||
self.mlp = Glm5NextVisionMLP(
|
||||
dim,
|
||||
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.models.glm4v import (
|
||||
Glm4vVisionMLP,
|
||||
Glm4vVisionModel,
|
||||
Glm4vVisionPatchEmbed,
|
||||
glm4v_vision_reduces_over_attn_tp,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_mm, get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
@@ -98,6 +99,9 @@ class GlmOcrVisionBlock(nn.Module):
|
||||
prefix=add_prefix("attn", prefix),
|
||||
num_dummy_heads=num_dummy_heads,
|
||||
use_data_parallel=use_data_parallel,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
)
|
||||
self.mlp = GlmOcrVisionMLP(
|
||||
dim,
|
||||
|
||||
Reference in New Issue
Block a user