[Refactor] Let vision encoders select their parallel group (#42586)

This commit is contained in:
Cheng Wan
2026-10-07 16:22:32 -07:00
committed by GitHub
parent b83827ea02
commit dec5d9d6fc
18 changed files with 942 additions and 154 deletions
@@ -16,7 +16,6 @@ from transformers.utils import TransformersKwargs, is_torchdynamo_compiling
from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig
from sglang.multimodal_gen.runtime.distributed import (
get_tp_rank,
get_tp_world_size,
model_parallel_is_initialized,
)
@@ -139,12 +138,6 @@ def _tp_world_size() -> int:
return get_tp_world_size()
def _tp_rank() -> int:
if not model_parallel_is_initialized():
return 0
return get_tp_rank()
def _linear_output(linear: nn.Module, x: torch.Tensor) -> torch.Tensor:
output = linear(x)
return output[0] if isinstance(output, tuple) else output
@@ -163,8 +156,7 @@ def _make_column_linear(
out_features,
bias=bias,
gather_output=False,
tp_size=_tp_world_size(),
tp_rank=_tp_rank(),
parallel_group="tp" if model_parallel_is_initialized() else "replicated",
)
return ReplicatedLinear(in_features, out_features, bias=bias)
@@ -181,8 +173,7 @@ def _make_row_linear(
in_features,
out_features,
bias=bias,
tp_size=_tp_world_size(),
tp_rank=_tp_rank(),
parallel_group="tp" if model_parallel_is_initialized() else "replicated",
)
return ReplicatedLinear(in_features, out_features, bias=bias)
@@ -360,8 +351,7 @@ class Qwen2_5_VLDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
prefix=f"model.language_model.layers.{layer_idx}.mlp",
fuse_gate_up=False,
tp_size=mlp_tp_size,
tp_rank=_tp_rank() if mlp_tp_size > 1 else 0,
parallel_group="tp" if mlp_tp_size > 1 else "replicated",
)
norm_kwargs = dict(
eps=config.rms_norm_eps,
@@ -19,7 +19,6 @@ from sglang.srt.models.qwen3_vl import (
Qwen3VLMoeVisionPatchMerger,
Qwen3VLVisionPatchEmbed,
)
from sglang.srt.runtime_context import get_parallel
@dataclass(frozen=True)
@@ -70,7 +69,6 @@ class Qwen3VLVisionBlock(nn.Module):
prefix: str = "visual",
) -> None:
super().__init__()
parallel = get_parallel()
self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)
self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)
self.attn = QwenVLVisionAttention(
@@ -86,8 +84,7 @@ class Qwen3VLVisionBlock(nn.Module):
hidden_act=config.hidden_act,
prefix=f"{prefix}.blocks.{layer_idx}.mlp",
quant_config=quant_config,
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
def forward(
@@ -210,7 +207,6 @@ class Qwen3VLVisionTransformer(nn.Module):
prefix: str = "visual",
) -> None:
super().__init__()
parallel = get_parallel()
self.config = config
self.spatial_merge_size = config.spatial_merge_size
self.spatial_merge_unit = config.spatial_merge_size**2
@@ -233,8 +229,7 @@ class Qwen3VLVisionTransformer(nn.Module):
spatial_merge_size=config.spatial_merge_size,
use_postshuffle_norm=False,
prefix="visual.merger",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
self.deepstack_visual_indexes = tuple(config.deepstack_visual_indexes)
self.deepstack_merger_list = nn.ModuleList(
@@ -245,8 +240,7 @@ class Qwen3VLVisionTransformer(nn.Module):
spatial_merge_size=config.spatial_merge_size,
use_postshuffle_norm=True,
prefix=f"visual.deepstack_merger_list.{merger_idx}",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
for merger_idx, _ in enumerate(self.deepstack_visual_indexes)
)
@@ -79,8 +79,7 @@ class QwenVLVisionAttention(nn.Module):
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
self.proj = RowParallelLinear(
input_size=config.hidden_size,
@@ -88,8 +87,7 @@ class QwenVLVisionAttention(nn.Module):
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.proj",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
backend = get_attn_backend(self.head_dim, torch.get_default_dtype())
@@ -75,7 +75,6 @@ def test_text_mlp_uses_single_rank_when_intermediate_size_is_not_tp_divisible(
):
monkeypatch.setattr(qwen2_5vl, "Qwen2_5_VLAttention", lambda *_args: nn.Identity())
monkeypatch.setattr(qwen2_5vl, "_tp_world_size", lambda: 3)
monkeypatch.setattr(qwen2_5vl, "_tp_rank", lambda: 2)
config = SimpleNamespace(
hidden_size=16,
intermediate_size=25,
@@ -0,0 +1,131 @@
"""Qwen generation projections tested in the diffusion dependency environment."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
import torch.nn.functional as F
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
from sglang.srt.layers.linear import QKVParallelLinear, ReplicatedLinear
from sglang.srt.runtime_context import SpawnRanks, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.parallel_groups import parallel_scope, publish
class TestQwenGenerationParallelGroups(unittest.TestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def test_generation_helpers_match_initialized_scope(self):
import sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl as generation
publish(
ServerArgs(model_path="dummy", device="cpu", tp_size=4),
role="test",
ranks=SpawnRanks(world_rank=3),
)
for initialized in (False, True):
with patch.object(
generation,
"model_parallel_is_initialized",
return_value=initialized,
):
for tensor_parallel in (False, True):
for maker in (
generation._make_column_linear,
generation._make_row_linear,
):
layer = maker(
8, 8, bias=True, use_tensor_parallel=tensor_parallel
)
expected = (3, 4) if initialized and tensor_parallel else (0, 1)
actual = (
(0, 1)
if isinstance(layer, ReplicatedLinear)
else (layer.tp_rank, layer.tp_size)
)
self.assertEqual(actual, expected)
self.assertEqual(
isinstance(layer, ReplicatedLinear), not tensor_parallel
)
def test_native_vision_projection_math_and_reload(self):
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend_context_manager,
)
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_vision import (
QwenVLVisionAttention,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
for dp in (1, 2):
with self.subTest(dp=dp):
reset_context()
server = ServerArgs(
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp
)
publish(server, role="test", ranks=SpawnRanks(world_rank=3))
initialize_dp_attention_flags(server)
with global_force_attn_backend_context_manager(
AttentionBackendEnum.TORCH_SDPA
):
module = QwenVLVisionAttention(
SimpleNamespace(hidden_size=32, num_heads=8),
prefix="visual",
model_name="Qwen",
)
tp = Mock(all_reduce=Mock(side_effect=lambda x: x * 2))
attn = Mock(all_reduce=Mock(side_effect=AssertionError("wrong group")))
for layer in (module.qkv_proj, module.proj):
rank, size = (
getattr(layer, "tp_rank", 0),
getattr(layer, "tp_size", 1),
)
self.assertEqual((rank, size), (3, 4))
full = (
torch.arange(32 * 32, dtype=layer.weight.dtype).reshape(32, 32)
/ 1024
)
bias = torch.arange(32, dtype=layer.weight.dtype) / 64
qkv = isinstance(layer, QKVParallelLinear)
with parallel_scope(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0):
for shard in ("q", "k", "v") if qkv else (None,):
args = (shard,) if qkv else ()
layer.weight.weight_loader(layer.weight, full, *args)
layer.bias.weight_loader(layer.bias, bias, *args)
weight = (
torch.cat([full.chunk(size)[rank]] * 3)
if qkv
else full.chunk(size, dim=1)[rank]
)
expected_bias = (
torch.cat([bias.chunk(size)[rank]] * 3) if qkv else None
)
inputs = (
torch.arange(2 * weight.shape[1], dtype=weight.dtype).reshape(
2, -1
)
/ 64
)
expected = F.linear(inputs, weight, expected_bias) * (
1 if qkv else 2
)
with (
parallel_scope(tp_group=tp, attn_tp_group=attn),
patch(
"sglang.srt.layers.linear.is_allocation_symmetric",
return_value=False,
),
patch(
"sglang.srt.layers.linear.use_symmetric_memory",
return_value=nullcontext(),
),
):
torch.testing.assert_close(layer(inputs)[0], expected)
torch.testing.assert_close(layer.weight, weight, rtol=0, atol=0)
attn.all_reduce.assert_not_called()
tp.all_reduce.assert_called_once()
+3 -6
View File
@@ -1196,8 +1196,7 @@ class VisionAttention(nn.Module):
total_num_kv_heads=num_dummy_heads + num_kv_heads,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="replicated" if use_data_parallel else "attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
else:
@@ -1206,8 +1205,7 @@ class VisionAttention(nn.Module):
output_size=3 * self.dummy_dim,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="replicated" if use_data_parallel else "attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
# TODO: this layer shards over attention TP but reduces over the full TP
@@ -1226,8 +1224,7 @@ class VisionAttention(nn.Module):
output_size=embed_dim,
bias=proj_bias,
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="replicated" if use_data_parallel else "attn_tp",
prefix=add_prefix("proj", prefix),
use_dp_attention_reduce=use_dp_attention_reduce,
)
+15 -8
View File
@@ -58,6 +58,20 @@ logger = logging.getLogger(__name__)
LinearParallelGroup = Literal["tp", "attn_tp", "replicated"]
def resolve_linear_parallel_group(
parallel_group: LinearParallelGroup,
) -> Tuple[int, int]:
"""Freeze a group's weight partition in the current construction scope."""
if parallel_group == "replicated":
return 0, 1
if parallel_group not in ("tp", "attn_tp"):
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
parallel = get_parallel()
if parallel_group == "attn_tp":
return parallel.attn_tp_rank, parallel.attn_tp_size
return parallel.tp_rank, parallel.tp_size
def _resolve_linear_partition(
parallel_group: Optional[LinearParallelGroup],
tp_rank: Optional[int],
@@ -74,14 +88,7 @@ def _resolve_linear_partition(
raise ValueError(
"parallel_group cannot be combined with tp_rank or tp_size"
)
if parallel_group == "replicated":
return 0, 1
if parallel_group not in ("tp", "attn_tp"):
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
parallel = get_parallel()
if parallel_group == "attn_tp":
return parallel.attn_tp_rank, parallel.attn_tp_size
return parallel.tp_rank, parallel.tp_size
return resolve_linear_parallel_group(parallel_group)
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
+10 -16
View File
@@ -76,18 +76,16 @@ class Glm4vRMSNorm(RMSNorm):
return x
# TODO: fold into parallel_group selection ("attn_tp" under DP attention).
def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool:
return is_dp_attention_enabled() and not use_data_parallel
def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]:
def _glm4v_vision_parallel_group(use_data_parallel: bool) -> str:
if use_data_parallel:
return 1, 0
parallel = get_parallel()
return "replicated"
if glm4v_vision_reduces_over_attn_tp(use_data_parallel):
return parallel.attn_tp_size, parallel.attn_tp_rank
return parallel.tp_size, parallel.tp_rank
return "attn_tp"
return "tp"
class Glm4vVisionMLP(nn.Module):
@@ -101,15 +99,14 @@ class Glm4vVisionMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel)
parallel_group = _glm4v_vision_parallel_group(use_data_parallel)
self.gate_up_proj = MergedColumnParallelLinear(
input_size=in_features,
output_sizes=[hidden_features] * 2, # [gate_proj, up_proj]
bias=bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
hidden_features,
@@ -117,8 +114,7 @@ class Glm4vVisionMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
@@ -257,7 +253,7 @@ class Glm4vPatchMerger(nn.Module):
) -> None:
super().__init__()
self.hidden_size = d_model
tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel)
parallel_group = _glm4v_vision_parallel_group(use_data_parallel)
self.proj = ReplicatedLinear(
self.hidden_size,
self.hidden_size,
@@ -272,8 +268,7 @@ class Glm4vPatchMerger(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
context_dim,
@@ -281,8 +276,7 @@ class Glm4vPatchMerger(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
+4 -7
View File
@@ -47,7 +47,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import (
InternViTCudaGraphRunner,
)
from sglang.srt.multimodal.mm_utils import run_dp_sharded_vision_model
from sglang.srt.runtime_context import get_mm, get_parallel
from sglang.srt.runtime_context import get_mm
from sglang.srt.utils import is_cuda
from sglang.utils import logger
@@ -200,8 +200,7 @@ class InternMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
self.tp_size = 1 if use_data_parallel else get_parallel().tp_size
self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
parallel_group = "replicated" if use_data_parallel else "tp"
self.config = config
self.act = get_act_fn(config.hidden_act)
self.fc1 = ColumnParallelLinear(
@@ -209,16 +208,14 @@ class InternMLP(nn.Module):
config.intermediate_size,
bias=True,
quant_config=None,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.fc2 = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
bias=True,
quant_config=None,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -5
View File
@@ -341,7 +341,6 @@ class MLP2(nn.Module):
self.quant_config = quant_config
use_tensor_parallel = use_tensor_parallel and not use_data_parallel
tp_size = get_parallel().attn_tp_size if use_tensor_parallel else 1
tp_rank = get_parallel().attn_tp_rank if use_tensor_parallel else 0
if isinstance(self.quant_config, ModelSlimConfig):
self.fc0 = ReplicatedLinear(
dims[0],
@@ -373,16 +372,14 @@ class MLP2(nn.Module):
dims[1],
bias=bias,
prefix=add_prefix("fc0", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group="attn_tp",
)
self.fc1 = RowParallelLinear(
dims[1],
dims[2],
bias=bias,
prefix=add_prefix("fc1", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group="attn_tp",
)
else:
self.fc0 = nn.Linear(dims[0], dims[1], bias=bias)
+6 -15
View File
@@ -85,7 +85,6 @@ class MiniMaxVLMultiModalProjector(nn.Module):
)
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -103,8 +102,7 @@ class MiniMaxVLMultiModalProjector(nn.Module):
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
)
assert projector_hidden_act == "gelu", (
f"Only gelu activation is supported, got {projector_hidden_act}"
@@ -116,8 +114,7 @@ class MiniMaxVLMultiModalProjector(nn.Module):
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
use_dp_attention_reduce=is_dp_attention_enabled(),
)
@@ -150,7 +147,6 @@ class MiniMaxVLPatchMerger(nn.Module):
)
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -168,8 +164,7 @@ class MiniMaxVLPatchMerger(nn.Module):
bias=patch_merge_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
)
assert projector_hidden_act == "gelu", (
f"Only gelu activation is supported, got {projector_hidden_act}"
@@ -181,8 +176,7 @@ class MiniMaxVLPatchMerger(nn.Module):
bias=patch_merge_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
use_dp_attention_reduce=is_dp_attention_enabled(),
)
@@ -282,7 +276,6 @@ class CLIPEncoderLayer(nn.Module):
self.embed_dim = config.hidden_size
self.use_data_parallel = use_data_parallel
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -314,8 +307,7 @@ class CLIPEncoderLayer(nn.Module):
config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.mlp.fc1",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
)
hidden_act = getattr(config, "hidden_act", "gelu")
assert hidden_act == "gelu", (
@@ -327,8 +319,7 @@ class CLIPEncoderLayer(nn.Module):
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.mlp.fc2",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
use_dp_attention_reduce=is_dp_attention_enabled(),
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+14 -24
View File
@@ -50,9 +50,11 @@ from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -140,21 +142,16 @@ class Qwen2_5_VLMLP(nn.Module):
prefix: str = "",
use_data_parallel: bool = False,
fuse_gate_up: bool = True,
tp_size: Optional[int] = None,
tp_rank: Optional[int] = None,
parallel_group: Optional[LinearParallelGroup] = None,
):
super().__init__()
if use_data_parallel and parallel_group is not None:
raise ValueError("Explicit MLP TP cannot be combined with data parallel")
if use_data_parallel:
if tp_size is not None or tp_rank is not None:
raise ValueError(
"Explicit MLP TP cannot be combined with data parallel"
)
self.tp_size, self.tp_rank = 1, 0
else:
if (tp_size is None) != (tp_rank is None):
raise ValueError("MLP tp_size and tp_rank must be set together")
self.tp_size = get_parallel().tp_size if tp_size is None else tp_size
self.tp_rank = get_parallel().tp_rank if tp_rank is None else tp_rank
parallel_group = "replicated"
elif parallel_group is None:
parallel_group = "tp"
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
self.fuse_gate_up = fuse_gate_up
if fuse_gate_up:
self.gate_up_proj = MergedColumnParallelLinear(
@@ -163,8 +160,7 @@ class Qwen2_5_VLMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
else:
projection_kwargs = dict(
@@ -172,8 +168,7 @@ class Qwen2_5_VLMLP(nn.Module):
output_size=hidden_features,
bias=bias,
quant_config=quant_config,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.gate_proj = ColumnParallelLinear(
**projection_kwargs,
@@ -198,8 +193,7 @@ class Qwen2_5_VLMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.hidden_act = hidden_act
if self.fuse_gate_up and self.hidden_act == "silu":
@@ -326,8 +320,6 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
cast_x_before_out_mul=cast_x_before_out_mul,
force_native=force_native_norm,
)
tp_size = 1 if use_data_parallel else get_parallel().tp_size
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
self.mlp = nn.ModuleList(
[
ColumnParallelLinear(
@@ -336,8 +328,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("mlp.0", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
),
nn.GELU(),
RowParallelLinear(
@@ -346,8 +337,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("mlp.2", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
),
]
)
+24 -34
View File
@@ -44,7 +44,12 @@ from sglang.srt.layers.dp_attention import (
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -111,23 +116,16 @@ _is_cpu = is_cpu()
_VECTORIZED_VL_POS_EMBED_MIN_IMAGES = 6
def _resolve_vision_tp(
def _resolve_vision_parallel_group(
*,
use_data_parallel: bool,
tp_size: Optional[int],
tp_rank: Optional[int],
) -> tuple[int, int]:
parallel_group: Optional[LinearParallelGroup],
) -> LinearParallelGroup:
if use_data_parallel:
if tp_size is not None or tp_rank is not None:
if parallel_group is not None:
raise ValueError("Explicit vision TP cannot be combined with data parallel")
return 1, 0
if (tp_size is None) != (tp_rank is None):
raise ValueError("Vision tp_size and tp_rank must be set together")
if tp_size is None:
parallel = get_parallel()
return parallel.attn_tp_size, parallel.attn_tp_rank
assert tp_rank is not None
return tp_size, tp_rank
return "replicated"
return "attn_tp" if parallel_group is None else parallel_group
class Qwen3_VisionMLP(nn.Module):
@@ -140,15 +138,13 @@ class Qwen3_VisionMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
use_data_parallel: bool = False,
tp_size: Optional[int] = None,
tp_rank: Optional[int] = None,
parallel_group: Optional[LinearParallelGroup] = None,
):
super().__init__()
self.tp_size, self.tp_rank = _resolve_vision_tp(
use_data_parallel=use_data_parallel,
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group = _resolve_vision_parallel_group(
use_data_parallel=use_data_parallel, parallel_group=parallel_group
)
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -165,8 +161,7 @@ class Qwen3_VisionMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("linear_fc1", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.linear_fc2 = RowParallelLinear(
hidden_features,
@@ -174,8 +169,7 @@ class Qwen3_VisionMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("linear_fc2", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=is_dp_attention_enabled(),
)
self.act = ACT2FN[hidden_act]
@@ -306,8 +300,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
use_data_parallel: bool = False,
tp_size: Optional[int] = None,
tp_rank: Optional[int] = None,
parallel_group: Optional[LinearParallelGroup] = None,
disable_merger_proj: bool = False,
) -> None:
super().__init__()
@@ -323,11 +316,10 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
self.hidden_size if use_postshuffle_norm else context_dim
)
if not disable_merger_proj:
self.tp_size, self.tp_rank = _resolve_vision_tp(
use_data_parallel=use_data_parallel,
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group = _resolve_vision_parallel_group(
use_data_parallel=use_data_parallel, parallel_group=parallel_group
)
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -344,8 +336,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("linear_fc1", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.act_fn = nn.GELU()
self.linear_fc2 = RowParallelLinear(
@@ -354,8 +345,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("linear_fc2", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=is_dp_attention_enabled(),
)
+2 -7
View File
@@ -14,7 +14,6 @@ from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix
@@ -71,15 +70,12 @@ class SiglipMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
tp_size = 1 if use_data_parallel else get_parallel().tp_size
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
self.fc1 = ColumnParallelLinear(
config.hidden_size,
config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("fc1", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
)
self.act = act_layer()
self.fc2 = RowParallelLinear(
@@ -87,8 +83,7 @@ class SiglipMLP(nn.Module):
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("fc2", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
+2 -5
View File
@@ -524,15 +524,13 @@ class Step3VisionMLP(nn.Module):
# Since this is a dense model,
# the MLP component likewise adopts a DP-MLP approach modeled after DP Attention.
# This choice may not represent the optimal solution and remains open to further deliberation.
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.fc1 = ColumnParallelLinear(
dim,
intermediate_size,
bias=bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("gate_proj", prefix),
)
self.act = ACT2FN[hidden_act] # quick_gelu
@@ -550,8 +548,7 @@ class Step3VisionMLP(nn.Module):
dim,
bias=bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("down_proj", prefix),
)
+51
View File
@@ -0,0 +1,51 @@
"""Group metadata fixtures for tests that do not initialize distributed groups."""
from contextlib import contextmanager
from types import SimpleNamespace
from sglang.srt.runtime_context import SpawnRanks, get_parallel
from sglang.srt.runtime_context import publish as publish_context
def _group_metadata(values):
parallel = get_parallel()
for name in ("tp", "attn_tp"):
rank_key, size_key, group_key = (
f"{name}_rank",
f"{name}_size",
f"{name}_group",
)
rank = values.get(rank_key, getattr(parallel, rank_key))
size = values.get(size_key, getattr(parallel, size_key))
if group_key not in values and (rank_key in values or size_key in values):
values[group_key] = SimpleNamespace(rank_in_group=rank, world_size=size)
group = values.get(group_key)
if group is not None:
# Execution mocks also need placement metadata when used to build layers.
if not isinstance(getattr(group, "rank_in_group", None), int):
group.rank_in_group = rank
if not isinstance(getattr(group, "world_size", None), int):
group.world_size = size
return values
def publish(*args, **kwargs):
if kwargs.get("ranks") is None:
kwargs["ranks"] = SpawnRanks(world_rank=0)
context = publish_context(*args, **kwargs)
parallel = get_parallel()
groups = {
f"{name}_group": SimpleNamespace(
rank_in_group=getattr(parallel, f"{name}_rank"),
world_size=getattr(parallel, f"{name}_size"),
)
for name in ("tp", "attn_tp")
}
parallel.override_permanently(**groups)
return context
@contextmanager
def parallel_scope(**values):
with get_parallel().override(**_group_metadata(values)) as parallel:
yield parallel
@@ -0,0 +1,294 @@
"""Qwen vision wrappers and generation helpers retain their placement policies."""
import inspect
import unittest
from contextlib import nullcontext
from unittest.mock import Mock, patch
import torch
import torch.nn.functional as F
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
from sglang.srt.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
def group_kwargs(cls, group):
if group is None:
return {}
if "parallel_group" in inspect.signature(cls).parameters:
return dict(parallel_group=group)
p = get_parallel()
rank, size = (
(0, 1)
if group == "replicated"
else (p.tp_rank, p.tp_size)
if group == "tp"
else (p.attn_tp_rank, p.attn_tp_size)
)
return dict(tp_rank=rank, tp_size=size)
def build_qwen(model, group=None, *, replicated=False, width=32, quant_config=None):
options = dict(quant_config=quant_config, use_data_parallel=replicated)
if model.startswith("qwen25"):
from sglang.srt.models.qwen2_5_vl import Qwen2_5_VLMLP
module = Qwen2_5_VLMLP(
width,
2 * width,
bias=True,
fuse_gate_up=model == "qwen25_fused",
**options,
**group_kwargs(Qwen2_5_VLMLP, group),
)
layers = (
(module.gate_up_proj, module.down_proj)
if module.fuse_gate_up
else (module.gate_proj, module.up_proj, module.down_proj)
)
elif model == "qwen3_mlp":
from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP
module = Qwen3_VisionMLP(
width,
2 * width,
bias=True,
**options,
**group_kwargs(Qwen3_VisionMLP, group),
)
layers = (module.linear_fc1, module.linear_fc2)
elif model == "qwen3_merger":
from sglang.srt.models.qwen3_vl import Qwen3VLMoeVisionPatchMerger
module = Qwen3VLMoeVisionPatchMerger(
width,
width,
2 * width,
spatial_merge_size=1,
**options,
**group_kwargs(Qwen3VLMoeVisionPatchMerger, group),
)
layers = (module.linear_fc1, module.linear_fc2)
else:
raise ValueError(model)
return module, layers
def load_projection(layer):
"""Load known full weights under rank zero, returning the expected owned shard."""
rank, size = getattr(layer, "tp_rank", 0), getattr(layer, "tp_size", 1)
row = isinstance(layer, RowParallelLinear)
dtype, device = layer.weight.dtype, layer.weight.device
def values(rows, columns, offset):
return (
(
torch.arange(rows * columns, device=device).reshape(rows, columns)
+ offset
)
% 23
/ 128
).to(dtype)
with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0):
if isinstance(layer, QKVParallelLinear):
weights, biases = [], []
for offset, shard_id in enumerate(("q", "k", "v")):
heads = (
layer.total_num_heads
if shard_id == "q"
else layer.total_num_kv_heads
)
weight = values(heads * layer.head_size, layer.input_size, offset)
partitions = size if shard_id == "q" else min(heads, size)
index = rank if shard_id == "q" else rank // max(size // heads, 1)
layer.weight.weight_loader(layer.weight, weight, shard_id)
weights.append(weight.chunk(partitions)[index])
if layer.bias is not None:
bias = values(heads * layer.head_size, 1, offset).flatten()
layer.bias.weight_loader(layer.bias, bias, shard_id)
biases.append(bias.chunk(partitions)[index])
shard = torch.cat(weights)
bias_shard = torch.cat(biases) if biases else None
else:
weight = values(layer.output_size, layer.input_size, 3)
layer.weight.weight_loader(layer.weight, weight)
shard = (
weight.chunk(size, dim=1)[rank]
if row
else torch.cat(
[
p.chunk(size)[rank]
for p in weight.split(
getattr(layer, "output_sizes", [layer.output_size])
)
]
)
)
bias_shard = None
if layer.bias is not None:
bias = values(layer.output_size, 1, 4).flatten()
layer.bias.weight_loader(layer.bias, bias)
bias_shard = (
bias
if row
else torch.cat(
[
p.chunk(size)[rank]
for p in bias.split(
getattr(layer, "output_sizes", [layer.output_size])
)
]
)
)
return shard, bias_shard
class TestQwenVisionWrapperGroups(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def test_native_projection_math_loaders_and_flags(self):
for dp in (1, 2):
reset_context()
server = ServerArgs(
model_path="dummy",
device="cpu",
tp_size=4,
attn_dp_size=dp,
mm_attention_backend="sdpa",
)
publish(server, role="test", ranks=SpawnRanks(world_rank=3))
initialize_dp_attention_flags(server)
tp, attn = Mock(), Mock()
tp.all_reduce.side_effect = lambda x: x * 2
attn.all_reduce.side_effect = lambda x: x * 3
for model in (
"qwen25_fused",
"qwen25_split",
"qwen3_mlp",
"qwen3_merger",
):
for group, replicated in (
(None, False),
(None, True),
("tp", False),
("attn_tp", False),
("replicated", False),
):
with self.subTest(
dp=dp, model=model, group=group, replicated=replicated
):
module, layers = build_qwen(model, group, replicated=replicated)
selected = (
"replicated"
if replicated
else group
or ("attn_tp" if model.startswith("qwen3") else "tp")
)
expected_rank, expected_size = (
(0, 1)
if selected == "replicated"
else (3, 4)
if selected == "tp"
else (3 % (4 // dp), 4 // dp)
)
for index, layer in enumerate(layers):
weight, bias = load_projection(layer)
row = isinstance(layer, RowParallelLinear)
inputs = (
torch.arange(2 * weight.shape[1]).reshape(2, -1).float()
/ 64
)
expected = F.linear(
inputs,
weight,
bias if not row or expected_rank == 0 else None,
)
dp_reduce = row and layer.use_dp_attention_reduce
if row and expected_size > 1:
expected *= 3 if dp_reduce else 2
tp.all_reduce.reset_mock()
attn.all_reduce.reset_mock()
with (
get_parallel().override(
tp_group=tp, attn_tp_group=attn
),
patch(
"sglang.srt.layers.linear.is_allocation_symmetric",
return_value=False,
),
patch(
"sglang.srt.layers.linear.use_symmetric_memory",
return_value=nullcontext(),
) as allocator,
):
actual = layer(inputs)[0]
torch.testing.assert_close(actual, expected)
if row:
allocator.assert_called_once_with(
attn
) if dp_reduce else allocator.assert_called_once_with(
tp, disabled=True
)
else:
allocator.assert_not_called()
self.assertEqual(
tp.all_reduce.call_count,
int(row and expected_size > 1 and not dp_reduce),
)
self.assertEqual(
attn.all_reduce.call_count,
int(row and expected_size > 1 and dp_reduce),
)
self.assertEqual(
(
getattr(layer, "tp_rank", 0),
getattr(layer, "tp_size", 1),
),
(expected_rank, expected_size),
)
if model == "qwen25_split":
self.assertEqual(
isinstance(module.down_proj, ReplicatedLinear),
expected_size == 1,
)
def test_wrapper_conflict_guard_and_disabled_merger(self):
publish(
ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_cp_size=2),
role="test",
ranks=SpawnRanks(world_rank=3),
)
for model in ("qwen25_fused", "qwen3_mlp", "qwen3_merger"):
with self.assertRaisesRegex(
ValueError, "cannot be combined with data parallel"
):
build_qwen(model, "replicated", replicated=True)
from sglang.srt.models.qwen3_vl import Qwen3VLMoeVisionPatchMerger
reset_context()
merger = Qwen3VLMoeVisionPatchMerger(
32,
32,
64,
disable_merger_proj=True,
**group_kwargs(Qwen3VLMoeVisionPatchMerger, "replicated"),
)
self.assertFalse(hasattr(merger, "linear_fc1"))
self.assertFalse(hasattr(merger, "tp_size"))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,376 @@
"""Vision projections retain their layout and execution policy across reload scopes."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
import torch.nn.functional as F
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
from sglang.srt.runtime_context import (
SpawnRanks,
get_disagg,
get_parallel,
publish,
reset_context,
)
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
VISION_MODELS = (
"vision_packed",
"vision_column",
"vision_dummy",
"vision_gqa",
"vision_legacy",
"siglip",
"glm4v_mlp",
"glm4v_merger",
"internvl_mlp",
"qwen_merger",
"minimax_projector",
"minimax_merger",
"minimax_encoder",
"moonvit",
"step3",
)
def build_vision(model, use_data_parallel=False, width=32, quant_config=None):
config = SimpleNamespace(
hidden_size=width,
intermediate_size=2 * width,
num_attention_heads=8,
layer_norm_eps=1e-6,
hidden_act="gelu",
)
options = dict(quant_config=quant_config, use_data_parallel=use_data_parallel)
if model.startswith("vision_"):
from sglang.srt.layers.attention.vision import VisionAttention
module = VisionAttention(
width,
8,
width,
use_qkv_parallel=model != "vision_column",
num_dummy_heads=4 if model == "vision_dummy" else 0,
num_kv_heads=4 if model == "vision_gqa" else 8,
use_dp_attention_reduce=model != "vision_legacy",
allow_tp_reduce_mismatch=model == "vision_legacy",
qkv_backend="sdpa",
**options,
)
layers = (module.qkv_proj, module.proj)
elif model in ("glm4v_mlp", "glm4v_merger"):
from sglang.srt.models.glm4v import Glm4vPatchMerger, Glm4vVisionMLP
constructor = Glm4vVisionMLP if model == "glm4v_mlp" else Glm4vPatchMerger
module = constructor(width, 2 * width, bias=True, **options)
layers = (module.gate_up_proj, module.down_proj)
elif model == "internvl_mlp":
from sglang.srt.models.internvl import InternMLP
module = InternMLP(config, use_data_parallel=use_data_parallel)
layers = (module.fc1, module.fc2)
elif model == "siglip":
from sglang.srt.models.siglip import SiglipMLP
module = SiglipMLP(config, **options)
layers = (module.fc1, module.fc2)
elif model == "qwen_merger":
from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionPatchMerger
module = Qwen2_5_VisionPatchMerger(
width,
width,
2 * width,
spatial_merge_size=1,
force_native_norm=True,
**options,
)
layers = (module.mlp[0], module.mlp[2])
elif model == "moonvit":
from sglang.srt.models.kimi_vl_moonvit import MLP2
module = MLP2(
[width, 2 * width, width],
torch.nn.GELU(),
use_tensor_parallel=True,
**options,
)
layers = (module.fc0, module.fc1)
elif model == "step3":
from sglang.srt.models.step3_vl import Step3VisionMLP
module = Step3VisionMLP(width, 2 * width, quant_config=quant_config)
layers = (module.fc1, module.fc2)
else:
from sglang.srt.models.minimax_vl_common import (
CLIPEncoderLayer,
MiniMaxVLMultiModalProjector,
MiniMaxVLPatchMerger,
)
if model == "minimax_projector":
module = MiniMaxVLMultiModalProjector(
width, width, "gelu", True, projector_hidden_size=2 * width, **options
)
layers = (module.linear_1, module.linear_2)
elif model == "minimax_merger":
module = MiniMaxVLPatchMerger(
1, width, "gelu", True, projector_hidden_size=2 * width, **options
)
layers = (module.linear_1, module.linear_2)
else:
module = CLIPEncoderLayer(config, **options)
layers = (
module.fc1,
module.fc2,
module.self_attn.qkv_proj,
module.self_attn.proj,
)
return module, layers
def load_projection(layer):
"""Load known full weights under rank zero, returning the expected owned shard."""
rank, size = layer.tp_rank, layer.tp_size
row = isinstance(layer, RowParallelLinear)
dtype, device = layer.weight.dtype, layer.weight.device
def values(rows, columns, offset):
return (
(
torch.arange(rows * columns, device=device).reshape(rows, columns)
+ offset
)
% 23
/ 128
).to(dtype)
with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0):
if isinstance(layer, QKVParallelLinear):
weights, biases = [], []
for offset, shard_id in enumerate(("q", "k", "v")):
heads = (
layer.total_num_heads
if shard_id == "q"
else layer.total_num_kv_heads
)
weight = values(heads * layer.head_size, layer.input_size, offset)
partitions = size if shard_id == "q" else min(heads, size)
index = rank if shard_id == "q" else rank // max(size // heads, 1)
layer.weight.weight_loader(layer.weight, weight, shard_id)
weights.append(weight.chunk(partitions)[index])
if layer.bias is not None:
bias = values(heads * layer.head_size, 1, offset).flatten()
layer.bias.weight_loader(layer.bias, bias, shard_id)
biases.append(bias.chunk(partitions)[index])
shard = torch.cat(weights)
bias_shard = torch.cat(biases) if biases else None
else:
weight = values(layer.output_size, layer.input_size, 3)
layer.weight.weight_loader(layer.weight, weight)
shard = (
weight.chunk(size, dim=1)[rank]
if row
else torch.cat(
[
p.chunk(size)[rank]
for p in weight.split(
getattr(layer, "output_sizes", [layer.output_size])
)
]
)
)
bias_shard = None
if layer.bias is not None:
bias = values(layer.output_size, 1, 4).flatten()
layer.bias.weight_loader(layer.bias, bias)
bias_shard = (
bias
if row
else torch.cat(
[
p.chunk(size)[rank]
for p in bias.split(
getattr(layer, "output_sizes", [layer.output_size])
)
]
)
)
return shard, bias_shard
class TestVisionParallelGroups(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def test_projection_layout_reload_and_row_policy(self):
for dp_size in (1, 2):
publish(
ServerArgs(
model_path="dummy",
device="cpu",
tp_size=4,
attn_dp_size=dp_size,
enable_dp_attention=dp_size > 1,
mm_attention_backend="sdpa",
),
role="test",
ranks=SpawnRanks(world_rank=3),
)
initialize_dp_attention_flags(
ServerArgs(
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp_size
)
)
tp, attn = Mock(), Mock()
tp.all_reduce.side_effect = lambda x: x * 4
attn.all_reduce.side_effect = lambda x: x * (4 // dp_size)
for model in VISION_MODELS:
for replicated in (
(False,) if model in ("moonvit", "step3") else (False, True)
):
with self.subTest(dp=dp_size, model=model, replicated=replicated):
if dp_size == 2 and model in ("moonvit", "step3"):
with self.assertRaisesRegex(
ValueError, "shards over the attention TP group"
):
build_vision(model, replicated)
continue
module, layers = build_vision(model, replicated)
group = (
"tp"
if model
in (
"siglip",
"qwen_merger",
"glm4v_mlp",
"glm4v_merger",
"internvl_mlp",
)
else "attn_tp"
)
rank, size = (
(0, 1)
if replicated
else (
(3, 4)
if group == "tp"
else (3 // dp_size, 4 // dp_size)
)
)
for layer in layers:
self.assertEqual(
(layer.tp_rank, layer.tp_size), (rank, size)
)
shard, bias = load_projection(layer)
torch.testing.assert_close(layer.weight, shard)
if bias is not None:
torch.testing.assert_close(layer.bias, bias)
row = isinstance(layer, RowParallelLinear)
inputs = (
torch.arange(
2 * shard.shape[1], dtype=shard.dtype
).reshape(2, -1)
/ 128
)
expected = F.linear(
inputs, shard, bias if not row or rank == 0 else None
)
if row and size > 1:
expected *= (
(4 // dp_size)
if layer.use_dp_attention_reduce
else 4
)
tp.all_reduce.reset_mock()
attn.all_reduce.reset_mock()
with (
get_parallel().override(
tp_group=tp, attn_tp_group=attn
),
patch(
"sglang.srt.layers.linear.is_allocation_symmetric",
return_value=False,
),
patch(
"sglang.srt.layers.linear.use_symmetric_memory",
return_value=nullcontext(),
) as allocator,
):
torch.testing.assert_close(layer(inputs)[0], expected)
if row:
self.assertTrue(layer.reduce_results)
reduction = (
attn if layer.use_dp_attention_reduce else tp
)
if layer.use_dp_attention_reduce:
allocator.assert_called_once_with(reduction)
else:
allocator.assert_called_once_with(
reduction, disabled=True
)
self.assertEqual(
reduction.all_reduce.call_count, int(size > 1)
)
self.assertEqual(
(
tp if reduction is attn else attn
).all_reduce.call_count,
0,
)
else:
allocator.assert_not_called()
self.assertEqual(
(layer.tp_rank, layer.tp_size), (rank, size)
)
if hasattr(module, "tp_rank"):
self.assertEqual(
(module.tp_rank, module.tp_size), (rank, size)
)
reset_context()
def test_guard_and_moonvit_dense_path(self):
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.models.kimi_vl_moonvit import MLP2
publish(
ServerArgs(
model_path="dummy",
device="cpu",
tp_size=4,
attn_dp_size=2,
mm_attention_backend="sdpa",
),
role="test",
ranks=SpawnRanks(world_rank=3),
)
with self.assertRaisesRegex(ValueError, "shards over the attention TP group"):
VisionAttention(32, 8, 32, True, qkv_backend="sdpa")
with get_disagg().override(language_model_only=True):
offloaded = VisionAttention(32, 8, 32, True, qkv_backend="sdpa")
self.assertEqual((offloaded.proj.tp_rank, offloaded.proj.tp_size), (1, 2))
self.assertFalse(offloaded.proj.use_dp_attention_reduce)
for data_parallel, tensor_parallel in ((False, False), (True, True)):
module = MLP2(
[32, 64, 32],
torch.nn.GELU(),
use_data_parallel=data_parallel,
use_tensor_parallel=tensor_parallel,
)
self.assertIsInstance(module.fc0, torch.nn.Linear)
self.assertIsInstance(module.fc1, torch.nn.Linear)
self.assertEqual(module(torch.zeros(2, 32)).shape, (2, 32))
if __name__ == "__main__":
unittest.main()