From dec5d9d6fc70cd341b793a8259fb346d792ade36 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:22:32 -0700 Subject: [PATCH] [Refactor] Let vision encoders select their parallel group (#42586) --- .../runtime/models/encoders/qwen2_5vl.py | 16 +- .../runtime/models/encoders/qwen3vl_vision.py | 12 +- .../runtime/models/encoders/qwen_vl_vision.py | 6 +- .../test/unit/test_qwen2_5vl_generation.py | 1 - .../unit/test_qwen_vision_parallel_groups.py | 131 ++++++ python/sglang/srt/layers/attention/vision.py | 9 +- python/sglang/srt/layers/linear.py | 23 +- python/sglang/srt/models/glm4v.py | 26 +- python/sglang/srt/models/internvl.py | 11 +- python/sglang/srt/models/kimi_vl_moonvit.py | 7 +- python/sglang/srt/models/minimax_vl_common.py | 21 +- python/sglang/srt/models/qwen2_5_vl.py | 38 +- python/sglang/srt/models/qwen3_vl.py | 58 ++- python/sglang/srt/models/siglip.py | 9 +- python/sglang/srt/models/step3_vl.py | 7 +- python/sglang/test/parallel_groups.py | 51 +++ ...est_qwen_vision_wrapper_parallel_groups.py | 294 ++++++++++++++ .../layers/test_vision_parallel_groups.py | 376 ++++++++++++++++++ 18 files changed, 942 insertions(+), 154 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_qwen_vision_parallel_groups.py create mode 100644 python/sglang/test/parallel_groups.py create mode 100644 test/registered/unit/layers/test_qwen_vision_wrapper_parallel_groups.py create mode 100644 test/registered/unit/layers/test_vision_parallel_groups.py diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py index a782df3b8a..5bd7d411a0 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py @@ -16,7 +16,6 @@ from transformers.utils import TransformersKwargs, is_torchdynamo_compiling from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig from sglang.multimodal_gen.runtime.distributed import ( - get_tp_rank, get_tp_world_size, model_parallel_is_initialized, ) @@ -139,12 +138,6 @@ def _tp_world_size() -> int: return get_tp_world_size() -def _tp_rank() -> int: - if not model_parallel_is_initialized(): - return 0 - return get_tp_rank() - - def _linear_output(linear: nn.Module, x: torch.Tensor) -> torch.Tensor: output = linear(x) return output[0] if isinstance(output, tuple) else output @@ -163,8 +156,7 @@ def _make_column_linear( out_features, bias=bias, gather_output=False, - tp_size=_tp_world_size(), - tp_rank=_tp_rank(), + parallel_group="tp" if model_parallel_is_initialized() else "replicated", ) return ReplicatedLinear(in_features, out_features, bias=bias) @@ -181,8 +173,7 @@ def _make_row_linear( in_features, out_features, bias=bias, - tp_size=_tp_world_size(), - tp_rank=_tp_rank(), + parallel_group="tp" if model_parallel_is_initialized() else "replicated", ) return ReplicatedLinear(in_features, out_features, bias=bias) @@ -360,8 +351,7 @@ class Qwen2_5_VLDecoderLayer(nn.Module): hidden_act=config.hidden_act, prefix=f"model.language_model.layers.{layer_idx}.mlp", fuse_gate_up=False, - tp_size=mlp_tp_size, - tp_rank=_tp_rank() if mlp_tp_size > 1 else 0, + parallel_group="tp" if mlp_tp_size > 1 else "replicated", ) norm_kwargs = dict( eps=config.rms_norm_eps, diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl_vision.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl_vision.py index c3fefc9d1a..e362467a44 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl_vision.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl_vision.py @@ -19,7 +19,6 @@ from sglang.srt.models.qwen3_vl import ( Qwen3VLMoeVisionPatchMerger, Qwen3VLVisionPatchEmbed, ) -from sglang.srt.runtime_context import get_parallel @dataclass(frozen=True) @@ -70,7 +69,6 @@ class Qwen3VLVisionBlock(nn.Module): prefix: str = "visual", ) -> None: super().__init__() - parallel = get_parallel() self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6) self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6) self.attn = QwenVLVisionAttention( @@ -86,8 +84,7 @@ class Qwen3VLVisionBlock(nn.Module): hidden_act=config.hidden_act, prefix=f"{prefix}.blocks.{layer_idx}.mlp", quant_config=quant_config, - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", ) def forward( @@ -210,7 +207,6 @@ class Qwen3VLVisionTransformer(nn.Module): prefix: str = "visual", ) -> None: super().__init__() - parallel = get_parallel() self.config = config self.spatial_merge_size = config.spatial_merge_size self.spatial_merge_unit = config.spatial_merge_size**2 @@ -233,8 +229,7 @@ class Qwen3VLVisionTransformer(nn.Module): spatial_merge_size=config.spatial_merge_size, use_postshuffle_norm=False, prefix="visual.merger", - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", ) self.deepstack_visual_indexes = tuple(config.deepstack_visual_indexes) self.deepstack_merger_list = nn.ModuleList( @@ -245,8 +240,7 @@ class Qwen3VLVisionTransformer(nn.Module): spatial_merge_size=config.spatial_merge_size, use_postshuffle_norm=True, prefix=f"visual.deepstack_merger_list.{merger_idx}", - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", ) for merger_idx, _ in enumerate(self.deepstack_visual_indexes) ) diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl_vision.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl_vision.py index 169605a770..6a46ce9cff 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl_vision.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl_vision.py @@ -79,8 +79,7 @@ class QwenVLVisionAttention(nn.Module): bias=True, quant_config=quant_config, prefix=f"{prefix}.qkv_proj", - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", ) self.proj = RowParallelLinear( input_size=config.hidden_size, @@ -88,8 +87,7 @@ class QwenVLVisionAttention(nn.Module): bias=True, quant_config=quant_config, prefix=f"{prefix}.proj", - tp_rank=parallel.tp_rank, - tp_size=parallel.tp_size, + parallel_group="tp", ) backend = get_attn_backend(self.head_dim, torch.get_default_dtype()) diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen2_5vl_generation.py b/python/sglang/multimodal_gen/test/unit/test_qwen2_5vl_generation.py index cc58a98764..c9711e83bb 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen2_5vl_generation.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen2_5vl_generation.py @@ -75,7 +75,6 @@ def test_text_mlp_uses_single_rank_when_intermediate_size_is_not_tp_divisible( ): monkeypatch.setattr(qwen2_5vl, "Qwen2_5_VLAttention", lambda *_args: nn.Identity()) monkeypatch.setattr(qwen2_5vl, "_tp_world_size", lambda: 3) - monkeypatch.setattr(qwen2_5vl, "_tp_rank", lambda: 2) config = SimpleNamespace( hidden_size=16, intermediate_size=25, diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_vision_parallel_groups.py b/python/sglang/multimodal_gen/test/unit/test_qwen_vision_parallel_groups.py new file mode 100644 index 0000000000..907b7e2c3b --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_vision_parallel_groups.py @@ -0,0 +1,131 @@ +"""Qwen generation projections tested in the diffusion dependency environment.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.linear import QKVParallelLinear, ReplicatedLinear +from sglang.srt.runtime_context import SpawnRanks, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.parallel_groups import parallel_scope, publish + + +class TestQwenGenerationParallelGroups(unittest.TestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_generation_helpers_match_initialized_scope(self): + import sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl as generation + + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + for initialized in (False, True): + with patch.object( + generation, + "model_parallel_is_initialized", + return_value=initialized, + ): + for tensor_parallel in (False, True): + for maker in ( + generation._make_column_linear, + generation._make_row_linear, + ): + layer = maker( + 8, 8, bias=True, use_tensor_parallel=tensor_parallel + ) + expected = (3, 4) if initialized and tensor_parallel else (0, 1) + actual = ( + (0, 1) + if isinstance(layer, ReplicatedLinear) + else (layer.tp_rank, layer.tp_size) + ) + self.assertEqual(actual, expected) + self.assertEqual( + isinstance(layer, ReplicatedLinear), not tensor_parallel + ) + + def test_native_vision_projection_math_and_reload(self): + from sglang.multimodal_gen.runtime.layers.attention.selector import ( + global_force_attn_backend_context_manager, + ) + from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_vision import ( + QwenVLVisionAttention, + ) + from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + for dp in (1, 2): + with self.subTest(dp=dp): + reset_context() + server = ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp + ) + publish(server, role="test", ranks=SpawnRanks(world_rank=3)) + initialize_dp_attention_flags(server) + with global_force_attn_backend_context_manager( + AttentionBackendEnum.TORCH_SDPA + ): + module = QwenVLVisionAttention( + SimpleNamespace(hidden_size=32, num_heads=8), + prefix="visual", + model_name="Qwen", + ) + tp = Mock(all_reduce=Mock(side_effect=lambda x: x * 2)) + attn = Mock(all_reduce=Mock(side_effect=AssertionError("wrong group"))) + for layer in (module.qkv_proj, module.proj): + rank, size = ( + getattr(layer, "tp_rank", 0), + getattr(layer, "tp_size", 1), + ) + self.assertEqual((rank, size), (3, 4)) + full = ( + torch.arange(32 * 32, dtype=layer.weight.dtype).reshape(32, 32) + / 1024 + ) + bias = torch.arange(32, dtype=layer.weight.dtype) / 64 + qkv = isinstance(layer, QKVParallelLinear) + with parallel_scope(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0): + for shard in ("q", "k", "v") if qkv else (None,): + args = (shard,) if qkv else () + layer.weight.weight_loader(layer.weight, full, *args) + layer.bias.weight_loader(layer.bias, bias, *args) + weight = ( + torch.cat([full.chunk(size)[rank]] * 3) + if qkv + else full.chunk(size, dim=1)[rank] + ) + expected_bias = ( + torch.cat([bias.chunk(size)[rank]] * 3) if qkv else None + ) + inputs = ( + torch.arange(2 * weight.shape[1], dtype=weight.dtype).reshape( + 2, -1 + ) + / 64 + ) + expected = F.linear(inputs, weight, expected_bias) * ( + 1 if qkv else 2 + ) + with ( + parallel_scope(tp_group=tp, attn_tp_group=attn), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ), + ): + torch.testing.assert_close(layer(inputs)[0], expected) + torch.testing.assert_close(layer.weight, weight, rtol=0, atol=0) + attn.all_reduce.assert_not_called() + tp.all_reduce.assert_called_once() diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index ceeaea811d..29c5bd02b9 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -1196,8 +1196,7 @@ class VisionAttention(nn.Module): total_num_kv_heads=num_dummy_heads + num_kv_heads, bias=qkv_bias, quant_config=quant_config, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="replicated" if use_data_parallel else "attn_tp", prefix=add_prefix("qkv_proj", prefix), ) else: @@ -1206,8 +1205,7 @@ class VisionAttention(nn.Module): output_size=3 * self.dummy_dim, bias=qkv_bias, quant_config=quant_config, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="replicated" if use_data_parallel else "attn_tp", prefix=add_prefix("qkv_proj", prefix), ) # TODO: this layer shards over attention TP but reduces over the full TP @@ -1226,8 +1224,7 @@ class VisionAttention(nn.Module): output_size=embed_dim, bias=proj_bias, quant_config=quant_config, - tp_rank=self.tp_rank, - tp_size=self.tp_size, + parallel_group="replicated" if use_data_parallel else "attn_tp", prefix=add_prefix("proj", prefix), use_dp_attention_reduce=use_dp_attention_reduce, ) diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py index 380ff4b941..a112723707 100644 --- a/python/sglang/srt/layers/linear.py +++ b/python/sglang/srt/layers/linear.py @@ -58,6 +58,20 @@ logger = logging.getLogger(__name__) LinearParallelGroup = Literal["tp", "attn_tp", "replicated"] +def resolve_linear_parallel_group( + parallel_group: LinearParallelGroup, +) -> Tuple[int, int]: + """Freeze a group's weight partition in the current construction scope.""" + if parallel_group == "replicated": + return 0, 1 + if parallel_group not in ("tp", "attn_tp"): + raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}") + parallel = get_parallel() + if parallel_group == "attn_tp": + return parallel.attn_tp_rank, parallel.attn_tp_size + return parallel.tp_rank, parallel.tp_size + + def _resolve_linear_partition( parallel_group: Optional[LinearParallelGroup], tp_rank: Optional[int], @@ -74,14 +88,7 @@ def _resolve_linear_partition( raise ValueError( "parallel_group cannot be combined with tp_rank or tp_size" ) - if parallel_group == "replicated": - return 0, 1 - if parallel_group not in ("tp", "attn_tp"): - raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}") - parallel = get_parallel() - if parallel_group == "attn_tp": - return parallel.attn_tp_rank, parallel.attn_tp_size - return parallel.tp_rank, parallel.tp_size + return resolve_linear_parallel_group(parallel_group) if tp_rank is None: tp_rank = get_parallel().tp_rank if tp_size is None: diff --git a/python/sglang/srt/models/glm4v.py b/python/sglang/srt/models/glm4v.py index 714065edd4..b20af5168a 100644 --- a/python/sglang/srt/models/glm4v.py +++ b/python/sglang/srt/models/glm4v.py @@ -76,18 +76,16 @@ class Glm4vRMSNorm(RMSNorm): return x -# TODO: fold into parallel_group selection ("attn_tp" under DP attention). def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool: return is_dp_attention_enabled() and not use_data_parallel -def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]: +def _glm4v_vision_parallel_group(use_data_parallel: bool) -> str: if use_data_parallel: - return 1, 0 - parallel = get_parallel() + return "replicated" if glm4v_vision_reduces_over_attn_tp(use_data_parallel): - return parallel.attn_tp_size, parallel.attn_tp_rank - return parallel.tp_size, parallel.tp_rank + return "attn_tp" + return "tp" class Glm4vVisionMLP(nn.Module): @@ -101,15 +99,14 @@ class Glm4vVisionMLP(nn.Module): use_data_parallel: bool = False, ): super().__init__() - self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel) + parallel_group = _glm4v_vision_parallel_group(use_data_parallel) self.gate_up_proj = MergedColumnParallelLinear( input_size=in_features, output_sizes=[hidden_features] * 2, # [gate_proj, up_proj] bias=bias, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( hidden_features, @@ -117,8 +114,7 @@ class Glm4vVisionMLP(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("down_proj", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( use_data_parallel ), @@ -257,7 +253,7 @@ class Glm4vPatchMerger(nn.Module): ) -> None: super().__init__() self.hidden_size = d_model - tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel) + parallel_group = _glm4v_vision_parallel_group(use_data_parallel) self.proj = ReplicatedLinear( self.hidden_size, self.hidden_size, @@ -272,8 +268,7 @@ class Glm4vPatchMerger(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( context_dim, @@ -281,8 +276,7 @@ class Glm4vPatchMerger(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("down_proj", prefix), - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group=parallel_group, use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp( use_data_parallel ), diff --git a/python/sglang/srt/models/internvl.py b/python/sglang/srt/models/internvl.py index ad2d033425..6f57a8cbe8 100644 --- a/python/sglang/srt/models/internvl.py +++ b/python/sglang/srt/models/internvl.py @@ -47,7 +47,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import ( InternViTCudaGraphRunner, ) from sglang.srt.multimodal.mm_utils import run_dp_sharded_vision_model -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_mm from sglang.srt.utils import is_cuda from sglang.utils import logger @@ -200,8 +200,7 @@ class InternMLP(nn.Module): use_data_parallel: bool = False, ): super().__init__() - self.tp_size = 1 if use_data_parallel else get_parallel().tp_size - self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank + parallel_group = "replicated" if use_data_parallel else "tp" self.config = config self.act = get_act_fn(config.hidden_act) self.fc1 = ColumnParallelLinear( @@ -209,16 +208,14 @@ class InternMLP(nn.Module): config.intermediate_size, bias=True, quant_config=None, - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) self.fc2 = RowParallelLinear( config.intermediate_size, config.hidden_size, bias=True, quant_config=None, - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/models/kimi_vl_moonvit.py b/python/sglang/srt/models/kimi_vl_moonvit.py index b90a213322..1d1e26575a 100644 --- a/python/sglang/srt/models/kimi_vl_moonvit.py +++ b/python/sglang/srt/models/kimi_vl_moonvit.py @@ -341,7 +341,6 @@ class MLP2(nn.Module): self.quant_config = quant_config use_tensor_parallel = use_tensor_parallel and not use_data_parallel tp_size = get_parallel().attn_tp_size if use_tensor_parallel else 1 - tp_rank = get_parallel().attn_tp_rank if use_tensor_parallel else 0 if isinstance(self.quant_config, ModelSlimConfig): self.fc0 = ReplicatedLinear( dims[0], @@ -373,16 +372,14 @@ class MLP2(nn.Module): dims[1], bias=bias, prefix=add_prefix("fc0", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group="attn_tp", ) self.fc1 = RowParallelLinear( dims[1], dims[2], bias=bias, prefix=add_prefix("fc1", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group="attn_tp", ) else: self.fc0 = nn.Linear(dims[0], dims[1], bias=bias) diff --git a/python/sglang/srt/models/minimax_vl_common.py b/python/sglang/srt/models/minimax_vl_common.py index 853cf14519..f4c766fac6 100644 --- a/python/sglang/srt/models/minimax_vl_common.py +++ b/python/sglang/srt/models/minimax_vl_common.py @@ -85,7 +85,6 @@ class MiniMaxVLMultiModalProjector(nn.Module): ) tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size - tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank # TODO: this layer shards over attention TP but reduces over the full TP # group without attention DP; reduce over the attention-TP group so # attention CP narrower than TP can run it. @@ -103,8 +102,7 @@ class MiniMaxVLMultiModalProjector(nn.Module): bias=multimodal_projector_bias, quant_config=quant_config, prefix=f"{prefix}.linear_1", - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "attn_tp", ) assert projector_hidden_act == "gelu", ( f"Only gelu activation is supported, got {projector_hidden_act}" @@ -116,8 +114,7 @@ class MiniMaxVLMultiModalProjector(nn.Module): bias=multimodal_projector_bias, quant_config=quant_config, prefix=f"{prefix}.linear_2", - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "attn_tp", use_dp_attention_reduce=is_dp_attention_enabled(), ) @@ -150,7 +147,6 @@ class MiniMaxVLPatchMerger(nn.Module): ) tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size - tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank # TODO: this layer shards over attention TP but reduces over the full TP # group without attention DP; reduce over the attention-TP group so # attention CP narrower than TP can run it. @@ -168,8 +164,7 @@ class MiniMaxVLPatchMerger(nn.Module): bias=patch_merge_bias, quant_config=quant_config, prefix=f"{prefix}.linear_1", - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "attn_tp", ) assert projector_hidden_act == "gelu", ( f"Only gelu activation is supported, got {projector_hidden_act}" @@ -181,8 +176,7 @@ class MiniMaxVLPatchMerger(nn.Module): bias=patch_merge_bias, quant_config=quant_config, prefix=f"{prefix}.linear_2", - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "attn_tp", use_dp_attention_reduce=is_dp_attention_enabled(), ) @@ -282,7 +276,6 @@ class CLIPEncoderLayer(nn.Module): self.embed_dim = config.hidden_size self.use_data_parallel = use_data_parallel tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size - tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank # TODO: this layer shards over attention TP but reduces over the full TP # group without attention DP; reduce over the attention-TP group so # attention CP narrower than TP can run it. @@ -314,8 +307,7 @@ class CLIPEncoderLayer(nn.Module): config.intermediate_size, quant_config=quant_config, prefix=f"{prefix}.mlp.fc1", - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "attn_tp", ) hidden_act = getattr(config, "hidden_act", "gelu") assert hidden_act == "gelu", ( @@ -327,8 +319,7 @@ class CLIPEncoderLayer(nn.Module): config.hidden_size, quant_config=quant_config, prefix=f"{prefix}.mlp.fc2", - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "attn_tp", use_dp_attention_reduce=is_dp_attention_enabled(), ) self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) diff --git a/python/sglang/srt/models/qwen2_5_vl.py b/python/sglang/srt/models/qwen2_5_vl.py index b0575a7242..7b898d67de 100644 --- a/python/sglang/srt/models/qwen2_5_vl.py +++ b/python/sglang/srt/models/qwen2_5_vl.py @@ -50,9 +50,11 @@ from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType @@ -140,21 +142,16 @@ class Qwen2_5_VLMLP(nn.Module): prefix: str = "", use_data_parallel: bool = False, fuse_gate_up: bool = True, - tp_size: Optional[int] = None, - tp_rank: Optional[int] = None, + parallel_group: Optional[LinearParallelGroup] = None, ): super().__init__() + if use_data_parallel and parallel_group is not None: + raise ValueError("Explicit MLP TP cannot be combined with data parallel") if use_data_parallel: - if tp_size is not None or tp_rank is not None: - raise ValueError( - "Explicit MLP TP cannot be combined with data parallel" - ) - self.tp_size, self.tp_rank = 1, 0 - else: - if (tp_size is None) != (tp_rank is None): - raise ValueError("MLP tp_size and tp_rank must be set together") - self.tp_size = get_parallel().tp_size if tp_size is None else tp_size - self.tp_rank = get_parallel().tp_rank if tp_rank is None else tp_rank + parallel_group = "replicated" + elif parallel_group is None: + parallel_group = "tp" + self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group) self.fuse_gate_up = fuse_gate_up if fuse_gate_up: self.gate_up_proj = MergedColumnParallelLinear( @@ -163,8 +160,7 @@ class Qwen2_5_VLMLP(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) else: projection_kwargs = dict( @@ -172,8 +168,7 @@ class Qwen2_5_VLMLP(nn.Module): output_size=hidden_features, bias=bias, quant_config=quant_config, - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) self.gate_proj = ColumnParallelLinear( **projection_kwargs, @@ -198,8 +193,7 @@ class Qwen2_5_VLMLP(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("down_proj", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) self.hidden_act = hidden_act if self.fuse_gate_up and self.hidden_act == "silu": @@ -326,8 +320,6 @@ class Qwen2_5_VisionPatchMerger(nn.Module): cast_x_before_out_mul=cast_x_before_out_mul, force_native=force_native_norm, ) - tp_size = 1 if use_data_parallel else get_parallel().tp_size - tp_rank = 0 if use_data_parallel else get_parallel().tp_rank self.mlp = nn.ModuleList( [ ColumnParallelLinear( @@ -336,8 +328,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module): bias=True, quant_config=quant_config, prefix=add_prefix("mlp.0", prefix), - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "tp", ), nn.GELU(), RowParallelLinear( @@ -346,8 +337,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module): bias=True, quant_config=quant_config, prefix=add_prefix("mlp.2", prefix), - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "tp", ), ] ) diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 6c38dc9b9c..c1af8f286a 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -44,7 +44,12 @@ from sglang.srt.layers.dp_attention import ( reject_attn_tp_shard_with_tp_reduce, ) from sglang.srt.layers.layer_boundary.residual import batch as residual_batch -from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + LinearParallelGroup, + RowParallelLinear, + resolve_linear_parallel_group, +) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -111,23 +116,16 @@ _is_cpu = is_cpu() _VECTORIZED_VL_POS_EMBED_MIN_IMAGES = 6 -def _resolve_vision_tp( +def _resolve_vision_parallel_group( *, use_data_parallel: bool, - tp_size: Optional[int], - tp_rank: Optional[int], -) -> tuple[int, int]: + parallel_group: Optional[LinearParallelGroup], +) -> LinearParallelGroup: if use_data_parallel: - if tp_size is not None or tp_rank is not None: + if parallel_group is not None: raise ValueError("Explicit vision TP cannot be combined with data parallel") - return 1, 0 - if (tp_size is None) != (tp_rank is None): - raise ValueError("Vision tp_size and tp_rank must be set together") - if tp_size is None: - parallel = get_parallel() - return parallel.attn_tp_size, parallel.attn_tp_rank - assert tp_rank is not None - return tp_size, tp_rank + return "replicated" + return "attn_tp" if parallel_group is None else parallel_group class Qwen3_VisionMLP(nn.Module): @@ -140,15 +138,13 @@ class Qwen3_VisionMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", use_data_parallel: bool = False, - tp_size: Optional[int] = None, - tp_rank: Optional[int] = None, + parallel_group: Optional[LinearParallelGroup] = None, ): super().__init__() - self.tp_size, self.tp_rank = _resolve_vision_tp( - use_data_parallel=use_data_parallel, - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group = _resolve_vision_parallel_group( + use_data_parallel=use_data_parallel, parallel_group=parallel_group ) + self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group) # TODO: this layer shards over attention TP but reduces over the full TP # group without attention DP; reduce over the attention-TP group so # attention CP narrower than TP can run it. @@ -165,8 +161,7 @@ class Qwen3_VisionMLP(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("linear_fc1", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) self.linear_fc2 = RowParallelLinear( hidden_features, @@ -174,8 +169,7 @@ class Qwen3_VisionMLP(nn.Module): bias=bias, quant_config=quant_config, prefix=add_prefix("linear_fc2", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, use_dp_attention_reduce=is_dp_attention_enabled(), ) self.act = ACT2FN[hidden_act] @@ -306,8 +300,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", use_data_parallel: bool = False, - tp_size: Optional[int] = None, - tp_rank: Optional[int] = None, + parallel_group: Optional[LinearParallelGroup] = None, disable_merger_proj: bool = False, ) -> None: super().__init__() @@ -323,11 +316,10 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module): self.hidden_size if use_postshuffle_norm else context_dim ) if not disable_merger_proj: - self.tp_size, self.tp_rank = _resolve_vision_tp( - use_data_parallel=use_data_parallel, - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group = _resolve_vision_parallel_group( + use_data_parallel=use_data_parallel, parallel_group=parallel_group ) + self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group) # TODO: this layer shards over attention TP but reduces over the full TP # group without attention DP; reduce over the attention-TP group so # attention CP narrower than TP can run it. @@ -344,8 +336,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module): bias=True, quant_config=quant_config, prefix=add_prefix("linear_fc1", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, ) self.act_fn = nn.GELU() self.linear_fc2 = RowParallelLinear( @@ -354,8 +345,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module): bias=True, quant_config=quant_config, prefix=add_prefix("linear_fc2", prefix), - tp_size=self.tp_size, - tp_rank=self.tp_rank, + parallel_group=parallel_group, use_dp_attention_reduce=is_dp_attention_enabled(), ) diff --git a/python/sglang/srt/models/siglip.py b/python/sglang/srt/models/siglip.py index 52d072cee7..124febb29e 100644 --- a/python/sglang/srt/models/siglip.py +++ b/python/sglang/srt/models/siglip.py @@ -14,7 +14,6 @@ from sglang.srt.layers.conv import Conv2dLayer from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding -from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import add_prefix @@ -71,15 +70,12 @@ class SiglipMLP(nn.Module): use_data_parallel: bool = False, ): super().__init__() - tp_size = 1 if use_data_parallel else get_parallel().tp_size - tp_rank = 0 if use_data_parallel else get_parallel().tp_rank self.fc1 = ColumnParallelLinear( config.hidden_size, config.intermediate_size, quant_config=quant_config, prefix=add_prefix("fc1", prefix), - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "tp", ) self.act = act_layer() self.fc2 = RowParallelLinear( @@ -87,8 +83,7 @@ class SiglipMLP(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("fc2", prefix), - tp_size=tp_size, - tp_rank=tp_rank, + parallel_group="replicated" if use_data_parallel else "tp", ) def forward(self, x: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 88711ed00f..e36dcfc26e 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -524,15 +524,13 @@ class Step3VisionMLP(nn.Module): # Since this is a dense model, # the MLP component likewise adopts a DP-MLP approach modeled after DP Attention. # This choice may not represent the optimal solution and remains open to further deliberation. - attn_tp_rank = get_parallel().attn_tp_rank attn_tp_size = get_parallel().attn_tp_size self.fc1 = ColumnParallelLinear( dim, intermediate_size, bias=bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("gate_proj", prefix), ) self.act = ACT2FN[hidden_act] # quick_gelu @@ -550,8 +548,7 @@ class Step3VisionMLP(nn.Module): dim, bias=bias, quant_config=quant_config, - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, + parallel_group="attn_tp", prefix=add_prefix("down_proj", prefix), ) diff --git a/python/sglang/test/parallel_groups.py b/python/sglang/test/parallel_groups.py new file mode 100644 index 0000000000..579960a1ca --- /dev/null +++ b/python/sglang/test/parallel_groups.py @@ -0,0 +1,51 @@ +"""Group metadata fixtures for tests that do not initialize distributed groups.""" + +from contextlib import contextmanager +from types import SimpleNamespace + +from sglang.srt.runtime_context import SpawnRanks, get_parallel +from sglang.srt.runtime_context import publish as publish_context + + +def _group_metadata(values): + parallel = get_parallel() + for name in ("tp", "attn_tp"): + rank_key, size_key, group_key = ( + f"{name}_rank", + f"{name}_size", + f"{name}_group", + ) + rank = values.get(rank_key, getattr(parallel, rank_key)) + size = values.get(size_key, getattr(parallel, size_key)) + if group_key not in values and (rank_key in values or size_key in values): + values[group_key] = SimpleNamespace(rank_in_group=rank, world_size=size) + group = values.get(group_key) + if group is not None: + # Execution mocks also need placement metadata when used to build layers. + if not isinstance(getattr(group, "rank_in_group", None), int): + group.rank_in_group = rank + if not isinstance(getattr(group, "world_size", None), int): + group.world_size = size + return values + + +def publish(*args, **kwargs): + if kwargs.get("ranks") is None: + kwargs["ranks"] = SpawnRanks(world_rank=0) + context = publish_context(*args, **kwargs) + parallel = get_parallel() + groups = { + f"{name}_group": SimpleNamespace( + rank_in_group=getattr(parallel, f"{name}_rank"), + world_size=getattr(parallel, f"{name}_size"), + ) + for name in ("tp", "attn_tp") + } + parallel.override_permanently(**groups) + return context + + +@contextmanager +def parallel_scope(**values): + with get_parallel().override(**_group_metadata(values)) as parallel: + yield parallel diff --git a/test/registered/unit/layers/test_qwen_vision_wrapper_parallel_groups.py b/test/registered/unit/layers/test_qwen_vision_wrapper_parallel_groups.py new file mode 100644 index 0000000000..9c9683f20d --- /dev/null +++ b/test/registered/unit/layers/test_qwen_vision_wrapper_parallel_groups.py @@ -0,0 +1,294 @@ +"""Qwen vision wrappers and generation helpers retain their placement policies.""" + +import inspect +import unittest +from contextlib import nullcontext +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.linear import ( + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def group_kwargs(cls, group): + if group is None: + return {} + if "parallel_group" in inspect.signature(cls).parameters: + return dict(parallel_group=group) + p = get_parallel() + rank, size = ( + (0, 1) + if group == "replicated" + else (p.tp_rank, p.tp_size) + if group == "tp" + else (p.attn_tp_rank, p.attn_tp_size) + ) + return dict(tp_rank=rank, tp_size=size) + + +def build_qwen(model, group=None, *, replicated=False, width=32, quant_config=None): + options = dict(quant_config=quant_config, use_data_parallel=replicated) + if model.startswith("qwen25"): + from sglang.srt.models.qwen2_5_vl import Qwen2_5_VLMLP + + module = Qwen2_5_VLMLP( + width, + 2 * width, + bias=True, + fuse_gate_up=model == "qwen25_fused", + **options, + **group_kwargs(Qwen2_5_VLMLP, group), + ) + layers = ( + (module.gate_up_proj, module.down_proj) + if module.fuse_gate_up + else (module.gate_proj, module.up_proj, module.down_proj) + ) + elif model == "qwen3_mlp": + from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP + + module = Qwen3_VisionMLP( + width, + 2 * width, + bias=True, + **options, + **group_kwargs(Qwen3_VisionMLP, group), + ) + layers = (module.linear_fc1, module.linear_fc2) + elif model == "qwen3_merger": + from sglang.srt.models.qwen3_vl import Qwen3VLMoeVisionPatchMerger + + module = Qwen3VLMoeVisionPatchMerger( + width, + width, + 2 * width, + spatial_merge_size=1, + **options, + **group_kwargs(Qwen3VLMoeVisionPatchMerger, group), + ) + layers = (module.linear_fc1, module.linear_fc2) + else: + raise ValueError(model) + return module, layers + + +def load_projection(layer): + """Load known full weights under rank zero, returning the expected owned shard.""" + rank, size = getattr(layer, "tp_rank", 0), getattr(layer, "tp_size", 1) + row = isinstance(layer, RowParallelLinear) + dtype, device = layer.weight.dtype, layer.weight.device + + def values(rows, columns, offset): + return ( + ( + torch.arange(rows * columns, device=device).reshape(rows, columns) + + offset + ) + % 23 + / 128 + ).to(dtype) + + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + if isinstance(layer, QKVParallelLinear): + weights, biases = [], [] + for offset, shard_id in enumerate(("q", "k", "v")): + heads = ( + layer.total_num_heads + if shard_id == "q" + else layer.total_num_kv_heads + ) + weight = values(heads * layer.head_size, layer.input_size, offset) + partitions = size if shard_id == "q" else min(heads, size) + index = rank if shard_id == "q" else rank // max(size // heads, 1) + layer.weight.weight_loader(layer.weight, weight, shard_id) + weights.append(weight.chunk(partitions)[index]) + if layer.bias is not None: + bias = values(heads * layer.head_size, 1, offset).flatten() + layer.bias.weight_loader(layer.bias, bias, shard_id) + biases.append(bias.chunk(partitions)[index]) + shard = torch.cat(weights) + bias_shard = torch.cat(biases) if biases else None + else: + weight = values(layer.output_size, layer.input_size, 3) + layer.weight.weight_loader(layer.weight, weight) + shard = ( + weight.chunk(size, dim=1)[rank] + if row + else torch.cat( + [ + p.chunk(size)[rank] + for p in weight.split( + getattr(layer, "output_sizes", [layer.output_size]) + ) + ] + ) + ) + bias_shard = None + if layer.bias is not None: + bias = values(layer.output_size, 1, 4).flatten() + layer.bias.weight_loader(layer.bias, bias) + bias_shard = ( + bias + if row + else torch.cat( + [ + p.chunk(size)[rank] + for p in bias.split( + getattr(layer, "output_sizes", [layer.output_size]) + ) + ] + ) + ) + return shard, bias_shard + + +class TestQwenVisionWrapperGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_native_projection_math_loaders_and_flags(self): + for dp in (1, 2): + reset_context() + server = ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + attn_dp_size=dp, + mm_attention_backend="sdpa", + ) + publish(server, role="test", ranks=SpawnRanks(world_rank=3)) + initialize_dp_attention_flags(server) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 2 + attn.all_reduce.side_effect = lambda x: x * 3 + for model in ( + "qwen25_fused", + "qwen25_split", + "qwen3_mlp", + "qwen3_merger", + ): + for group, replicated in ( + (None, False), + (None, True), + ("tp", False), + ("attn_tp", False), + ("replicated", False), + ): + with self.subTest( + dp=dp, model=model, group=group, replicated=replicated + ): + module, layers = build_qwen(model, group, replicated=replicated) + selected = ( + "replicated" + if replicated + else group + or ("attn_tp" if model.startswith("qwen3") else "tp") + ) + expected_rank, expected_size = ( + (0, 1) + if selected == "replicated" + else (3, 4) + if selected == "tp" + else (3 % (4 // dp), 4 // dp) + ) + for index, layer in enumerate(layers): + weight, bias = load_projection(layer) + row = isinstance(layer, RowParallelLinear) + inputs = ( + torch.arange(2 * weight.shape[1]).reshape(2, -1).float() + / 64 + ) + expected = F.linear( + inputs, + weight, + bias if not row or expected_rank == 0 else None, + ) + dp_reduce = row and layer.use_dp_attention_reduce + if row and expected_size > 1: + expected *= 3 if dp_reduce else 2 + tp.all_reduce.reset_mock() + attn.all_reduce.reset_mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + actual = layer(inputs)[0] + torch.testing.assert_close(actual, expected) + if row: + allocator.assert_called_once_with( + attn + ) if dp_reduce else allocator.assert_called_once_with( + tp, disabled=True + ) + else: + allocator.assert_not_called() + self.assertEqual( + tp.all_reduce.call_count, + int(row and expected_size > 1 and not dp_reduce), + ) + self.assertEqual( + attn.all_reduce.call_count, + int(row and expected_size > 1 and dp_reduce), + ) + self.assertEqual( + ( + getattr(layer, "tp_rank", 0), + getattr(layer, "tp_size", 1), + ), + (expected_rank, expected_size), + ) + if model == "qwen25_split": + self.assertEqual( + isinstance(module.down_proj, ReplicatedLinear), + expected_size == 1, + ) + + def test_wrapper_conflict_guard_and_disabled_merger(self): + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_cp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + for model in ("qwen25_fused", "qwen3_mlp", "qwen3_merger"): + with self.assertRaisesRegex( + ValueError, "cannot be combined with data parallel" + ): + build_qwen(model, "replicated", replicated=True) + from sglang.srt.models.qwen3_vl import Qwen3VLMoeVisionPatchMerger + + reset_context() + merger = Qwen3VLMoeVisionPatchMerger( + 32, + 32, + 64, + disable_merger_proj=True, + **group_kwargs(Qwen3VLMoeVisionPatchMerger, "replicated"), + ) + self.assertFalse(hasattr(merger, "linear_fc1")) + self.assertFalse(hasattr(merger, "tp_size")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_vision_parallel_groups.py b/test/registered/unit/layers/test_vision_parallel_groups.py new file mode 100644 index 0000000000..43f647f637 --- /dev/null +++ b/test/registered/unit/layers/test_vision_parallel_groups.py @@ -0,0 +1,376 @@ +"""Vision projections retain their layout and execution policy across reload scopes.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear +from sglang.srt.runtime_context import ( + SpawnRanks, + get_disagg, + get_parallel, + publish, + reset_context, +) +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +VISION_MODELS = ( + "vision_packed", + "vision_column", + "vision_dummy", + "vision_gqa", + "vision_legacy", + "siglip", + "glm4v_mlp", + "glm4v_merger", + "internvl_mlp", + "qwen_merger", + "minimax_projector", + "minimax_merger", + "minimax_encoder", + "moonvit", + "step3", +) + + +def build_vision(model, use_data_parallel=False, width=32, quant_config=None): + config = SimpleNamespace( + hidden_size=width, + intermediate_size=2 * width, + num_attention_heads=8, + layer_norm_eps=1e-6, + hidden_act="gelu", + ) + options = dict(quant_config=quant_config, use_data_parallel=use_data_parallel) + if model.startswith("vision_"): + from sglang.srt.layers.attention.vision import VisionAttention + + module = VisionAttention( + width, + 8, + width, + use_qkv_parallel=model != "vision_column", + num_dummy_heads=4 if model == "vision_dummy" else 0, + num_kv_heads=4 if model == "vision_gqa" else 8, + use_dp_attention_reduce=model != "vision_legacy", + allow_tp_reduce_mismatch=model == "vision_legacy", + qkv_backend="sdpa", + **options, + ) + layers = (module.qkv_proj, module.proj) + elif model in ("glm4v_mlp", "glm4v_merger"): + from sglang.srt.models.glm4v import Glm4vPatchMerger, Glm4vVisionMLP + + constructor = Glm4vVisionMLP if model == "glm4v_mlp" else Glm4vPatchMerger + module = constructor(width, 2 * width, bias=True, **options) + layers = (module.gate_up_proj, module.down_proj) + elif model == "internvl_mlp": + from sglang.srt.models.internvl import InternMLP + + module = InternMLP(config, use_data_parallel=use_data_parallel) + layers = (module.fc1, module.fc2) + elif model == "siglip": + from sglang.srt.models.siglip import SiglipMLP + + module = SiglipMLP(config, **options) + layers = (module.fc1, module.fc2) + elif model == "qwen_merger": + from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionPatchMerger + + module = Qwen2_5_VisionPatchMerger( + width, + width, + 2 * width, + spatial_merge_size=1, + force_native_norm=True, + **options, + ) + layers = (module.mlp[0], module.mlp[2]) + elif model == "moonvit": + from sglang.srt.models.kimi_vl_moonvit import MLP2 + + module = MLP2( + [width, 2 * width, width], + torch.nn.GELU(), + use_tensor_parallel=True, + **options, + ) + layers = (module.fc0, module.fc1) + elif model == "step3": + from sglang.srt.models.step3_vl import Step3VisionMLP + + module = Step3VisionMLP(width, 2 * width, quant_config=quant_config) + layers = (module.fc1, module.fc2) + else: + from sglang.srt.models.minimax_vl_common import ( + CLIPEncoderLayer, + MiniMaxVLMultiModalProjector, + MiniMaxVLPatchMerger, + ) + + if model == "minimax_projector": + module = MiniMaxVLMultiModalProjector( + width, width, "gelu", True, projector_hidden_size=2 * width, **options + ) + layers = (module.linear_1, module.linear_2) + elif model == "minimax_merger": + module = MiniMaxVLPatchMerger( + 1, width, "gelu", True, projector_hidden_size=2 * width, **options + ) + layers = (module.linear_1, module.linear_2) + else: + module = CLIPEncoderLayer(config, **options) + layers = ( + module.fc1, + module.fc2, + module.self_attn.qkv_proj, + module.self_attn.proj, + ) + return module, layers + + +def load_projection(layer): + """Load known full weights under rank zero, returning the expected owned shard.""" + rank, size = layer.tp_rank, layer.tp_size + row = isinstance(layer, RowParallelLinear) + dtype, device = layer.weight.dtype, layer.weight.device + + def values(rows, columns, offset): + return ( + ( + torch.arange(rows * columns, device=device).reshape(rows, columns) + + offset + ) + % 23 + / 128 + ).to(dtype) + + with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0): + if isinstance(layer, QKVParallelLinear): + weights, biases = [], [] + for offset, shard_id in enumerate(("q", "k", "v")): + heads = ( + layer.total_num_heads + if shard_id == "q" + else layer.total_num_kv_heads + ) + weight = values(heads * layer.head_size, layer.input_size, offset) + partitions = size if shard_id == "q" else min(heads, size) + index = rank if shard_id == "q" else rank // max(size // heads, 1) + layer.weight.weight_loader(layer.weight, weight, shard_id) + weights.append(weight.chunk(partitions)[index]) + if layer.bias is not None: + bias = values(heads * layer.head_size, 1, offset).flatten() + layer.bias.weight_loader(layer.bias, bias, shard_id) + biases.append(bias.chunk(partitions)[index]) + shard = torch.cat(weights) + bias_shard = torch.cat(biases) if biases else None + else: + weight = values(layer.output_size, layer.input_size, 3) + layer.weight.weight_loader(layer.weight, weight) + shard = ( + weight.chunk(size, dim=1)[rank] + if row + else torch.cat( + [ + p.chunk(size)[rank] + for p in weight.split( + getattr(layer, "output_sizes", [layer.output_size]) + ) + ] + ) + ) + bias_shard = None + if layer.bias is not None: + bias = values(layer.output_size, 1, 4).flatten() + layer.bias.weight_loader(layer.bias, bias) + bias_shard = ( + bias + if row + else torch.cat( + [ + p.chunk(size)[rank] + for p in bias.split( + getattr(layer, "output_sizes", [layer.output_size]) + ) + ] + ) + ) + return shard, bias_shard + + +class TestVisionParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def test_projection_layout_reload_and_row_policy(self): + for dp_size in (1, 2): + publish( + ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + attn_dp_size=dp_size, + enable_dp_attention=dp_size > 1, + mm_attention_backend="sdpa", + ), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + initialize_dp_attention_flags( + ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp_size + ) + ) + tp, attn = Mock(), Mock() + tp.all_reduce.side_effect = lambda x: x * 4 + attn.all_reduce.side_effect = lambda x: x * (4 // dp_size) + for model in VISION_MODELS: + for replicated in ( + (False,) if model in ("moonvit", "step3") else (False, True) + ): + with self.subTest(dp=dp_size, model=model, replicated=replicated): + if dp_size == 2 and model in ("moonvit", "step3"): + with self.assertRaisesRegex( + ValueError, "shards over the attention TP group" + ): + build_vision(model, replicated) + continue + module, layers = build_vision(model, replicated) + group = ( + "tp" + if model + in ( + "siglip", + "qwen_merger", + "glm4v_mlp", + "glm4v_merger", + "internvl_mlp", + ) + else "attn_tp" + ) + rank, size = ( + (0, 1) + if replicated + else ( + (3, 4) + if group == "tp" + else (3 // dp_size, 4 // dp_size) + ) + ) + for layer in layers: + self.assertEqual( + (layer.tp_rank, layer.tp_size), (rank, size) + ) + shard, bias = load_projection(layer) + torch.testing.assert_close(layer.weight, shard) + if bias is not None: + torch.testing.assert_close(layer.bias, bias) + row = isinstance(layer, RowParallelLinear) + inputs = ( + torch.arange( + 2 * shard.shape[1], dtype=shard.dtype + ).reshape(2, -1) + / 128 + ) + expected = F.linear( + inputs, shard, bias if not row or rank == 0 else None + ) + if row and size > 1: + expected *= ( + (4 // dp_size) + if layer.use_dp_attention_reduce + else 4 + ) + tp.all_reduce.reset_mock() + attn.all_reduce.reset_mock() + with ( + get_parallel().override( + tp_group=tp, attn_tp_group=attn + ), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + torch.testing.assert_close(layer(inputs)[0], expected) + if row: + self.assertTrue(layer.reduce_results) + reduction = ( + attn if layer.use_dp_attention_reduce else tp + ) + if layer.use_dp_attention_reduce: + allocator.assert_called_once_with(reduction) + else: + allocator.assert_called_once_with( + reduction, disabled=True + ) + self.assertEqual( + reduction.all_reduce.call_count, int(size > 1) + ) + self.assertEqual( + ( + tp if reduction is attn else attn + ).all_reduce.call_count, + 0, + ) + else: + allocator.assert_not_called() + self.assertEqual( + (layer.tp_rank, layer.tp_size), (rank, size) + ) + if hasattr(module, "tp_rank"): + self.assertEqual( + (module.tp_rank, module.tp_size), (rank, size) + ) + reset_context() + + def test_guard_and_moonvit_dense_path(self): + from sglang.srt.layers.attention.vision import VisionAttention + from sglang.srt.models.kimi_vl_moonvit import MLP2 + + publish( + ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + attn_dp_size=2, + mm_attention_backend="sdpa", + ), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + with self.assertRaisesRegex(ValueError, "shards over the attention TP group"): + VisionAttention(32, 8, 32, True, qkv_backend="sdpa") + with get_disagg().override(language_model_only=True): + offloaded = VisionAttention(32, 8, 32, True, qkv_backend="sdpa") + self.assertEqual((offloaded.proj.tp_rank, offloaded.proj.tp_size), (1, 2)) + self.assertFalse(offloaded.proj.use_dp_attention_reduce) + for data_parallel, tensor_parallel in ((False, False), (True, True)): + module = MLP2( + [32, 64, 32], + torch.nn.GELU(), + use_data_parallel=data_parallel, + use_tensor_parallel=tensor_parallel, + ) + self.assertIsInstance(module.fc0, torch.nn.Linear) + self.assertIsInstance(module.fc1, torch.nn.Linear) + self.assertEqual(module(torch.zeros(2, 32)).shape, (2, 32)) + + +if __name__ == "__main__": + unittest.main()