[Refactor] Let vision encoders select their parallel group (#42586)

This commit is contained in:
Cheng Wan
2026-10-07 16:22:32 -07:00
committed by GitHub
parent b83827ea02
commit dec5d9d6fc
18 changed files with 942 additions and 154 deletions
@@ -16,7 +16,6 @@ from transformers.utils import TransformersKwargs, is_torchdynamo_compiling
from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig
from sglang.multimodal_gen.runtime.distributed import (
get_tp_rank,
get_tp_world_size,
model_parallel_is_initialized,
)
@@ -139,12 +138,6 @@ def _tp_world_size() -> int:
return get_tp_world_size()
def _tp_rank() -> int:
if not model_parallel_is_initialized():
return 0
return get_tp_rank()
def _linear_output(linear: nn.Module, x: torch.Tensor) -> torch.Tensor:
output = linear(x)
return output[0] if isinstance(output, tuple) else output
@@ -163,8 +156,7 @@ def _make_column_linear(
out_features,
bias=bias,
gather_output=False,
tp_size=_tp_world_size(),
tp_rank=_tp_rank(),
parallel_group="tp" if model_parallel_is_initialized() else "replicated",
)
return ReplicatedLinear(in_features, out_features, bias=bias)
@@ -181,8 +173,7 @@ def _make_row_linear(
in_features,
out_features,
bias=bias,
tp_size=_tp_world_size(),
tp_rank=_tp_rank(),
parallel_group="tp" if model_parallel_is_initialized() else "replicated",
)
return ReplicatedLinear(in_features, out_features, bias=bias)
@@ -360,8 +351,7 @@ class Qwen2_5_VLDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
prefix=f"model.language_model.layers.{layer_idx}.mlp",
fuse_gate_up=False,
tp_size=mlp_tp_size,
tp_rank=_tp_rank() if mlp_tp_size > 1 else 0,
parallel_group="tp" if mlp_tp_size > 1 else "replicated",
)
norm_kwargs = dict(
eps=config.rms_norm_eps,
@@ -19,7 +19,6 @@ from sglang.srt.models.qwen3_vl import (
Qwen3VLMoeVisionPatchMerger,
Qwen3VLVisionPatchEmbed,
)
from sglang.srt.runtime_context import get_parallel
@dataclass(frozen=True)
@@ -70,7 +69,6 @@ class Qwen3VLVisionBlock(nn.Module):
prefix: str = "visual",
) -> None:
super().__init__()
parallel = get_parallel()
self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)
self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)
self.attn = QwenVLVisionAttention(
@@ -86,8 +84,7 @@ class Qwen3VLVisionBlock(nn.Module):
hidden_act=config.hidden_act,
prefix=f"{prefix}.blocks.{layer_idx}.mlp",
quant_config=quant_config,
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
def forward(
@@ -210,7 +207,6 @@ class Qwen3VLVisionTransformer(nn.Module):
prefix: str = "visual",
) -> None:
super().__init__()
parallel = get_parallel()
self.config = config
self.spatial_merge_size = config.spatial_merge_size
self.spatial_merge_unit = config.spatial_merge_size**2
@@ -233,8 +229,7 @@ class Qwen3VLVisionTransformer(nn.Module):
spatial_merge_size=config.spatial_merge_size,
use_postshuffle_norm=False,
prefix="visual.merger",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
self.deepstack_visual_indexes = tuple(config.deepstack_visual_indexes)
self.deepstack_merger_list = nn.ModuleList(
@@ -245,8 +240,7 @@ class Qwen3VLVisionTransformer(nn.Module):
spatial_merge_size=config.spatial_merge_size,
use_postshuffle_norm=True,
prefix=f"visual.deepstack_merger_list.{merger_idx}",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
for merger_idx, _ in enumerate(self.deepstack_visual_indexes)
)
@@ -79,8 +79,7 @@ class QwenVLVisionAttention(nn.Module):
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
self.proj = RowParallelLinear(
input_size=config.hidden_size,
@@ -88,8 +87,7 @@ class QwenVLVisionAttention(nn.Module):
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.proj",
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
parallel_group="tp",
)
backend = get_attn_backend(self.head_dim, torch.get_default_dtype())
@@ -75,7 +75,6 @@ def test_text_mlp_uses_single_rank_when_intermediate_size_is_not_tp_divisible(
):
monkeypatch.setattr(qwen2_5vl, "Qwen2_5_VLAttention", lambda *_args: nn.Identity())
monkeypatch.setattr(qwen2_5vl, "_tp_world_size", lambda: 3)
monkeypatch.setattr(qwen2_5vl, "_tp_rank", lambda: 2)
config = SimpleNamespace(
hidden_size=16,
intermediate_size=25,
@@ -0,0 +1,131 @@
"""Qwen generation projections tested in the diffusion dependency environment."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
import torch.nn.functional as F
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
from sglang.srt.layers.linear import QKVParallelLinear, ReplicatedLinear
from sglang.srt.runtime_context import SpawnRanks, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.parallel_groups import parallel_scope, publish
class TestQwenGenerationParallelGroups(unittest.TestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def test_generation_helpers_match_initialized_scope(self):
import sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl as generation
publish(
ServerArgs(model_path="dummy", device="cpu", tp_size=4),
role="test",
ranks=SpawnRanks(world_rank=3),
)
for initialized in (False, True):
with patch.object(
generation,
"model_parallel_is_initialized",
return_value=initialized,
):
for tensor_parallel in (False, True):
for maker in (
generation._make_column_linear,
generation._make_row_linear,
):
layer = maker(
8, 8, bias=True, use_tensor_parallel=tensor_parallel
)
expected = (3, 4) if initialized and tensor_parallel else (0, 1)
actual = (
(0, 1)
if isinstance(layer, ReplicatedLinear)
else (layer.tp_rank, layer.tp_size)
)
self.assertEqual(actual, expected)
self.assertEqual(
isinstance(layer, ReplicatedLinear), not tensor_parallel
)
def test_native_vision_projection_math_and_reload(self):
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend_context_manager,
)
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_vision import (
QwenVLVisionAttention,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
for dp in (1, 2):
with self.subTest(dp=dp):
reset_context()
server = ServerArgs(
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp
)
publish(server, role="test", ranks=SpawnRanks(world_rank=3))
initialize_dp_attention_flags(server)
with global_force_attn_backend_context_manager(
AttentionBackendEnum.TORCH_SDPA
):
module = QwenVLVisionAttention(
SimpleNamespace(hidden_size=32, num_heads=8),
prefix="visual",
model_name="Qwen",
)
tp = Mock(all_reduce=Mock(side_effect=lambda x: x * 2))
attn = Mock(all_reduce=Mock(side_effect=AssertionError("wrong group")))
for layer in (module.qkv_proj, module.proj):
rank, size = (
getattr(layer, "tp_rank", 0),
getattr(layer, "tp_size", 1),
)
self.assertEqual((rank, size), (3, 4))
full = (
torch.arange(32 * 32, dtype=layer.weight.dtype).reshape(32, 32)
/ 1024
)
bias = torch.arange(32, dtype=layer.weight.dtype) / 64
qkv = isinstance(layer, QKVParallelLinear)
with parallel_scope(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0):
for shard in ("q", "k", "v") if qkv else (None,):
args = (shard,) if qkv else ()
layer.weight.weight_loader(layer.weight, full, *args)
layer.bias.weight_loader(layer.bias, bias, *args)
weight = (
torch.cat([full.chunk(size)[rank]] * 3)
if qkv
else full.chunk(size, dim=1)[rank]
)
expected_bias = (
torch.cat([bias.chunk(size)[rank]] * 3) if qkv else None
)
inputs = (
torch.arange(2 * weight.shape[1], dtype=weight.dtype).reshape(
2, -1
)
/ 64
)
expected = F.linear(inputs, weight, expected_bias) * (
1 if qkv else 2
)
with (
parallel_scope(tp_group=tp, attn_tp_group=attn),
patch(
"sglang.srt.layers.linear.is_allocation_symmetric",
return_value=False,
),
patch(
"sglang.srt.layers.linear.use_symmetric_memory",
return_value=nullcontext(),
),
):
torch.testing.assert_close(layer(inputs)[0], expected)
torch.testing.assert_close(layer.weight, weight, rtol=0, atol=0)
attn.all_reduce.assert_not_called()
tp.all_reduce.assert_called_once()
+3 -6
View File
@@ -1196,8 +1196,7 @@ class VisionAttention(nn.Module):
total_num_kv_heads=num_dummy_heads + num_kv_heads,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="replicated" if use_data_parallel else "attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
else:
@@ -1206,8 +1205,7 @@ class VisionAttention(nn.Module):
output_size=3 * self.dummy_dim,
bias=qkv_bias,
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="replicated" if use_data_parallel else "attn_tp",
prefix=add_prefix("qkv_proj", prefix),
)
# TODO: this layer shards over attention TP but reduces over the full TP
@@ -1226,8 +1224,7 @@ class VisionAttention(nn.Module):
output_size=embed_dim,
bias=proj_bias,
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
parallel_group="replicated" if use_data_parallel else "attn_tp",
prefix=add_prefix("proj", prefix),
use_dp_attention_reduce=use_dp_attention_reduce,
)
+15 -8
View File
@@ -58,6 +58,20 @@ logger = logging.getLogger(__name__)
LinearParallelGroup = Literal["tp", "attn_tp", "replicated"]
def resolve_linear_parallel_group(
parallel_group: LinearParallelGroup,
) -> Tuple[int, int]:
"""Freeze a group's weight partition in the current construction scope."""
if parallel_group == "replicated":
return 0, 1
if parallel_group not in ("tp", "attn_tp"):
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
parallel = get_parallel()
if parallel_group == "attn_tp":
return parallel.attn_tp_rank, parallel.attn_tp_size
return parallel.tp_rank, parallel.tp_size
def _resolve_linear_partition(
parallel_group: Optional[LinearParallelGroup],
tp_rank: Optional[int],
@@ -74,14 +88,7 @@ def _resolve_linear_partition(
raise ValueError(
"parallel_group cannot be combined with tp_rank or tp_size"
)
if parallel_group == "replicated":
return 0, 1
if parallel_group not in ("tp", "attn_tp"):
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
parallel = get_parallel()
if parallel_group == "attn_tp":
return parallel.attn_tp_rank, parallel.attn_tp_size
return parallel.tp_rank, parallel.tp_size
return resolve_linear_parallel_group(parallel_group)
if tp_rank is None:
tp_rank = get_parallel().tp_rank
if tp_size is None:
+10 -16
View File
@@ -76,18 +76,16 @@ class Glm4vRMSNorm(RMSNorm):
return x
# TODO: fold into parallel_group selection ("attn_tp" under DP attention).
def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool:
return is_dp_attention_enabled() and not use_data_parallel
def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]:
def _glm4v_vision_parallel_group(use_data_parallel: bool) -> str:
if use_data_parallel:
return 1, 0
parallel = get_parallel()
return "replicated"
if glm4v_vision_reduces_over_attn_tp(use_data_parallel):
return parallel.attn_tp_size, parallel.attn_tp_rank
return parallel.tp_size, parallel.tp_rank
return "attn_tp"
return "tp"
class Glm4vVisionMLP(nn.Module):
@@ -101,15 +99,14 @@ class Glm4vVisionMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel)
parallel_group = _glm4v_vision_parallel_group(use_data_parallel)
self.gate_up_proj = MergedColumnParallelLinear(
input_size=in_features,
output_sizes=[hidden_features] * 2, # [gate_proj, up_proj]
bias=bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
hidden_features,
@@ -117,8 +114,7 @@ class Glm4vVisionMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
@@ -257,7 +253,7 @@ class Glm4vPatchMerger(nn.Module):
) -> None:
super().__init__()
self.hidden_size = d_model
tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel)
parallel_group = _glm4v_vision_parallel_group(use_data_parallel)
self.proj = ReplicatedLinear(
self.hidden_size,
self.hidden_size,
@@ -272,8 +268,7 @@ class Glm4vPatchMerger(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
context_dim,
@@ -281,8 +276,7 @@ class Glm4vPatchMerger(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
use_data_parallel
),
+4 -7
View File
@@ -47,7 +47,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import (
InternViTCudaGraphRunner,
)
from sglang.srt.multimodal.mm_utils import run_dp_sharded_vision_model
from sglang.srt.runtime_context import get_mm, get_parallel
from sglang.srt.runtime_context import get_mm
from sglang.srt.utils import is_cuda
from sglang.utils import logger
@@ -200,8 +200,7 @@ class InternMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
self.tp_size = 1 if use_data_parallel else get_parallel().tp_size
self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
parallel_group = "replicated" if use_data_parallel else "tp"
self.config = config
self.act = get_act_fn(config.hidden_act)
self.fc1 = ColumnParallelLinear(
@@ -209,16 +208,14 @@ class InternMLP(nn.Module):
config.intermediate_size,
bias=True,
quant_config=None,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.fc2 = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
bias=True,
quant_config=None,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -5
View File
@@ -341,7 +341,6 @@ class MLP2(nn.Module):
self.quant_config = quant_config
use_tensor_parallel = use_tensor_parallel and not use_data_parallel
tp_size = get_parallel().attn_tp_size if use_tensor_parallel else 1
tp_rank = get_parallel().attn_tp_rank if use_tensor_parallel else 0
if isinstance(self.quant_config, ModelSlimConfig):
self.fc0 = ReplicatedLinear(
dims[0],
@@ -373,16 +372,14 @@ class MLP2(nn.Module):
dims[1],
bias=bias,
prefix=add_prefix("fc0", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group="attn_tp",
)
self.fc1 = RowParallelLinear(
dims[1],
dims[2],
bias=bias,
prefix=add_prefix("fc1", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group="attn_tp",
)
else:
self.fc0 = nn.Linear(dims[0], dims[1], bias=bias)
+6 -15
View File
@@ -85,7 +85,6 @@ class MiniMaxVLMultiModalProjector(nn.Module):
)
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -103,8 +102,7 @@ class MiniMaxVLMultiModalProjector(nn.Module):
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
)
assert projector_hidden_act == "gelu", (
f"Only gelu activation is supported, got {projector_hidden_act}"
@@ -116,8 +114,7 @@ class MiniMaxVLMultiModalProjector(nn.Module):
bias=multimodal_projector_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
use_dp_attention_reduce=is_dp_attention_enabled(),
)
@@ -150,7 +147,6 @@ class MiniMaxVLPatchMerger(nn.Module):
)
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -168,8 +164,7 @@ class MiniMaxVLPatchMerger(nn.Module):
bias=patch_merge_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
)
assert projector_hidden_act == "gelu", (
f"Only gelu activation is supported, got {projector_hidden_act}"
@@ -181,8 +176,7 @@ class MiniMaxVLPatchMerger(nn.Module):
bias=patch_merge_bias,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
use_dp_attention_reduce=is_dp_attention_enabled(),
)
@@ -282,7 +276,6 @@ class CLIPEncoderLayer(nn.Module):
self.embed_dim = config.hidden_size
self.use_data_parallel = use_data_parallel
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -314,8 +307,7 @@ class CLIPEncoderLayer(nn.Module):
config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.mlp.fc1",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
)
hidden_act = getattr(config, "hidden_act", "gelu")
assert hidden_act == "gelu", (
@@ -327,8 +319,7 @@ class CLIPEncoderLayer(nn.Module):
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.mlp.fc2",
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "attn_tp",
use_dp_attention_reduce=is_dp_attention_enabled(),
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+14 -24
View File
@@ -50,9 +50,11 @@ from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -140,21 +142,16 @@ class Qwen2_5_VLMLP(nn.Module):
prefix: str = "",
use_data_parallel: bool = False,
fuse_gate_up: bool = True,
tp_size: Optional[int] = None,
tp_rank: Optional[int] = None,
parallel_group: Optional[LinearParallelGroup] = None,
):
super().__init__()
if use_data_parallel and parallel_group is not None:
raise ValueError("Explicit MLP TP cannot be combined with data parallel")
if use_data_parallel:
if tp_size is not None or tp_rank is not None:
raise ValueError(
"Explicit MLP TP cannot be combined with data parallel"
)
self.tp_size, self.tp_rank = 1, 0
else:
if (tp_size is None) != (tp_rank is None):
raise ValueError("MLP tp_size and tp_rank must be set together")
self.tp_size = get_parallel().tp_size if tp_size is None else tp_size
self.tp_rank = get_parallel().tp_rank if tp_rank is None else tp_rank
parallel_group = "replicated"
elif parallel_group is None:
parallel_group = "tp"
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
self.fuse_gate_up = fuse_gate_up
if fuse_gate_up:
self.gate_up_proj = MergedColumnParallelLinear(
@@ -163,8 +160,7 @@ class Qwen2_5_VLMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
else:
projection_kwargs = dict(
@@ -172,8 +168,7 @@ class Qwen2_5_VLMLP(nn.Module):
output_size=hidden_features,
bias=bias,
quant_config=quant_config,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.gate_proj = ColumnParallelLinear(
**projection_kwargs,
@@ -198,8 +193,7 @@ class Qwen2_5_VLMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.hidden_act = hidden_act
if self.fuse_gate_up and self.hidden_act == "silu":
@@ -326,8 +320,6 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
cast_x_before_out_mul=cast_x_before_out_mul,
force_native=force_native_norm,
)
tp_size = 1 if use_data_parallel else get_parallel().tp_size
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
self.mlp = nn.ModuleList(
[
ColumnParallelLinear(
@@ -336,8 +328,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("mlp.0", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
),
nn.GELU(),
RowParallelLinear(
@@ -346,8 +337,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("mlp.2", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
),
]
)
+24 -34
View File
@@ -44,7 +44,12 @@ from sglang.srt.layers.dp_attention import (
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -111,23 +116,16 @@ _is_cpu = is_cpu()
_VECTORIZED_VL_POS_EMBED_MIN_IMAGES = 6
def _resolve_vision_tp(
def _resolve_vision_parallel_group(
*,
use_data_parallel: bool,
tp_size: Optional[int],
tp_rank: Optional[int],
) -> tuple[int, int]:
parallel_group: Optional[LinearParallelGroup],
) -> LinearParallelGroup:
if use_data_parallel:
if tp_size is not None or tp_rank is not None:
if parallel_group is not None:
raise ValueError("Explicit vision TP cannot be combined with data parallel")
return 1, 0
if (tp_size is None) != (tp_rank is None):
raise ValueError("Vision tp_size and tp_rank must be set together")
if tp_size is None:
parallel = get_parallel()
return parallel.attn_tp_size, parallel.attn_tp_rank
assert tp_rank is not None
return tp_size, tp_rank
return "replicated"
return "attn_tp" if parallel_group is None else parallel_group
class Qwen3_VisionMLP(nn.Module):
@@ -140,15 +138,13 @@ class Qwen3_VisionMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
use_data_parallel: bool = False,
tp_size: Optional[int] = None,
tp_rank: Optional[int] = None,
parallel_group: Optional[LinearParallelGroup] = None,
):
super().__init__()
self.tp_size, self.tp_rank = _resolve_vision_tp(
use_data_parallel=use_data_parallel,
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group = _resolve_vision_parallel_group(
use_data_parallel=use_data_parallel, parallel_group=parallel_group
)
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -165,8 +161,7 @@ class Qwen3_VisionMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("linear_fc1", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.linear_fc2 = RowParallelLinear(
hidden_features,
@@ -174,8 +169,7 @@ class Qwen3_VisionMLP(nn.Module):
bias=bias,
quant_config=quant_config,
prefix=add_prefix("linear_fc2", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=is_dp_attention_enabled(),
)
self.act = ACT2FN[hidden_act]
@@ -306,8 +300,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
use_data_parallel: bool = False,
tp_size: Optional[int] = None,
tp_rank: Optional[int] = None,
parallel_group: Optional[LinearParallelGroup] = None,
disable_merger_proj: bool = False,
) -> None:
super().__init__()
@@ -323,11 +316,10 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
self.hidden_size if use_postshuffle_norm else context_dim
)
if not disable_merger_proj:
self.tp_size, self.tp_rank = _resolve_vision_tp(
use_data_parallel=use_data_parallel,
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group = _resolve_vision_parallel_group(
use_data_parallel=use_data_parallel, parallel_group=parallel_group
)
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
@@ -344,8 +336,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("linear_fc1", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
)
self.act_fn = nn.GELU()
self.linear_fc2 = RowParallelLinear(
@@ -354,8 +345,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
bias=True,
quant_config=quant_config,
prefix=add_prefix("linear_fc2", prefix),
tp_size=self.tp_size,
tp_rank=self.tp_rank,
parallel_group=parallel_group,
use_dp_attention_reduce=is_dp_attention_enabled(),
)
+2 -7
View File
@@ -14,7 +14,6 @@ from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix
@@ -71,15 +70,12 @@ class SiglipMLP(nn.Module):
use_data_parallel: bool = False,
):
super().__init__()
tp_size = 1 if use_data_parallel else get_parallel().tp_size
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
self.fc1 = ColumnParallelLinear(
config.hidden_size,
config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("fc1", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
)
self.act = act_layer()
self.fc2 = RowParallelLinear(
@@ -87,8 +83,7 @@ class SiglipMLP(nn.Module):
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("fc2", prefix),
tp_size=tp_size,
tp_rank=tp_rank,
parallel_group="replicated" if use_data_parallel else "tp",
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
+2 -5
View File
@@ -524,15 +524,13 @@ class Step3VisionMLP(nn.Module):
# Since this is a dense model,
# the MLP component likewise adopts a DP-MLP approach modeled after DP Attention.
# This choice may not represent the optimal solution and remains open to further deliberation.
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
self.fc1 = ColumnParallelLinear(
dim,
intermediate_size,
bias=bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("gate_proj", prefix),
)
self.act = ACT2FN[hidden_act] # quick_gelu
@@ -550,8 +548,7 @@ class Step3VisionMLP(nn.Module):
dim,
bias=bias,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
parallel_group="attn_tp",
prefix=add_prefix("down_proj", prefix),
)
+51
View File
@@ -0,0 +1,51 @@
"""Group metadata fixtures for tests that do not initialize distributed groups."""
from contextlib import contextmanager
from types import SimpleNamespace
from sglang.srt.runtime_context import SpawnRanks, get_parallel
from sglang.srt.runtime_context import publish as publish_context
def _group_metadata(values):
parallel = get_parallel()
for name in ("tp", "attn_tp"):
rank_key, size_key, group_key = (
f"{name}_rank",
f"{name}_size",
f"{name}_group",
)
rank = values.get(rank_key, getattr(parallel, rank_key))
size = values.get(size_key, getattr(parallel, size_key))
if group_key not in values and (rank_key in values or size_key in values):
values[group_key] = SimpleNamespace(rank_in_group=rank, world_size=size)
group = values.get(group_key)
if group is not None:
# Execution mocks also need placement metadata when used to build layers.
if not isinstance(getattr(group, "rank_in_group", None), int):
group.rank_in_group = rank
if not isinstance(getattr(group, "world_size", None), int):
group.world_size = size
return values
def publish(*args, **kwargs):
if kwargs.get("ranks") is None:
kwargs["ranks"] = SpawnRanks(world_rank=0)
context = publish_context(*args, **kwargs)
parallel = get_parallel()
groups = {
f"{name}_group": SimpleNamespace(
rank_in_group=getattr(parallel, f"{name}_rank"),
world_size=getattr(parallel, f"{name}_size"),
)
for name in ("tp", "attn_tp")
}
parallel.override_permanently(**groups)
return context
@contextmanager
def parallel_scope(**values):
with get_parallel().override(**_group_metadata(values)) as parallel:
yield parallel