[Refactor] Let vision encoders select their parallel group (#42586)
This commit is contained in:
@@ -16,7 +16,6 @@ from transformers.utils import TransformersKwargs, is_torchdynamo_compiling
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_tp_rank,
|
||||
get_tp_world_size,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
@@ -139,12 +138,6 @@ def _tp_world_size() -> int:
|
||||
return get_tp_world_size()
|
||||
|
||||
|
||||
def _tp_rank() -> int:
|
||||
if not model_parallel_is_initialized():
|
||||
return 0
|
||||
return get_tp_rank()
|
||||
|
||||
|
||||
def _linear_output(linear: nn.Module, x: torch.Tensor) -> torch.Tensor:
|
||||
output = linear(x)
|
||||
return output[0] if isinstance(output, tuple) else output
|
||||
@@ -163,8 +156,7 @@ def _make_column_linear(
|
||||
out_features,
|
||||
bias=bias,
|
||||
gather_output=False,
|
||||
tp_size=_tp_world_size(),
|
||||
tp_rank=_tp_rank(),
|
||||
parallel_group="tp" if model_parallel_is_initialized() else "replicated",
|
||||
)
|
||||
return ReplicatedLinear(in_features, out_features, bias=bias)
|
||||
|
||||
@@ -181,8 +173,7 @@ def _make_row_linear(
|
||||
in_features,
|
||||
out_features,
|
||||
bias=bias,
|
||||
tp_size=_tp_world_size(),
|
||||
tp_rank=_tp_rank(),
|
||||
parallel_group="tp" if model_parallel_is_initialized() else "replicated",
|
||||
)
|
||||
return ReplicatedLinear(in_features, out_features, bias=bias)
|
||||
|
||||
@@ -360,8 +351,7 @@ class Qwen2_5_VLDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
prefix=f"model.language_model.layers.{layer_idx}.mlp",
|
||||
fuse_gate_up=False,
|
||||
tp_size=mlp_tp_size,
|
||||
tp_rank=_tp_rank() if mlp_tp_size > 1 else 0,
|
||||
parallel_group="tp" if mlp_tp_size > 1 else "replicated",
|
||||
)
|
||||
norm_kwargs = dict(
|
||||
eps=config.rms_norm_eps,
|
||||
|
||||
@@ -19,7 +19,6 @@ from sglang.srt.models.qwen3_vl import (
|
||||
Qwen3VLMoeVisionPatchMerger,
|
||||
Qwen3VLVisionPatchEmbed,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -70,7 +69,6 @@ class Qwen3VLVisionBlock(nn.Module):
|
||||
prefix: str = "visual",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
parallel = get_parallel()
|
||||
self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)
|
||||
self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)
|
||||
self.attn = QwenVLVisionAttention(
|
||||
@@ -86,8 +84,7 @@ class Qwen3VLVisionBlock(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
prefix=f"{prefix}.blocks.{layer_idx}.mlp",
|
||||
quant_config=quant_config,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -210,7 +207,6 @@ class Qwen3VLVisionTransformer(nn.Module):
|
||||
prefix: str = "visual",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
parallel = get_parallel()
|
||||
self.config = config
|
||||
self.spatial_merge_size = config.spatial_merge_size
|
||||
self.spatial_merge_unit = config.spatial_merge_size**2
|
||||
@@ -233,8 +229,7 @@ class Qwen3VLVisionTransformer(nn.Module):
|
||||
spatial_merge_size=config.spatial_merge_size,
|
||||
use_postshuffle_norm=False,
|
||||
prefix="visual.merger",
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
)
|
||||
self.deepstack_visual_indexes = tuple(config.deepstack_visual_indexes)
|
||||
self.deepstack_merger_list = nn.ModuleList(
|
||||
@@ -245,8 +240,7 @@ class Qwen3VLVisionTransformer(nn.Module):
|
||||
spatial_merge_size=config.spatial_merge_size,
|
||||
use_postshuffle_norm=True,
|
||||
prefix=f"visual.deepstack_merger_list.{merger_idx}",
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
)
|
||||
for merger_idx, _ in enumerate(self.deepstack_visual_indexes)
|
||||
)
|
||||
|
||||
@@ -79,8 +79,7 @@ class QwenVLVisionAttention(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
)
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=config.hidden_size,
|
||||
@@ -88,8 +87,7 @@ class QwenVLVisionAttention(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.proj",
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
)
|
||||
|
||||
backend = get_attn_backend(self.head_dim, torch.get_default_dtype())
|
||||
|
||||
@@ -75,7 +75,6 @@ def test_text_mlp_uses_single_rank_when_intermediate_size_is_not_tp_divisible(
|
||||
):
|
||||
monkeypatch.setattr(qwen2_5vl, "Qwen2_5_VLAttention", lambda *_args: nn.Identity())
|
||||
monkeypatch.setattr(qwen2_5vl, "_tp_world_size", lambda: 3)
|
||||
monkeypatch.setattr(qwen2_5vl, "_tp_rank", lambda: 2)
|
||||
config = SimpleNamespace(
|
||||
hidden_size=16,
|
||||
intermediate_size=25,
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Qwen generation projections tested in the diffusion dependency environment."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, ReplicatedLinear
|
||||
from sglang.srt.runtime_context import SpawnRanks, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.parallel_groups import parallel_scope, publish
|
||||
|
||||
|
||||
class TestQwenGenerationParallelGroups(unittest.TestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def test_generation_helpers_match_initialized_scope(self):
|
||||
import sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl as generation
|
||||
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", device="cpu", tp_size=4),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=3),
|
||||
)
|
||||
for initialized in (False, True):
|
||||
with patch.object(
|
||||
generation,
|
||||
"model_parallel_is_initialized",
|
||||
return_value=initialized,
|
||||
):
|
||||
for tensor_parallel in (False, True):
|
||||
for maker in (
|
||||
generation._make_column_linear,
|
||||
generation._make_row_linear,
|
||||
):
|
||||
layer = maker(
|
||||
8, 8, bias=True, use_tensor_parallel=tensor_parallel
|
||||
)
|
||||
expected = (3, 4) if initialized and tensor_parallel else (0, 1)
|
||||
actual = (
|
||||
(0, 1)
|
||||
if isinstance(layer, ReplicatedLinear)
|
||||
else (layer.tp_rank, layer.tp_size)
|
||||
)
|
||||
self.assertEqual(actual, expected)
|
||||
self.assertEqual(
|
||||
isinstance(layer, ReplicatedLinear), not tensor_parallel
|
||||
)
|
||||
|
||||
def test_native_vision_projection_math_and_reload(self):
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend_context_manager,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_vision import (
|
||||
QwenVLVisionAttention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
for dp in (1, 2):
|
||||
with self.subTest(dp=dp):
|
||||
reset_context()
|
||||
server = ServerArgs(
|
||||
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp
|
||||
)
|
||||
publish(server, role="test", ranks=SpawnRanks(world_rank=3))
|
||||
initialize_dp_attention_flags(server)
|
||||
with global_force_attn_backend_context_manager(
|
||||
AttentionBackendEnum.TORCH_SDPA
|
||||
):
|
||||
module = QwenVLVisionAttention(
|
||||
SimpleNamespace(hidden_size=32, num_heads=8),
|
||||
prefix="visual",
|
||||
model_name="Qwen",
|
||||
)
|
||||
tp = Mock(all_reduce=Mock(side_effect=lambda x: x * 2))
|
||||
attn = Mock(all_reduce=Mock(side_effect=AssertionError("wrong group")))
|
||||
for layer in (module.qkv_proj, module.proj):
|
||||
rank, size = (
|
||||
getattr(layer, "tp_rank", 0),
|
||||
getattr(layer, "tp_size", 1),
|
||||
)
|
||||
self.assertEqual((rank, size), (3, 4))
|
||||
full = (
|
||||
torch.arange(32 * 32, dtype=layer.weight.dtype).reshape(32, 32)
|
||||
/ 1024
|
||||
)
|
||||
bias = torch.arange(32, dtype=layer.weight.dtype) / 64
|
||||
qkv = isinstance(layer, QKVParallelLinear)
|
||||
with parallel_scope(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0):
|
||||
for shard in ("q", "k", "v") if qkv else (None,):
|
||||
args = (shard,) if qkv else ()
|
||||
layer.weight.weight_loader(layer.weight, full, *args)
|
||||
layer.bias.weight_loader(layer.bias, bias, *args)
|
||||
weight = (
|
||||
torch.cat([full.chunk(size)[rank]] * 3)
|
||||
if qkv
|
||||
else full.chunk(size, dim=1)[rank]
|
||||
)
|
||||
expected_bias = (
|
||||
torch.cat([bias.chunk(size)[rank]] * 3) if qkv else None
|
||||
)
|
||||
inputs = (
|
||||
torch.arange(2 * weight.shape[1], dtype=weight.dtype).reshape(
|
||||
2, -1
|
||||
)
|
||||
/ 64
|
||||
)
|
||||
expected = F.linear(inputs, weight, expected_bias) * (
|
||||
1 if qkv else 2
|
||||
)
|
||||
with (
|
||||
parallel_scope(tp_group=tp, attn_tp_group=attn),
|
||||
patch(
|
||||
"sglang.srt.layers.linear.is_allocation_symmetric",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.linear.use_symmetric_memory",
|
||||
return_value=nullcontext(),
|
||||
),
|
||||
):
|
||||
torch.testing.assert_close(layer(inputs)[0], expected)
|
||||
torch.testing.assert_close(layer.weight, weight, rtol=0, atol=0)
|
||||
attn.all_reduce.assert_not_called()
|
||||
tp.all_reduce.assert_called_once()
|
||||
@@ -1196,8 +1196,7 @@ class VisionAttention(nn.Module):
|
||||
total_num_kv_heads=num_dummy_heads + num_kv_heads,
|
||||
bias=qkv_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
else:
|
||||
@@ -1206,8 +1205,7 @@ class VisionAttention(nn.Module):
|
||||
output_size=3 * self.dummy_dim,
|
||||
bias=qkv_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
@@ -1226,8 +1224,7 @@ class VisionAttention(nn.Module):
|
||||
output_size=embed_dim,
|
||||
bias=proj_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
prefix=add_prefix("proj", prefix),
|
||||
use_dp_attention_reduce=use_dp_attention_reduce,
|
||||
)
|
||||
|
||||
@@ -58,6 +58,20 @@ logger = logging.getLogger(__name__)
|
||||
LinearParallelGroup = Literal["tp", "attn_tp", "replicated"]
|
||||
|
||||
|
||||
def resolve_linear_parallel_group(
|
||||
parallel_group: LinearParallelGroup,
|
||||
) -> Tuple[int, int]:
|
||||
"""Freeze a group's weight partition in the current construction scope."""
|
||||
if parallel_group == "replicated":
|
||||
return 0, 1
|
||||
if parallel_group not in ("tp", "attn_tp"):
|
||||
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
|
||||
parallel = get_parallel()
|
||||
if parallel_group == "attn_tp":
|
||||
return parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
return parallel.tp_rank, parallel.tp_size
|
||||
|
||||
|
||||
def _resolve_linear_partition(
|
||||
parallel_group: Optional[LinearParallelGroup],
|
||||
tp_rank: Optional[int],
|
||||
@@ -74,14 +88,7 @@ def _resolve_linear_partition(
|
||||
raise ValueError(
|
||||
"parallel_group cannot be combined with tp_rank or tp_size"
|
||||
)
|
||||
if parallel_group == "replicated":
|
||||
return 0, 1
|
||||
if parallel_group not in ("tp", "attn_tp"):
|
||||
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
|
||||
parallel = get_parallel()
|
||||
if parallel_group == "attn_tp":
|
||||
return parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
return parallel.tp_rank, parallel.tp_size
|
||||
return resolve_linear_parallel_group(parallel_group)
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
|
||||
@@ -76,18 +76,16 @@ class Glm4vRMSNorm(RMSNorm):
|
||||
return x
|
||||
|
||||
|
||||
# TODO: fold into parallel_group selection ("attn_tp" under DP attention).
|
||||
def glm4v_vision_reduces_over_attn_tp(use_data_parallel: bool) -> bool:
|
||||
return is_dp_attention_enabled() and not use_data_parallel
|
||||
|
||||
|
||||
def _glm4v_vision_tp(use_data_parallel: bool) -> Tuple[int, int]:
|
||||
def _glm4v_vision_parallel_group(use_data_parallel: bool) -> str:
|
||||
if use_data_parallel:
|
||||
return 1, 0
|
||||
parallel = get_parallel()
|
||||
return "replicated"
|
||||
if glm4v_vision_reduces_over_attn_tp(use_data_parallel):
|
||||
return parallel.attn_tp_size, parallel.attn_tp_rank
|
||||
return parallel.tp_size, parallel.tp_rank
|
||||
return "attn_tp"
|
||||
return "tp"
|
||||
|
||||
|
||||
class Glm4vVisionMLP(nn.Module):
|
||||
@@ -101,15 +99,14 @@ class Glm4vVisionMLP(nn.Module):
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size, self.tp_rank = _glm4v_vision_tp(use_data_parallel)
|
||||
parallel_group = _glm4v_vision_parallel_group(use_data_parallel)
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=in_features,
|
||||
output_sizes=[hidden_features] * 2, # [gate_proj, up_proj]
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
hidden_features,
|
||||
@@ -117,8 +114,7 @@ class Glm4vVisionMLP(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
@@ -257,7 +253,7 @@ class Glm4vPatchMerger(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = d_model
|
||||
tp_size, tp_rank = _glm4v_vision_tp(use_data_parallel)
|
||||
parallel_group = _glm4v_vision_parallel_group(use_data_parallel)
|
||||
self.proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
@@ -272,8 +268,7 @@ class Glm4vPatchMerger(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
context_dim,
|
||||
@@ -281,8 +276,7 @@ class Glm4vPatchMerger(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
use_dp_attention_reduce=glm4v_vision_reduces_over_attn_tp(
|
||||
use_data_parallel
|
||||
),
|
||||
|
||||
@@ -47,7 +47,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import (
|
||||
InternViTCudaGraphRunner,
|
||||
)
|
||||
from sglang.srt.multimodal.mm_utils import run_dp_sharded_vision_model
|
||||
from sglang.srt.runtime_context import get_mm, get_parallel
|
||||
from sglang.srt.runtime_context import get_mm
|
||||
from sglang.srt.utils import is_cuda
|
||||
from sglang.utils import logger
|
||||
|
||||
@@ -200,8 +200,7 @@ class InternMLP(nn.Module):
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = 1 if use_data_parallel else get_parallel().tp_size
|
||||
self.tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
|
||||
parallel_group = "replicated" if use_data_parallel else "tp"
|
||||
self.config = config
|
||||
self.act = get_act_fn(config.hidden_act)
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
@@ -209,16 +208,14 @@ class InternMLP(nn.Module):
|
||||
config.intermediate_size,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -341,7 +341,6 @@ class MLP2(nn.Module):
|
||||
self.quant_config = quant_config
|
||||
use_tensor_parallel = use_tensor_parallel and not use_data_parallel
|
||||
tp_size = get_parallel().attn_tp_size if use_tensor_parallel else 1
|
||||
tp_rank = get_parallel().attn_tp_rank if use_tensor_parallel else 0
|
||||
if isinstance(self.quant_config, ModelSlimConfig):
|
||||
self.fc0 = ReplicatedLinear(
|
||||
dims[0],
|
||||
@@ -373,16 +372,14 @@ class MLP2(nn.Module):
|
||||
dims[1],
|
||||
bias=bias,
|
||||
prefix=add_prefix("fc0", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
self.fc1 = RowParallelLinear(
|
||||
dims[1],
|
||||
dims[2],
|
||||
bias=bias,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
else:
|
||||
self.fc0 = nn.Linear(dims[0], dims[1], bias=bias)
|
||||
|
||||
@@ -85,7 +85,6 @@ class MiniMaxVLMultiModalProjector(nn.Module):
|
||||
)
|
||||
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
@@ -103,8 +102,7 @@ class MiniMaxVLMultiModalProjector(nn.Module):
|
||||
bias=multimodal_projector_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
)
|
||||
assert projector_hidden_act == "gelu", (
|
||||
f"Only gelu activation is supported, got {projector_hidden_act}"
|
||||
@@ -116,8 +114,7 @@ class MiniMaxVLMultiModalProjector(nn.Module):
|
||||
bias=multimodal_projector_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
)
|
||||
|
||||
@@ -150,7 +147,6 @@ class MiniMaxVLPatchMerger(nn.Module):
|
||||
)
|
||||
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
@@ -168,8 +164,7 @@ class MiniMaxVLPatchMerger(nn.Module):
|
||||
bias=patch_merge_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
)
|
||||
assert projector_hidden_act == "gelu", (
|
||||
f"Only gelu activation is supported, got {projector_hidden_act}"
|
||||
@@ -181,8 +176,7 @@ class MiniMaxVLPatchMerger(nn.Module):
|
||||
bias=patch_merge_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
)
|
||||
|
||||
@@ -282,7 +276,6 @@ class CLIPEncoderLayer(nn.Module):
|
||||
self.embed_dim = config.hidden_size
|
||||
self.use_data_parallel = use_data_parallel
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
@@ -314,8 +307,7 @@ class CLIPEncoderLayer(nn.Module):
|
||||
config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp.fc1",
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
)
|
||||
hidden_act = getattr(config, "hidden_act", "gelu")
|
||||
assert hidden_act == "gelu", (
|
||||
@@ -327,8 +319,7 @@ class CLIPEncoderLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp.fc2",
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "attn_tp",
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
)
|
||||
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
|
||||
@@ -50,9 +50,11 @@ from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -140,21 +142,16 @@ class Qwen2_5_VLMLP(nn.Module):
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
fuse_gate_up: bool = True,
|
||||
tp_size: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
super().__init__()
|
||||
if use_data_parallel and parallel_group is not None:
|
||||
raise ValueError("Explicit MLP TP cannot be combined with data parallel")
|
||||
if use_data_parallel:
|
||||
if tp_size is not None or tp_rank is not None:
|
||||
raise ValueError(
|
||||
"Explicit MLP TP cannot be combined with data parallel"
|
||||
)
|
||||
self.tp_size, self.tp_rank = 1, 0
|
||||
else:
|
||||
if (tp_size is None) != (tp_rank is None):
|
||||
raise ValueError("MLP tp_size and tp_rank must be set together")
|
||||
self.tp_size = get_parallel().tp_size if tp_size is None else tp_size
|
||||
self.tp_rank = get_parallel().tp_rank if tp_rank is None else tp_rank
|
||||
parallel_group = "replicated"
|
||||
elif parallel_group is None:
|
||||
parallel_group = "tp"
|
||||
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
|
||||
self.fuse_gate_up = fuse_gate_up
|
||||
if fuse_gate_up:
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -163,8 +160,7 @@ class Qwen2_5_VLMLP(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
else:
|
||||
projection_kwargs = dict(
|
||||
@@ -172,8 +168,7 @@ class Qwen2_5_VLMLP(nn.Module):
|
||||
output_size=hidden_features,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.gate_proj = ColumnParallelLinear(
|
||||
**projection_kwargs,
|
||||
@@ -198,8 +193,7 @@ class Qwen2_5_VLMLP(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.hidden_act = hidden_act
|
||||
if self.fuse_gate_up and self.hidden_act == "silu":
|
||||
@@ -326,8 +320,6 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
|
||||
cast_x_before_out_mul=cast_x_before_out_mul,
|
||||
force_native=force_native_norm,
|
||||
)
|
||||
tp_size = 1 if use_data_parallel else get_parallel().tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
|
||||
self.mlp = nn.ModuleList(
|
||||
[
|
||||
ColumnParallelLinear(
|
||||
@@ -336,8 +328,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp.0", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "tp",
|
||||
),
|
||||
nn.GELU(),
|
||||
RowParallelLinear(
|
||||
@@ -346,8 +337,7 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp.2", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "tp",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -44,7 +44,12 @@ from sglang.srt.layers.dp_attention import (
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -111,23 +116,16 @@ _is_cpu = is_cpu()
|
||||
_VECTORIZED_VL_POS_EMBED_MIN_IMAGES = 6
|
||||
|
||||
|
||||
def _resolve_vision_tp(
|
||||
def _resolve_vision_parallel_group(
|
||||
*,
|
||||
use_data_parallel: bool,
|
||||
tp_size: Optional[int],
|
||||
tp_rank: Optional[int],
|
||||
) -> tuple[int, int]:
|
||||
parallel_group: Optional[LinearParallelGroup],
|
||||
) -> LinearParallelGroup:
|
||||
if use_data_parallel:
|
||||
if tp_size is not None or tp_rank is not None:
|
||||
if parallel_group is not None:
|
||||
raise ValueError("Explicit vision TP cannot be combined with data parallel")
|
||||
return 1, 0
|
||||
if (tp_size is None) != (tp_rank is None):
|
||||
raise ValueError("Vision tp_size and tp_rank must be set together")
|
||||
if tp_size is None:
|
||||
parallel = get_parallel()
|
||||
return parallel.attn_tp_size, parallel.attn_tp_rank
|
||||
assert tp_rank is not None
|
||||
return tp_size, tp_rank
|
||||
return "replicated"
|
||||
return "attn_tp" if parallel_group is None else parallel_group
|
||||
|
||||
|
||||
class Qwen3_VisionMLP(nn.Module):
|
||||
@@ -140,15 +138,13 @@ class Qwen3_VisionMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
tp_size: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size, self.tp_rank = _resolve_vision_tp(
|
||||
use_data_parallel=use_data_parallel,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group = _resolve_vision_parallel_group(
|
||||
use_data_parallel=use_data_parallel, parallel_group=parallel_group
|
||||
)
|
||||
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
@@ -165,8 +161,7 @@ class Qwen3_VisionMLP(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("linear_fc1", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.linear_fc2 = RowParallelLinear(
|
||||
hidden_features,
|
||||
@@ -174,8 +169,7 @@ class Qwen3_VisionMLP(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("linear_fc2", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
)
|
||||
self.act = ACT2FN[hidden_act]
|
||||
@@ -306,8 +300,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
tp_size: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
disable_merger_proj: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -323,11 +316,10 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
|
||||
self.hidden_size if use_postshuffle_norm else context_dim
|
||||
)
|
||||
if not disable_merger_proj:
|
||||
self.tp_size, self.tp_rank = _resolve_vision_tp(
|
||||
use_data_parallel=use_data_parallel,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group = _resolve_vision_parallel_group(
|
||||
use_data_parallel=use_data_parallel, parallel_group=parallel_group
|
||||
)
|
||||
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
@@ -344,8 +336,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("linear_fc1", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.act_fn = nn.GELU()
|
||||
self.linear_fc2 = RowParallelLinear(
|
||||
@@ -354,8 +345,7 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("linear_fc2", prefix),
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
parallel_group=parallel_group,
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
)
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
|
||||
@@ -71,15 +70,12 @@ class SiglipMLP(nn.Module):
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
tp_size = 1 if use_data_parallel else get_parallel().tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().tp_rank
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "tp",
|
||||
)
|
||||
self.act = act_layer()
|
||||
self.fc2 = RowParallelLinear(
|
||||
@@ -87,8 +83,7 @@ class SiglipMLP(nn.Module):
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("fc2", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
parallel_group="replicated" if use_data_parallel else "tp",
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -524,15 +524,13 @@ class Step3VisionMLP(nn.Module):
|
||||
# Since this is a dense model,
|
||||
# the MLP component likewise adopts a DP-MLP approach modeled after DP Attention.
|
||||
# This choice may not represent the optimal solution and remains open to further deliberation.
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
dim,
|
||||
intermediate_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("gate_proj", prefix),
|
||||
)
|
||||
self.act = ACT2FN[hidden_act] # quick_gelu
|
||||
@@ -550,8 +548,7 @@ class Step3VisionMLP(nn.Module):
|
||||
dim,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Group metadata fixtures for tests that do not initialize distributed groups."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel
|
||||
from sglang.srt.runtime_context import publish as publish_context
|
||||
|
||||
|
||||
def _group_metadata(values):
|
||||
parallel = get_parallel()
|
||||
for name in ("tp", "attn_tp"):
|
||||
rank_key, size_key, group_key = (
|
||||
f"{name}_rank",
|
||||
f"{name}_size",
|
||||
f"{name}_group",
|
||||
)
|
||||
rank = values.get(rank_key, getattr(parallel, rank_key))
|
||||
size = values.get(size_key, getattr(parallel, size_key))
|
||||
if group_key not in values and (rank_key in values or size_key in values):
|
||||
values[group_key] = SimpleNamespace(rank_in_group=rank, world_size=size)
|
||||
group = values.get(group_key)
|
||||
if group is not None:
|
||||
# Execution mocks also need placement metadata when used to build layers.
|
||||
if not isinstance(getattr(group, "rank_in_group", None), int):
|
||||
group.rank_in_group = rank
|
||||
if not isinstance(getattr(group, "world_size", None), int):
|
||||
group.world_size = size
|
||||
return values
|
||||
|
||||
|
||||
def publish(*args, **kwargs):
|
||||
if kwargs.get("ranks") is None:
|
||||
kwargs["ranks"] = SpawnRanks(world_rank=0)
|
||||
context = publish_context(*args, **kwargs)
|
||||
parallel = get_parallel()
|
||||
groups = {
|
||||
f"{name}_group": SimpleNamespace(
|
||||
rank_in_group=getattr(parallel, f"{name}_rank"),
|
||||
world_size=getattr(parallel, f"{name}_size"),
|
||||
)
|
||||
for name in ("tp", "attn_tp")
|
||||
}
|
||||
parallel.override_permanently(**groups)
|
||||
return context
|
||||
|
||||
|
||||
@contextmanager
|
||||
def parallel_scope(**values):
|
||||
with get_parallel().override(**_group_metadata(values)) as parallel:
|
||||
yield parallel
|
||||
@@ -0,0 +1,294 @@
|
||||
"""Qwen vision wrappers and generation helpers retain their placement policies."""
|
||||
|
||||
import inspect
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def group_kwargs(cls, group):
|
||||
if group is None:
|
||||
return {}
|
||||
if "parallel_group" in inspect.signature(cls).parameters:
|
||||
return dict(parallel_group=group)
|
||||
p = get_parallel()
|
||||
rank, size = (
|
||||
(0, 1)
|
||||
if group == "replicated"
|
||||
else (p.tp_rank, p.tp_size)
|
||||
if group == "tp"
|
||||
else (p.attn_tp_rank, p.attn_tp_size)
|
||||
)
|
||||
return dict(tp_rank=rank, tp_size=size)
|
||||
|
||||
|
||||
def build_qwen(model, group=None, *, replicated=False, width=32, quant_config=None):
|
||||
options = dict(quant_config=quant_config, use_data_parallel=replicated)
|
||||
if model.startswith("qwen25"):
|
||||
from sglang.srt.models.qwen2_5_vl import Qwen2_5_VLMLP
|
||||
|
||||
module = Qwen2_5_VLMLP(
|
||||
width,
|
||||
2 * width,
|
||||
bias=True,
|
||||
fuse_gate_up=model == "qwen25_fused",
|
||||
**options,
|
||||
**group_kwargs(Qwen2_5_VLMLP, group),
|
||||
)
|
||||
layers = (
|
||||
(module.gate_up_proj, module.down_proj)
|
||||
if module.fuse_gate_up
|
||||
else (module.gate_proj, module.up_proj, module.down_proj)
|
||||
)
|
||||
elif model == "qwen3_mlp":
|
||||
from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP
|
||||
|
||||
module = Qwen3_VisionMLP(
|
||||
width,
|
||||
2 * width,
|
||||
bias=True,
|
||||
**options,
|
||||
**group_kwargs(Qwen3_VisionMLP, group),
|
||||
)
|
||||
layers = (module.linear_fc1, module.linear_fc2)
|
||||
elif model == "qwen3_merger":
|
||||
from sglang.srt.models.qwen3_vl import Qwen3VLMoeVisionPatchMerger
|
||||
|
||||
module = Qwen3VLMoeVisionPatchMerger(
|
||||
width,
|
||||
width,
|
||||
2 * width,
|
||||
spatial_merge_size=1,
|
||||
**options,
|
||||
**group_kwargs(Qwen3VLMoeVisionPatchMerger, group),
|
||||
)
|
||||
layers = (module.linear_fc1, module.linear_fc2)
|
||||
else:
|
||||
raise ValueError(model)
|
||||
return module, layers
|
||||
|
||||
|
||||
def load_projection(layer):
|
||||
"""Load known full weights under rank zero, returning the expected owned shard."""
|
||||
rank, size = getattr(layer, "tp_rank", 0), getattr(layer, "tp_size", 1)
|
||||
row = isinstance(layer, RowParallelLinear)
|
||||
dtype, device = layer.weight.dtype, layer.weight.device
|
||||
|
||||
def values(rows, columns, offset):
|
||||
return (
|
||||
(
|
||||
torch.arange(rows * columns, device=device).reshape(rows, columns)
|
||||
+ offset
|
||||
)
|
||||
% 23
|
||||
/ 128
|
||||
).to(dtype)
|
||||
|
||||
with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0):
|
||||
if isinstance(layer, QKVParallelLinear):
|
||||
weights, biases = [], []
|
||||
for offset, shard_id in enumerate(("q", "k", "v")):
|
||||
heads = (
|
||||
layer.total_num_heads
|
||||
if shard_id == "q"
|
||||
else layer.total_num_kv_heads
|
||||
)
|
||||
weight = values(heads * layer.head_size, layer.input_size, offset)
|
||||
partitions = size if shard_id == "q" else min(heads, size)
|
||||
index = rank if shard_id == "q" else rank // max(size // heads, 1)
|
||||
layer.weight.weight_loader(layer.weight, weight, shard_id)
|
||||
weights.append(weight.chunk(partitions)[index])
|
||||
if layer.bias is not None:
|
||||
bias = values(heads * layer.head_size, 1, offset).flatten()
|
||||
layer.bias.weight_loader(layer.bias, bias, shard_id)
|
||||
biases.append(bias.chunk(partitions)[index])
|
||||
shard = torch.cat(weights)
|
||||
bias_shard = torch.cat(biases) if biases else None
|
||||
else:
|
||||
weight = values(layer.output_size, layer.input_size, 3)
|
||||
layer.weight.weight_loader(layer.weight, weight)
|
||||
shard = (
|
||||
weight.chunk(size, dim=1)[rank]
|
||||
if row
|
||||
else torch.cat(
|
||||
[
|
||||
p.chunk(size)[rank]
|
||||
for p in weight.split(
|
||||
getattr(layer, "output_sizes", [layer.output_size])
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
bias_shard = None
|
||||
if layer.bias is not None:
|
||||
bias = values(layer.output_size, 1, 4).flatten()
|
||||
layer.bias.weight_loader(layer.bias, bias)
|
||||
bias_shard = (
|
||||
bias
|
||||
if row
|
||||
else torch.cat(
|
||||
[
|
||||
p.chunk(size)[rank]
|
||||
for p in bias.split(
|
||||
getattr(layer, "output_sizes", [layer.output_size])
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
return shard, bias_shard
|
||||
|
||||
|
||||
class TestQwenVisionWrapperGroups(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def test_native_projection_math_loaders_and_flags(self):
|
||||
for dp in (1, 2):
|
||||
reset_context()
|
||||
server = ServerArgs(
|
||||
model_path="dummy",
|
||||
device="cpu",
|
||||
tp_size=4,
|
||||
attn_dp_size=dp,
|
||||
mm_attention_backend="sdpa",
|
||||
)
|
||||
publish(server, role="test", ranks=SpawnRanks(world_rank=3))
|
||||
initialize_dp_attention_flags(server)
|
||||
tp, attn = Mock(), Mock()
|
||||
tp.all_reduce.side_effect = lambda x: x * 2
|
||||
attn.all_reduce.side_effect = lambda x: x * 3
|
||||
for model in (
|
||||
"qwen25_fused",
|
||||
"qwen25_split",
|
||||
"qwen3_mlp",
|
||||
"qwen3_merger",
|
||||
):
|
||||
for group, replicated in (
|
||||
(None, False),
|
||||
(None, True),
|
||||
("tp", False),
|
||||
("attn_tp", False),
|
||||
("replicated", False),
|
||||
):
|
||||
with self.subTest(
|
||||
dp=dp, model=model, group=group, replicated=replicated
|
||||
):
|
||||
module, layers = build_qwen(model, group, replicated=replicated)
|
||||
selected = (
|
||||
"replicated"
|
||||
if replicated
|
||||
else group
|
||||
or ("attn_tp" if model.startswith("qwen3") else "tp")
|
||||
)
|
||||
expected_rank, expected_size = (
|
||||
(0, 1)
|
||||
if selected == "replicated"
|
||||
else (3, 4)
|
||||
if selected == "tp"
|
||||
else (3 % (4 // dp), 4 // dp)
|
||||
)
|
||||
for index, layer in enumerate(layers):
|
||||
weight, bias = load_projection(layer)
|
||||
row = isinstance(layer, RowParallelLinear)
|
||||
inputs = (
|
||||
torch.arange(2 * weight.shape[1]).reshape(2, -1).float()
|
||||
/ 64
|
||||
)
|
||||
expected = F.linear(
|
||||
inputs,
|
||||
weight,
|
||||
bias if not row or expected_rank == 0 else None,
|
||||
)
|
||||
dp_reduce = row and layer.use_dp_attention_reduce
|
||||
if row and expected_size > 1:
|
||||
expected *= 3 if dp_reduce else 2
|
||||
tp.all_reduce.reset_mock()
|
||||
attn.all_reduce.reset_mock()
|
||||
with (
|
||||
get_parallel().override(
|
||||
tp_group=tp, attn_tp_group=attn
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.linear.is_allocation_symmetric",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.linear.use_symmetric_memory",
|
||||
return_value=nullcontext(),
|
||||
) as allocator,
|
||||
):
|
||||
actual = layer(inputs)[0]
|
||||
torch.testing.assert_close(actual, expected)
|
||||
if row:
|
||||
allocator.assert_called_once_with(
|
||||
attn
|
||||
) if dp_reduce else allocator.assert_called_once_with(
|
||||
tp, disabled=True
|
||||
)
|
||||
else:
|
||||
allocator.assert_not_called()
|
||||
self.assertEqual(
|
||||
tp.all_reduce.call_count,
|
||||
int(row and expected_size > 1 and not dp_reduce),
|
||||
)
|
||||
self.assertEqual(
|
||||
attn.all_reduce.call_count,
|
||||
int(row and expected_size > 1 and dp_reduce),
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
getattr(layer, "tp_rank", 0),
|
||||
getattr(layer, "tp_size", 1),
|
||||
),
|
||||
(expected_rank, expected_size),
|
||||
)
|
||||
if model == "qwen25_split":
|
||||
self.assertEqual(
|
||||
isinstance(module.down_proj, ReplicatedLinear),
|
||||
expected_size == 1,
|
||||
)
|
||||
|
||||
def test_wrapper_conflict_guard_and_disabled_merger(self):
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_cp_size=2),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=3),
|
||||
)
|
||||
for model in ("qwen25_fused", "qwen3_mlp", "qwen3_merger"):
|
||||
with self.assertRaisesRegex(
|
||||
ValueError, "cannot be combined with data parallel"
|
||||
):
|
||||
build_qwen(model, "replicated", replicated=True)
|
||||
from sglang.srt.models.qwen3_vl import Qwen3VLMoeVisionPatchMerger
|
||||
|
||||
reset_context()
|
||||
merger = Qwen3VLMoeVisionPatchMerger(
|
||||
32,
|
||||
32,
|
||||
64,
|
||||
disable_merger_proj=True,
|
||||
**group_kwargs(Qwen3VLMoeVisionPatchMerger, "replicated"),
|
||||
)
|
||||
self.assertFalse(hasattr(merger, "linear_fc1"))
|
||||
self.assertFalse(hasattr(merger, "tp_size"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Vision projections retain their layout and execution policy across reload scopes."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention_flags
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from sglang.srt.runtime_context import (
|
||||
SpawnRanks,
|
||||
get_disagg,
|
||||
get_parallel,
|
||||
publish,
|
||||
reset_context,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
VISION_MODELS = (
|
||||
"vision_packed",
|
||||
"vision_column",
|
||||
"vision_dummy",
|
||||
"vision_gqa",
|
||||
"vision_legacy",
|
||||
"siglip",
|
||||
"glm4v_mlp",
|
||||
"glm4v_merger",
|
||||
"internvl_mlp",
|
||||
"qwen_merger",
|
||||
"minimax_projector",
|
||||
"minimax_merger",
|
||||
"minimax_encoder",
|
||||
"moonvit",
|
||||
"step3",
|
||||
)
|
||||
|
||||
|
||||
def build_vision(model, use_data_parallel=False, width=32, quant_config=None):
|
||||
config = SimpleNamespace(
|
||||
hidden_size=width,
|
||||
intermediate_size=2 * width,
|
||||
num_attention_heads=8,
|
||||
layer_norm_eps=1e-6,
|
||||
hidden_act="gelu",
|
||||
)
|
||||
options = dict(quant_config=quant_config, use_data_parallel=use_data_parallel)
|
||||
if model.startswith("vision_"):
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
|
||||
module = VisionAttention(
|
||||
width,
|
||||
8,
|
||||
width,
|
||||
use_qkv_parallel=model != "vision_column",
|
||||
num_dummy_heads=4 if model == "vision_dummy" else 0,
|
||||
num_kv_heads=4 if model == "vision_gqa" else 8,
|
||||
use_dp_attention_reduce=model != "vision_legacy",
|
||||
allow_tp_reduce_mismatch=model == "vision_legacy",
|
||||
qkv_backend="sdpa",
|
||||
**options,
|
||||
)
|
||||
layers = (module.qkv_proj, module.proj)
|
||||
elif model in ("glm4v_mlp", "glm4v_merger"):
|
||||
from sglang.srt.models.glm4v import Glm4vPatchMerger, Glm4vVisionMLP
|
||||
|
||||
constructor = Glm4vVisionMLP if model == "glm4v_mlp" else Glm4vPatchMerger
|
||||
module = constructor(width, 2 * width, bias=True, **options)
|
||||
layers = (module.gate_up_proj, module.down_proj)
|
||||
elif model == "internvl_mlp":
|
||||
from sglang.srt.models.internvl import InternMLP
|
||||
|
||||
module = InternMLP(config, use_data_parallel=use_data_parallel)
|
||||
layers = (module.fc1, module.fc2)
|
||||
elif model == "siglip":
|
||||
from sglang.srt.models.siglip import SiglipMLP
|
||||
|
||||
module = SiglipMLP(config, **options)
|
||||
layers = (module.fc1, module.fc2)
|
||||
elif model == "qwen_merger":
|
||||
from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionPatchMerger
|
||||
|
||||
module = Qwen2_5_VisionPatchMerger(
|
||||
width,
|
||||
width,
|
||||
2 * width,
|
||||
spatial_merge_size=1,
|
||||
force_native_norm=True,
|
||||
**options,
|
||||
)
|
||||
layers = (module.mlp[0], module.mlp[2])
|
||||
elif model == "moonvit":
|
||||
from sglang.srt.models.kimi_vl_moonvit import MLP2
|
||||
|
||||
module = MLP2(
|
||||
[width, 2 * width, width],
|
||||
torch.nn.GELU(),
|
||||
use_tensor_parallel=True,
|
||||
**options,
|
||||
)
|
||||
layers = (module.fc0, module.fc1)
|
||||
elif model == "step3":
|
||||
from sglang.srt.models.step3_vl import Step3VisionMLP
|
||||
|
||||
module = Step3VisionMLP(width, 2 * width, quant_config=quant_config)
|
||||
layers = (module.fc1, module.fc2)
|
||||
else:
|
||||
from sglang.srt.models.minimax_vl_common import (
|
||||
CLIPEncoderLayer,
|
||||
MiniMaxVLMultiModalProjector,
|
||||
MiniMaxVLPatchMerger,
|
||||
)
|
||||
|
||||
if model == "minimax_projector":
|
||||
module = MiniMaxVLMultiModalProjector(
|
||||
width, width, "gelu", True, projector_hidden_size=2 * width, **options
|
||||
)
|
||||
layers = (module.linear_1, module.linear_2)
|
||||
elif model == "minimax_merger":
|
||||
module = MiniMaxVLPatchMerger(
|
||||
1, width, "gelu", True, projector_hidden_size=2 * width, **options
|
||||
)
|
||||
layers = (module.linear_1, module.linear_2)
|
||||
else:
|
||||
module = CLIPEncoderLayer(config, **options)
|
||||
layers = (
|
||||
module.fc1,
|
||||
module.fc2,
|
||||
module.self_attn.qkv_proj,
|
||||
module.self_attn.proj,
|
||||
)
|
||||
return module, layers
|
||||
|
||||
|
||||
def load_projection(layer):
|
||||
"""Load known full weights under rank zero, returning the expected owned shard."""
|
||||
rank, size = layer.tp_rank, layer.tp_size
|
||||
row = isinstance(layer, RowParallelLinear)
|
||||
dtype, device = layer.weight.dtype, layer.weight.device
|
||||
|
||||
def values(rows, columns, offset):
|
||||
return (
|
||||
(
|
||||
torch.arange(rows * columns, device=device).reshape(rows, columns)
|
||||
+ offset
|
||||
)
|
||||
% 23
|
||||
/ 128
|
||||
).to(dtype)
|
||||
|
||||
with get_parallel().override(tp_rank=0, attn_dp_rank=0, attn_tp_rank=0):
|
||||
if isinstance(layer, QKVParallelLinear):
|
||||
weights, biases = [], []
|
||||
for offset, shard_id in enumerate(("q", "k", "v")):
|
||||
heads = (
|
||||
layer.total_num_heads
|
||||
if shard_id == "q"
|
||||
else layer.total_num_kv_heads
|
||||
)
|
||||
weight = values(heads * layer.head_size, layer.input_size, offset)
|
||||
partitions = size if shard_id == "q" else min(heads, size)
|
||||
index = rank if shard_id == "q" else rank // max(size // heads, 1)
|
||||
layer.weight.weight_loader(layer.weight, weight, shard_id)
|
||||
weights.append(weight.chunk(partitions)[index])
|
||||
if layer.bias is not None:
|
||||
bias = values(heads * layer.head_size, 1, offset).flatten()
|
||||
layer.bias.weight_loader(layer.bias, bias, shard_id)
|
||||
biases.append(bias.chunk(partitions)[index])
|
||||
shard = torch.cat(weights)
|
||||
bias_shard = torch.cat(biases) if biases else None
|
||||
else:
|
||||
weight = values(layer.output_size, layer.input_size, 3)
|
||||
layer.weight.weight_loader(layer.weight, weight)
|
||||
shard = (
|
||||
weight.chunk(size, dim=1)[rank]
|
||||
if row
|
||||
else torch.cat(
|
||||
[
|
||||
p.chunk(size)[rank]
|
||||
for p in weight.split(
|
||||
getattr(layer, "output_sizes", [layer.output_size])
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
bias_shard = None
|
||||
if layer.bias is not None:
|
||||
bias = values(layer.output_size, 1, 4).flatten()
|
||||
layer.bias.weight_loader(layer.bias, bias)
|
||||
bias_shard = (
|
||||
bias
|
||||
if row
|
||||
else torch.cat(
|
||||
[
|
||||
p.chunk(size)[rank]
|
||||
for p in bias.split(
|
||||
getattr(layer, "output_sizes", [layer.output_size])
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
return shard, bias_shard
|
||||
|
||||
|
||||
class TestVisionParallelGroups(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def test_projection_layout_reload_and_row_policy(self):
|
||||
for dp_size in (1, 2):
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy",
|
||||
device="cpu",
|
||||
tp_size=4,
|
||||
attn_dp_size=dp_size,
|
||||
enable_dp_attention=dp_size > 1,
|
||||
mm_attention_backend="sdpa",
|
||||
),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=3),
|
||||
)
|
||||
initialize_dp_attention_flags(
|
||||
ServerArgs(
|
||||
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp_size
|
||||
)
|
||||
)
|
||||
tp, attn = Mock(), Mock()
|
||||
tp.all_reduce.side_effect = lambda x: x * 4
|
||||
attn.all_reduce.side_effect = lambda x: x * (4 // dp_size)
|
||||
for model in VISION_MODELS:
|
||||
for replicated in (
|
||||
(False,) if model in ("moonvit", "step3") else (False, True)
|
||||
):
|
||||
with self.subTest(dp=dp_size, model=model, replicated=replicated):
|
||||
if dp_size == 2 and model in ("moonvit", "step3"):
|
||||
with self.assertRaisesRegex(
|
||||
ValueError, "shards over the attention TP group"
|
||||
):
|
||||
build_vision(model, replicated)
|
||||
continue
|
||||
module, layers = build_vision(model, replicated)
|
||||
group = (
|
||||
"tp"
|
||||
if model
|
||||
in (
|
||||
"siglip",
|
||||
"qwen_merger",
|
||||
"glm4v_mlp",
|
||||
"glm4v_merger",
|
||||
"internvl_mlp",
|
||||
)
|
||||
else "attn_tp"
|
||||
)
|
||||
rank, size = (
|
||||
(0, 1)
|
||||
if replicated
|
||||
else (
|
||||
(3, 4)
|
||||
if group == "tp"
|
||||
else (3 // dp_size, 4 // dp_size)
|
||||
)
|
||||
)
|
||||
for layer in layers:
|
||||
self.assertEqual(
|
||||
(layer.tp_rank, layer.tp_size), (rank, size)
|
||||
)
|
||||
shard, bias = load_projection(layer)
|
||||
torch.testing.assert_close(layer.weight, shard)
|
||||
if bias is not None:
|
||||
torch.testing.assert_close(layer.bias, bias)
|
||||
row = isinstance(layer, RowParallelLinear)
|
||||
inputs = (
|
||||
torch.arange(
|
||||
2 * shard.shape[1], dtype=shard.dtype
|
||||
).reshape(2, -1)
|
||||
/ 128
|
||||
)
|
||||
expected = F.linear(
|
||||
inputs, shard, bias if not row or rank == 0 else None
|
||||
)
|
||||
if row and size > 1:
|
||||
expected *= (
|
||||
(4 // dp_size)
|
||||
if layer.use_dp_attention_reduce
|
||||
else 4
|
||||
)
|
||||
tp.all_reduce.reset_mock()
|
||||
attn.all_reduce.reset_mock()
|
||||
with (
|
||||
get_parallel().override(
|
||||
tp_group=tp, attn_tp_group=attn
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.linear.is_allocation_symmetric",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.linear.use_symmetric_memory",
|
||||
return_value=nullcontext(),
|
||||
) as allocator,
|
||||
):
|
||||
torch.testing.assert_close(layer(inputs)[0], expected)
|
||||
if row:
|
||||
self.assertTrue(layer.reduce_results)
|
||||
reduction = (
|
||||
attn if layer.use_dp_attention_reduce else tp
|
||||
)
|
||||
if layer.use_dp_attention_reduce:
|
||||
allocator.assert_called_once_with(reduction)
|
||||
else:
|
||||
allocator.assert_called_once_with(
|
||||
reduction, disabled=True
|
||||
)
|
||||
self.assertEqual(
|
||||
reduction.all_reduce.call_count, int(size > 1)
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
tp if reduction is attn else attn
|
||||
).all_reduce.call_count,
|
||||
0,
|
||||
)
|
||||
else:
|
||||
allocator.assert_not_called()
|
||||
self.assertEqual(
|
||||
(layer.tp_rank, layer.tp_size), (rank, size)
|
||||
)
|
||||
if hasattr(module, "tp_rank"):
|
||||
self.assertEqual(
|
||||
(module.tp_rank, module.tp_size), (rank, size)
|
||||
)
|
||||
reset_context()
|
||||
|
||||
def test_guard_and_moonvit_dense_path(self):
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.models.kimi_vl_moonvit import MLP2
|
||||
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy",
|
||||
device="cpu",
|
||||
tp_size=4,
|
||||
attn_dp_size=2,
|
||||
mm_attention_backend="sdpa",
|
||||
),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=3),
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "shards over the attention TP group"):
|
||||
VisionAttention(32, 8, 32, True, qkv_backend="sdpa")
|
||||
with get_disagg().override(language_model_only=True):
|
||||
offloaded = VisionAttention(32, 8, 32, True, qkv_backend="sdpa")
|
||||
self.assertEqual((offloaded.proj.tp_rank, offloaded.proj.tp_size), (1, 2))
|
||||
self.assertFalse(offloaded.proj.use_dp_attention_reduce)
|
||||
for data_parallel, tensor_parallel in ((False, False), (True, True)):
|
||||
module = MLP2(
|
||||
[32, 64, 32],
|
||||
torch.nn.GELU(),
|
||||
use_data_parallel=data_parallel,
|
||||
use_tensor_parallel=tensor_parallel,
|
||||
)
|
||||
self.assertIsInstance(module.fc0, torch.nn.Linear)
|
||||
self.assertIsInstance(module.fc1, torch.nn.Linear)
|
||||
self.assertEqual(module(torch.zeros(2, 32)).shape, (2, 32))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user