[Refactor] Let the remaining placement consumers read the parallel context (#42348)

This commit is contained in:
Cheng Wan
2026-10-03 14:59:32 -07:00
committed by GitHub
parent 07064fd4a4
commit 10edb05e72
73 changed files with 875 additions and 362 deletions
@@ -120,12 +120,14 @@ python -m sglang.launch_server \
**Important**: `--attn-dp-size` must be greater than 1 for DPA to work. When `attn_dp_size == 1` (default), DPA is disabled. The constraint `tp_size % attn_dp_size == 0` must also be satisfied. `--dp-size` only sets the number of data-parallel replicas of the model; combining replicas (`--dp-size` > 1) with DPA is not supported.
> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead.
> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead. The flag will be removed after 2026-12-31.
>
> Under DeepSeek context parallelism (DSA, MLA, and DeepSeek-V4 prefill CP), `--dp-size N` alone still runs N attention-DP groups, and logs a deprecation warning; it will be rejected after 2026-12-31. Use `--attn-dp-size N` there too.
**Elastic EP scale joiners.** A rank that joins an existing DPA deployment through elastic EP scale-up runs DPA even when it is launched with `--attn-dp-size 1`: its own attention DP group is one rank wide.
**DP rank count.** The number of DP ranks (one scheduler each) is `dp_size × attn_dp_size`; one of the two is always 1.
- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Its deprecated `enable_dp_attention` field is always `false`; read `attn_dp_size` instead.
- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Until the flag is removed, its deprecated `enable_dp_attention` field still reports whether DPA runs; new clients should read `attn_dp_size`.
- Wire formats that predate `--attn-dp-size` keep the key `dp_size` for the DP rank count: KV-event descriptors, the PD prefill server info, and load snapshots.
### TP LMHead with all-to-all
@@ -3,7 +3,10 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from sglang.srt.arg_groups.model_override_base import attention_backends_of
from sglang.srt.arg_groups.model_override_base import (
attention_backends_of,
context_parallel_attn_dp_size,
)
from sglang.srt.arg_groups.overrides import (
_deepseek_v4_kv_cache_dtype,
declare_resolution,
@@ -157,13 +160,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
f"DeepSeekV4 prefill CP on HIP does not support {feature} yet."
)
# DeepSeek-V4 CP runs data-parallel groups as attention DP.
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
attn_dp_size = context_parallel_attn_dp_size(cfg, "DeepSeek-V4 context parallelism")
declare_resolution(
server_args,
"validate_deepseek_v4_cp",
@@ -171,7 +171,7 @@ class Parallel(msgspec.Struct):
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
] = False
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
# resolves to `attn_dp_size = N`, `dp_size = 1`.
# resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31.
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
enable_dp_attention_local_control_broadcast: A[
bool,
@@ -226,6 +226,27 @@ def ep_joiner_of(cfg: Any) -> bool:
return cfg.ep_join_mode in ("scale", "recover")
def context_parallel_attn_dp_size(cfg: Any, layout: str) -> int:
"""The attention-DP width a DeepSeek context-parallel layout runs.
These layouts run data-parallel groups as attention DP, so `--dp-size N`
alone still means N attention-DP groups there. That reading is deprecated.
"""
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
if cfg.dp_size > 1:
# TODO: reject `--dp-size > 1` here after 2026-12-31.
logger.warning(
f"--dp-size {cfg.dp_size} under {layout} runs {cfg.dp_size} attention "
"data-parallel groups. This is deprecated and will be rejected after "
f"2026-12-31; use --attn-dp-size {cfg.dp_size} instead."
)
return cfg.attn_dp_size * cfg.dp_size
def ep_scale_joiner_of(cfg: Any) -> bool:
"""The scale-up arm of :func:`ep_joiner_of`."""
return cfg.ep_join_mode == "scale"
@@ -5,6 +5,7 @@ from typing import Any, Dict
from sglang.srt.arg_groups.model_override_base import (
_register_for,
context_parallel_attn_dp_size,
is_attention_backend_not_set,
resolving_view,
use_mla_backend,
@@ -85,13 +86,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
logger.warning(
"Context parallel feature is still under experiment. It has only been verified on Hopper platform."
)
# DSA CP runs data-parallel groups as attention DP.
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
attn_dp_size = context_parallel_attn_dp_size(
cfg, "DSA context parallelism"
)
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
overrides["attn_dp_size"] = attn_dp_size
overrides["dp_size"] = 1
overrides["moe_dense_tp_size"] = 1
@@ -166,13 +163,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
"MLA prefill context parallel is still experimental. "
"Verified on Hopper with the fa3 backend."
)
# MLA CP runs data-parallel groups as attention DP.
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
"data-parallel replicas combined with attention data parallelism "
"are not supported."
)
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
attn_dp_size = context_parallel_attn_dp_size(cfg, "MLA context parallelism")
overrides["attn_dp_size"] = attn_dp_size
overrides["dp_size"] = 1
# TODO(kpham-sgl) Supports moe_dense_tp_size != 1.
@@ -39,22 +39,23 @@ def handle_deprecated_dp_attention(server_args: Any):
Runs before any handler reads the DP layout. The old spelling counted
attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and
`dp_size` counts replicas again.
`dp_size` counts replicas again. Next to `--attn-dp-size` the flag adds
nothing: a serialized config reports it set whenever attention DP runs
(`ServerArgs.resolved_dict`), so reading one back resolves the same layout.
"""
cfg = resolving_view(server_args)
if cfg.enable_dp_attention:
if cfg.attn_dp_size != 1:
raise ValueError(
"--enable-dp-attention is the deprecated spelling of "
"--attn-dp-size; pass --attn-dp-size alone."
)
if not cfg.enable_dp_attention:
return
if cfg.attn_dp_size == 1:
declare_resolution(
server_args,
"_handle_deprecated_dp_attention",
attn_dp_size=cfg.dp_size,
dp_size=1,
enable_dp_attention=False,
)
declare_resolution(
server_args, "_handle_deprecated_dp_attention", enable_dp_attention=False
)
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
@@ -158,8 +158,6 @@ class BaseKVSender(ABC):
mgr: BaseKVManager,
bootstrap_addr: str,
bootstrap_room: int,
dest_tp_ranks: List[int],
pp_rank: int,
req_has_disagg_prefill_dp_rank: bool = False,
): ...
@@ -1766,8 +1766,6 @@ class CommonKVSender(BaseKVSender):
mgr: CommonKVManager,
bootstrap_addr: str,
bootstrap_room: int,
dest_tp_ranks: List[int],
pp_rank: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
self.kv_mgr = mgr
+2 -5
View File
@@ -31,7 +31,6 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
import numpy as np
import torch
from torch.distributed import ProcessGroup
from sglang.srt.configs.mamba_utils import Mamba2CacheParams
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
@@ -382,7 +381,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
scheduler: Scheduler,
transfer_queue: DecodeTransferQueue,
tree_cache: BasePrefixCache,
gloo_group: ProcessGroup,
gpu_id: int,
bootstrap_port: int,
max_total_num_tokens: int,
@@ -402,7 +400,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.tree_cache = tree_cache
self.host_pool = None
self.host_reserved_tokens = 0
self.gloo_group = gloo_group
self.gloo_group = parallel.attn_tp_group.cpu_group
# Destinations visible to prefill but not yet on the transfer queue.
self._num_published_destinations = 0
self.tp_rank = parallel.tp_rank
@@ -2379,14 +2377,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
def __init__(
self,
gloo_group: ProcessGroup,
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
metadata_buffers: MetadataBuffers,
scheduler: Scheduler,
tree_cache: BasePrefixCache,
):
self.queue: List[DecodeRequest] = []
self.gloo_group = gloo_group
self.gloo_group = get_parallel().attn_tp_group.cpu_group
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
self.tp_rank = get_parallel().tp_rank
self.metadata_buffers = metadata_buffers
@@ -25,10 +25,7 @@ from fastapi import FastAPI
from fastapi.responses import ORJSONResponse, Response
from transformers import PretrainedConfig
from sglang.srt.distributed.parallel_state import (
GroupCoordinator,
get_mooncake_transfer_engine,
)
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
from sglang.srt.environ import envs
from sglang.srt.managers.io_struct import (
EncoderDispatchErrorReq,
@@ -1870,9 +1867,6 @@ class MMReceiverBase(ABC):
server_args: ServerArgs,
dtype: Optional[torch.dtype] = None,
hf_config: Optional[PretrainedConfig] = None,
pp_rank: Optional[int] = None,
tp_rank: Optional[int] = None,
tp_group: Optional[GroupCoordinator] = None,
scheduler: Optional["Scheduler"] = None,
encode_urls: Optional[List[str]] = None,
):
@@ -1892,10 +1886,13 @@ class MMReceiverBase(ABC):
)
self.recv_timeout = envs.SGLANG_ENCODER_RECV_TIMEOUT.get()
self.host = get_local_ip_auto(get_serving().host)
self.pp_rank = pp_rank
self.tp_rank = tp_rank
self.tp_size = get_parallel().tp_size
self.tp_group = tp_group
# A scheduler-side receiver works across its TP group; the tokenizer
# side has no placement.
parallel = get_parallel()
self.pp_rank = parallel.pp_rank if scheduler is not None else None
self.tp_rank = parallel.tp_rank if scheduler is not None else None
self.tp_size = parallel.tp_size
self.tp_group = parallel.tp_group if scheduler is not None else None
self.nnodes = get_parallel().nnodes
self.hostname = get_local_ip_auto()
self.waiting_list: List[WaitingMMRequestBase] = []
@@ -1908,7 +1905,7 @@ class MMReceiverBase(ABC):
and scheduler is not None
):
self.registration_runner = _ReceiveRegistrationRunner(
f"encoder-receive-registration-{tp_rank}"
f"encoder-receive-registration-{self.tp_rank}"
)
(
self.scheduler_embedding_port,
@@ -2674,9 +2671,6 @@ class MMReceiverHTTP(MMReceiverBase):
server_args: ServerArgs,
dtype: Optional[torch.dtype] = None,
hf_config: Optional[PretrainedConfig] = None,
pp_rank: Optional[int] = None,
tp_rank: Optional[int] = None,
tp_group: Optional[GroupCoordinator] = None,
scheduler: Optional["Scheduler"] = None,
encode_urls: Optional[List[str]] = None,
):
@@ -2684,9 +2678,6 @@ class MMReceiverHTTP(MMReceiverBase):
server_args,
dtype=dtype,
hf_config=hf_config,
pp_rank=pp_rank,
tp_rank=tp_rank,
tp_group=tp_group,
scheduler=scheduler,
encode_urls=encode_urls,
)
@@ -2813,9 +2804,6 @@ class MMReceiverGrpc(MMReceiverBase):
server_args: ServerArgs,
dtype: Optional[torch.dtype] = None,
hf_config: Optional[PretrainedConfig] = None,
pp_rank: Optional[int] = None,
tp_rank: Optional[int] = None,
tp_group: Optional[GroupCoordinator] = None,
scheduler: Optional["Scheduler"] = None,
encode_urls: Optional[List[str]] = None,
):
@@ -2830,9 +2818,6 @@ class MMReceiverGrpc(MMReceiverBase):
server_args,
dtype=dtype,
hf_config=hf_config,
pp_rank=pp_rank,
tp_rank=tp_rank,
tp_group=tp_group,
scheduler=scheduler,
encode_urls=encode_urls,
)
@@ -2946,9 +2931,6 @@ def create_mm_receiver(
server_args: ServerArgs,
dtype: Optional[torch.dtype] = None,
hf_config: Optional[PretrainedConfig] = None,
pp_rank: Optional[int] = None,
tp_rank: Optional[int] = None,
tp_group: Optional[GroupCoordinator] = None,
scheduler: Optional["Scheduler"] = None,
transport_mode: Optional[str] = None,
encode_urls: Optional[List[str]] = None,
@@ -2967,9 +2949,6 @@ def create_mm_receiver(
server_args,
dtype=dtype,
hf_config=hf_config,
pp_rank=pp_rank,
tp_rank=tp_rank,
tp_group=tp_group,
scheduler=scheduler,
encode_urls=encode_urls,
)
@@ -43,8 +43,6 @@ class FakeKVSender(BaseKVSender):
mgr: BaseKVManager,
bootstrap_addr: str,
bootstrap_room: int,
dest_tp_ranks: List[int],
pp_rank: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
self.kv_mgr = mgr
@@ -2813,16 +2813,12 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
mgr: MooncakeKVManager,
bootstrap_addr: str,
bootstrap_room: int,
dest_tp_ranks: List[int],
pp_rank: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
super().__init__(
mgr,
bootstrap_addr,
bootstrap_room,
dest_tp_ranks,
pp_rank,
req_has_disagg_prefill_dp_rank,
)
self.init_time = time.time()
@@ -1785,16 +1785,12 @@ class MoriKVSender(CommonKVSender):
mgr: MoriKVManager,
bootstrap_addr: str,
bootstrap_room: int,
dest_tp_ranks: List[int],
pp_rank: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
super().__init__(
mgr,
bootstrap_addr,
bootstrap_room,
dest_tp_ranks,
pp_rank,
req_has_disagg_prefill_dp_rank,
)
self.conclude_state: Optional[KVPoll] = None
@@ -3133,16 +3133,12 @@ class NixlKVSender(CommonKVSender):
mgr: NixlKVManager,
bootstrap_addr: str,
bootstrap_room: int,
dest_tp_ranks: List[int],
pp_rank: int,
req_has_disagg_prefill_dp_rank: bool = False,
):
super().__init__(
mgr,
bootstrap_addr,
bootstrap_room,
dest_tp_ranks,
pp_rank,
req_has_disagg_prefill_dp_rank,
)
self.init_time = time.time()
@@ -382,14 +382,10 @@ class PrefillBootstrapQueue:
)
kv_sender_class = get_kv_class(backend, KVClassType.SENDER)
dest_tp_ranks = [self.tp_rank]
req.disagg_kv_sender = kv_sender_class(
mgr=self.kv_manager,
bootstrap_addr=f"{req.bootstrap_host}:{self.bootstrap_port}",
bootstrap_room=req.bootstrap_room,
dest_tp_ranks=dest_tp_ranks,
pp_rank=self.pp_rank,
req_has_disagg_prefill_dp_rank=req.disagg_prefill_dp_rank is not None,
)
self._process_req(req)
@@ -83,6 +83,7 @@ from sglang.srt.distributed import (
tensor_model_parallel_all_gather,
)
from sglang.srt.distributed import utils as dist_utils
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -1099,6 +1100,7 @@ class VisionAttention(nn.Module):
] = None,
use_data_parallel: bool = False,
use_dp_attention_reduce: bool = False,
allow_tp_reduce_mismatch: bool = False,
aux_stream: Optional[torch.cuda.Stream] = None,
workspace_buffer: Optional[torch.Tensor] = None,
use_sink: bool = False,
@@ -1208,6 +1210,17 @@ class VisionAttention(nn.Module):
tp_size=self.tp_size,
prefix=add_prefix("qkv_proj", prefix),
)
# TODO: this layer shards over attention TP but reduces over the full TP
# group; reduce over the attention-TP group so attention DP and attention
# CP narrower than TP can run it.
if not allow_tp_reduce_mismatch:
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=self.tp_size,
reduces_over_attn_tp=use_dp_attention_reduce,
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.proj = RowParallelLinear(
input_size=self.dummy_dim,
output_size=embed_dim,
+73
View File
@@ -455,6 +455,79 @@ def is_dp_attention_enabled() -> bool:
return get_flags().dp.enabled
def get_dp_tp_group() -> GroupCoordinator:
"""The TP ranks that run one DP rank's batch: the attention-TP group with
attention DP, otherwise the whole TP group.
Attention-CP peers of a DP rank are members only in the second case.
"""
parallel = get_parallel()
return parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
def multimodal_encoder_runs_here() -> bool:
"""Whether a multimodal tower built on this worker is also forwarded here.
A tower is built unconditionally by most models, but an encoder-disaggregated
language instance forwards it on the encoder instance, a language-model-only
instance rejects multimodal requests, and a PD decode instance embeds nothing
(`general_mm_embed_routine` skips decode and target-verify forwards).
Adaptive dispatch is the exception on the first of those: it keeps the
requests it does not send to the encoder, and forwards the tower for them.
"""
from sglang.srt.runtime_context import get_disagg
disagg = get_disagg()
offloaded_to_an_encoder_instance = (
disagg.language_only and not disagg.enable_adaptive_dispatch_to_encoder
)
return not (
offloaded_to_an_encoder_instance
or disagg.language_model_only
or disagg.disaggregation_mode == "decode"
)
def reject_attn_tp_shard_with_tp_reduce(
layer: str,
*,
shard_tp_size: int,
reduces_over_attn_tp: bool,
multimodal_encoder: bool = False,
hint: str = "",
) -> None:
"""Reject a layer that shards over attention TP but all-reduces over the TP group.
The two groups differ only when attention DP or attention CP makes attention
TP narrower than TP. There the all-reduce mixes ranks that hold other
requests or replicated inputs, so the output is wrong; a one-rank shard does
not reduce at all.
"""
tp_size = get_parallel().tp_size
if reduces_over_attn_tp or not 1 < shard_tp_size < tp_size:
return
if multimodal_encoder and not multimodal_encoder_runs_here():
return
# Name only the widths that are actually narrowing the group here, so the
# remedy is one the operator can apply.
parallel = get_parallel()
narrowed_by = []
if parallel.attn_dp_size > 1:
narrowed_by.append(f"--attn-dp-size {parallel.attn_dp_size}")
if parallel.attn_cp_size > 1:
narrowed_by.append(f"--attn-cp-size {parallel.attn_cp_size}")
remedy = " and ".join(f"drop {flag}" for flag in narrowed_by) or (
"widen the attention TP group"
)
raise ValueError(
f"{layer} shards over the attention TP group ({shard_tp_size} ranks) "
f"but all-reduces over the full TP group ({tp_size} ranks), so it does "
f"not support {' with '.join(narrowed_by) or 'this layout'} yet. "
f"Use --attn-dp-size equal to --tp-size, or {remedy}{hint}."
)
def is_allocation_symmetric() -> bool:
return not is_dp_attention_enabled() or is_dp_max_padding()
@@ -464,6 +464,7 @@ class FusedMoE(torch.nn.Module):
is_gated=is_gated,
routing_method_type=routing_method_type,
gate_up_interleaved=gate_up_interleaved,
layer=self,
)
self.quant_method = quant_method
@@ -68,6 +68,35 @@ class MoeRunnerConfig:
# False preserves backend defaults, including their existing FP32 paths.
silu_mul_keep_fp32: bool = False
# The MoE placement a fused func needs is the layer's own, not the
# deployment's: DWDP's `FusedMoE.bind_full_expert_weights` collapses a
# layer's EP view after load. Read it off the layer so there is one
# address to keep correct.
@property
def moe_tp_size(self) -> int:
return self._layer_placement("moe_tp_size")
@property
def moe_tp_rank(self) -> int:
return self._layer_placement("moe_tp_rank")
@property
def moe_ep_size(self) -> int:
return self._layer_placement("moe_ep_size")
@property
def moe_ep_rank(self) -> int:
return self._layer_placement("moe_ep_rank")
def _layer_placement(self, name: str) -> int:
if self.layer is None:
raise AttributeError(
f"{name} is the layer's MoE placement, but this runner config "
"has no layer. A layer that runs fused MoE funcs states itself "
"on its runner config."
)
return getattr(self.layer, name)
@dataclass
class RunnerInput(ABC):
@@ -58,10 +58,6 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
swiglu_alpha: Optional[torch.Tensor] = None
swiglu_beta: Optional[torch.Tensor] = None
swiglu_limit: Optional[torch.Tensor] = None
moe_tp_size: int = 1
moe_tp_rank: int = 0
moe_ep_size: int = 1
moe_ep_rank: int = 0
apply_routed_scaling_factor: bool = True
@@ -144,12 +140,6 @@ class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo):
# SwigluStep variant.
use_swiglu_step: bool = False
# TP/EP topology (forwarded to the FlashInfer kernel)
moe_tp_size: int = 1
moe_tp_rank: int = 0
moe_ep_size: int = 1
moe_ep_rank: int = 0
# GPT-OSS pads its input hidden dim up to the (pre-padded) loaded weight
# width and trims the output back. DSv4 leaves this as ``None`` (no pad).
padded_hidden: Optional[int] = None
@@ -318,10 +308,10 @@ def _run_flashinfer_cutlass(
swiglu_alpha=quant_info.swiglu_alpha,
swiglu_beta=quant_info.swiglu_beta,
swiglu_limit=quant_info.swiglu_limit,
ep_size=quant_info.moe_ep_size,
ep_rank=quant_info.moe_ep_rank,
tp_size=quant_info.moe_tp_size,
tp_rank=quant_info.moe_tp_rank,
ep_size=runner_config.moe_ep_size,
ep_rank=runner_config.moe_ep_rank,
tp_size=runner_config.moe_tp_size,
tp_rank=runner_config.moe_tp_rank,
tune_max_num_tokens=next_power_of_2(x.shape[0]),
activation_type=_activation_type(runner_config),
enable_alltoall=enable_alltoall,
@@ -525,10 +515,10 @@ def _fused_experts_flashinfer_mxfp4_cutlass(
swiglu_alpha=quant_info.swiglu_alpha,
swiglu_beta=quant_info.swiglu_beta,
swiglu_limit=quant_info.swiglu_limit,
tp_size=quant_info.moe_tp_size,
tp_rank=quant_info.moe_tp_rank,
ep_size=quant_info.moe_ep_size,
ep_rank=quant_info.moe_ep_rank,
tp_size=runner_config.moe_tp_size,
tp_rank=runner_config.moe_tp_rank,
ep_size=runner_config.moe_ep_size,
ep_rank=runner_config.moe_ep_rank,
use_w4_group_scaling=not use_mxfp8_act_scaling,
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
activation_type=(
@@ -80,7 +80,6 @@ class HpcOpsMoeQuantInfo(MoeQuantInfo):
w2_weight: torch.Tensor
block_quant: bool
global_num_experts: int
moe_ep_rank: int
# Blockwise path
w13_weight_scale_inv: Optional[torch.Tensor] = None
w2_weight_scale_inv: Optional[torch.Tensor] = None
@@ -183,7 +182,7 @@ def fused_experts_none_to_hpc_ops(
quant_info.w2_weight_scale_inv,
topk_ids,
topk_weights,
quant_info.moe_ep_rank,
runner_config.moe_ep_rank,
quant_info.global_num_experts,
)
else:
@@ -198,7 +197,7 @@ def fused_experts_none_to_hpc_ops(
act_and_mul_scale,
topk_ids,
topk_weights,
quant_info.moe_ep_rank,
runner_config.moe_ep_rank,
quant_info.global_num_experts,
)
@@ -71,7 +71,6 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
assert not config.actorder, (
"Actorder is not supported by flashinfer_trtllm backend"
)
self.moe_ep_rank = get_parallel().moe_ep_rank
if self.quant_config.quant_format != CompressionFormat.pack_quantized.value:
raise ValueError(
@@ -348,7 +347,7 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
n_group=topk_config.num_expert_group,
topk_group=topk_config.topk_group,
intermediate_size=self.moe_runner_config.intermediate_size_per_partition,
local_expert_offset=self.moe_ep_rank * local_num_experts,
local_expert_offset=self.moe_runner_config.moe_ep_rank * local_num_experts,
local_num_experts=local_num_experts,
routed_scaling_factor=routed_scaling_factor,
routing_method_type=routing_method_type,
@@ -368,10 +368,6 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
apply_routed_scaling_factor=False,
)
return self.runner.run(dispatch_output, quant_info)
@@ -2701,7 +2701,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_weight=layer.w2_weight,
block_quant=True,
global_num_experts=int(layer.num_experts),
moe_ep_rank=int(layer.moe_ep_rank),
w13_weight_scale_inv=layer.hpc_ops_w13_weight_scale,
w2_weight_scale_inv=layer.hpc_ops_w2_weight_scale,
block_shape=self.quant_config.weight_block_size,
@@ -2712,7 +2711,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_weight=layer.w2_weight,
block_quant=False,
global_num_experts=int(layer.num_experts),
moe_ep_rank=int(layer.moe_ep_rank),
gate_up_alphas=layer.hpc_ops_gate_up_alphas,
down_alphas=layer.hpc_ops_down_alphas,
w13_input_scale=layer.w13_input_scale,
@@ -1426,10 +1426,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk,
)
return self.runner.run(dispatch_output, quant_info)
@@ -3253,10 +3249,6 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
apply_routed_scaling_factor=False,
)
return self.runner.run(dispatch_output, quant_info)
@@ -1482,10 +1482,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
swiglu_alpha=layer.swiglu_alpha,
swiglu_beta=layer.swiglu_beta,
swiglu_limit=layer.swiglu_limit,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
padded_hidden=self._padded_hidden,
)
return self.runner.run(dispatch_output, quant_info)
@@ -1529,10 +1525,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
swiglu_alpha=layer.swiglu_alpha,
swiglu_beta=layer.swiglu_beta,
swiglu_limit=layer.swiglu_limit,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
padded_hidden=self._padded_hidden,
)
return self.runner.run(dispatch_output, quant_info)
@@ -87,6 +87,9 @@ class Mxfp4FlashinferCutlassMoEMethod:
from sglang.srt.layers.moe.utils import MoeRunnerBackend
self.moe_runner_config = moe_runner_config
# The fused func reads the layer's MoE placement off the config.
if moe_runner_config.layer is None:
moe_runner_config.layer = layer
E = layer.num_local_experts
device = layer.w13_weight.device
@@ -249,10 +252,6 @@ class Mxfp4FlashinferCutlassMoEMethod:
swiglu_beta=None,
swiglu_limit=self._swiglu_limit_tensor,
use_swiglu_step=self._use_swiglu_step,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
padded_hidden=None,
)
return self.runner.run(dispatch_output, quant_info)
@@ -1105,10 +1105,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
moe_tp_rank=layer.moe_tp_rank,
apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk,
)
return self.runner.run(dispatch_output, quant_info)
-3
View File
@@ -1149,9 +1149,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
has_active_lora=has_active_lora,
experts_shared_outer_loras=self.experts_shared_outer_loras,
cg_buffers=cg_buffers,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
hidden_size=getattr(self.base_layer, "hidden_size", 0),
lora_use_virtual_experts=self.lora_use_virtual_experts,
)
-7
View File
@@ -75,8 +75,6 @@ class LoRAManager:
dtype: torch.dtype,
server_args: ServerArgs,
lora_backend: str = "triton",
tp_size: int = 1,
tp_rank: int = 0,
max_lora_rank: Optional[int] = None,
target_modules: Optional[Iterable[str]] = None,
lora_paths: Optional[List[LoRARef]] = None,
@@ -90,8 +88,6 @@ class LoRAManager:
self.load_config: LoadConfig = load_config
self.dtype: torch.dtype = dtype
self.device: torch.device = next(self.base_model.parameters()).device
self.tp_size: int = tp_size
self.tp_rank: int = tp_rank
# Attention projections shard on the attn-TP group; extracted once
# here (parallel groups are frozen after init_torch_distributed).
self.attn_tp_size: int = get_parallel().attn_tp_size
@@ -1004,9 +1000,6 @@ class LoRAManager:
base_hf_config=self.base_hf_config,
max_loras_per_batch=self.max_loras_per_batch,
dtype=self.dtype,
tp_size=self.tp_size,
tp_rank=self.tp_rank,
attn_tp_size=self.attn_tp_size,
max_lora_rank=self.max_lora_rank,
target_modules=self.target_modules,
base_model=self.base_model,
@@ -184,10 +184,6 @@ class LoRAInfo:
experts_shared_outer_loras: bool = False
cg_buffers: dict | None = None
fully_sharded: bool = False
tp_size: int = 1
tp_rank: int = 0
hidden_size: int = 0
lora_use_virtual_experts: bool = False
@@ -391,7 +387,6 @@ def _add_lora_gate_up_delta(
expand_num_warps=4,
expand_num_stages=2,
expand_split_k=1,
fully_sharded=lora_info.fully_sharded,
)
@@ -424,12 +419,6 @@ def _add_lora_down_delta(
if lora_info.experts_shared_outer_loras and not lora_info.lora_use_virtual_experts:
down_lora_b = down_lora_b.expand(-1, lora_info.num_experts, -1, -1)
if lora_info.fully_sharded and lora_info.tp_size > 1:
shard_size = lora_info.hidden_size // lora_info.tp_size
offset = shard_size * lora_info.tp_rank
else:
offset = 0
if lora_info.lora_use_virtual_experts:
merged_experts_fused_moe_lora_add(
output=intermediate_cache,
@@ -474,8 +463,6 @@ def _add_lora_down_delta(
expand_num_stages=2,
expand_split_k=1,
mul_routed_weight=True,
fully_sharded=lora_info.fully_sharded,
offset=offset,
)
+5 -7
View File
@@ -138,9 +138,6 @@ class LoRAMemoryPool:
base_hf_config: AutoConfig,
max_loras_per_batch: int,
dtype: torch.dtype,
tp_size: int,
tp_rank: int,
attn_tp_size: int,
max_lora_rank: int,
target_modules: Set[str],
base_model: torch.nn.Module,
@@ -154,8 +151,9 @@ class LoRAMemoryPool:
self.num_layer: int = base_hf_config.num_hidden_layers
self.max_loras_per_batch: int = max_loras_per_batch
self.dtype: torch.dtype = dtype
self.tp_size: int = tp_size
self.tp_rank: int = tp_rank
parallel = get_parallel()
self.tp_size: int = parallel.tp_size
self.tp_rank: int = parallel.tp_rank
self.lora_added_tokens_size: int = lora_added_tokens_size
self.max_lora_rank: int = max_lora_rank
self.target_modules: Set[str] = target_modules
@@ -193,7 +191,7 @@ class LoRAMemoryPool:
# under attention DP is `attn_tp_size = tp_size // attn_dp_size`.
# The corresponding LoRA wrappers slice weights by the base layer's
# attn_tp-local rank, so the buffer shapes must match that shard.
self.attn_tp_size: int = attn_tp_size
self.attn_tp_size: int = parallel.attn_tp_size
# Initialize eviction policy
self.eviction_policy = get_eviction_policy(eviction_policy)
@@ -227,7 +225,7 @@ class LoRAMemoryPool:
# Cache lm_head shard_indices from the base model so that buffer
# allocation uses the same sharding as the base ParallelLMHead layer.
self.lm_head_shard_indices = None
if "lm_head" in target_modules and tp_size > 1:
if "lm_head" in target_modules and self.tp_size > 1:
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
for _, module in base_model.named_modules():
+14 -16
View File
@@ -705,17 +705,15 @@ class HiCacheController:
if storage_backend_extra_config is None:
storage_backend_extra_config = {}
parallel = get_parallel()
if is_dp_attention_enabled():
self.tp_rank = get_parallel().attn_tp_rank
self.tp_size = get_parallel().attn_tp_size
self.dp_rank = get_parallel().attn_dp_rank
tp_rank = parallel.attn_tp_rank
tp_size = parallel.attn_tp_size
dp_rank = parallel.attn_dp_rank
else:
self.tp_rank = get_parallel().tp_rank
self.tp_size = get_parallel().tp_size
self.dp_rank = 0
self.pp_rank = get_parallel().pp_rank
self.pp_size = get_parallel().pp_size
tp_rank = parallel.tp_rank
tp_size = parallel.tp_size
dp_rank = 0
# Currently, NPUMLATokenToKVPool is the subclass of MLATokenToKVPool.
# DeepSeekV4TokenToKVPool has compressed MLA-style rank-replicated cache
@@ -732,23 +730,23 @@ class HiCacheController:
should_split_heads = False
if tp_lcm_size:
assert tp_lcm_size % self.tp_size == 0, (
assert tp_lcm_size % tp_size == 0, (
"tp_lcm_size must be divisible by tp_size."
)
should_split_heads = (
not is_rank_replicated
and self.mem_pool_host.layout == "page_head"
and tp_lcm_size > self.tp_size
and tp_lcm_size > tp_size
)
attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
model_name = storage_model_name(model_name, self.storage_host_pool)
return HiCacheStorageConfig(
tp_rank=self.tp_rank,
tp_size=self.tp_size,
pp_rank=self.pp_rank,
pp_size=self.pp_size,
tp_rank=tp_rank,
tp_size=tp_size,
pp_rank=parallel.pp_rank,
pp_size=parallel.pp_size,
attn_cp_rank=attn_cp_rank,
attn_cp_size=attn_cp_size,
# TODO(hzh): Rename is_mla_model to is_rank_replicated.
@@ -758,7 +756,7 @@ class HiCacheController:
model_name=model_name,
tp_lcm_size=tp_lcm_size,
should_split_heads=should_split_heads,
dp_rank=self.dp_rank,
dp_rank=dp_rank,
extra_config=storage_backend_extra_config,
)
@@ -76,12 +76,9 @@ class _NegotiateOutput(NamedTuple):
class PrefillDelayer:
def __init__(
self,
cpu_group,
max_delay_passes: int,
token_usage_low_watermark: Optional[float],
metrics_collector: Optional["SchedulerMetricsCollector"] = None,
device: Optional["torch.device"] = "cpu",
device_group=None,
debug_log_enabled: bool = True,
):
self._max_delay_passes = max_delay_passes
@@ -117,14 +114,12 @@ class PrefillDelayer:
get_schedule().disable_overlap_schedule
or envs.SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH.get()
)
tp_group = parallel.tp_group
if use_nccl:
assert device_group is not None, (
"device_group is required when using NCCL for PrefillDelayer all-gather"
)
self._gather_group = device_group
self._gather_device = device
self._gather_group = tp_group.device_group
self._gather_device = tp_group.device
else:
self._gather_group = cpu_group
self._gather_group = tp_group.cpu_group
self._gather_device = "cpu"
# Fields packed per rank into the all-gather tensor: prefillable,
+11 -36
View File
@@ -109,6 +109,7 @@ from sglang.srt.dllm.mixin.scheduler import SchedulerDllmMixin
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.layers.moe import initialize_moe_config
from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config
from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config
@@ -568,9 +569,6 @@ class Scheduler(
tp_worker=self.tp_worker,
page_size=self.page_size,
spec_algorithm=self.spec_algorithm,
attn_tp_cpu_group=self.attn_tp_cpu_group,
tp_cpu_group=self.tp_cpu_group,
attn_cp_cpu_group=self.attn_cp_cpu_group,
enable_metrics=get_observability().enable_metrics,
enable_kv_cache_events=bool(
get_observability().kv_events_config
@@ -578,8 +576,6 @@ class Scheduler(
and get_parallel().attn_tp_rank == 0
and get_parallel().attn_cp_rank == 0
),
tp_group=self.tp_group,
pp_group=self.pp_group,
enable_hierarchical_cache=self.enable_hierarchical_cache,
hicache_draft_plan=(
self.draft_worker.hicache_draft_plan
@@ -626,11 +622,7 @@ class Scheduler(
self.decode_offload_manager = DecodeKVCacheOffloadManager(
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
tp_group=(
self.attn_tp_cpu_group
if self.attn_dp_enabled
else self.tp_cpu_group
),
tp_group=self.dp_tp_cpu_group,
tree_cache=self.tree_cache,
)
else:
@@ -747,16 +739,16 @@ class Scheduler(
):
return
parallel = get_parallel()
tp_group = parallel.tp_group
rank = (
get_parallel().dp_rank
if get_parallel().dp_rank is not None
else self.tp_group.rank_in_group
parallel.dp_rank if parallel.dp_rank is not None else tp_group.rank_in_group
)
logger.info("HCCL DP prewarm start: rank=%s", rank)
_prewarm_hccl_group(
device=self.tp_group.device,
group=self.tp_group.device_group,
device_module=self.tp_group.device_module,
device=tp_group.device,
group=tp_group.device_group,
device_module=tp_group.device_module,
)
logger.info("HCCL DP prewarm done: rank=%s", rank)
@@ -1213,11 +1205,9 @@ class Scheduler(
self.world_group = get_parallel().world_group
# NOTE: dp_tp_* are request/data-plane coordination groups (not tensor collectives).
# When DP attention is enabled, scope to the attention-TP group; otherwise use
# the base TP group. Entry rank is the local rank 0 in that group.
# Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA
# stream/device coupling (#11910).
self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group
# Entry rank is the local rank 0 in that group. Use the CPU (gloo) group to
# broadcast VLM Python objects and avoid CUDA stream/device coupling (#11910).
self.dp_tp_group = get_dp_tp_group()
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
@@ -1408,8 +1398,6 @@ class Scheduler(
)
else:
self.prefill_delayer = PrefillDelayer(
cpu_group=self.tp_cpu_group,
device_group=self.tp_group.device_group,
metrics_collector=(
self.metrics_collector
if self.metrics_reporter.enable_metrics
@@ -1417,7 +1405,6 @@ class Scheduler(
),
max_delay_passes=get_schedule().prefill_delayer_max_delay_passes,
token_usage_low_watermark=get_schedule().prefill_delayer_token_usage_low_watermark,
device=self.tp_group.device,
debug_log_enabled=get_parallel().attn_tp_rank == 0,
)
@@ -1536,7 +1523,6 @@ class Scheduler(
# The decode requests polling kv cache
self.disagg_decode_transfer_queue = DecodeTransferQueue(
gloo_group=self.attn_tp_cpu_group,
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
metadata_buffers=self.disagg_metadata_buffers,
scheduler=self,
@@ -1553,7 +1539,6 @@ class Scheduler(
scheduler=self,
transfer_queue=self.disagg_decode_transfer_queue,
tree_cache=self.tree_cache,
gloo_group=self.attn_tp_cpu_group,
gpu_id=get_device().gpu_id,
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
max_total_num_tokens=self.max_total_num_tokens,
@@ -1613,9 +1598,6 @@ class Scheduler(
self.server_args,
dtype=self.model_config.dtype,
hf_config=self.model_config.hf_config,
pp_rank=get_parallel().pp_rank,
tp_rank=get_parallel().tp_rank,
tp_group=self.tp_group,
scheduler=self,
)
@@ -2242,7 +2224,6 @@ class Scheduler(
def init_profiler(self) -> None:
self.profiler_manager = SchedulerProfilerManager(
dp_tp_cpu_group=self.dp_tp_cpu_group,
get_forward_ct=lambda: self.forward_ct,
)
@@ -2250,7 +2231,6 @@ class Scheduler(
self.weight_updater = SchedulerWeightUpdaterManager(
tp_worker=self.tp_worker,
draft_worker=self.draft_worker,
tp_cpu_group=self.tp_cpu_group,
memory_saver_adapter=self.memory_saver_adapter,
flush_cache=self.flush_cache,
is_fully_idle=self.is_fully_idle,
@@ -2333,11 +2313,6 @@ class Scheduler(
recv_skipper=self.recv_skipper,
input_blocker=self.input_blocker,
mm_receiver=self.mm_receiver,
tp_group=self.tp_group,
tp_cpu_group=self.tp_cpu_group,
attn_tp_cpu_group=self.attn_tp_cpu_group,
attn_cp_cpu_group=self.attn_cp_cpu_group,
world_group=self.world_group,
server_args=self.server_args,
model_config=self.model_config,
max_recv_per_poll=self.max_recv_per_poll,
@@ -4,7 +4,7 @@ import gc
import logging
import os
import time
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
from typing import (
TYPE_CHECKING,
@@ -17,6 +17,7 @@ from typing import (
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.managers.io_struct import ProfileReq, ProfileReqOutput, ProfileReqType
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.step_span_utils import set_detailed_annotations_enabled
@@ -51,10 +52,13 @@ logger = logging.getLogger(__name__)
@dataclass(kw_only=True)
class SchedulerProfilerManager:
dp_tp_cpu_group: Any
get_forward_ct: Callable[[], int]
# Taken at construction: a batch can stop profiling while a scoped
# override has moved the TP group.
dp_tp_cpu_group: Any = field(init=False)
def __post_init__(self) -> None:
self.dp_tp_cpu_group = get_dp_tp_group().cpu_group
if envs.SGLANG_PROFILE_V2.get():
self._profile_manager = ProfileManager(
cpu_group=self.dp_tp_cpu_group,
@@ -63,11 +63,6 @@ class SchedulerRequestReceiver:
recv_skipper: Any
input_blocker: Any
mm_receiver: Any
tp_group: Any
tp_cpu_group: Any
attn_tp_cpu_group: Any
attn_cp_cpu_group: Any
world_group: Any
server_args: ServerArgs
model_config: ModelConfig
max_recv_per_poll: int
@@ -157,7 +152,7 @@ class SchedulerRequestReceiver:
recv_reqs = point_to_point_pyobj(
[],
get_parallel().pp_rank * get_parallel().tp_size + dp_offset,
self.world_group.cpu_group,
get_parallel().world_group.cpu_group,
(get_parallel().pp_rank - 1) * get_parallel().tp_size + dp_offset,
get_parallel().pp_rank * get_parallel().tp_size + dp_offset,
)
@@ -195,22 +190,24 @@ class SchedulerRequestReceiver:
if _local_ctrl:
control_reqs = attn_cp_tp_broadcast_pyobj(control_reqs)
elif get_parallel().tp_size != 1:
tp_group = get_parallel().tp_group
control_reqs = broadcast_pyobj(
control_reqs,
self.tp_group.rank,
self.tp_cpu_group,
src=self.tp_group.ranks[0],
tp_group.rank,
tp_group.cpu_group,
src=tp_group.ranks[0],
)
recv_reqs = work_reqs + control_reqs
else:
if recv_reqs is not None:
recv_reqs = [*recv_reqs, *local_reqs]
if get_parallel().tp_size != 1:
tp_group = get_parallel().tp_group
recv_reqs = broadcast_pyobj(
recv_reqs,
self.tp_group.rank,
self.tp_cpu_group,
src=self.tp_group.ranks[0],
tp_group.rank,
tp_group.cpu_group,
src=tp_group.ranks[0],
)
return recv_reqs
@@ -268,11 +265,11 @@ class SchedulerRequestReceiver:
parallel = get_parallel()
if parallel.attn_dp_enabled:
if parallel.attn_tp_size > 1:
barrier(group=self.attn_tp_cpu_group)
barrier(group=parallel.attn_tp_group.cpu_group)
if parallel.attn_cp_size > 1:
barrier(group=self.attn_cp_cpu_group)
barrier(group=parallel.attn_cp_group.cpu_group)
elif parallel.tp_size > 1:
barrier(group=self.tp_cpu_group)
barrier(group=parallel.tp_group.cpu_group)
# 2. materialize independently so one bad VLM request does not stop the loop
failed = torch.zeros(len(tokenized_reqs), dtype=torch.int32)
@@ -292,11 +289,15 @@ class SchedulerRequestReceiver:
# 3. all ranks reject the same requests before entering model collectives
if parallel.attn_dp_enabled:
if parallel.attn_tp_size > 1:
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group)
all_reduce(
failed, op=ReduceOp.MAX, group=parallel.attn_tp_group.cpu_group
)
if parallel.attn_cp_size > 1:
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_cp_cpu_group)
all_reduce(
failed, op=ReduceOp.MAX, group=parallel.attn_cp_group.cpu_group
)
elif parallel.tp_size > 1:
all_reduce(failed, op=ReduceOp.MAX, group=self.tp_cpu_group)
all_reduce(failed, op=ReduceOp.MAX, group=parallel.tp_group.cpu_group)
error = MMInputsProcessError(
"Failed to materialize shared-memory multimodal features on a scheduler rank."
@@ -44,7 +44,7 @@ from sglang.srt.managers.io_struct import (
UpdateWeightsFromTensorReqInput,
UpdateWeightsFromTensorReqOutput,
)
from sglang.srt.runtime_context import get_model
from sglang.srt.runtime_context import get_model, get_parallel
from sglang.srt.utils.weight_checker import overall_checksum
logger = logging.getLogger(__name__)
@@ -88,7 +88,6 @@ class _WeightUpdateSession(msgspec.Struct, frozen=True):
class SchedulerWeightUpdaterManager:
tp_worker: Any
draft_worker: Any
tp_cpu_group: Any
memory_saver_adapter: Any
flush_cache: Callable[..., bool]
is_fully_idle: Callable[..., bool]
@@ -98,6 +97,10 @@ class SchedulerWeightUpdaterManager:
stashed_model_static_state: Any = None
# replicated on every TP rank, so a rejected call returns on all ranks before any barrier
_session: Optional[_WeightUpdateSession] = None
tp_cpu_group: Any = field(init=False)
def __post_init__(self) -> None:
self.tp_cpu_group = get_parallel().tp_group.cpu_group
@contextmanager
def _observe_weight_load(self, source: str) -> Iterator[None]:
@@ -865,7 +865,7 @@ class HybridCacheController(BaseHiCacheController):
def get_prefetch_submission(self, rid: str) -> Optional[PrefetchSubmission]:
if self.pp_prefetch_command_group is None:
return None
if self.pp_rank != 0:
if self.storage_config.pp_rank != 0:
return PrefetchSubmission()
with self.pp_prefetch_state_lock:
state = self.pp_prefetch_states.get(rid)
@@ -898,7 +898,7 @@ class HybridCacheController(BaseHiCacheController):
)
)
if self.pp_rank != 0:
if self.storage_config.pp_rank != 0:
raise RuntimeError("Only PP0 can submit a PP prefetch ticket.")
rid = handle.rid
@@ -928,7 +928,8 @@ class HybridCacheController(BaseHiCacheController):
storage_hit_count = len(ticket.prefetch_key.token_ids)
try:
for pp_rank in range(self.tp_rank, self.pp_size, self.tp_size):
config = self.storage_config
for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size):
_, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank)
storage_hit_count = min(storage_hit_count, rank_hit_count)
except Exception:
@@ -1096,7 +1097,7 @@ class HybridCacheController(BaseHiCacheController):
assert group is not None
rank = torch.distributed.get_rank()
source = torch.distributed.get_process_group_ranks(group)[0]
is_source = self.pp_rank == 0
is_source = self.storage_config.pp_rank == 0
while True:
objects = []
@@ -2,6 +2,7 @@ from __future__ import annotations
import logging
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.runtime_context import get_exec
logger = logging.getLogger(__name__)
@@ -58,10 +59,7 @@ from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
from sglang.srt.utils import ceil_align, is_hip
if TYPE_CHECKING:
from torch.distributed import ProcessGroup
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.managers.tp_worker import BaseTpWorker
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
@@ -266,13 +264,8 @@ def build_kv_cache(
tp_worker: BaseTpWorker,
page_size: int,
spec_algorithm: SpeculativeAlgorithm,
attn_tp_cpu_group: ProcessGroup,
tp_cpu_group: ProcessGroup,
attn_cp_cpu_group: ProcessGroup,
enable_metrics: bool,
enable_kv_cache_events: bool,
tp_group: GroupCoordinator,
pp_group: GroupCoordinator,
enable_hierarchical_cache: bool,
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
) -> KVCacheBuildResult:
@@ -358,12 +351,10 @@ def build_kv_cache(
else token_to_kv_pool_allocator.page_size
),
is_eagle=spec_algorithm.is_eagle(),
tp_cache_group=(
attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group
),
attn_cp_cache_group=attn_cp_cpu_group,
attn_tp_cache_group=attn_tp_cpu_group,
pp_cache_group=pp_group.cpu_group,
tp_cache_group=get_dp_tp_group().cpu_group,
attn_cp_cache_group=parallel.attn_cp_group.cpu_group,
attn_tp_cache_group=parallel.attn_tp_group.cpu_group,
pp_cache_group=parallel.pp_group.cpu_group,
eviction_policy=get_memory().radix_eviction_policy,
eviction_policy_config=get_memory().radix_eviction_policy_config,
enable_metrics=enable_metrics,
@@ -394,7 +385,7 @@ def build_kv_cache(
model_config=model_config,
tp_size=parallel.tp_size,
tp_rank=parallel.tp_rank,
tp_group=tp_group,
tp_group=parallel.tp_group,
)
with auto_size_hicache(
params,
@@ -123,6 +123,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
extra_config, *_ = HybridCacheController.parse_storage_backend_extra_config(
get_memory().hicache_storage_backend_extra_config
)
# TODO: unlike `HiCacheController._generate_storage_config`, this sets no
# `dp_rank`. Aligning the storage-config producers changes persisted
# storage keys, so it waits for a storage-key format change.
storage_config = HiCacheStorageConfig(
tp_rank=tp_rank,
tp_size=tp_size,
@@ -295,6 +295,11 @@ class UMBPDirectLinker(UnifiedCacheLinker):
)
extra_config["dram_page_size"] = dram_page_size
# TODO: `tp_rank` is the rank in the cache TP group but `tp_size` is the
# full TP width, and no `dp_rank` is set, unlike
# `HiCacheController._generate_storage_config`. Aligning the
# storage-config producers changes persisted storage keys, so it waits
# for a storage-key format change.
storage_config = HiCacheStorageConfig(
tp_rank=tp_rank,
tp_size=get_parallel().tp_size,
@@ -276,13 +276,14 @@ class StorageAttachment:
"""
cache = self._cache
controller = cache.cache_controller
config = controller.storage_config
attn_cp_rank, attn_cp_size = controller.get_attn_cp_rank_and_size()
labels = {
"storage_backend": storage_backend,
"tp_rank": controller.tp_rank,
"dp_rank": controller.dp_rank,
"pp_rank": controller.pp_rank,
"pp_size": controller.pp_size,
"tp_rank": config.tp_rank,
"dp_rank": config.dp_rank,
"pp_rank": config.pp_rank,
"pp_size": config.pp_size,
"attn_cp_rank": attn_cp_rank,
"attn_cp_size": attn_cp_size,
}
@@ -76,6 +76,7 @@ from sglang.srt.layers.cp.utils import (
is_cp_active,
is_mla_cp_enabled,
)
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.sampler import create_sampler
from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers
@@ -963,11 +964,7 @@ class ModelRunner:
top_k=hisparse_top_k,
device_buffer_size=hisparse_cfg.device_buffer_size,
device=self.device,
tp_group=(
get_parallel().attn_tp_group.cpu_group
if get_parallel().attn_dp_enabled
else get_parallel().tp_group.cpu_group
),
tp_group=get_dp_tp_group().cpu_group,
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
swap_in_block_size=hisparse_cfg.swap_in_block_size,
shared_index_layers=resolve_shared_index_layers(
@@ -1368,8 +1365,6 @@ class ModelRunner:
dtype=self.dtype,
server_args=self.server_args,
lora_backend=get_lora().lora_backend,
tp_size=get_parallel().tp_size,
tp_rank=get_parallel().tp_rank,
max_lora_rank=get_lora().max_lora_rank,
target_modules=get_lora().lora_target_modules,
lora_paths=get_lora().lora_paths,
+10
View File
@@ -12,6 +12,7 @@ from transformers import CLIPConfig, CLIPTextConfig, CLIPVisionConfig
from sglang.srt.layers.activation import QuickGELU, get_act_fn
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
from sglang.srt.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -200,6 +201,15 @@ class CLIPAttention(nn.Module):
tp_rank=parallel.attn_tp_rank,
tp_size=parallel.attn_tp_size,
)
# TODO: this layer shards over attention TP but reduces over the full TP
# group; reduce over the attention-TP group so attention DP and attention
# CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=parallel.attn_tp_size,
reduces_over_attn_tp=False,
multimodal_encoder=True,
)
self.proj = RowParallelLinear(
input_size=config.hidden_size,
output_size=config.hidden_size,
+1 -3
View File
@@ -29,9 +29,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
@@ -209,6 +209,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin):
gemm1_alpha=None,
gemm1_clamp_limit=None,
is_gated=True,
layer=self,
)
FusedMoELoadingMixin.__init__(
+9
View File
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
from triton.language.extra import libdevice
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
@@ -313,6 +314,14 @@ class IQuestQ1Attention(nn.Module):
tp_size=attn_tp_size,
prefix=add_prefix("qkv_proj", prefix),
)
# The decoder boundary reduces partial attention outputs. Only a
# standalone attention that owns its TP reduction needs this guard.
if reduce_results:
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=attn_tp_size,
reduces_over_attn_tp=False,
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
config.hidden_size,
@@ -59,6 +59,7 @@ from sglang.srt.layers.attention.vision import (
prepare_vision_attention_metadata,
)
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
from sglang.srt.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
@@ -357,6 +358,16 @@ class MLP2(nn.Module):
prefix=add_prefix("fc1", prefix),
)
elif use_tensor_parallel:
# TODO: these layers shard over attention TP but reduce over the full
# TP group; reduce over the attention-TP group so attention DP and
# attention CP narrower than TP can run them.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=tp_size,
reduces_over_attn_tp=False,
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.fc0 = ColumnParallelLinear(
dims[0],
dims[1],
+6
View File
@@ -514,6 +514,12 @@ class AudioEncoderAttention(nn.Module):
window_size=window_size,
customized_position_embedding_applier=_audio_rope_applier,
prefix="attn",
# TODO: the audio encoder does not follow --mm-enable-dp-encoder and
# reduces over the full TP group, so its output is wrong under
# attention DP narrower than TP. Rejecting that would reject the
# documented MiMo-V2.5 `--tp 8 --attn-dp-size 2` launch; fix the
# reduce group or run the encoder data-parallel instead.
allow_tp_reduce_mismatch=True,
)
def forward(
+34 -1
View File
@@ -17,7 +17,10 @@ from sglang.srt.layers.attention.vision import (
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
@@ -83,6 +86,16 @@ class MiniMaxVLMultiModalProjector(nn.Module):
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=tp_size,
reduces_over_attn_tp=is_dp_attention_enabled(),
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.linear_1 = ColumnParallelLinear(
vision_hidden_size,
@@ -138,6 +151,16 @@ class MiniMaxVLPatchMerger(nn.Module):
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=tp_size,
reduces_over_attn_tp=is_dp_attention_enabled(),
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.linear_1 = ColumnParallelLinear(
text_hidden_size * spatial_merge_size**2,
@@ -260,6 +283,16 @@ class CLIPEncoderLayer(nn.Module):
self.use_data_parallel = use_data_parallel
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=tp_size,
reduces_over_attn_tp=is_dp_attention_enabled(),
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.self_attn = VisionAttention(
embed_dim=config.hidden_size,
+7
View File
@@ -4,6 +4,7 @@ import torch
from torch import nn
from transformers.configuration_utils import PretrainedConfig
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
from sglang.srt.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
@@ -282,6 +283,12 @@ class PhiMoEAttention(nn.Module):
tp_size=attn_tp_size,
prefix=add_prefix("qkv_proj", prefix),
)
# TODO: this layer shards over attention TP but reduces over the full TP
# group; reduce over the attention-TP group so attention DP and attention
# CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__, shard_tp_size=attn_tp_size, reduces_over_attn_tp=False
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
hidden_size,
+21
View File
@@ -41,6 +41,7 @@ from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.conv import Conv3dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -148,6 +149,16 @@ class Qwen3_VisionMLP(nn.Module):
tp_size=tp_size,
tp_rank=tp_rank,
)
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=self.tp_size,
reduces_over_attn_tp=is_dp_attention_enabled(),
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.linear_fc1 = ColumnParallelLinear(
in_features,
hidden_features,
@@ -317,6 +328,16 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
tp_size=tp_size,
tp_rank=tp_rank,
)
# TODO: this layer shards over attention TP but reduces over the full TP
# group without attention DP; reduce over the attention-TP group so
# attention CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=self.tp_size,
reduces_over_attn_tp=is_dp_attention_enabled(),
multimodal_encoder=True,
hint=", or --mm-enable-dp-encoder where the model supports it",
)
self.linear_fc1 = ColumnParallelLinear(
self.hidden_size,
self.padded_context_dim,
+10
View File
@@ -22,6 +22,7 @@ from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.layers.layer_boundary import (
declare_attn,
@@ -545,6 +546,15 @@ class Step3VisionMLP(nn.Module):
prefix=add_prefix("gate_proj", prefix),
)
self.act = ACT2FN[hidden_act] # quick_gelu
# TODO: this layer shards over attention TP but reduces over the full TP
# group; reduce over the attention-TP group so attention DP and attention
# CP narrower than TP can run it.
reject_attn_tp_shard_with_tp_reduce(
type(self).__name__,
shard_tp_size=attn_tp_size,
reduces_over_attn_tp=False,
multimodal_encoder=True,
)
self.fc2 = RowParallelLinear(
intermediate_size,
dim,
+14 -3
View File
@@ -263,12 +263,23 @@ class ServerArgs:
return getattr(self, "_launch_command", None)
def resolved_dict(self) -> dict[str, Any]:
"""Serialize resolved field values, expanding nested records and excluding bookkeeping."""
"""Serialize resolved field values, expanding nested records and excluding bookkeeping.
return {
One exception: the deprecated `enable_dp_attention` reports whether an
attention-DP width was configured, for clients that still read it from
`/server_info`. Resolution leaves the field false. It reports the width
rather than `attn_dp_enabled`, whose elastic scale-joiner arm is true at
width one: a dump carrying that would fold the joiner's replicas into
attention-DP groups when it is read back.
"""
resolved = {
field.name: _plain(resolution_result(self, field.name))
for field in record_fields(type(self))
}
# TODO: drop together with `--enable-dp-attention` after 2026-12-31.
resolved["enable_dp_attention"] = resolving_view(self).attn_dp_size > 1
return resolved
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
"MuseGlimmerForConditionalGeneration",
@@ -351,7 +362,7 @@ class ServerArgs:
help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.",
)
# `enable_dp_attention` is `no_cli=True` too; resolution turns it into
# `attn_dp_size`.
# `attn_dp_size`. TODO: remove the flag and the field after 2026-12-31.
parser.add_argument(
"--enable-dp-attention",
action=DeprecatedStoreTrueAction,
@@ -19,6 +19,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
)
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
from sglang.srt.lora.layers import unwrap_lora_layer
@@ -402,11 +403,7 @@ class DFlashWorkerV2(BaseSpecWorker):
self._full_embed_gpu: Optional[torch.Tensor] = None
# Under dp attention, peer DP ranks run different (idle) paths, so
# spec broadcasts must stay within the attn-TP group.
self._tp_sync = SpecTpSync(
get_parallel().attn_tp_group
if get_parallel().attn_dp_enabled
else get_parallel().tp_group
)
self._tp_sync = SpecTpSync(get_dp_tp_group())
# Under dp attention, the draft worker runs on the per-DP attn-TP
# group, independent of idle peer DP ranks; it is built and run under
@@ -8,6 +8,7 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
)
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.schedule_batch import ScheduleBatch
@@ -233,9 +234,7 @@ class DSparkWorkerV2(BaseSpecWorker):
self._mask_token_id = runtime_config.mask_token_id
parallel = get_parallel()
self._tp_sync = SpecTpSync(
parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
)
self._tp_sync = SpecTpSync(get_dp_tp_group())
self._draft_graph_group = (
parallel.attn_tp_group
if self._draft_dp_context_enabled
+1 -3
View File
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
from sglang.srt.lora.lora_manager import LoRAManager
from sglang.srt.lora.lora_registry import LoRARef
from sglang.srt.runtime_context import get_parallel, get_spec
from sglang.srt.runtime_context import get_spec
if TYPE_CHECKING:
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -45,8 +45,6 @@ def init_uno_lora_manager(
dtype=model_runner.dtype,
server_args=model_runner.server_args,
lora_backend="uno_cublas", # fast path
tp_size=get_parallel().tp_size,
tp_rank=get_parallel().tp_rank,
# Infer these from the one trained adapter.
max_lora_rank=None,
target_modules=None,
+5 -2
View File
@@ -7,6 +7,7 @@ is not installed or the GPU is not SM90 (the kernels ship sm90a only).
"""
import unittest
from types import SimpleNamespace
import torch
@@ -91,6 +92,10 @@ class TestHpcOpsMoeBlockwise(CustomTestCase):
activation="silu",
is_gated=True,
inplace=False,
# The fused func reads the layer's MoE placement off the config.
layer=SimpleNamespace(
moe_tp_size=1, moe_tp_rank=0, moe_ep_size=1, moe_ep_rank=0
),
)
def _run_hpc_ops(self, x, w13, w2, w13_scale, w2_scale, topk_weights, topk_ids):
@@ -104,7 +109,6 @@ class TestHpcOpsMoeBlockwise(CustomTestCase):
w2_weight=w2,
block_quant=True,
global_num_experts=E,
moe_ep_rank=0,
w13_weight_scale_inv=pad_hpc_ops_block_scale(w13_scale),
w2_weight_scale_inv=pad_hpc_ops_block_scale(w2_scale),
block_shape=[128, 128],
@@ -175,7 +179,6 @@ class TestHpcOpsMoeBlockwise(CustomTestCase):
w2_weight=w2,
block_quant=False,
global_num_experts=E,
moe_ep_rank=0,
gate_up_alphas=w13_scale * a1_scale,
down_alphas=w2_scale * a2_scale,
w13_input_scale=a1_scale,
@@ -244,7 +244,7 @@ class TestDcpCachedPrefixSend(CustomTestCase):
}
mgr.add_transfer_request = Mock()
with get_context().override_server_args(dp_size=1):
sender = NixlKVSender(mgr, "unused", 1, [0], 0)
sender = NixlKVSender(mgr, "unused", 1)
sender.init((total - prefix + page_size - 1) // page_size, 3)
req = SimpleNamespace(
rid="cached-prefix",
@@ -33,8 +33,6 @@ class TestFakeKVSender(unittest.TestCase):
mgr=self.mgr if mgr is None else mgr,
bootstrap_addr="fake_addr:1234",
bootstrap_room=42,
dest_tp_ranks=[0],
pp_rank=0,
)
def _expire_deadline(self, sender: FakeKVSender, by: float = 1.0) -> None:
@@ -18,6 +18,7 @@ import platform
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.runtime_context import get_parallel
@@ -164,7 +165,10 @@ class TestSchedulerProfilerManagerMPS(unittest.TestCase):
SchedulerProfilerManager,
)
mgr = SchedulerProfilerManager(dp_tp_cpu_group=None, get_forward_ct=lambda: 0)
with get_parallel().override(
tp_group=SimpleNamespace(cpu_group=None), attn_dp_enabled=False
):
mgr = SchedulerProfilerManager(get_forward_ct=lambda: 0)
mgr._init_profile(output_dir, None, None, None, None, None, False, "test")
return mgr
@@ -0,0 +1,147 @@
"""A MoE layer owns its placement, and the fused funcs read it through the
layer's `MoeRunnerConfig`."""
import sys
from types import SimpleNamespace
import pytest
import torch
from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend
from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.moe_runner import flashinfer_cutlass
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
FlashInferCutlassMoeQuantInfo,
)
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="stage-b-test-cpu-intel")
def _layer(monkeypatch) -> FusedMoE:
method = UnquantizedFusedMoEMethod()
monkeypatch.setattr(method, "create_weights", lambda **kwargs: None)
monkeypatch.setattr(method, "create_moe_runner", lambda layer, config: None)
monkeypatch.setattr(
fused_moe_layer_module,
"create_moe_dispatcher",
lambda config, quant_method: SimpleNamespace(),
)
with (
get_context().override_server_args(model_path="dummy"),
get_flags().moe.override(
runner_backend=MoeRunnerBackend.AUTO,
a2a_backend=MoeA2ABackend.NONE,
),
get_parallel().override(
moe_ep_size=2,
moe_ep_rank=1,
moe_tp_size=2,
moe_tp_rank=1,
tp_size=4,
tp_rank=3,
attn_tp_size=4,
attn_tp_rank=3,
),
):
return FusedMoE(
num_experts=4,
hidden_size=4,
intermediate_size=8,
layer_id=0,
quant_method=method,
)
def _placement(obj):
return (obj.moe_tp_size, obj.moe_tp_rank, obj.moe_ep_size, obj.moe_ep_rank)
def test_a_config_without_a_layer_has_no_placement() -> None:
config = MoeRunnerConfig()
for name in ("moe_tp_size", "moe_tp_rank", "moe_ep_size", "moe_ep_rank"):
with pytest.raises(AttributeError, match="has no layer"):
getattr(config, name)
def test_the_config_reads_the_layer_placement(monkeypatch) -> None:
layer = _layer(monkeypatch)
assert _placement(layer) == (2, 1, 2, 1)
assert _placement(layer.moe_runner_config) == _placement(layer)
def test_a_full_expert_bind_collapses_ep_on_the_config(monkeypatch) -> None:
layer = _layer(monkeypatch)
layer.bind_full_expert_weights({})
assert _placement(layer) == (2, 1, 1, 0)
assert _placement(layer.moe_runner_config) == (2, 1, 1, 0)
assert layer.moe_runner_config.num_local_experts == 4
def test_the_kernel_gets_the_bound_placement(monkeypatch) -> None:
"""The reason the placement lives on the layer: after a full-expert bind
the kernel must be told this rank owns every expert."""
layer = _layer(monkeypatch)
before = _kernel_placement(monkeypatch, layer.moe_runner_config)
assert before == (2, 1, 2, 1)
layer.bind_full_expert_weights({})
# The layer kept its TP shard and now owns every expert.
assert _kernel_placement(monkeypatch, layer.moe_runner_config) == (2, 1, 1, 0)
def _kernel_placement(monkeypatch, runner_config):
"""Run the CUTLASS fused func against a fake kernel and report the
(tp_size, tp_rank, ep_size, ep_rank) it was handed."""
calls = []
def fake_kernel(**kwargs):
calls.append(kwargs)
return [kwargs["output"]]
monkeypatch.setattr(
flashinfer_cutlass,
"_flashinfer_cutlass_fused_moe",
lambda: (fake_kernel, None),
)
monkeypatch.setattr(flashinfer_cutlass, "_activation_type", lambda config: None)
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
flashinfer_cutlass._run_flashinfer_cutlass(
dispatch_output=SimpleNamespace(
hidden_states=hidden_states,
hidden_states_scale=None,
topk_output=SimpleNamespace(
topk_weights=torch.ones(2, 1),
topk_ids=torch.zeros(2, 1, dtype=torch.int64),
),
),
quant_info=FlashInferCutlassMoeQuantInfo(
quant_type="bf16", w13_weight=torch.zeros(1), w2_weight=torch.zeros(1)
),
runner_config=runner_config,
output=torch.empty_like(hidden_states),
)
(call,) = calls
return (call["tp_size"], call["tp_rank"], call["ep_size"], call["ep_rank"])
def test_cutlass_passes_the_runner_config_placement(monkeypatch) -> None:
placement = _kernel_placement(
monkeypatch,
MoeRunnerConfig(
layer=SimpleNamespace(
moe_tp_size=2, moe_tp_rank=1, moe_ep_size=4, moe_ep_rank=3
)
),
)
assert placement == (2, 1, 4, 3)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
@@ -342,6 +342,8 @@ def test_gpt_oss_sm120_padding_layout_and_kernel(monkeypatch, stated_tp_group):
gemm1_alpha=1.702,
gemm1_clamp_limit=7.0,
)
# The fused func reads the layer's MoE placement off the config.
config.layer = layer
method.moe_runner_config = config
method.runner = MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, config)
method._process_weights_for_sm120_cutlass(layer)
@@ -173,7 +173,7 @@ def _round_up(x, base):
return ((x + base - 1) // base) * base
def _build_method(num_experts, hidden, inter, *, use_humming=False):
def _build_method(num_experts, hidden, inter, *, layer=None, use_humming=False):
from sglang.srt.layers.quantization.mxfp4 import Mxfp4MoEMethod
method = Mxfp4MoEMethod.__new__(Mxfp4MoEMethod)
@@ -188,16 +188,17 @@ def _build_method(num_experts, hidden, inter, *, use_humming=False):
method._padded_hidden = _round_up(hidden, 128)
method._padded_intermediate = _round_up(inter, 128)
method.use_flashinfer = True
method.runner = _build_flashinfer_mxfp4_runner(num_experts, hidden, inter)
method.runner = _build_flashinfer_mxfp4_runner(num_experts, hidden, inter, layer)
return method
def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter):
def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter, layer=None):
"""Construct a real MoeRunner bound to the flashinfer_mxfp4 fused func.
Bypasses ``create_moe_runner`` (which needs a live server arg context)
and wires the runner with a minimal MoeRunnerConfig sufficient for the
cutlass SM90 fused func, which only reads dispatch_output / quant_info.
cutlass SM90 fused func. The func reads the layer's MoE placement off the
config, so the layer states itself here the way `FusedMoE` does.
"""
import sglang.srt.layers.moe.moe_runner.flashinfer_cutlass # noqa: F401
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
@@ -212,6 +213,7 @@ def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter):
top_k=None,
activation="silu",
is_gated=True,
layer=layer,
)
return MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, cfg)
@@ -299,7 +301,7 @@ def test_process_weights_matches_direct_interleave(num_experts, hidden, inter):
layer = _build_mock_layer(
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
)
method = _build_method(num_experts, hidden, inter)
method = _build_method(num_experts, hidden, inter, layer=layer)
method._process_weights_for_sm90_cutlass(layer)
N_pad = _round_up(inter, 128)
@@ -375,7 +377,7 @@ def test_apply_sm90_cutlass_matches_flashinfer_direct(
layer = _build_mock_layer(
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
)
method = _build_method(num_experts, hidden, inter)
method = _build_method(num_experts, hidden, inter, layer=layer)
method._process_weights_for_sm90_cutlass(layer)
out_sglang = method._apply_sm90_cutlass(
@@ -444,7 +446,7 @@ def test_process_weights_humming_matches_flashinfer_direct():
layer = _build_mock_layer(
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
)
method = _build_method(num_experts, hidden, inter, use_humming=True)
method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True)
method._process_weights_for_sm90_cutlass(layer)
# GPT-OSS loads pair-wise [gate, up]; FlashInfer consumes halved [up; gate].
@@ -484,7 +486,7 @@ def test_humming_padding_preserves_per_expert_residual():
layer = _build_mock_layer(
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
)
method = _build_method(num_experts, hidden, inter, use_humming=True)
method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True)
method._process_weights_for_sm90_cutlass(layer)
assert torch.equal(layer.w13_humming_residual_scale, expected_w13_residual * 64.0)
@@ -572,7 +574,7 @@ def test_humming_range_ignores_prerounded_hidden_tail():
layer = _build_mock_layer(
E, hidden_rounded, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
)
method = _build_method(E, hidden_rounded, inter, use_humming=True)
method = _build_method(E, hidden_rounded, inter, layer=layer, use_humming=True)
# What create_weights records from layer.hidden_size_unpadded.
method._unpadded_hidden = hidden_real
method._process_weights_for_sm90_cutlass(layer)
@@ -627,7 +629,7 @@ def test_apply_sm90_humming_matches_flashinfer_direct(
)
layer.moe_ep_size = ep_size
layer.moe_ep_rank = ep_rank
method = _build_method(num_experts, hidden, inter, use_humming=True)
method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True)
method._process_weights_for_sm90_cutlass(layer)
out_sglang = method._apply_sm90_cutlass(
layer, _MockDispatchOutput(x.clone(), topk_w, topk_i)
@@ -784,6 +786,7 @@ def test_dsv4_apply_matches_flashinfer_direct(
layer.moe_tp_rank = 0
layer.moe_ep_size = 1
layer.moe_ep_rank = 0
method.runner.config.layer = layer
method.process_weights_after_loading(layer)
@@ -0,0 +1,165 @@
"""Layers that shard over attention TP but all-reduce over the full TP group
refuse the layouts where the two groups differ."""
import unittest
from types import SimpleNamespace
from sglang.srt.layers.dp_attention import (
initialize_dp_attention_flags,
reject_attn_tp_shard_with_tp_reduce,
)
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
class TestAttnTpReduceRejection(CustomTestCase):
def publish(self, **fields):
reset_context()
self.addCleanup(reset_context)
server_args = ServerArgs(model_path="dummy", device="cpu", **fields)
publish(server_args, role="test", ranks=SpawnRanks(world_rank=0))
initialize_dp_attention_flags(server_args)
def test_only_a_narrower_wider_than_one_shard_with_a_tp_reduce_is_refused(self):
self.publish(tp_size=4)
with get_parallel().override(tp_size=4):
with self.assertRaisesRegex(ValueError, "Layer shards over the attention"):
reject_attn_tp_shard_with_tp_reduce(
"Layer", shard_tp_size=2, reduces_over_attn_tp=False
)
for shard_tp_size, reduces_over_attn_tp in (
(2, True),
(1, False),
(4, False),
):
with self.subTest(shard=shard_tp_size, reduces=reduces_over_attn_tp):
reject_attn_tp_shard_with_tp_reduce(
"Layer",
shard_tp_size=shard_tp_size,
reduces_over_attn_tp=reduces_over_attn_tp,
)
def test_an_attention_that_always_reduces_over_tp(self):
from sglang.srt.models.clip import CLIPAttention
config = SimpleNamespace(
hidden_size=16, num_attention_heads=4, attention_dropout=0.0
)
self.publish(tp_size=4, attn_dp_size=2)
with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"):
CLIPAttention(config)
# A one-rank attention-TP shard does not reduce at all.
self.publish(tp_size=4, attn_dp_size=4)
CLIPAttention(config)
self.publish(tp_size=4)
CLIPAttention(config)
def test_a_layer_that_reduces_over_attention_tp_only_with_attention_dp(self):
from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP
def build():
return Qwen3_VisionMLP(16, 32, bias=False, hidden_act="relu")
self.publish(tp_size=4, attn_cp_size=2)
with self.assertRaisesRegex(ValueError, "Qwen3_VisionMLP shards over"):
build()
for fields in (
dict(tp_size=4, attn_dp_size=2),
dict(tp_size=8, attn_dp_size=2, attn_cp_size=2),
dict(tp_size=4),
):
with self.subTest(**fields):
self.publish(**fields)
build()
# The data-parallel encoder runs one-rank shards.
self.publish(tp_size=4, attn_cp_size=2)
Qwen3_VisionMLP(16, 32, bias=False, hidden_act="relu", use_data_parallel=True)
def test_a_worker_that_never_forwards_its_tower(self):
"""Most models build their multimodal tower unconditionally. Where the
tower is not forwarded, the layout it would need must not be refused."""
from sglang.srt.models.clip import CLIPAttention
config = SimpleNamespace(
hidden_size=16, num_attention_heads=4, attention_dropout=0.0
)
self.publish(tp_size=4, attn_dp_size=2)
with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"):
CLIPAttention(config)
for fields in (
# The encoder instance forwards the tower instead.
dict(language_only=True),
# Adaptive dispatch is off, so nothing is kept back for local work.
dict(language_only=True, enable_adaptive_dispatch_to_encoder=False),
# Multimodal requests are rejected outright.
dict(language_model_only=True),
# A decode instance embeds nothing.
dict(disaggregation_mode="decode"),
):
with self.subTest(**fields):
self.publish(tp_size=4, attn_dp_size=2, **fields)
CLIPAttention(config)
def test_a_tower_kept_for_local_work_is_still_checked(self):
"""Adaptive dispatch sends only some requests to the encoder instance
and forwards the tower for the rest, so the layout still has to hold."""
from sglang.srt.models.clip import CLIPAttention
config = SimpleNamespace(
hidden_size=16, num_attention_heads=4, attention_dropout=0.0
)
self.publish(
tp_size=4,
attn_dp_size=2,
language_only=True,
enable_adaptive_dispatch_to_encoder=True,
)
with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"):
CLIPAttention(config)
def test_the_message_names_the_width_that_narrowed_the_group(self):
self.publish(tp_size=4, attn_dp_size=2)
with self.assertRaisesRegex(ValueError, r"drop --attn-dp-size 2"):
reject_attn_tp_shard_with_tp_reduce(
"Layer", shard_tp_size=2, reduces_over_attn_tp=False
)
self.publish(tp_size=4, attn_cp_size=2)
with self.assertRaisesRegex(ValueError, r"drop --attn-cp-size 2"):
reject_attn_tp_shard_with_tp_reduce(
"Layer", shard_tp_size=2, reduces_over_attn_tp=False
)
def test_a_text_layer_is_checked_on_every_worker(self):
"""The exemption belongs to towers that do not run, not to the worker:
a text attention still reduces on a language-only instance."""
from sglang.srt.models.phimoe import PhiMoEAttention
for fields in (
{},
dict(language_only=True),
dict(language_model_only=True),
dict(disaggregation_mode="decode"),
):
with self.subTest(**fields):
self.publish(tp_size=4, attn_dp_size=2, **fields)
with self.assertRaisesRegex(ValueError, "PhiMoEAttention shards over"):
PhiMoEAttention(hidden_size=16, num_heads=4, num_kv_heads=4)
def test_a_replicated_branch_is_not_gated(self):
"""MoonViT's tensor-parallel MLP is refused, while its ModelSlim branch
builds replicated layers that neither shard nor reduce."""
from sglang.srt.models.kimi_vl_moonvit import MLP2
self.publish(tp_size=4, attn_dp_size=2)
with self.assertRaisesRegex(ValueError, "MLP2 shards over"):
MLP2(dims=[16, 32, 16], activation=None, use_tensor_parallel=True)
# Without tensor parallelism the shard is one rank wide.
MLP2(dims=[16, 32, 16], activation=None)
if __name__ == "__main__":
unittest.main()
@@ -68,6 +68,8 @@ with mock.patch.dict(
_moe_runner_keeps_global_expert_ids,
)
from sglang.srt.runtime_context import get_parallel
class _IdentityMoeSlices:
def slice_moe_lora_a_weights(self, weights, _rank, _target):
@@ -982,6 +984,21 @@ class TestPoolInitPicksUpEpContext(unittest.TestCase):
return_value=keeps_global,
),
mock.patch.object(LoRAMemoryPool, "init_buffers", lambda self, _m: None),
# The pool reads its TP placement from the parallel context; state
# a whole layout so the topology check sees a consistent one.
get_parallel().override(
tp_size=tp_size,
tp_rank=tp_rank,
attn_tp_size=tp_size,
attn_tp_rank=tp_rank,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_tp_size=tp_size,
moe_ep_size=1,
moe_dp_size=1,
),
):
hf_cfg = types.SimpleNamespace(
num_hidden_layers=1,
@@ -995,9 +1012,6 @@ class TestPoolInitPicksUpEpContext(unittest.TestCase):
base_hf_config=hf_cfg,
max_loras_per_batch=1,
dtype=torch.bfloat16,
tp_size=tp_size,
tp_rank=tp_rank,
attn_tp_size=tp_size,
max_lora_rank=8,
target_modules={"qkv_proj"},
base_model=base_model,
@@ -92,18 +92,12 @@ def _request(feature, rid: str = "vlm-request") -> TokenizedEmbeddingReqInput:
def _receiver() -> SchedulerRequestReceiver:
group = SimpleNamespace(rank=0, ranks=[0], cpu_group=object())
return SchedulerRequestReceiver(
recv_from_tokenizer=None,
recv_from_rpc=None,
recv_skipper=None,
input_blocker=None,
mm_receiver=None,
tp_group=group,
tp_cpu_group=group,
attn_tp_cpu_group=group,
attn_cp_cpu_group=group,
world_group=group,
server_args=SimpleNamespace(),
model_config=SimpleNamespace(is_multimodal=True),
max_recv_per_poll=-1,
@@ -121,9 +115,12 @@ def _run_consensus_rank(rank: int, world_size: int, init_file: str) -> None:
)
try:
req = _request(_failed_pointer() if rank == 1 else _successful_pointer())
parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=world_size)
parallel = SimpleNamespace(
attn_dp_enabled=False,
tp_size=world_size,
tp_group=SimpleNamespace(cpu_group=torch.distributed.group.WORLD),
)
receiver = _receiver()
object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD)
with (
patch(
"sglang.srt.managers.mm_utils._get_is_default_transport",
@@ -152,7 +149,6 @@ def _run_image_receiver(rank, init_file, pipe):
)
try:
receiver = _receiver()
object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD)
torch.distributed.barrier()
torch.distributed.all_reduce(torch.zeros(1))
initial_fds = len(os.listdir("/proc/self/fd"))
@@ -169,7 +165,11 @@ def _run_image_receiver(rank, init_file, pipe):
),
patch(
"sglang.srt.managers.scheduler_components.request_receiver.get_parallel",
return_value=SimpleNamespace(attn_dp_enabled=False, tp_size=2),
return_value=SimpleNamespace(
attn_dp_enabled=False,
tp_size=2,
tp_group=SimpleNamespace(cpu_group=torch.distributed.group.WORLD),
),
),
):
for base in [30, 90]:
@@ -396,7 +396,11 @@ class TestShmRequestFailureConsensus(unittest.TestCase):
def test_peer_failure_rejects_the_local_request(self):
req = _request(torch.zeros(1))
parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=2)
parallel = SimpleNamespace(
attn_dp_enabled=False,
tp_size=2,
tp_group=SimpleNamespace(cpu_group=object()),
)
def inject_peer_failure(mask, **kwargs):
mask.fill_(1)
@@ -22,6 +22,7 @@ from sglang.srt.managers.scheduler_pp_mixin import ( # noqa: E402
_pp_exchange_outputs_before_forward,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402
from sglang.srt.runtime_context import get_parallel # noqa: E402
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
@@ -46,21 +47,12 @@ def _fake_group() -> SimpleNamespace:
def _make_receiver() -> SchedulerRequestReceiver:
tp_group = _fake_group()
attn_tp_group = _fake_group()
attn_cp_group = _fake_group()
world_group = _fake_group()
return SchedulerRequestReceiver(
recv_from_tokenizer=None,
recv_from_rpc=None,
recv_skipper=None,
input_blocker=None,
mm_receiver=None,
tp_group=tp_group,
tp_cpu_group=tp_group,
attn_tp_cpu_group=attn_tp_group,
attn_cp_cpu_group=attn_cp_group,
world_group=world_group,
server_args=SimpleNamespace(
enable_dp_attention_local_control_broadcast=False,
),
@@ -120,6 +112,7 @@ class TestRequestReceiverBroadcast(unittest.TestCase):
attn_tp_size=1,
attn_cp_size=1,
tp_size=32,
tp_group=_fake_group(),
)
with (
@@ -150,15 +143,16 @@ class TestRequestReceiverBroadcast(unittest.TestCase):
self.assertEqual(result, [control_req])
broadcast.assert_called_once_with(
[control_req],
receiver.tp_group.rank,
receiver.tp_cpu_group,
src=receiver.tp_group.ranks[0],
parallel.tp_group.rank,
parallel.tp_group.cpu_group,
src=parallel.tp_group.ranks[0],
)
class TestPPCPRankOffsets(unittest.TestCase):
def test_request_receiver_uses_cp_size_for_pp_recv_rank(self):
enter_scope(self, _published_topology())
enter_scope(self, get_parallel().override(world_group=_fake_group()))
calls = []
def fake_point_to_point_pyobj(data, rank, group, src, dst, **kwargs):
@@ -1,4 +1,5 @@
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
@@ -41,9 +42,12 @@ class TestPrefillDelayerGather(CustomTestCase):
"sglang.srt.managers.prefill_delayer.all_gather_single",
side_effect=_all_gather_over_tp,
),
get_parallel().override(
tp_group=SimpleNamespace(cpu_group=None, device_group=None)
),
):
delayer = PrefillDelayer(
cpu_group=None, max_delay_passes=4, token_usage_low_watermark=None
max_delay_passes=4, token_usage_low_watermark=None
)
info = delayer._gather_info(
local_prefillable=True,
@@ -11,6 +11,7 @@ from sglang.srt.managers.scheduler_components.weight_updater import (
SchedulerWeightUpdaterManager,
_WeightUpdateSession,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -140,8 +141,8 @@ class _WeightUpdaterManagerTestBase(CustomTestCase):
self.addCleanup(patcher.stop)
self.recorded = []
def _manager(self, target, draft=None, *, session=True):
manager = SchedulerWeightUpdaterManager(
def _build_manager(self, target, draft):
return SchedulerWeightUpdaterManager(
tp_worker=SimpleNamespace(
model_runner=target,
weight_update_runners=lambda: [("target", target)],
@@ -152,7 +153,6 @@ class _WeightUpdaterManagerTestBase(CustomTestCase):
if draft is None
else SimpleNamespace(weight_update_runners=lambda: [("draft", draft)])
),
tp_cpu_group=None,
memory_saver_adapter=None,
flush_cache=lambda **kwargs: True,
is_fully_idle=lambda **kwargs: True,
@@ -162,6 +162,10 @@ class _WeightUpdaterManagerTestBase(CustomTestCase):
)
),
)
def _manager(self, target, draft=None, *, session=True):
with get_parallel().override(tp_group=SimpleNamespace(cpu_group=None)):
manager = self._build_manager(target, draft)
if session:
# update_weights_from_* require an open session
manager._session = _WeightUpdateSession(selector="all")
@@ -16,7 +16,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestOutcome,
)
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.hicache_storage import (
HiCacheStorageConfig,
PoolName,
PoolTransfer,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
PPPrefetchDecision,
@@ -34,8 +38,18 @@ class TestPPPrefetchTicket(unittest.TestCase):
def setUp(self):
self.c = c = HybridCacheController.__new__(HybridCacheController)
c.page_size = c.prefetch_threshold = 4
c.pp_rank = c.tp_rank = 0
c.pp_size, c.tp_size = 4, 2
c.storage_config = HiCacheStorageConfig(
tp_rank=0,
tp_size=2,
pp_rank=0,
pp_size=4,
attn_cp_rank=0,
attn_cp_size=1,
is_mla_model=False,
enable_storage_metrics=False,
is_page_first_layout=False,
model_name=None,
)
c.pp_group, c.pp_prefetch_command_group = "pp", "command"
c.pp_prefetch_command_thread = None
c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"]
@@ -61,7 +75,9 @@ class TestPPPrefetchTicket(unittest.TestCase):
patcher.start()
self.addCleanup(patcher.stop)
patcher = patch.object(
torch.distributed, "get_rank", side_effect=lambda: c.pp_rank * 2 + c.tp_rank
torch.distributed,
"get_rank",
side_effect=lambda: c.storage_config.pp_rank * 2 + c.storage_config.tp_rank,
)
patcher.start()
self.addCleanup(patcher.stop)
@@ -98,7 +114,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
self.c.pp_prefetch_command_queue.put(None)
def broadcast(objects, *args, **kwargs):
return [next(commands)] if self.c.pp_rank else objects
return [next(commands)] if self.c.storage_config.pp_rank else objects
with patch(f"{HybridCacheController.__module__}.broadcast_pyobj", broadcast):
self.c.pp_prefetch_command_thread_func()
@@ -179,7 +195,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
handle = CacheRequestHandle("hit", 3)
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
ticket.last_hash = None
c.pp_rank = 1
c.storage_config.pp_rank = 1
c.pp_prefetch_states.clear()
cache.bind_prefetch_ticket("hit")
self.assertFalse(cache.check_prefetch_progress(handle))
@@ -216,7 +232,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
ticket = c.pp_prefetch_states.pop("hit").ticket
following = pickle.loads(pickle.dumps(ticket))
following.handle = CacheRequestHandle("next", 0)
c.pp_rank = 1
c.storage_config.pp_rank = 1
kv = torch.arange(8)
c.mem_pool_host.alloc.side_effect = [
kv,
@@ -257,7 +273,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
c, cache = self.c, self.cache
self.submit(pools=[PoolTransfer(PoolName.SWA, host_indices=torch.arange(4))])
ticket = c.pp_prefetch_states.pop("hit").ticket
c.pp_rank = 1
c.storage_config.pp_rank = 1
cache.bind_prefetch_ticket("hit")
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
@@ -290,7 +306,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
for rank in (0, 1):
with self.subTest(rank=rank):
c.pp_rank = rank
c.storage_config.pp_rank = rank
if rank:
c.pp_prefetch_states.clear()
c.mem_pool_host.alloc.reset_mock()
@@ -358,7 +374,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
def test_idle_source_broadcasts_empty_then_processes_ticket_and_stop(self):
c = self.c
c.tp_rank = 1 # The source is a global rank, not pp_rank=0.
c.storage_config.tp_rank = 1 # The source is a global rank, not pp_rank=0.
operation = self.submit().operation
c.pp_prefetch_command_queue.put(None)
get = c.pp_prefetch_command_queue.get
@@ -397,7 +413,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
c = self.c
self.submit()
ticket = c.pp_prefetch_states.pop("hit").ticket
c.pp_rank = 1
c.storage_config.pp_rank = 1
with (
patch(
f"{HybridCacheController.__module__}.broadcast_pyobj",
@@ -89,8 +89,13 @@ class TestAttnDpSize(CustomTestCase):
"replicas combined with attention data parallelism",
),
(
{"tp_size": 4, "attn_dp_size": 2, "enable_dp_attention": True},
"pass --attn-dp-size alone",
{
"tp_size": 4,
"dp_size": 2,
"attn_dp_size": 2,
"enable_dp_attention": True,
},
"replicas combined with attention data parallelism",
),
({"tp_size": 2, "attn_dp_size": 0}, "must be positive"),
):
@@ -98,6 +103,55 @@ class TestAttnDpSize(CustomTestCase):
with self.assertRaisesRegex(ValueError, message):
self.resolve(**fields)
def test_the_deprecated_flag_next_to_attn_dp_size_is_redundant(self):
both = self.resolve(tp_size=4, attn_dp_size=2, enable_dp_attention=True)
new = self.resolve(tp_size=4, attn_dp_size=2)
self.assertEqual(self.layout(both), (1, 2, False))
self.assertEqual(both.resolved_dict(), new.resolved_dict())
def test_the_readback_reports_the_deprecated_field_as_attention_dp(self):
for fields, attention_dp in (
({"tp_size": 4, "attn_dp_size": 2}, True),
({"tp_size": 4, "dp_size": 2, "enable_dp_attention": True}, True),
({"tp_size": 4, "dp_size": 2}, False),
({"tp_size": 4}, False),
):
with self.subTest(**fields):
server_args = self.resolve(**fields)
self.assertFalse(resolution_result(server_args, "enable_dp_attention"))
readback = server_args.resolved_dict()
self.assertIs(readback["enable_dp_attention"], attention_dp)
# A read-back config resolves to the same layout.
again = self.resolve(
**{
name: readback[name]
for name in (
"tp_size",
"dp_size",
"attn_dp_size",
"enable_dp_attention",
)
}
)
self.assertEqual(self.layout(again), self.layout(server_args))
def test_the_readback_reports_the_width_not_the_joiner_arm(self):
"""`attn_dp_enabled` is also true for an elastic scale joiner at width
one. The readback reports the configured width instead, so a joiner's
replicas are not folded into attention-DP groups when it is read back."""
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.runtime_context import attn_dp_enabled_of
joiner = ServerArgs(
model_path="dummy", tp_size=2, dp_size=2, ep_join_mode="scale"
)
joiner.resolve_once()
self.assertTrue(attn_dp_enabled_of(resolving_view(joiner)))
readback = joiner.resolved_dict()
self.assertFalse(readback["enable_dp_attention"])
self.assertEqual(readback["dp_size"], 2)
self.assertEqual(readback["attn_dp_size"], 1)
def test_both_cli_spellings_parse(self):
parser = argparse.ArgumentParser()
ServerArgs.add_cli_args(parser)