[Refactor] Let the remaining placement consumers read the parallel context (#42348)
This commit is contained in:
@@ -120,12 +120,14 @@ python -m sglang.launch_server \
|
||||
|
||||
**Important**: `--attn-dp-size` must be greater than 1 for DPA to work. When `attn_dp_size == 1` (default), DPA is disabled. The constraint `tp_size % attn_dp_size == 0` must also be satisfied. `--dp-size` only sets the number of data-parallel replicas of the model; combining replicas (`--dp-size` > 1) with DPA is not supported.
|
||||
|
||||
> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead.
|
||||
> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead. The flag will be removed after 2026-12-31.
|
||||
>
|
||||
> Under DeepSeek context parallelism (DSA, MLA, and DeepSeek-V4 prefill CP), `--dp-size N` alone still runs N attention-DP groups, and logs a deprecation warning; it will be rejected after 2026-12-31. Use `--attn-dp-size N` there too.
|
||||
|
||||
**Elastic EP scale joiners.** A rank that joins an existing DPA deployment through elastic EP scale-up runs DPA even when it is launched with `--attn-dp-size 1`: its own attention DP group is one rank wide.
|
||||
|
||||
**DP rank count.** The number of DP ranks (one scheduler each) is `dp_size × attn_dp_size`; one of the two is always 1.
|
||||
- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Its deprecated `enable_dp_attention` field is always `false`; read `attn_dp_size` instead.
|
||||
- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Until the flag is removed, its deprecated `enable_dp_attention` field still reports whether DPA runs; new clients should read `attn_dp_size`.
|
||||
- Wire formats that predate `--attn-dp-size` keep the key `dp_size` for the DP rank count: KV-event descriptors, the PD prefill server info, and load snapshots.
|
||||
|
||||
### TP LMHead with all-to-all
|
||||
|
||||
@@ -3,7 +3,10 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import attention_backends_of
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
attention_backends_of,
|
||||
context_parallel_attn_dp_size,
|
||||
)
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
_deepseek_v4_kv_cache_dtype,
|
||||
declare_resolution,
|
||||
@@ -157,13 +160,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
||||
f"DeepSeekV4 prefill CP on HIP does not support {feature} yet."
|
||||
)
|
||||
|
||||
# DeepSeek-V4 CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
attn_dp_size = context_parallel_attn_dp_size(cfg, "DeepSeek-V4 context parallelism")
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v4_cp",
|
||||
|
||||
@@ -171,7 +171,7 @@ class Parallel(msgspec.Struct):
|
||||
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
|
||||
] = False
|
||||
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
|
||||
# resolves to `attn_dp_size = N`, `dp_size = 1`.
|
||||
# resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31.
|
||||
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
|
||||
enable_dp_attention_local_control_broadcast: A[
|
||||
bool,
|
||||
|
||||
@@ -226,6 +226,27 @@ def ep_joiner_of(cfg: Any) -> bool:
|
||||
return cfg.ep_join_mode in ("scale", "recover")
|
||||
|
||||
|
||||
def context_parallel_attn_dp_size(cfg: Any, layout: str) -> int:
|
||||
"""The attention-DP width a DeepSeek context-parallel layout runs.
|
||||
|
||||
These layouts run data-parallel groups as attention DP, so `--dp-size N`
|
||||
alone still means N attention-DP groups there. That reading is deprecated.
|
||||
"""
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
if cfg.dp_size > 1:
|
||||
# TODO: reject `--dp-size > 1` here after 2026-12-31.
|
||||
logger.warning(
|
||||
f"--dp-size {cfg.dp_size} under {layout} runs {cfg.dp_size} attention "
|
||||
"data-parallel groups. This is deprecated and will be rejected after "
|
||||
f"2026-12-31; use --attn-dp-size {cfg.dp_size} instead."
|
||||
)
|
||||
return cfg.attn_dp_size * cfg.dp_size
|
||||
|
||||
|
||||
def ep_scale_joiner_of(cfg: Any) -> bool:
|
||||
"""The scale-up arm of :func:`ep_joiner_of`."""
|
||||
return cfg.ep_join_mode == "scale"
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Any, Dict
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
context_parallel_attn_dp_size,
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
use_mla_backend,
|
||||
@@ -85,13 +86,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
logger.warning(
|
||||
"Context parallel feature is still under experiment. It has only been verified on Hopper platform."
|
||||
)
|
||||
# DSA CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
attn_dp_size = context_parallel_attn_dp_size(
|
||||
cfg, "DSA context parallelism"
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
overrides["moe_dense_tp_size"] = 1
|
||||
@@ -166,13 +163,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"MLA prefill context parallel is still experimental. "
|
||||
"Verified on Hopper with the fa3 backend."
|
||||
)
|
||||
# MLA CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
attn_dp_size = context_parallel_attn_dp_size(cfg, "MLA context parallelism")
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
# TODO(kpham-sgl) Supports moe_dense_tp_size != 1.
|
||||
|
||||
@@ -39,22 +39,23 @@ def handle_deprecated_dp_attention(server_args: Any):
|
||||
|
||||
Runs before any handler reads the DP layout. The old spelling counted
|
||||
attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and
|
||||
`dp_size` counts replicas again.
|
||||
`dp_size` counts replicas again. Next to `--attn-dp-size` the flag adds
|
||||
nothing: a serialized config reports it set whenever attention DP runs
|
||||
(`ServerArgs.resolved_dict`), so reading one back resolves the same layout.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.enable_dp_attention:
|
||||
if cfg.attn_dp_size != 1:
|
||||
raise ValueError(
|
||||
"--enable-dp-attention is the deprecated spelling of "
|
||||
"--attn-dp-size; pass --attn-dp-size alone."
|
||||
)
|
||||
if not cfg.enable_dp_attention:
|
||||
return
|
||||
if cfg.attn_dp_size == 1:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_deprecated_dp_attention",
|
||||
attn_dp_size=cfg.dp_size,
|
||||
dp_size=1,
|
||||
enable_dp_attention=False,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args, "_handle_deprecated_dp_attention", enable_dp_attention=False
|
||||
)
|
||||
|
||||
|
||||
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
|
||||
|
||||
@@ -158,8 +158,6 @@ class BaseKVSender(ABC):
|
||||
mgr: BaseKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
): ...
|
||||
|
||||
|
||||
@@ -1766,8 +1766,6 @@ class CommonKVSender(BaseKVSender):
|
||||
mgr: CommonKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
self.kv_mgr = mgr
|
||||
|
||||
@@ -31,7 +31,6 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.srt.configs.mamba_utils import Mamba2CacheParams
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
@@ -382,7 +381,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
scheduler: Scheduler,
|
||||
transfer_queue: DecodeTransferQueue,
|
||||
tree_cache: BasePrefixCache,
|
||||
gloo_group: ProcessGroup,
|
||||
gpu_id: int,
|
||||
bootstrap_port: int,
|
||||
max_total_num_tokens: int,
|
||||
@@ -402,7 +400,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.tree_cache = tree_cache
|
||||
self.host_pool = None
|
||||
self.host_reserved_tokens = 0
|
||||
self.gloo_group = gloo_group
|
||||
self.gloo_group = parallel.attn_tp_group.cpu_group
|
||||
# Destinations visible to prefill but not yet on the transfer queue.
|
||||
self._num_published_destinations = 0
|
||||
self.tp_rank = parallel.tp_rank
|
||||
@@ -2379,14 +2377,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
gloo_group: ProcessGroup,
|
||||
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
|
||||
metadata_buffers: MetadataBuffers,
|
||||
scheduler: Scheduler,
|
||||
tree_cache: BasePrefixCache,
|
||||
):
|
||||
self.queue: List[DecodeRequest] = []
|
||||
self.gloo_group = gloo_group
|
||||
self.gloo_group = get_parallel().attn_tp_group.cpu_group
|
||||
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.metadata_buffers = metadata_buffers
|
||||
|
||||
@@ -25,10 +25,7 @@ from fastapi import FastAPI
|
||||
from fastapi.responses import ORJSONResponse, Response
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
GroupCoordinator,
|
||||
get_mooncake_transfer_engine,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.io_struct import (
|
||||
EncoderDispatchErrorReq,
|
||||
@@ -1870,9 +1867,6 @@ class MMReceiverBase(ABC):
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
):
|
||||
@@ -1892,10 +1886,13 @@ class MMReceiverBase(ABC):
|
||||
)
|
||||
self.recv_timeout = envs.SGLANG_ENCODER_RECV_TIMEOUT.get()
|
||||
self.host = get_local_ip_auto(get_serving().host)
|
||||
self.pp_rank = pp_rank
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_group = tp_group
|
||||
# A scheduler-side receiver works across its TP group; the tokenizer
|
||||
# side has no placement.
|
||||
parallel = get_parallel()
|
||||
self.pp_rank = parallel.pp_rank if scheduler is not None else None
|
||||
self.tp_rank = parallel.tp_rank if scheduler is not None else None
|
||||
self.tp_size = parallel.tp_size
|
||||
self.tp_group = parallel.tp_group if scheduler is not None else None
|
||||
self.nnodes = get_parallel().nnodes
|
||||
self.hostname = get_local_ip_auto()
|
||||
self.waiting_list: List[WaitingMMRequestBase] = []
|
||||
@@ -1908,7 +1905,7 @@ class MMReceiverBase(ABC):
|
||||
and scheduler is not None
|
||||
):
|
||||
self.registration_runner = _ReceiveRegistrationRunner(
|
||||
f"encoder-receive-registration-{tp_rank}"
|
||||
f"encoder-receive-registration-{self.tp_rank}"
|
||||
)
|
||||
(
|
||||
self.scheduler_embedding_port,
|
||||
@@ -2674,9 +2671,6 @@ class MMReceiverHTTP(MMReceiverBase):
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
):
|
||||
@@ -2684,9 +2678,6 @@ class MMReceiverHTTP(MMReceiverBase):
|
||||
server_args,
|
||||
dtype=dtype,
|
||||
hf_config=hf_config,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=tp_group,
|
||||
scheduler=scheduler,
|
||||
encode_urls=encode_urls,
|
||||
)
|
||||
@@ -2813,9 +2804,6 @@ class MMReceiverGrpc(MMReceiverBase):
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
):
|
||||
@@ -2830,9 +2818,6 @@ class MMReceiverGrpc(MMReceiverBase):
|
||||
server_args,
|
||||
dtype=dtype,
|
||||
hf_config=hf_config,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=tp_group,
|
||||
scheduler=scheduler,
|
||||
encode_urls=encode_urls,
|
||||
)
|
||||
@@ -2946,9 +2931,6 @@ def create_mm_receiver(
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
transport_mode: Optional[str] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
@@ -2967,9 +2949,6 @@ def create_mm_receiver(
|
||||
server_args,
|
||||
dtype=dtype,
|
||||
hf_config=hf_config,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=tp_group,
|
||||
scheduler=scheduler,
|
||||
encode_urls=encode_urls,
|
||||
)
|
||||
|
||||
@@ -43,8 +43,6 @@ class FakeKVSender(BaseKVSender):
|
||||
mgr: BaseKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
self.kv_mgr = mgr
|
||||
|
||||
@@ -2813,16 +2813,12 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
mgr: MooncakeKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
dest_tp_ranks,
|
||||
pp_rank,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.init_time = time.time()
|
||||
|
||||
@@ -1785,16 +1785,12 @@ class MoriKVSender(CommonKVSender):
|
||||
mgr: MoriKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
dest_tp_ranks,
|
||||
pp_rank,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.conclude_state: Optional[KVPoll] = None
|
||||
|
||||
@@ -3133,16 +3133,12 @@ class NixlKVSender(CommonKVSender):
|
||||
mgr: NixlKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
dest_tp_ranks,
|
||||
pp_rank,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.init_time = time.time()
|
||||
|
||||
@@ -382,14 +382,10 @@ class PrefillBootstrapQueue:
|
||||
)
|
||||
kv_sender_class = get_kv_class(backend, KVClassType.SENDER)
|
||||
|
||||
dest_tp_ranks = [self.tp_rank]
|
||||
|
||||
req.disagg_kv_sender = kv_sender_class(
|
||||
mgr=self.kv_manager,
|
||||
bootstrap_addr=f"{req.bootstrap_host}:{self.bootstrap_port}",
|
||||
bootstrap_room=req.bootstrap_room,
|
||||
dest_tp_ranks=dest_tp_ranks,
|
||||
pp_rank=self.pp_rank,
|
||||
req_has_disagg_prefill_dp_rank=req.disagg_prefill_dp_rank is not None,
|
||||
)
|
||||
self._process_req(req)
|
||||
|
||||
@@ -83,6 +83,7 @@ from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from sglang.srt.distributed import utils as dist_utils
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1099,6 +1100,7 @@ class VisionAttention(nn.Module):
|
||||
] = None,
|
||||
use_data_parallel: bool = False,
|
||||
use_dp_attention_reduce: bool = False,
|
||||
allow_tp_reduce_mismatch: bool = False,
|
||||
aux_stream: Optional[torch.cuda.Stream] = None,
|
||||
workspace_buffer: Optional[torch.Tensor] = None,
|
||||
use_sink: bool = False,
|
||||
@@ -1208,6 +1210,17 @@ class VisionAttention(nn.Module):
|
||||
tp_size=self.tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
if not allow_tp_reduce_mismatch:
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=self.tp_size,
|
||||
reduces_over_attn_tp=use_dp_attention_reduce,
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=self.dummy_dim,
|
||||
output_size=embed_dim,
|
||||
|
||||
@@ -455,6 +455,79 @@ def is_dp_attention_enabled() -> bool:
|
||||
return get_flags().dp.enabled
|
||||
|
||||
|
||||
def get_dp_tp_group() -> GroupCoordinator:
|
||||
"""The TP ranks that run one DP rank's batch: the attention-TP group with
|
||||
attention DP, otherwise the whole TP group.
|
||||
|
||||
Attention-CP peers of a DP rank are members only in the second case.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
|
||||
|
||||
|
||||
def multimodal_encoder_runs_here() -> bool:
|
||||
"""Whether a multimodal tower built on this worker is also forwarded here.
|
||||
|
||||
A tower is built unconditionally by most models, but an encoder-disaggregated
|
||||
language instance forwards it on the encoder instance, a language-model-only
|
||||
instance rejects multimodal requests, and a PD decode instance embeds nothing
|
||||
(`general_mm_embed_routine` skips decode and target-verify forwards).
|
||||
|
||||
Adaptive dispatch is the exception on the first of those: it keeps the
|
||||
requests it does not send to the encoder, and forwards the tower for them.
|
||||
"""
|
||||
from sglang.srt.runtime_context import get_disagg
|
||||
|
||||
disagg = get_disagg()
|
||||
offloaded_to_an_encoder_instance = (
|
||||
disagg.language_only and not disagg.enable_adaptive_dispatch_to_encoder
|
||||
)
|
||||
return not (
|
||||
offloaded_to_an_encoder_instance
|
||||
or disagg.language_model_only
|
||||
or disagg.disaggregation_mode == "decode"
|
||||
)
|
||||
|
||||
|
||||
def reject_attn_tp_shard_with_tp_reduce(
|
||||
layer: str,
|
||||
*,
|
||||
shard_tp_size: int,
|
||||
reduces_over_attn_tp: bool,
|
||||
multimodal_encoder: bool = False,
|
||||
hint: str = "",
|
||||
) -> None:
|
||||
"""Reject a layer that shards over attention TP but all-reduces over the TP group.
|
||||
|
||||
The two groups differ only when attention DP or attention CP makes attention
|
||||
TP narrower than TP. There the all-reduce mixes ranks that hold other
|
||||
requests or replicated inputs, so the output is wrong; a one-rank shard does
|
||||
not reduce at all.
|
||||
"""
|
||||
tp_size = get_parallel().tp_size
|
||||
if reduces_over_attn_tp or not 1 < shard_tp_size < tp_size:
|
||||
return
|
||||
if multimodal_encoder and not multimodal_encoder_runs_here():
|
||||
return
|
||||
# Name only the widths that are actually narrowing the group here, so the
|
||||
# remedy is one the operator can apply.
|
||||
parallel = get_parallel()
|
||||
narrowed_by = []
|
||||
if parallel.attn_dp_size > 1:
|
||||
narrowed_by.append(f"--attn-dp-size {parallel.attn_dp_size}")
|
||||
if parallel.attn_cp_size > 1:
|
||||
narrowed_by.append(f"--attn-cp-size {parallel.attn_cp_size}")
|
||||
remedy = " and ".join(f"drop {flag}" for flag in narrowed_by) or (
|
||||
"widen the attention TP group"
|
||||
)
|
||||
raise ValueError(
|
||||
f"{layer} shards over the attention TP group ({shard_tp_size} ranks) "
|
||||
f"but all-reduces over the full TP group ({tp_size} ranks), so it does "
|
||||
f"not support {' with '.join(narrowed_by) or 'this layout'} yet. "
|
||||
f"Use --attn-dp-size equal to --tp-size, or {remedy}{hint}."
|
||||
)
|
||||
|
||||
|
||||
def is_allocation_symmetric() -> bool:
|
||||
return not is_dp_attention_enabled() or is_dp_max_padding()
|
||||
|
||||
|
||||
@@ -464,6 +464,7 @@ class FusedMoE(torch.nn.Module):
|
||||
is_gated=is_gated,
|
||||
routing_method_type=routing_method_type,
|
||||
gate_up_interleaved=gate_up_interleaved,
|
||||
layer=self,
|
||||
)
|
||||
|
||||
self.quant_method = quant_method
|
||||
|
||||
@@ -68,6 +68,35 @@ class MoeRunnerConfig:
|
||||
# False preserves backend defaults, including their existing FP32 paths.
|
||||
silu_mul_keep_fp32: bool = False
|
||||
|
||||
# The MoE placement a fused func needs is the layer's own, not the
|
||||
# deployment's: DWDP's `FusedMoE.bind_full_expert_weights` collapses a
|
||||
# layer's EP view after load. Read it off the layer so there is one
|
||||
# address to keep correct.
|
||||
@property
|
||||
def moe_tp_size(self) -> int:
|
||||
return self._layer_placement("moe_tp_size")
|
||||
|
||||
@property
|
||||
def moe_tp_rank(self) -> int:
|
||||
return self._layer_placement("moe_tp_rank")
|
||||
|
||||
@property
|
||||
def moe_ep_size(self) -> int:
|
||||
return self._layer_placement("moe_ep_size")
|
||||
|
||||
@property
|
||||
def moe_ep_rank(self) -> int:
|
||||
return self._layer_placement("moe_ep_rank")
|
||||
|
||||
def _layer_placement(self, name: str) -> int:
|
||||
if self.layer is None:
|
||||
raise AttributeError(
|
||||
f"{name} is the layer's MoE placement, but this runner config "
|
||||
"has no layer. A layer that runs fused MoE funcs states itself "
|
||||
"on its runner config."
|
||||
)
|
||||
return getattr(self.layer, name)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunnerInput(ABC):
|
||||
|
||||
@@ -58,10 +58,6 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
|
||||
swiglu_alpha: Optional[torch.Tensor] = None
|
||||
swiglu_beta: Optional[torch.Tensor] = None
|
||||
swiglu_limit: Optional[torch.Tensor] = None
|
||||
moe_tp_size: int = 1
|
||||
moe_tp_rank: int = 0
|
||||
moe_ep_size: int = 1
|
||||
moe_ep_rank: int = 0
|
||||
apply_routed_scaling_factor: bool = True
|
||||
|
||||
|
||||
@@ -144,12 +140,6 @@ class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo):
|
||||
# SwigluStep variant.
|
||||
use_swiglu_step: bool = False
|
||||
|
||||
# TP/EP topology (forwarded to the FlashInfer kernel)
|
||||
moe_tp_size: int = 1
|
||||
moe_tp_rank: int = 0
|
||||
moe_ep_size: int = 1
|
||||
moe_ep_rank: int = 0
|
||||
|
||||
# GPT-OSS pads its input hidden dim up to the (pre-padded) loaded weight
|
||||
# width and trims the output back. DSv4 leaves this as ``None`` (no pad).
|
||||
padded_hidden: Optional[int] = None
|
||||
@@ -318,10 +308,10 @@ def _run_flashinfer_cutlass(
|
||||
swiglu_alpha=quant_info.swiglu_alpha,
|
||||
swiglu_beta=quant_info.swiglu_beta,
|
||||
swiglu_limit=quant_info.swiglu_limit,
|
||||
ep_size=quant_info.moe_ep_size,
|
||||
ep_rank=quant_info.moe_ep_rank,
|
||||
tp_size=quant_info.moe_tp_size,
|
||||
tp_rank=quant_info.moe_tp_rank,
|
||||
ep_size=runner_config.moe_ep_size,
|
||||
ep_rank=runner_config.moe_ep_rank,
|
||||
tp_size=runner_config.moe_tp_size,
|
||||
tp_rank=runner_config.moe_tp_rank,
|
||||
tune_max_num_tokens=next_power_of_2(x.shape[0]),
|
||||
activation_type=_activation_type(runner_config),
|
||||
enable_alltoall=enable_alltoall,
|
||||
@@ -525,10 +515,10 @@ def _fused_experts_flashinfer_mxfp4_cutlass(
|
||||
swiglu_alpha=quant_info.swiglu_alpha,
|
||||
swiglu_beta=quant_info.swiglu_beta,
|
||||
swiglu_limit=quant_info.swiglu_limit,
|
||||
tp_size=quant_info.moe_tp_size,
|
||||
tp_rank=quant_info.moe_tp_rank,
|
||||
ep_size=quant_info.moe_ep_size,
|
||||
ep_rank=quant_info.moe_ep_rank,
|
||||
tp_size=runner_config.moe_tp_size,
|
||||
tp_rank=runner_config.moe_tp_rank,
|
||||
ep_size=runner_config.moe_ep_size,
|
||||
ep_rank=runner_config.moe_ep_rank,
|
||||
use_w4_group_scaling=not use_mxfp8_act_scaling,
|
||||
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
|
||||
activation_type=(
|
||||
|
||||
@@ -80,7 +80,6 @@ class HpcOpsMoeQuantInfo(MoeQuantInfo):
|
||||
w2_weight: torch.Tensor
|
||||
block_quant: bool
|
||||
global_num_experts: int
|
||||
moe_ep_rank: int
|
||||
# Blockwise path
|
||||
w13_weight_scale_inv: Optional[torch.Tensor] = None
|
||||
w2_weight_scale_inv: Optional[torch.Tensor] = None
|
||||
@@ -183,7 +182,7 @@ def fused_experts_none_to_hpc_ops(
|
||||
quant_info.w2_weight_scale_inv,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
quant_info.moe_ep_rank,
|
||||
runner_config.moe_ep_rank,
|
||||
quant_info.global_num_experts,
|
||||
)
|
||||
else:
|
||||
@@ -198,7 +197,7 @@ def fused_experts_none_to_hpc_ops(
|
||||
act_and_mul_scale,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
quant_info.moe_ep_rank,
|
||||
runner_config.moe_ep_rank,
|
||||
quant_info.global_num_experts,
|
||||
)
|
||||
|
||||
|
||||
+1
-2
@@ -71,7 +71,6 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
|
||||
assert not config.actorder, (
|
||||
"Actorder is not supported by flashinfer_trtllm backend"
|
||||
)
|
||||
self.moe_ep_rank = get_parallel().moe_ep_rank
|
||||
|
||||
if self.quant_config.quant_format != CompressionFormat.pack_quantized.value:
|
||||
raise ValueError(
|
||||
@@ -348,7 +347,7 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
|
||||
n_group=topk_config.num_expert_group,
|
||||
topk_group=topk_config.topk_group,
|
||||
intermediate_size=self.moe_runner_config.intermediate_size_per_partition,
|
||||
local_expert_offset=self.moe_ep_rank * local_num_experts,
|
||||
local_expert_offset=self.moe_runner_config.moe_ep_rank * local_num_experts,
|
||||
local_num_experts=local_num_experts,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_method_type=routing_method_type,
|
||||
|
||||
-4
@@ -368,10 +368,6 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=False,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -2701,7 +2701,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_weight=layer.w2_weight,
|
||||
block_quant=True,
|
||||
global_num_experts=int(layer.num_experts),
|
||||
moe_ep_rank=int(layer.moe_ep_rank),
|
||||
w13_weight_scale_inv=layer.hpc_ops_w13_weight_scale,
|
||||
w2_weight_scale_inv=layer.hpc_ops_w2_weight_scale,
|
||||
block_shape=self.quant_config.weight_block_size,
|
||||
@@ -2712,7 +2711,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_weight=layer.w2_weight,
|
||||
block_quant=False,
|
||||
global_num_experts=int(layer.num_experts),
|
||||
moe_ep_rank=int(layer.moe_ep_rank),
|
||||
gate_up_alphas=layer.hpc_ops_gate_up_alphas,
|
||||
down_alphas=layer.hpc_ops_down_alphas,
|
||||
w13_input_scale=layer.w13_input_scale,
|
||||
|
||||
@@ -1426,10 +1426,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
@@ -3253,10 +3249,6 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=False,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -1482,10 +1482,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=layer.swiglu_alpha,
|
||||
swiglu_beta=layer.swiglu_beta,
|
||||
swiglu_limit=layer.swiglu_limit,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
padded_hidden=self._padded_hidden,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
@@ -1529,10 +1525,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=layer.swiglu_alpha,
|
||||
swiglu_beta=layer.swiglu_beta,
|
||||
swiglu_limit=layer.swiglu_limit,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
padded_hidden=self._padded_hidden,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -87,6 +87,9 @@ class Mxfp4FlashinferCutlassMoEMethod:
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend
|
||||
|
||||
self.moe_runner_config = moe_runner_config
|
||||
# The fused func reads the layer's MoE placement off the config.
|
||||
if moe_runner_config.layer is None:
|
||||
moe_runner_config.layer = layer
|
||||
|
||||
E = layer.num_local_experts
|
||||
device = layer.w13_weight.device
|
||||
@@ -249,10 +252,6 @@ class Mxfp4FlashinferCutlassMoEMethod:
|
||||
swiglu_beta=None,
|
||||
swiglu_limit=self._swiglu_limit_tensor,
|
||||
use_swiglu_step=self._use_swiglu_step,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
padded_hidden=None,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -1105,10 +1105,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -1149,9 +1149,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
has_active_lora=has_active_lora,
|
||||
experts_shared_outer_loras=self.experts_shared_outer_loras,
|
||||
cg_buffers=cg_buffers,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
hidden_size=getattr(self.base_layer, "hidden_size", 0),
|
||||
lora_use_virtual_experts=self.lora_use_virtual_experts,
|
||||
)
|
||||
|
||||
|
||||
@@ -75,8 +75,6 @@ class LoRAManager:
|
||||
dtype: torch.dtype,
|
||||
server_args: ServerArgs,
|
||||
lora_backend: str = "triton",
|
||||
tp_size: int = 1,
|
||||
tp_rank: int = 0,
|
||||
max_lora_rank: Optional[int] = None,
|
||||
target_modules: Optional[Iterable[str]] = None,
|
||||
lora_paths: Optional[List[LoRARef]] = None,
|
||||
@@ -90,8 +88,6 @@ class LoRAManager:
|
||||
self.load_config: LoadConfig = load_config
|
||||
self.dtype: torch.dtype = dtype
|
||||
self.device: torch.device = next(self.base_model.parameters()).device
|
||||
self.tp_size: int = tp_size
|
||||
self.tp_rank: int = tp_rank
|
||||
# Attention projections shard on the attn-TP group; extracted once
|
||||
# here (parallel groups are frozen after init_torch_distributed).
|
||||
self.attn_tp_size: int = get_parallel().attn_tp_size
|
||||
@@ -1004,9 +1000,6 @@ class LoRAManager:
|
||||
base_hf_config=self.base_hf_config,
|
||||
max_loras_per_batch=self.max_loras_per_batch,
|
||||
dtype=self.dtype,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
attn_tp_size=self.attn_tp_size,
|
||||
max_lora_rank=self.max_lora_rank,
|
||||
target_modules=self.target_modules,
|
||||
base_model=self.base_model,
|
||||
|
||||
@@ -184,10 +184,6 @@ class LoRAInfo:
|
||||
experts_shared_outer_loras: bool = False
|
||||
cg_buffers: dict | None = None
|
||||
|
||||
fully_sharded: bool = False
|
||||
tp_size: int = 1
|
||||
tp_rank: int = 0
|
||||
hidden_size: int = 0
|
||||
lora_use_virtual_experts: bool = False
|
||||
|
||||
|
||||
@@ -391,7 +387,6 @@ def _add_lora_gate_up_delta(
|
||||
expand_num_warps=4,
|
||||
expand_num_stages=2,
|
||||
expand_split_k=1,
|
||||
fully_sharded=lora_info.fully_sharded,
|
||||
)
|
||||
|
||||
|
||||
@@ -424,12 +419,6 @@ def _add_lora_down_delta(
|
||||
if lora_info.experts_shared_outer_loras and not lora_info.lora_use_virtual_experts:
|
||||
down_lora_b = down_lora_b.expand(-1, lora_info.num_experts, -1, -1)
|
||||
|
||||
if lora_info.fully_sharded and lora_info.tp_size > 1:
|
||||
shard_size = lora_info.hidden_size // lora_info.tp_size
|
||||
offset = shard_size * lora_info.tp_rank
|
||||
else:
|
||||
offset = 0
|
||||
|
||||
if lora_info.lora_use_virtual_experts:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
output=intermediate_cache,
|
||||
@@ -474,8 +463,6 @@ def _add_lora_down_delta(
|
||||
expand_num_stages=2,
|
||||
expand_split_k=1,
|
||||
mul_routed_weight=True,
|
||||
fully_sharded=lora_info.fully_sharded,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -138,9 +138,6 @@ class LoRAMemoryPool:
|
||||
base_hf_config: AutoConfig,
|
||||
max_loras_per_batch: int,
|
||||
dtype: torch.dtype,
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
attn_tp_size: int,
|
||||
max_lora_rank: int,
|
||||
target_modules: Set[str],
|
||||
base_model: torch.nn.Module,
|
||||
@@ -154,8 +151,9 @@ class LoRAMemoryPool:
|
||||
self.num_layer: int = base_hf_config.num_hidden_layers
|
||||
self.max_loras_per_batch: int = max_loras_per_batch
|
||||
self.dtype: torch.dtype = dtype
|
||||
self.tp_size: int = tp_size
|
||||
self.tp_rank: int = tp_rank
|
||||
parallel = get_parallel()
|
||||
self.tp_size: int = parallel.tp_size
|
||||
self.tp_rank: int = parallel.tp_rank
|
||||
self.lora_added_tokens_size: int = lora_added_tokens_size
|
||||
self.max_lora_rank: int = max_lora_rank
|
||||
self.target_modules: Set[str] = target_modules
|
||||
@@ -193,7 +191,7 @@ class LoRAMemoryPool:
|
||||
# under attention DP is `attn_tp_size = tp_size // attn_dp_size`.
|
||||
# The corresponding LoRA wrappers slice weights by the base layer's
|
||||
# attn_tp-local rank, so the buffer shapes must match that shard.
|
||||
self.attn_tp_size: int = attn_tp_size
|
||||
self.attn_tp_size: int = parallel.attn_tp_size
|
||||
|
||||
# Initialize eviction policy
|
||||
self.eviction_policy = get_eviction_policy(eviction_policy)
|
||||
@@ -227,7 +225,7 @@ class LoRAMemoryPool:
|
||||
# Cache lm_head shard_indices from the base model so that buffer
|
||||
# allocation uses the same sharding as the base ParallelLMHead layer.
|
||||
self.lm_head_shard_indices = None
|
||||
if "lm_head" in target_modules and tp_size > 1:
|
||||
if "lm_head" in target_modules and self.tp_size > 1:
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
for _, module in base_model.named_modules():
|
||||
|
||||
@@ -705,17 +705,15 @@ class HiCacheController:
|
||||
if storage_backend_extra_config is None:
|
||||
storage_backend_extra_config = {}
|
||||
|
||||
parallel = get_parallel()
|
||||
if is_dp_attention_enabled():
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.tp_size = get_parallel().attn_tp_size
|
||||
self.dp_rank = get_parallel().attn_dp_rank
|
||||
tp_rank = parallel.attn_tp_rank
|
||||
tp_size = parallel.attn_tp_size
|
||||
dp_rank = parallel.attn_dp_rank
|
||||
else:
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.dp_rank = 0
|
||||
|
||||
self.pp_rank = get_parallel().pp_rank
|
||||
self.pp_size = get_parallel().pp_size
|
||||
tp_rank = parallel.tp_rank
|
||||
tp_size = parallel.tp_size
|
||||
dp_rank = 0
|
||||
|
||||
# Currently, NPUMLATokenToKVPool is the subclass of MLATokenToKVPool.
|
||||
# DeepSeekV4TokenToKVPool has compressed MLA-style rank-replicated cache
|
||||
@@ -732,23 +730,23 @@ class HiCacheController:
|
||||
should_split_heads = False
|
||||
|
||||
if tp_lcm_size:
|
||||
assert tp_lcm_size % self.tp_size == 0, (
|
||||
assert tp_lcm_size % tp_size == 0, (
|
||||
"tp_lcm_size must be divisible by tp_size."
|
||||
)
|
||||
should_split_heads = (
|
||||
not is_rank_replicated
|
||||
and self.mem_pool_host.layout == "page_head"
|
||||
and tp_lcm_size > self.tp_size
|
||||
and tp_lcm_size > tp_size
|
||||
)
|
||||
|
||||
attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
|
||||
model_name = storage_model_name(model_name, self.storage_host_pool)
|
||||
|
||||
return HiCacheStorageConfig(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
pp_rank=self.pp_rank,
|
||||
pp_size=self.pp_size,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
pp_rank=parallel.pp_rank,
|
||||
pp_size=parallel.pp_size,
|
||||
attn_cp_rank=attn_cp_rank,
|
||||
attn_cp_size=attn_cp_size,
|
||||
# TODO(hzh): Rename is_mla_model to is_rank_replicated.
|
||||
@@ -758,7 +756,7 @@ class HiCacheController:
|
||||
model_name=model_name,
|
||||
tp_lcm_size=tp_lcm_size,
|
||||
should_split_heads=should_split_heads,
|
||||
dp_rank=self.dp_rank,
|
||||
dp_rank=dp_rank,
|
||||
extra_config=storage_backend_extra_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -76,12 +76,9 @@ class _NegotiateOutput(NamedTuple):
|
||||
class PrefillDelayer:
|
||||
def __init__(
|
||||
self,
|
||||
cpu_group,
|
||||
max_delay_passes: int,
|
||||
token_usage_low_watermark: Optional[float],
|
||||
metrics_collector: Optional["SchedulerMetricsCollector"] = None,
|
||||
device: Optional["torch.device"] = "cpu",
|
||||
device_group=None,
|
||||
debug_log_enabled: bool = True,
|
||||
):
|
||||
self._max_delay_passes = max_delay_passes
|
||||
@@ -117,14 +114,12 @@ class PrefillDelayer:
|
||||
get_schedule().disable_overlap_schedule
|
||||
or envs.SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH.get()
|
||||
)
|
||||
tp_group = parallel.tp_group
|
||||
if use_nccl:
|
||||
assert device_group is not None, (
|
||||
"device_group is required when using NCCL for PrefillDelayer all-gather"
|
||||
)
|
||||
self._gather_group = device_group
|
||||
self._gather_device = device
|
||||
self._gather_group = tp_group.device_group
|
||||
self._gather_device = tp_group.device
|
||||
else:
|
||||
self._gather_group = cpu_group
|
||||
self._gather_group = tp_group.cpu_group
|
||||
self._gather_device = "cpu"
|
||||
|
||||
# Fields packed per rank into the all-gather tensor: prefillable,
|
||||
|
||||
@@ -109,6 +109,7 @@ from sglang.srt.dllm.mixin.scheduler import SchedulerDllmMixin
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.moe import initialize_moe_config
|
||||
from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config
|
||||
from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config
|
||||
@@ -568,9 +569,6 @@ class Scheduler(
|
||||
tp_worker=self.tp_worker,
|
||||
page_size=self.page_size,
|
||||
spec_algorithm=self.spec_algorithm,
|
||||
attn_tp_cpu_group=self.attn_tp_cpu_group,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
attn_cp_cpu_group=self.attn_cp_cpu_group,
|
||||
enable_metrics=get_observability().enable_metrics,
|
||||
enable_kv_cache_events=bool(
|
||||
get_observability().kv_events_config
|
||||
@@ -578,8 +576,6 @@ class Scheduler(
|
||||
and get_parallel().attn_tp_rank == 0
|
||||
and get_parallel().attn_cp_rank == 0
|
||||
),
|
||||
tp_group=self.tp_group,
|
||||
pp_group=self.pp_group,
|
||||
enable_hierarchical_cache=self.enable_hierarchical_cache,
|
||||
hicache_draft_plan=(
|
||||
self.draft_worker.hicache_draft_plan
|
||||
@@ -626,11 +622,7 @@ class Scheduler(
|
||||
self.decode_offload_manager = DecodeKVCacheOffloadManager(
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
tp_group=(
|
||||
self.attn_tp_cpu_group
|
||||
if self.attn_dp_enabled
|
||||
else self.tp_cpu_group
|
||||
),
|
||||
tp_group=self.dp_tp_cpu_group,
|
||||
tree_cache=self.tree_cache,
|
||||
)
|
||||
else:
|
||||
@@ -747,16 +739,16 @@ class Scheduler(
|
||||
):
|
||||
return
|
||||
|
||||
parallel = get_parallel()
|
||||
tp_group = parallel.tp_group
|
||||
rank = (
|
||||
get_parallel().dp_rank
|
||||
if get_parallel().dp_rank is not None
|
||||
else self.tp_group.rank_in_group
|
||||
parallel.dp_rank if parallel.dp_rank is not None else tp_group.rank_in_group
|
||||
)
|
||||
logger.info("HCCL DP prewarm start: rank=%s", rank)
|
||||
_prewarm_hccl_group(
|
||||
device=self.tp_group.device,
|
||||
group=self.tp_group.device_group,
|
||||
device_module=self.tp_group.device_module,
|
||||
device=tp_group.device,
|
||||
group=tp_group.device_group,
|
||||
device_module=tp_group.device_module,
|
||||
)
|
||||
logger.info("HCCL DP prewarm done: rank=%s", rank)
|
||||
|
||||
@@ -1213,11 +1205,9 @@ class Scheduler(
|
||||
self.world_group = get_parallel().world_group
|
||||
|
||||
# NOTE: dp_tp_* are request/data-plane coordination groups (not tensor collectives).
|
||||
# When DP attention is enabled, scope to the attention-TP group; otherwise use
|
||||
# the base TP group. Entry rank is the local rank 0 in that group.
|
||||
# Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA
|
||||
# stream/device coupling (#11910).
|
||||
self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group
|
||||
# Entry rank is the local rank 0 in that group. Use the CPU (gloo) group to
|
||||
# broadcast VLM Python objects and avoid CUDA stream/device coupling (#11910).
|
||||
self.dp_tp_group = get_dp_tp_group()
|
||||
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
|
||||
|
||||
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
|
||||
@@ -1408,8 +1398,6 @@ class Scheduler(
|
||||
)
|
||||
else:
|
||||
self.prefill_delayer = PrefillDelayer(
|
||||
cpu_group=self.tp_cpu_group,
|
||||
device_group=self.tp_group.device_group,
|
||||
metrics_collector=(
|
||||
self.metrics_collector
|
||||
if self.metrics_reporter.enable_metrics
|
||||
@@ -1417,7 +1405,6 @@ class Scheduler(
|
||||
),
|
||||
max_delay_passes=get_schedule().prefill_delayer_max_delay_passes,
|
||||
token_usage_low_watermark=get_schedule().prefill_delayer_token_usage_low_watermark,
|
||||
device=self.tp_group.device,
|
||||
debug_log_enabled=get_parallel().attn_tp_rank == 0,
|
||||
)
|
||||
|
||||
@@ -1536,7 +1523,6 @@ class Scheduler(
|
||||
|
||||
# The decode requests polling kv cache
|
||||
self.disagg_decode_transfer_queue = DecodeTransferQueue(
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
|
||||
metadata_buffers=self.disagg_metadata_buffers,
|
||||
scheduler=self,
|
||||
@@ -1553,7 +1539,6 @@ class Scheduler(
|
||||
scheduler=self,
|
||||
transfer_queue=self.disagg_decode_transfer_queue,
|
||||
tree_cache=self.tree_cache,
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
gpu_id=get_device().gpu_id,
|
||||
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
@@ -1613,9 +1598,6 @@ class Scheduler(
|
||||
self.server_args,
|
||||
dtype=self.model_config.dtype,
|
||||
hf_config=self.model_config.hf_config,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
tp_group=self.tp_group,
|
||||
scheduler=self,
|
||||
)
|
||||
|
||||
@@ -2242,7 +2224,6 @@ class Scheduler(
|
||||
|
||||
def init_profiler(self) -> None:
|
||||
self.profiler_manager = SchedulerProfilerManager(
|
||||
dp_tp_cpu_group=self.dp_tp_cpu_group,
|
||||
get_forward_ct=lambda: self.forward_ct,
|
||||
)
|
||||
|
||||
@@ -2250,7 +2231,6 @@ class Scheduler(
|
||||
self.weight_updater = SchedulerWeightUpdaterManager(
|
||||
tp_worker=self.tp_worker,
|
||||
draft_worker=self.draft_worker,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
memory_saver_adapter=self.memory_saver_adapter,
|
||||
flush_cache=self.flush_cache,
|
||||
is_fully_idle=self.is_fully_idle,
|
||||
@@ -2333,11 +2313,6 @@ class Scheduler(
|
||||
recv_skipper=self.recv_skipper,
|
||||
input_blocker=self.input_blocker,
|
||||
mm_receiver=self.mm_receiver,
|
||||
tp_group=self.tp_group,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
attn_tp_cpu_group=self.attn_tp_cpu_group,
|
||||
attn_cp_cpu_group=self.attn_cp_cpu_group,
|
||||
world_group=self.world_group,
|
||||
server_args=self.server_args,
|
||||
model_config=self.model_config,
|
||||
max_recv_per_poll=self.max_recv_per_poll,
|
||||
|
||||
@@ -4,7 +4,7 @@ import gc
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
@@ -17,6 +17,7 @@ from typing import (
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.managers.io_struct import ProfileReq, ProfileReqOutput, ProfileReqType
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.step_span_utils import set_detailed_annotations_enabled
|
||||
@@ -51,10 +52,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class SchedulerProfilerManager:
|
||||
dp_tp_cpu_group: Any
|
||||
get_forward_ct: Callable[[], int]
|
||||
# Taken at construction: a batch can stop profiling while a scoped
|
||||
# override has moved the TP group.
|
||||
dp_tp_cpu_group: Any = field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.dp_tp_cpu_group = get_dp_tp_group().cpu_group
|
||||
if envs.SGLANG_PROFILE_V2.get():
|
||||
self._profile_manager = ProfileManager(
|
||||
cpu_group=self.dp_tp_cpu_group,
|
||||
|
||||
@@ -63,11 +63,6 @@ class SchedulerRequestReceiver:
|
||||
recv_skipper: Any
|
||||
input_blocker: Any
|
||||
mm_receiver: Any
|
||||
tp_group: Any
|
||||
tp_cpu_group: Any
|
||||
attn_tp_cpu_group: Any
|
||||
attn_cp_cpu_group: Any
|
||||
world_group: Any
|
||||
server_args: ServerArgs
|
||||
model_config: ModelConfig
|
||||
max_recv_per_poll: int
|
||||
@@ -157,7 +152,7 @@ class SchedulerRequestReceiver:
|
||||
recv_reqs = point_to_point_pyobj(
|
||||
[],
|
||||
get_parallel().pp_rank * get_parallel().tp_size + dp_offset,
|
||||
self.world_group.cpu_group,
|
||||
get_parallel().world_group.cpu_group,
|
||||
(get_parallel().pp_rank - 1) * get_parallel().tp_size + dp_offset,
|
||||
get_parallel().pp_rank * get_parallel().tp_size + dp_offset,
|
||||
)
|
||||
@@ -195,22 +190,24 @@ class SchedulerRequestReceiver:
|
||||
if _local_ctrl:
|
||||
control_reqs = attn_cp_tp_broadcast_pyobj(control_reqs)
|
||||
elif get_parallel().tp_size != 1:
|
||||
tp_group = get_parallel().tp_group
|
||||
control_reqs = broadcast_pyobj(
|
||||
control_reqs,
|
||||
self.tp_group.rank,
|
||||
self.tp_cpu_group,
|
||||
src=self.tp_group.ranks[0],
|
||||
tp_group.rank,
|
||||
tp_group.cpu_group,
|
||||
src=tp_group.ranks[0],
|
||||
)
|
||||
recv_reqs = work_reqs + control_reqs
|
||||
else:
|
||||
if recv_reqs is not None:
|
||||
recv_reqs = [*recv_reqs, *local_reqs]
|
||||
if get_parallel().tp_size != 1:
|
||||
tp_group = get_parallel().tp_group
|
||||
recv_reqs = broadcast_pyobj(
|
||||
recv_reqs,
|
||||
self.tp_group.rank,
|
||||
self.tp_cpu_group,
|
||||
src=self.tp_group.ranks[0],
|
||||
tp_group.rank,
|
||||
tp_group.cpu_group,
|
||||
src=tp_group.ranks[0],
|
||||
)
|
||||
return recv_reqs
|
||||
|
||||
@@ -268,11 +265,11 @@ class SchedulerRequestReceiver:
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_dp_enabled:
|
||||
if parallel.attn_tp_size > 1:
|
||||
barrier(group=self.attn_tp_cpu_group)
|
||||
barrier(group=parallel.attn_tp_group.cpu_group)
|
||||
if parallel.attn_cp_size > 1:
|
||||
barrier(group=self.attn_cp_cpu_group)
|
||||
barrier(group=parallel.attn_cp_group.cpu_group)
|
||||
elif parallel.tp_size > 1:
|
||||
barrier(group=self.tp_cpu_group)
|
||||
barrier(group=parallel.tp_group.cpu_group)
|
||||
|
||||
# 2. materialize independently so one bad VLM request does not stop the loop
|
||||
failed = torch.zeros(len(tokenized_reqs), dtype=torch.int32)
|
||||
@@ -292,11 +289,15 @@ class SchedulerRequestReceiver:
|
||||
# 3. all ranks reject the same requests before entering model collectives
|
||||
if parallel.attn_dp_enabled:
|
||||
if parallel.attn_tp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group)
|
||||
all_reduce(
|
||||
failed, op=ReduceOp.MAX, group=parallel.attn_tp_group.cpu_group
|
||||
)
|
||||
if parallel.attn_cp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_cp_cpu_group)
|
||||
all_reduce(
|
||||
failed, op=ReduceOp.MAX, group=parallel.attn_cp_group.cpu_group
|
||||
)
|
||||
elif parallel.tp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.tp_cpu_group)
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=parallel.tp_group.cpu_group)
|
||||
|
||||
error = MMInputsProcessError(
|
||||
"Failed to materialize shared-memory multimodal features on a scheduler rank."
|
||||
|
||||
@@ -44,7 +44,7 @@ from sglang.srt.managers.io_struct import (
|
||||
UpdateWeightsFromTensorReqInput,
|
||||
UpdateWeightsFromTensorReqOutput,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.runtime_context import get_model, get_parallel
|
||||
from sglang.srt.utils.weight_checker import overall_checksum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -88,7 +88,6 @@ class _WeightUpdateSession(msgspec.Struct, frozen=True):
|
||||
class SchedulerWeightUpdaterManager:
|
||||
tp_worker: Any
|
||||
draft_worker: Any
|
||||
tp_cpu_group: Any
|
||||
memory_saver_adapter: Any
|
||||
flush_cache: Callable[..., bool]
|
||||
is_fully_idle: Callable[..., bool]
|
||||
@@ -98,6 +97,10 @@ class SchedulerWeightUpdaterManager:
|
||||
stashed_model_static_state: Any = None
|
||||
# replicated on every TP rank, so a rejected call returns on all ranks before any barrier
|
||||
_session: Optional[_WeightUpdateSession] = None
|
||||
tp_cpu_group: Any = field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.tp_cpu_group = get_parallel().tp_group.cpu_group
|
||||
|
||||
@contextmanager
|
||||
def _observe_weight_load(self, source: str) -> Iterator[None]:
|
||||
|
||||
@@ -865,7 +865,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
def get_prefetch_submission(self, rid: str) -> Optional[PrefetchSubmission]:
|
||||
if self.pp_prefetch_command_group is None:
|
||||
return None
|
||||
if self.pp_rank != 0:
|
||||
if self.storage_config.pp_rank != 0:
|
||||
return PrefetchSubmission()
|
||||
with self.pp_prefetch_state_lock:
|
||||
state = self.pp_prefetch_states.get(rid)
|
||||
@@ -898,7 +898,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
)
|
||||
|
||||
if self.pp_rank != 0:
|
||||
if self.storage_config.pp_rank != 0:
|
||||
raise RuntimeError("Only PP0 can submit a PP prefetch ticket.")
|
||||
|
||||
rid = handle.rid
|
||||
@@ -928,7 +928,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
storage_hit_count = len(ticket.prefetch_key.token_ids)
|
||||
try:
|
||||
for pp_rank in range(self.tp_rank, self.pp_size, self.tp_size):
|
||||
config = self.storage_config
|
||||
for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size):
|
||||
_, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank)
|
||||
storage_hit_count = min(storage_hit_count, rank_hit_count)
|
||||
except Exception:
|
||||
@@ -1096,7 +1097,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
assert group is not None
|
||||
rank = torch.distributed.get_rank()
|
||||
source = torch.distributed.get_process_group_ranks(group)[0]
|
||||
is_source = self.pp_rank == 0
|
||||
is_source = self.storage_config.pp_rank == 0
|
||||
|
||||
while True:
|
||||
objects = []
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -58,10 +59,7 @@ from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
|
||||
from sglang.srt.utils import ceil_align, is_hip
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.managers.tp_worker import BaseTpWorker
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
|
||||
@@ -266,13 +264,8 @@ def build_kv_cache(
|
||||
tp_worker: BaseTpWorker,
|
||||
page_size: int,
|
||||
spec_algorithm: SpeculativeAlgorithm,
|
||||
attn_tp_cpu_group: ProcessGroup,
|
||||
tp_cpu_group: ProcessGroup,
|
||||
attn_cp_cpu_group: ProcessGroup,
|
||||
enable_metrics: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
tp_group: GroupCoordinator,
|
||||
pp_group: GroupCoordinator,
|
||||
enable_hierarchical_cache: bool,
|
||||
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
|
||||
) -> KVCacheBuildResult:
|
||||
@@ -358,12 +351,10 @@ def build_kv_cache(
|
||||
else token_to_kv_pool_allocator.page_size
|
||||
),
|
||||
is_eagle=spec_algorithm.is_eagle(),
|
||||
tp_cache_group=(
|
||||
attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group
|
||||
),
|
||||
attn_cp_cache_group=attn_cp_cpu_group,
|
||||
attn_tp_cache_group=attn_tp_cpu_group,
|
||||
pp_cache_group=pp_group.cpu_group,
|
||||
tp_cache_group=get_dp_tp_group().cpu_group,
|
||||
attn_cp_cache_group=parallel.attn_cp_group.cpu_group,
|
||||
attn_tp_cache_group=parallel.attn_tp_group.cpu_group,
|
||||
pp_cache_group=parallel.pp_group.cpu_group,
|
||||
eviction_policy=get_memory().radix_eviction_policy,
|
||||
eviction_policy_config=get_memory().radix_eviction_policy_config,
|
||||
enable_metrics=enable_metrics,
|
||||
@@ -394,7 +385,7 @@ def build_kv_cache(
|
||||
model_config=model_config,
|
||||
tp_size=parallel.tp_size,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_group=tp_group,
|
||||
tp_group=parallel.tp_group,
|
||||
)
|
||||
with auto_size_hicache(
|
||||
params,
|
||||
|
||||
@@ -123,6 +123,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
|
||||
extra_config, *_ = HybridCacheController.parse_storage_backend_extra_config(
|
||||
get_memory().hicache_storage_backend_extra_config
|
||||
)
|
||||
# TODO: unlike `HiCacheController._generate_storage_config`, this sets no
|
||||
# `dp_rank`. Aligning the storage-config producers changes persisted
|
||||
# storage keys, so it waits for a storage-key format change.
|
||||
storage_config = HiCacheStorageConfig(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
|
||||
@@ -295,6 +295,11 @@ class UMBPDirectLinker(UnifiedCacheLinker):
|
||||
)
|
||||
extra_config["dram_page_size"] = dram_page_size
|
||||
|
||||
# TODO: `tp_rank` is the rank in the cache TP group but `tp_size` is the
|
||||
# full TP width, and no `dp_rank` is set, unlike
|
||||
# `HiCacheController._generate_storage_config`. Aligning the
|
||||
# storage-config producers changes persisted storage keys, so it waits
|
||||
# for a storage-key format change.
|
||||
storage_config = HiCacheStorageConfig(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=get_parallel().tp_size,
|
||||
|
||||
@@ -276,13 +276,14 @@ class StorageAttachment:
|
||||
"""
|
||||
cache = self._cache
|
||||
controller = cache.cache_controller
|
||||
config = controller.storage_config
|
||||
attn_cp_rank, attn_cp_size = controller.get_attn_cp_rank_and_size()
|
||||
labels = {
|
||||
"storage_backend": storage_backend,
|
||||
"tp_rank": controller.tp_rank,
|
||||
"dp_rank": controller.dp_rank,
|
||||
"pp_rank": controller.pp_rank,
|
||||
"pp_size": controller.pp_size,
|
||||
"tp_rank": config.tp_rank,
|
||||
"dp_rank": config.dp_rank,
|
||||
"pp_rank": config.pp_rank,
|
||||
"pp_size": config.pp_size,
|
||||
"attn_cp_rank": attn_cp_rank,
|
||||
"attn_cp_size": attn_cp_size,
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ from sglang.srt.layers.cp.utils import (
|
||||
is_cp_active,
|
||||
is_mla_cp_enabled,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.sampler import create_sampler
|
||||
from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers
|
||||
@@ -963,11 +964,7 @@ class ModelRunner:
|
||||
top_k=hisparse_top_k,
|
||||
device_buffer_size=hisparse_cfg.device_buffer_size,
|
||||
device=self.device,
|
||||
tp_group=(
|
||||
get_parallel().attn_tp_group.cpu_group
|
||||
if get_parallel().attn_dp_enabled
|
||||
else get_parallel().tp_group.cpu_group
|
||||
),
|
||||
tp_group=get_dp_tp_group().cpu_group,
|
||||
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
|
||||
swap_in_block_size=hisparse_cfg.swap_in_block_size,
|
||||
shared_index_layers=resolve_shared_index_layers(
|
||||
@@ -1368,8 +1365,6 @@ class ModelRunner:
|
||||
dtype=self.dtype,
|
||||
server_args=self.server_args,
|
||||
lora_backend=get_lora().lora_backend,
|
||||
tp_size=get_parallel().tp_size,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
max_lora_rank=get_lora().max_lora_rank,
|
||||
target_modules=get_lora().lora_target_modules,
|
||||
lora_paths=get_lora().lora_paths,
|
||||
|
||||
@@ -12,6 +12,7 @@ from transformers import CLIPConfig, CLIPTextConfig, CLIPVisionConfig
|
||||
|
||||
from sglang.srt.layers.activation import QuickGELU, get_act_fn
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -200,6 +201,15 @@ class CLIPAttention(nn.Module):
|
||||
tp_rank=parallel.attn_tp_rank,
|
||||
tp_size=parallel.attn_tp_size,
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=parallel.attn_tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
multimodal_encoder=True,
|
||||
)
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=config.hidden_size,
|
||||
|
||||
@@ -29,9 +29,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
|
||||
@@ -209,6 +209,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin):
|
||||
gemm1_alpha=None,
|
||||
gemm1_clamp_limit=None,
|
||||
is_gated=True,
|
||||
layer=self,
|
||||
)
|
||||
|
||||
FusedMoELoadingMixin.__init__(
|
||||
|
||||
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
@@ -313,6 +314,14 @@ class IQuestQ1Attention(nn.Module):
|
||||
tp_size=attn_tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# The decoder boundary reduces partial attention outputs. Only a
|
||||
# standalone attention that owns its TP reduction needs this guard.
|
||||
if reduce_results:
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=attn_tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
|
||||
@@ -59,6 +59,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -357,6 +358,16 @@ class MLP2(nn.Module):
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
elif use_tensor_parallel:
|
||||
# TODO: these layers shard over attention TP but reduce over the full
|
||||
# TP group; reduce over the attention-TP group so attention DP and
|
||||
# attention CP narrower than TP can run them.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.fc0 = ColumnParallelLinear(
|
||||
dims[0],
|
||||
dims[1],
|
||||
|
||||
@@ -514,6 +514,12 @@ class AudioEncoderAttention(nn.Module):
|
||||
window_size=window_size,
|
||||
customized_position_embedding_applier=_audio_rope_applier,
|
||||
prefix="attn",
|
||||
# TODO: the audio encoder does not follow --mm-enable-dp-encoder and
|
||||
# reduces over the full TP group, so its output is wrong under
|
||||
# attention DP narrower than TP. Rejecting that would reject the
|
||||
# documented MiMo-V2.5 `--tp 8 --attn-dp-size 2` launch; fix the
|
||||
# reduce group or run the encoder data-parallel instead.
|
||||
allow_tp_reduce_mismatch=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -17,7 +17,10 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -83,6 +86,16 @@ class MiniMaxVLMultiModalProjector(nn.Module):
|
||||
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
|
||||
self.linear_1 = ColumnParallelLinear(
|
||||
vision_hidden_size,
|
||||
@@ -138,6 +151,16 @@ class MiniMaxVLPatchMerger(nn.Module):
|
||||
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
|
||||
self.linear_1 = ColumnParallelLinear(
|
||||
text_hidden_size * spatial_merge_size**2,
|
||||
@@ -260,6 +283,16 @@ class CLIPEncoderLayer(nn.Module):
|
||||
self.use_data_parallel = use_data_parallel
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
|
||||
self.self_attn = VisionAttention(
|
||||
embed_dim=config.hidden_size,
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -282,6 +283,12 @@ class PhiMoEAttention(nn.Module):
|
||||
tp_size=attn_tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__, shard_tp_size=attn_tp_size, reduces_over_attn_tp=False
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
hidden_size,
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
@@ -148,6 +149,16 @@ class Qwen3_VisionMLP(nn.Module):
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=self.tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.linear_fc1 = ColumnParallelLinear(
|
||||
in_features,
|
||||
hidden_features,
|
||||
@@ -317,6 +328,16 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=self.tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.linear_fc1 = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.padded_context_dim,
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
@@ -545,6 +546,15 @@ class Step3VisionMLP(nn.Module):
|
||||
prefix=add_prefix("gate_proj", prefix),
|
||||
)
|
||||
self.act = ACT2FN[hidden_act] # quick_gelu
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=attn_tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
multimodal_encoder=True,
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
intermediate_size,
|
||||
dim,
|
||||
|
||||
@@ -263,12 +263,23 @@ class ServerArgs:
|
||||
return getattr(self, "_launch_command", None)
|
||||
|
||||
def resolved_dict(self) -> dict[str, Any]:
|
||||
"""Serialize resolved field values, expanding nested records and excluding bookkeeping."""
|
||||
"""Serialize resolved field values, expanding nested records and excluding bookkeeping.
|
||||
|
||||
return {
|
||||
One exception: the deprecated `enable_dp_attention` reports whether an
|
||||
attention-DP width was configured, for clients that still read it from
|
||||
`/server_info`. Resolution leaves the field false. It reports the width
|
||||
rather than `attn_dp_enabled`, whose elastic scale-joiner arm is true at
|
||||
width one: a dump carrying that would fold the joiner's replicas into
|
||||
attention-DP groups when it is read back.
|
||||
"""
|
||||
|
||||
resolved = {
|
||||
field.name: _plain(resolution_result(self, field.name))
|
||||
for field in record_fields(type(self))
|
||||
}
|
||||
# TODO: drop together with `--enable-dp-attention` after 2026-12-31.
|
||||
resolved["enable_dp_attention"] = resolving_view(self).attn_dp_size > 1
|
||||
return resolved
|
||||
|
||||
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
@@ -351,7 +362,7 @@ class ServerArgs:
|
||||
help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.",
|
||||
)
|
||||
# `enable_dp_attention` is `no_cli=True` too; resolution turns it into
|
||||
# `attn_dp_size`.
|
||||
# `attn_dp_size`. TODO: remove the flag and the field after 2026-12-31.
|
||||
parser.add_argument(
|
||||
"--enable-dp-attention",
|
||||
action=DeprecatedStoreTrueAction,
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
|
||||
)
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
@@ -402,11 +403,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._full_embed_gpu: Optional[torch.Tensor] = None
|
||||
# Under dp attention, peer DP ranks run different (idle) paths, so
|
||||
# spec broadcasts must stay within the attn-TP group.
|
||||
self._tp_sync = SpecTpSync(
|
||||
get_parallel().attn_tp_group
|
||||
if get_parallel().attn_dp_enabled
|
||||
else get_parallel().tp_group
|
||||
)
|
||||
self._tp_sync = SpecTpSync(get_dp_tp_group())
|
||||
|
||||
# Under dp attention, the draft worker runs on the per-DP attn-TP
|
||||
# group, independent of idle peer DP ranks; it is built and run under
|
||||
|
||||
@@ -8,6 +8,7 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
)
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -233,9 +234,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
self._mask_token_id = runtime_config.mask_token_id
|
||||
|
||||
parallel = get_parallel()
|
||||
self._tp_sync = SpecTpSync(
|
||||
parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
|
||||
)
|
||||
self._tp_sync = SpecTpSync(get_dp_tp_group())
|
||||
self._draft_graph_group = (
|
||||
parallel.attn_tp_group
|
||||
if self._draft_dp_context_enabled
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.lora.lora_manager import LoRAManager
|
||||
from sglang.srt.lora.lora_registry import LoRARef
|
||||
from sglang.srt.runtime_context import get_parallel, get_spec
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -45,8 +45,6 @@ def init_uno_lora_manager(
|
||||
dtype=model_runner.dtype,
|
||||
server_args=model_runner.server_args,
|
||||
lora_backend="uno_cublas", # fast path
|
||||
tp_size=get_parallel().tp_size,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
# Infer these from the one trained adapter.
|
||||
max_lora_rank=None,
|
||||
target_modules=None,
|
||||
|
||||
@@ -7,6 +7,7 @@ is not installed or the GPU is not SM90 (the kernels ship sm90a only).
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
@@ -91,6 +92,10 @@ class TestHpcOpsMoeBlockwise(CustomTestCase):
|
||||
activation="silu",
|
||||
is_gated=True,
|
||||
inplace=False,
|
||||
# The fused func reads the layer's MoE placement off the config.
|
||||
layer=SimpleNamespace(
|
||||
moe_tp_size=1, moe_tp_rank=0, moe_ep_size=1, moe_ep_rank=0
|
||||
),
|
||||
)
|
||||
|
||||
def _run_hpc_ops(self, x, w13, w2, w13_scale, w2_scale, topk_weights, topk_ids):
|
||||
@@ -104,7 +109,6 @@ class TestHpcOpsMoeBlockwise(CustomTestCase):
|
||||
w2_weight=w2,
|
||||
block_quant=True,
|
||||
global_num_experts=E,
|
||||
moe_ep_rank=0,
|
||||
w13_weight_scale_inv=pad_hpc_ops_block_scale(w13_scale),
|
||||
w2_weight_scale_inv=pad_hpc_ops_block_scale(w2_scale),
|
||||
block_shape=[128, 128],
|
||||
@@ -175,7 +179,6 @@ class TestHpcOpsMoeBlockwise(CustomTestCase):
|
||||
w2_weight=w2,
|
||||
block_quant=False,
|
||||
global_num_experts=E,
|
||||
moe_ep_rank=0,
|
||||
gate_up_alphas=w13_scale * a1_scale,
|
||||
down_alphas=w2_scale * a2_scale,
|
||||
w13_input_scale=a1_scale,
|
||||
|
||||
@@ -244,7 +244,7 @@ class TestDcpCachedPrefixSend(CustomTestCase):
|
||||
}
|
||||
mgr.add_transfer_request = Mock()
|
||||
with get_context().override_server_args(dp_size=1):
|
||||
sender = NixlKVSender(mgr, "unused", 1, [0], 0)
|
||||
sender = NixlKVSender(mgr, "unused", 1)
|
||||
sender.init((total - prefix + page_size - 1) // page_size, 3)
|
||||
req = SimpleNamespace(
|
||||
rid="cached-prefix",
|
||||
|
||||
@@ -33,8 +33,6 @@ class TestFakeKVSender(unittest.TestCase):
|
||||
mgr=self.mgr if mgr is None else mgr,
|
||||
bootstrap_addr="fake_addr:1234",
|
||||
bootstrap_room=42,
|
||||
dest_tp_ranks=[0],
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
def _expire_deadline(self, sender: FakeKVSender, by: float = 1.0) -> None:
|
||||
|
||||
@@ -18,6 +18,7 @@ import platform
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
@@ -164,7 +165,10 @@ class TestSchedulerProfilerManagerMPS(unittest.TestCase):
|
||||
SchedulerProfilerManager,
|
||||
)
|
||||
|
||||
mgr = SchedulerProfilerManager(dp_tp_cpu_group=None, get_forward_ct=lambda: 0)
|
||||
with get_parallel().override(
|
||||
tp_group=SimpleNamespace(cpu_group=None), attn_dp_enabled=False
|
||||
):
|
||||
mgr = SchedulerProfilerManager(get_forward_ct=lambda: 0)
|
||||
mgr._init_profile(output_dir, None, None, None, None, None, False, "test")
|
||||
return mgr
|
||||
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""A MoE layer owns its placement, and the fused funcs read it through the
|
||||
layer's `MoeRunnerConfig`."""
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend
|
||||
from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.moe_runner import flashinfer_cutlass
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
FlashInferCutlassMoeQuantInfo,
|
||||
)
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=10, suite="stage-b-test-cpu-intel")
|
||||
|
||||
|
||||
def _layer(monkeypatch) -> FusedMoE:
|
||||
method = UnquantizedFusedMoEMethod()
|
||||
monkeypatch.setattr(method, "create_weights", lambda **kwargs: None)
|
||||
monkeypatch.setattr(method, "create_moe_runner", lambda layer, config: None)
|
||||
monkeypatch.setattr(
|
||||
fused_moe_layer_module,
|
||||
"create_moe_dispatcher",
|
||||
lambda config, quant_method: SimpleNamespace(),
|
||||
)
|
||||
with (
|
||||
get_context().override_server_args(model_path="dummy"),
|
||||
get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend.AUTO,
|
||||
a2a_backend=MoeA2ABackend.NONE,
|
||||
),
|
||||
get_parallel().override(
|
||||
moe_ep_size=2,
|
||||
moe_ep_rank=1,
|
||||
moe_tp_size=2,
|
||||
moe_tp_rank=1,
|
||||
tp_size=4,
|
||||
tp_rank=3,
|
||||
attn_tp_size=4,
|
||||
attn_tp_rank=3,
|
||||
),
|
||||
):
|
||||
return FusedMoE(
|
||||
num_experts=4,
|
||||
hidden_size=4,
|
||||
intermediate_size=8,
|
||||
layer_id=0,
|
||||
quant_method=method,
|
||||
)
|
||||
|
||||
|
||||
def _placement(obj):
|
||||
return (obj.moe_tp_size, obj.moe_tp_rank, obj.moe_ep_size, obj.moe_ep_rank)
|
||||
|
||||
|
||||
def test_a_config_without_a_layer_has_no_placement() -> None:
|
||||
config = MoeRunnerConfig()
|
||||
for name in ("moe_tp_size", "moe_tp_rank", "moe_ep_size", "moe_ep_rank"):
|
||||
with pytest.raises(AttributeError, match="has no layer"):
|
||||
getattr(config, name)
|
||||
|
||||
|
||||
def test_the_config_reads_the_layer_placement(monkeypatch) -> None:
|
||||
layer = _layer(monkeypatch)
|
||||
assert _placement(layer) == (2, 1, 2, 1)
|
||||
assert _placement(layer.moe_runner_config) == _placement(layer)
|
||||
|
||||
|
||||
def test_a_full_expert_bind_collapses_ep_on_the_config(monkeypatch) -> None:
|
||||
layer = _layer(monkeypatch)
|
||||
layer.bind_full_expert_weights({})
|
||||
assert _placement(layer) == (2, 1, 1, 0)
|
||||
assert _placement(layer.moe_runner_config) == (2, 1, 1, 0)
|
||||
assert layer.moe_runner_config.num_local_experts == 4
|
||||
|
||||
|
||||
def test_the_kernel_gets_the_bound_placement(monkeypatch) -> None:
|
||||
"""The reason the placement lives on the layer: after a full-expert bind
|
||||
the kernel must be told this rank owns every expert."""
|
||||
layer = _layer(monkeypatch)
|
||||
before = _kernel_placement(monkeypatch, layer.moe_runner_config)
|
||||
assert before == (2, 1, 2, 1)
|
||||
|
||||
layer.bind_full_expert_weights({})
|
||||
|
||||
# The layer kept its TP shard and now owns every expert.
|
||||
assert _kernel_placement(monkeypatch, layer.moe_runner_config) == (2, 1, 1, 0)
|
||||
|
||||
|
||||
def _kernel_placement(monkeypatch, runner_config):
|
||||
"""Run the CUTLASS fused func against a fake kernel and report the
|
||||
(tp_size, tp_rank, ep_size, ep_rank) it was handed."""
|
||||
calls = []
|
||||
|
||||
def fake_kernel(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return [kwargs["output"]]
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_cutlass,
|
||||
"_flashinfer_cutlass_fused_moe",
|
||||
lambda: (fake_kernel, None),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_cutlass, "_activation_type", lambda config: None)
|
||||
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
flashinfer_cutlass._run_flashinfer_cutlass(
|
||||
dispatch_output=SimpleNamespace(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_scale=None,
|
||||
topk_output=SimpleNamespace(
|
||||
topk_weights=torch.ones(2, 1),
|
||||
topk_ids=torch.zeros(2, 1, dtype=torch.int64),
|
||||
),
|
||||
),
|
||||
quant_info=FlashInferCutlassMoeQuantInfo(
|
||||
quant_type="bf16", w13_weight=torch.zeros(1), w2_weight=torch.zeros(1)
|
||||
),
|
||||
runner_config=runner_config,
|
||||
output=torch.empty_like(hidden_states),
|
||||
)
|
||||
(call,) = calls
|
||||
return (call["tp_size"], call["tp_rank"], call["ep_size"], call["ep_rank"])
|
||||
|
||||
|
||||
def test_cutlass_passes_the_runner_config_placement(monkeypatch) -> None:
|
||||
placement = _kernel_placement(
|
||||
monkeypatch,
|
||||
MoeRunnerConfig(
|
||||
layer=SimpleNamespace(
|
||||
moe_tp_size=2, moe_tp_rank=1, moe_ep_size=4, moe_ep_rank=3
|
||||
)
|
||||
),
|
||||
)
|
||||
assert placement == (2, 1, 4, 3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -342,6 +342,8 @@ def test_gpt_oss_sm120_padding_layout_and_kernel(monkeypatch, stated_tp_group):
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_clamp_limit=7.0,
|
||||
)
|
||||
# The fused func reads the layer's MoE placement off the config.
|
||||
config.layer = layer
|
||||
method.moe_runner_config = config
|
||||
method.runner = MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, config)
|
||||
method._process_weights_for_sm120_cutlass(layer)
|
||||
|
||||
@@ -173,7 +173,7 @@ def _round_up(x, base):
|
||||
return ((x + base - 1) // base) * base
|
||||
|
||||
|
||||
def _build_method(num_experts, hidden, inter, *, use_humming=False):
|
||||
def _build_method(num_experts, hidden, inter, *, layer=None, use_humming=False):
|
||||
from sglang.srt.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
|
||||
method = Mxfp4MoEMethod.__new__(Mxfp4MoEMethod)
|
||||
@@ -188,16 +188,17 @@ def _build_method(num_experts, hidden, inter, *, use_humming=False):
|
||||
method._padded_hidden = _round_up(hidden, 128)
|
||||
method._padded_intermediate = _round_up(inter, 128)
|
||||
method.use_flashinfer = True
|
||||
method.runner = _build_flashinfer_mxfp4_runner(num_experts, hidden, inter)
|
||||
method.runner = _build_flashinfer_mxfp4_runner(num_experts, hidden, inter, layer)
|
||||
return method
|
||||
|
||||
|
||||
def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter):
|
||||
def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter, layer=None):
|
||||
"""Construct a real MoeRunner bound to the flashinfer_mxfp4 fused func.
|
||||
|
||||
Bypasses ``create_moe_runner`` (which needs a live server arg context)
|
||||
and wires the runner with a minimal MoeRunnerConfig sufficient for the
|
||||
cutlass SM90 fused func, which only reads dispatch_output / quant_info.
|
||||
cutlass SM90 fused func. The func reads the layer's MoE placement off the
|
||||
config, so the layer states itself here the way `FusedMoE` does.
|
||||
"""
|
||||
import sglang.srt.layers.moe.moe_runner.flashinfer_cutlass # noqa: F401
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
@@ -212,6 +213,7 @@ def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter):
|
||||
top_k=None,
|
||||
activation="silu",
|
||||
is_gated=True,
|
||||
layer=layer,
|
||||
)
|
||||
return MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, cfg)
|
||||
|
||||
@@ -299,7 +301,7 @@ def test_process_weights_matches_direct_interleave(num_experts, hidden, inter):
|
||||
layer = _build_mock_layer(
|
||||
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
|
||||
)
|
||||
method = _build_method(num_experts, hidden, inter)
|
||||
method = _build_method(num_experts, hidden, inter, layer=layer)
|
||||
method._process_weights_for_sm90_cutlass(layer)
|
||||
|
||||
N_pad = _round_up(inter, 128)
|
||||
@@ -375,7 +377,7 @@ def test_apply_sm90_cutlass_matches_flashinfer_direct(
|
||||
layer = _build_mock_layer(
|
||||
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
|
||||
)
|
||||
method = _build_method(num_experts, hidden, inter)
|
||||
method = _build_method(num_experts, hidden, inter, layer=layer)
|
||||
method._process_weights_for_sm90_cutlass(layer)
|
||||
|
||||
out_sglang = method._apply_sm90_cutlass(
|
||||
@@ -444,7 +446,7 @@ def test_process_weights_humming_matches_flashinfer_direct():
|
||||
layer = _build_mock_layer(
|
||||
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
|
||||
)
|
||||
method = _build_method(num_experts, hidden, inter, use_humming=True)
|
||||
method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True)
|
||||
method._process_weights_for_sm90_cutlass(layer)
|
||||
|
||||
# GPT-OSS loads pair-wise [gate, up]; FlashInfer consumes halved [up; gate].
|
||||
@@ -484,7 +486,7 @@ def test_humming_padding_preserves_per_expert_residual():
|
||||
layer = _build_mock_layer(
|
||||
num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
|
||||
)
|
||||
method = _build_method(num_experts, hidden, inter, use_humming=True)
|
||||
method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True)
|
||||
method._process_weights_for_sm90_cutlass(layer)
|
||||
|
||||
assert torch.equal(layer.w13_humming_residual_scale, expected_w13_residual * 64.0)
|
||||
@@ -572,7 +574,7 @@ def test_humming_range_ignores_prerounded_hidden_tail():
|
||||
layer = _build_mock_layer(
|
||||
E, hidden_rounded, inter, w13, w2, w13_s, w2_s, w13_b, w2_b
|
||||
)
|
||||
method = _build_method(E, hidden_rounded, inter, use_humming=True)
|
||||
method = _build_method(E, hidden_rounded, inter, layer=layer, use_humming=True)
|
||||
# What create_weights records from layer.hidden_size_unpadded.
|
||||
method._unpadded_hidden = hidden_real
|
||||
method._process_weights_for_sm90_cutlass(layer)
|
||||
@@ -627,7 +629,7 @@ def test_apply_sm90_humming_matches_flashinfer_direct(
|
||||
)
|
||||
layer.moe_ep_size = ep_size
|
||||
layer.moe_ep_rank = ep_rank
|
||||
method = _build_method(num_experts, hidden, inter, use_humming=True)
|
||||
method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True)
|
||||
method._process_weights_for_sm90_cutlass(layer)
|
||||
out_sglang = method._apply_sm90_cutlass(
|
||||
layer, _MockDispatchOutput(x.clone(), topk_w, topk_i)
|
||||
@@ -784,6 +786,7 @@ def test_dsv4_apply_matches_flashinfer_direct(
|
||||
layer.moe_tp_rank = 0
|
||||
layer.moe_ep_size = 1
|
||||
layer.moe_ep_rank = 0
|
||||
method.runner.config.layer = layer
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Layers that shard over attention TP but all-reduce over the full TP group
|
||||
refuse the layouts where the two groups differ."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
initialize_dp_attention_flags,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestAttnTpReduceRejection(CustomTestCase):
|
||||
def publish(self, **fields):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
server_args = ServerArgs(model_path="dummy", device="cpu", **fields)
|
||||
publish(server_args, role="test", ranks=SpawnRanks(world_rank=0))
|
||||
initialize_dp_attention_flags(server_args)
|
||||
|
||||
def test_only_a_narrower_wider_than_one_shard_with_a_tp_reduce_is_refused(self):
|
||||
self.publish(tp_size=4)
|
||||
with get_parallel().override(tp_size=4):
|
||||
with self.assertRaisesRegex(ValueError, "Layer shards over the attention"):
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
"Layer", shard_tp_size=2, reduces_over_attn_tp=False
|
||||
)
|
||||
for shard_tp_size, reduces_over_attn_tp in (
|
||||
(2, True),
|
||||
(1, False),
|
||||
(4, False),
|
||||
):
|
||||
with self.subTest(shard=shard_tp_size, reduces=reduces_over_attn_tp):
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
"Layer",
|
||||
shard_tp_size=shard_tp_size,
|
||||
reduces_over_attn_tp=reduces_over_attn_tp,
|
||||
)
|
||||
|
||||
def test_an_attention_that_always_reduces_over_tp(self):
|
||||
from sglang.srt.models.clip import CLIPAttention
|
||||
|
||||
config = SimpleNamespace(
|
||||
hidden_size=16, num_attention_heads=4, attention_dropout=0.0
|
||||
)
|
||||
self.publish(tp_size=4, attn_dp_size=2)
|
||||
with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"):
|
||||
CLIPAttention(config)
|
||||
# A one-rank attention-TP shard does not reduce at all.
|
||||
self.publish(tp_size=4, attn_dp_size=4)
|
||||
CLIPAttention(config)
|
||||
self.publish(tp_size=4)
|
||||
CLIPAttention(config)
|
||||
|
||||
def test_a_layer_that_reduces_over_attention_tp_only_with_attention_dp(self):
|
||||
from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP
|
||||
|
||||
def build():
|
||||
return Qwen3_VisionMLP(16, 32, bias=False, hidden_act="relu")
|
||||
|
||||
self.publish(tp_size=4, attn_cp_size=2)
|
||||
with self.assertRaisesRegex(ValueError, "Qwen3_VisionMLP shards over"):
|
||||
build()
|
||||
for fields in (
|
||||
dict(tp_size=4, attn_dp_size=2),
|
||||
dict(tp_size=8, attn_dp_size=2, attn_cp_size=2),
|
||||
dict(tp_size=4),
|
||||
):
|
||||
with self.subTest(**fields):
|
||||
self.publish(**fields)
|
||||
build()
|
||||
# The data-parallel encoder runs one-rank shards.
|
||||
self.publish(tp_size=4, attn_cp_size=2)
|
||||
Qwen3_VisionMLP(16, 32, bias=False, hidden_act="relu", use_data_parallel=True)
|
||||
|
||||
def test_a_worker_that_never_forwards_its_tower(self):
|
||||
"""Most models build their multimodal tower unconditionally. Where the
|
||||
tower is not forwarded, the layout it would need must not be refused."""
|
||||
from sglang.srt.models.clip import CLIPAttention
|
||||
|
||||
config = SimpleNamespace(
|
||||
hidden_size=16, num_attention_heads=4, attention_dropout=0.0
|
||||
)
|
||||
self.publish(tp_size=4, attn_dp_size=2)
|
||||
with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"):
|
||||
CLIPAttention(config)
|
||||
for fields in (
|
||||
# The encoder instance forwards the tower instead.
|
||||
dict(language_only=True),
|
||||
# Adaptive dispatch is off, so nothing is kept back for local work.
|
||||
dict(language_only=True, enable_adaptive_dispatch_to_encoder=False),
|
||||
# Multimodal requests are rejected outright.
|
||||
dict(language_model_only=True),
|
||||
# A decode instance embeds nothing.
|
||||
dict(disaggregation_mode="decode"),
|
||||
):
|
||||
with self.subTest(**fields):
|
||||
self.publish(tp_size=4, attn_dp_size=2, **fields)
|
||||
CLIPAttention(config)
|
||||
|
||||
def test_a_tower_kept_for_local_work_is_still_checked(self):
|
||||
"""Adaptive dispatch sends only some requests to the encoder instance
|
||||
and forwards the tower for the rest, so the layout still has to hold."""
|
||||
from sglang.srt.models.clip import CLIPAttention
|
||||
|
||||
config = SimpleNamespace(
|
||||
hidden_size=16, num_attention_heads=4, attention_dropout=0.0
|
||||
)
|
||||
self.publish(
|
||||
tp_size=4,
|
||||
attn_dp_size=2,
|
||||
language_only=True,
|
||||
enable_adaptive_dispatch_to_encoder=True,
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"):
|
||||
CLIPAttention(config)
|
||||
|
||||
def test_the_message_names_the_width_that_narrowed_the_group(self):
|
||||
self.publish(tp_size=4, attn_dp_size=2)
|
||||
with self.assertRaisesRegex(ValueError, r"drop --attn-dp-size 2"):
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
"Layer", shard_tp_size=2, reduces_over_attn_tp=False
|
||||
)
|
||||
self.publish(tp_size=4, attn_cp_size=2)
|
||||
with self.assertRaisesRegex(ValueError, r"drop --attn-cp-size 2"):
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
"Layer", shard_tp_size=2, reduces_over_attn_tp=False
|
||||
)
|
||||
|
||||
def test_a_text_layer_is_checked_on_every_worker(self):
|
||||
"""The exemption belongs to towers that do not run, not to the worker:
|
||||
a text attention still reduces on a language-only instance."""
|
||||
from sglang.srt.models.phimoe import PhiMoEAttention
|
||||
|
||||
for fields in (
|
||||
{},
|
||||
dict(language_only=True),
|
||||
dict(language_model_only=True),
|
||||
dict(disaggregation_mode="decode"),
|
||||
):
|
||||
with self.subTest(**fields):
|
||||
self.publish(tp_size=4, attn_dp_size=2, **fields)
|
||||
with self.assertRaisesRegex(ValueError, "PhiMoEAttention shards over"):
|
||||
PhiMoEAttention(hidden_size=16, num_heads=4, num_kv_heads=4)
|
||||
|
||||
def test_a_replicated_branch_is_not_gated(self):
|
||||
"""MoonViT's tensor-parallel MLP is refused, while its ModelSlim branch
|
||||
builds replicated layers that neither shard nor reduce."""
|
||||
from sglang.srt.models.kimi_vl_moonvit import MLP2
|
||||
|
||||
self.publish(tp_size=4, attn_dp_size=2)
|
||||
with self.assertRaisesRegex(ValueError, "MLP2 shards over"):
|
||||
MLP2(dims=[16, 32, 16], activation=None, use_tensor_parallel=True)
|
||||
# Without tensor parallelism the shard is one rank wide.
|
||||
MLP2(dims=[16, 32, 16], activation=None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -68,6 +68,8 @@ with mock.patch.dict(
|
||||
_moe_runner_keeps_global_expert_ids,
|
||||
)
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class _IdentityMoeSlices:
|
||||
def slice_moe_lora_a_weights(self, weights, _rank, _target):
|
||||
@@ -982,6 +984,21 @@ class TestPoolInitPicksUpEpContext(unittest.TestCase):
|
||||
return_value=keeps_global,
|
||||
),
|
||||
mock.patch.object(LoRAMemoryPool, "init_buffers", lambda self, _m: None),
|
||||
# The pool reads its TP placement from the parallel context; state
|
||||
# a whole layout so the topology check sees a consistent one.
|
||||
get_parallel().override(
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
attn_tp_size=tp_size,
|
||||
attn_tp_rank=tp_rank,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_tp_size=tp_size,
|
||||
moe_ep_size=1,
|
||||
moe_dp_size=1,
|
||||
),
|
||||
):
|
||||
hf_cfg = types.SimpleNamespace(
|
||||
num_hidden_layers=1,
|
||||
@@ -995,9 +1012,6 @@ class TestPoolInitPicksUpEpContext(unittest.TestCase):
|
||||
base_hf_config=hf_cfg,
|
||||
max_loras_per_batch=1,
|
||||
dtype=torch.bfloat16,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
attn_tp_size=tp_size,
|
||||
max_lora_rank=8,
|
||||
target_modules={"qkv_proj"},
|
||||
base_model=base_model,
|
||||
|
||||
@@ -92,18 +92,12 @@ def _request(feature, rid: str = "vlm-request") -> TokenizedEmbeddingReqInput:
|
||||
|
||||
|
||||
def _receiver() -> SchedulerRequestReceiver:
|
||||
group = SimpleNamespace(rank=0, ranks=[0], cpu_group=object())
|
||||
return SchedulerRequestReceiver(
|
||||
recv_from_tokenizer=None,
|
||||
recv_from_rpc=None,
|
||||
recv_skipper=None,
|
||||
input_blocker=None,
|
||||
mm_receiver=None,
|
||||
tp_group=group,
|
||||
tp_cpu_group=group,
|
||||
attn_tp_cpu_group=group,
|
||||
attn_cp_cpu_group=group,
|
||||
world_group=group,
|
||||
server_args=SimpleNamespace(),
|
||||
model_config=SimpleNamespace(is_multimodal=True),
|
||||
max_recv_per_poll=-1,
|
||||
@@ -121,9 +115,12 @@ def _run_consensus_rank(rank: int, world_size: int, init_file: str) -> None:
|
||||
)
|
||||
try:
|
||||
req = _request(_failed_pointer() if rank == 1 else _successful_pointer())
|
||||
parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=world_size)
|
||||
parallel = SimpleNamespace(
|
||||
attn_dp_enabled=False,
|
||||
tp_size=world_size,
|
||||
tp_group=SimpleNamespace(cpu_group=torch.distributed.group.WORLD),
|
||||
)
|
||||
receiver = _receiver()
|
||||
object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD)
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.managers.mm_utils._get_is_default_transport",
|
||||
@@ -152,7 +149,6 @@ def _run_image_receiver(rank, init_file, pipe):
|
||||
)
|
||||
try:
|
||||
receiver = _receiver()
|
||||
object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD)
|
||||
torch.distributed.barrier()
|
||||
torch.distributed.all_reduce(torch.zeros(1))
|
||||
initial_fds = len(os.listdir("/proc/self/fd"))
|
||||
@@ -169,7 +165,11 @@ def _run_image_receiver(rank, init_file, pipe):
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.request_receiver.get_parallel",
|
||||
return_value=SimpleNamespace(attn_dp_enabled=False, tp_size=2),
|
||||
return_value=SimpleNamespace(
|
||||
attn_dp_enabled=False,
|
||||
tp_size=2,
|
||||
tp_group=SimpleNamespace(cpu_group=torch.distributed.group.WORLD),
|
||||
),
|
||||
),
|
||||
):
|
||||
for base in [30, 90]:
|
||||
@@ -396,7 +396,11 @@ class TestShmRequestFailureConsensus(unittest.TestCase):
|
||||
|
||||
def test_peer_failure_rejects_the_local_request(self):
|
||||
req = _request(torch.zeros(1))
|
||||
parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=2)
|
||||
parallel = SimpleNamespace(
|
||||
attn_dp_enabled=False,
|
||||
tp_size=2,
|
||||
tp_group=SimpleNamespace(cpu_group=object()),
|
||||
)
|
||||
|
||||
def inject_peer_failure(mask, **kwargs):
|
||||
mask.fill_(1)
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.managers.scheduler_pp_mixin import ( # noqa: E402
|
||||
_pp_exchange_outputs_before_forward,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402
|
||||
from sglang.srt.runtime_context import get_parallel # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
@@ -46,21 +47,12 @@ def _fake_group() -> SimpleNamespace:
|
||||
|
||||
|
||||
def _make_receiver() -> SchedulerRequestReceiver:
|
||||
tp_group = _fake_group()
|
||||
attn_tp_group = _fake_group()
|
||||
attn_cp_group = _fake_group()
|
||||
world_group = _fake_group()
|
||||
return SchedulerRequestReceiver(
|
||||
recv_from_tokenizer=None,
|
||||
recv_from_rpc=None,
|
||||
recv_skipper=None,
|
||||
input_blocker=None,
|
||||
mm_receiver=None,
|
||||
tp_group=tp_group,
|
||||
tp_cpu_group=tp_group,
|
||||
attn_tp_cpu_group=attn_tp_group,
|
||||
attn_cp_cpu_group=attn_cp_group,
|
||||
world_group=world_group,
|
||||
server_args=SimpleNamespace(
|
||||
enable_dp_attention_local_control_broadcast=False,
|
||||
),
|
||||
@@ -120,6 +112,7 @@ class TestRequestReceiverBroadcast(unittest.TestCase):
|
||||
attn_tp_size=1,
|
||||
attn_cp_size=1,
|
||||
tp_size=32,
|
||||
tp_group=_fake_group(),
|
||||
)
|
||||
|
||||
with (
|
||||
@@ -150,15 +143,16 @@ class TestRequestReceiverBroadcast(unittest.TestCase):
|
||||
self.assertEqual(result, [control_req])
|
||||
broadcast.assert_called_once_with(
|
||||
[control_req],
|
||||
receiver.tp_group.rank,
|
||||
receiver.tp_cpu_group,
|
||||
src=receiver.tp_group.ranks[0],
|
||||
parallel.tp_group.rank,
|
||||
parallel.tp_group.cpu_group,
|
||||
src=parallel.tp_group.ranks[0],
|
||||
)
|
||||
|
||||
|
||||
class TestPPCPRankOffsets(unittest.TestCase):
|
||||
def test_request_receiver_uses_cp_size_for_pp_recv_rank(self):
|
||||
enter_scope(self, _published_topology())
|
||||
enter_scope(self, get_parallel().override(world_group=_fake_group()))
|
||||
calls = []
|
||||
|
||||
def fake_point_to_point_pyobj(data, rank, group, src, dst, **kwargs):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
@@ -41,9 +42,12 @@ class TestPrefillDelayerGather(CustomTestCase):
|
||||
"sglang.srt.managers.prefill_delayer.all_gather_single",
|
||||
side_effect=_all_gather_over_tp,
|
||||
),
|
||||
get_parallel().override(
|
||||
tp_group=SimpleNamespace(cpu_group=None, device_group=None)
|
||||
),
|
||||
):
|
||||
delayer = PrefillDelayer(
|
||||
cpu_group=None, max_delay_passes=4, token_usage_low_watermark=None
|
||||
max_delay_passes=4, token_usage_low_watermark=None
|
||||
)
|
||||
info = delayer._gather_info(
|
||||
local_prefillable=True,
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.srt.managers.scheduler_components.weight_updater import (
|
||||
SchedulerWeightUpdaterManager,
|
||||
_WeightUpdateSession,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -140,8 +141,8 @@ class _WeightUpdaterManagerTestBase(CustomTestCase):
|
||||
self.addCleanup(patcher.stop)
|
||||
self.recorded = []
|
||||
|
||||
def _manager(self, target, draft=None, *, session=True):
|
||||
manager = SchedulerWeightUpdaterManager(
|
||||
def _build_manager(self, target, draft):
|
||||
return SchedulerWeightUpdaterManager(
|
||||
tp_worker=SimpleNamespace(
|
||||
model_runner=target,
|
||||
weight_update_runners=lambda: [("target", target)],
|
||||
@@ -152,7 +153,6 @@ class _WeightUpdaterManagerTestBase(CustomTestCase):
|
||||
if draft is None
|
||||
else SimpleNamespace(weight_update_runners=lambda: [("draft", draft)])
|
||||
),
|
||||
tp_cpu_group=None,
|
||||
memory_saver_adapter=None,
|
||||
flush_cache=lambda **kwargs: True,
|
||||
is_fully_idle=lambda **kwargs: True,
|
||||
@@ -162,6 +162,10 @@ class _WeightUpdaterManagerTestBase(CustomTestCase):
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
def _manager(self, target, draft=None, *, session=True):
|
||||
with get_parallel().override(tp_group=SimpleNamespace(cpu_group=None)):
|
||||
manager = self._build_manager(target, draft)
|
||||
if session:
|
||||
# update_weights_from_* require an open session
|
||||
manager._session = _WeightUpdateSession(selector="all")
|
||||
|
||||
@@ -16,7 +16,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestOutcome,
|
||||
)
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
HiCacheStorageConfig,
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
PPPrefetchDecision,
|
||||
@@ -34,8 +38,18 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.c = c = HybridCacheController.__new__(HybridCacheController)
|
||||
c.page_size = c.prefetch_threshold = 4
|
||||
c.pp_rank = c.tp_rank = 0
|
||||
c.pp_size, c.tp_size = 4, 2
|
||||
c.storage_config = HiCacheStorageConfig(
|
||||
tp_rank=0,
|
||||
tp_size=2,
|
||||
pp_rank=0,
|
||||
pp_size=4,
|
||||
attn_cp_rank=0,
|
||||
attn_cp_size=1,
|
||||
is_mla_model=False,
|
||||
enable_storage_metrics=False,
|
||||
is_page_first_layout=False,
|
||||
model_name=None,
|
||||
)
|
||||
c.pp_group, c.pp_prefetch_command_group = "pp", "command"
|
||||
c.pp_prefetch_command_thread = None
|
||||
c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"]
|
||||
@@ -61,7 +75,9 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
patcher = patch.object(
|
||||
torch.distributed, "get_rank", side_effect=lambda: c.pp_rank * 2 + c.tp_rank
|
||||
torch.distributed,
|
||||
"get_rank",
|
||||
side_effect=lambda: c.storage_config.pp_rank * 2 + c.storage_config.tp_rank,
|
||||
)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
@@ -98,7 +114,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
self.c.pp_prefetch_command_queue.put(None)
|
||||
|
||||
def broadcast(objects, *args, **kwargs):
|
||||
return [next(commands)] if self.c.pp_rank else objects
|
||||
return [next(commands)] if self.c.storage_config.pp_rank else objects
|
||||
|
||||
with patch(f"{HybridCacheController.__module__}.broadcast_pyobj", broadcast):
|
||||
self.c.pp_prefetch_command_thread_func()
|
||||
@@ -179,7 +195,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
handle = CacheRequestHandle("hit", 3)
|
||||
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
|
||||
ticket.last_hash = None
|
||||
c.pp_rank = 1
|
||||
c.storage_config.pp_rank = 1
|
||||
c.pp_prefetch_states.clear()
|
||||
cache.bind_prefetch_ticket("hit")
|
||||
self.assertFalse(cache.check_prefetch_progress(handle))
|
||||
@@ -216,7 +232,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
ticket = c.pp_prefetch_states.pop("hit").ticket
|
||||
following = pickle.loads(pickle.dumps(ticket))
|
||||
following.handle = CacheRequestHandle("next", 0)
|
||||
c.pp_rank = 1
|
||||
c.storage_config.pp_rank = 1
|
||||
kv = torch.arange(8)
|
||||
c.mem_pool_host.alloc.side_effect = [
|
||||
kv,
|
||||
@@ -257,7 +273,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c, cache = self.c, self.cache
|
||||
self.submit(pools=[PoolTransfer(PoolName.SWA, host_indices=torch.arange(4))])
|
||||
ticket = c.pp_prefetch_states.pop("hit").ticket
|
||||
c.pp_rank = 1
|
||||
c.storage_config.pp_rank = 1
|
||||
cache.bind_prefetch_ticket("hit")
|
||||
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
||||
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
||||
@@ -290,7 +306,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
|
||||
for rank in (0, 1):
|
||||
with self.subTest(rank=rank):
|
||||
c.pp_rank = rank
|
||||
c.storage_config.pp_rank = rank
|
||||
if rank:
|
||||
c.pp_prefetch_states.clear()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
@@ -358,7 +374,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
|
||||
def test_idle_source_broadcasts_empty_then_processes_ticket_and_stop(self):
|
||||
c = self.c
|
||||
c.tp_rank = 1 # The source is a global rank, not pp_rank=0.
|
||||
c.storage_config.tp_rank = 1 # The source is a global rank, not pp_rank=0.
|
||||
operation = self.submit().operation
|
||||
c.pp_prefetch_command_queue.put(None)
|
||||
get = c.pp_prefetch_command_queue.get
|
||||
@@ -397,7 +413,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c = self.c
|
||||
self.submit()
|
||||
ticket = c.pp_prefetch_states.pop("hit").ticket
|
||||
c.pp_rank = 1
|
||||
c.storage_config.pp_rank = 1
|
||||
with (
|
||||
patch(
|
||||
f"{HybridCacheController.__module__}.broadcast_pyobj",
|
||||
|
||||
@@ -89,8 +89,13 @@ class TestAttnDpSize(CustomTestCase):
|
||||
"replicas combined with attention data parallelism",
|
||||
),
|
||||
(
|
||||
{"tp_size": 4, "attn_dp_size": 2, "enable_dp_attention": True},
|
||||
"pass --attn-dp-size alone",
|
||||
{
|
||||
"tp_size": 4,
|
||||
"dp_size": 2,
|
||||
"attn_dp_size": 2,
|
||||
"enable_dp_attention": True,
|
||||
},
|
||||
"replicas combined with attention data parallelism",
|
||||
),
|
||||
({"tp_size": 2, "attn_dp_size": 0}, "must be positive"),
|
||||
):
|
||||
@@ -98,6 +103,55 @@ class TestAttnDpSize(CustomTestCase):
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
self.resolve(**fields)
|
||||
|
||||
def test_the_deprecated_flag_next_to_attn_dp_size_is_redundant(self):
|
||||
both = self.resolve(tp_size=4, attn_dp_size=2, enable_dp_attention=True)
|
||||
new = self.resolve(tp_size=4, attn_dp_size=2)
|
||||
self.assertEqual(self.layout(both), (1, 2, False))
|
||||
self.assertEqual(both.resolved_dict(), new.resolved_dict())
|
||||
|
||||
def test_the_readback_reports_the_deprecated_field_as_attention_dp(self):
|
||||
for fields, attention_dp in (
|
||||
({"tp_size": 4, "attn_dp_size": 2}, True),
|
||||
({"tp_size": 4, "dp_size": 2, "enable_dp_attention": True}, True),
|
||||
({"tp_size": 4, "dp_size": 2}, False),
|
||||
({"tp_size": 4}, False),
|
||||
):
|
||||
with self.subTest(**fields):
|
||||
server_args = self.resolve(**fields)
|
||||
self.assertFalse(resolution_result(server_args, "enable_dp_attention"))
|
||||
readback = server_args.resolved_dict()
|
||||
self.assertIs(readback["enable_dp_attention"], attention_dp)
|
||||
# A read-back config resolves to the same layout.
|
||||
again = self.resolve(
|
||||
**{
|
||||
name: readback[name]
|
||||
for name in (
|
||||
"tp_size",
|
||||
"dp_size",
|
||||
"attn_dp_size",
|
||||
"enable_dp_attention",
|
||||
)
|
||||
}
|
||||
)
|
||||
self.assertEqual(self.layout(again), self.layout(server_args))
|
||||
|
||||
def test_the_readback_reports_the_width_not_the_joiner_arm(self):
|
||||
"""`attn_dp_enabled` is also true for an elastic scale joiner at width
|
||||
one. The readback reports the configured width instead, so a joiner's
|
||||
replicas are not folded into attention-DP groups when it is read back."""
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
|
||||
joiner = ServerArgs(
|
||||
model_path="dummy", tp_size=2, dp_size=2, ep_join_mode="scale"
|
||||
)
|
||||
joiner.resolve_once()
|
||||
self.assertTrue(attn_dp_enabled_of(resolving_view(joiner)))
|
||||
readback = joiner.resolved_dict()
|
||||
self.assertFalse(readback["enable_dp_attention"])
|
||||
self.assertEqual(readback["dp_size"], 2)
|
||||
self.assertEqual(readback["attn_dp_size"], 1)
|
||||
|
||||
def test_both_cli_spellings_parse(self):
|
||||
parser = argparse.ArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
|
||||
Reference in New Issue
Block a user