[Refactor] Let the remaining placement consumers read the parallel context (#42348)
This commit is contained in:
@@ -3,7 +3,10 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import attention_backends_of
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
attention_backends_of,
|
||||
context_parallel_attn_dp_size,
|
||||
)
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
_deepseek_v4_kv_cache_dtype,
|
||||
declare_resolution,
|
||||
@@ -157,13 +160,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None:
|
||||
f"DeepSeekV4 prefill CP on HIP does not support {feature} yet."
|
||||
)
|
||||
|
||||
# DeepSeek-V4 CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
attn_dp_size = context_parallel_attn_dp_size(cfg, "DeepSeek-V4 context parallelism")
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v4_cp",
|
||||
|
||||
@@ -171,7 +171,7 @@ class Parallel(msgspec.Struct):
|
||||
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
|
||||
] = False
|
||||
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
|
||||
# resolves to `attn_dp_size = N`, `dp_size = 1`.
|
||||
# resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31.
|
||||
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
|
||||
enable_dp_attention_local_control_broadcast: A[
|
||||
bool,
|
||||
|
||||
@@ -226,6 +226,27 @@ def ep_joiner_of(cfg: Any) -> bool:
|
||||
return cfg.ep_join_mode in ("scale", "recover")
|
||||
|
||||
|
||||
def context_parallel_attn_dp_size(cfg: Any, layout: str) -> int:
|
||||
"""The attention-DP width a DeepSeek context-parallel layout runs.
|
||||
|
||||
These layouts run data-parallel groups as attention DP, so `--dp-size N`
|
||||
alone still means N attention-DP groups there. That reading is deprecated.
|
||||
"""
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
if cfg.dp_size > 1:
|
||||
# TODO: reject `--dp-size > 1` here after 2026-12-31.
|
||||
logger.warning(
|
||||
f"--dp-size {cfg.dp_size} under {layout} runs {cfg.dp_size} attention "
|
||||
"data-parallel groups. This is deprecated and will be rejected after "
|
||||
f"2026-12-31; use --attn-dp-size {cfg.dp_size} instead."
|
||||
)
|
||||
return cfg.attn_dp_size * cfg.dp_size
|
||||
|
||||
|
||||
def ep_scale_joiner_of(cfg: Any) -> bool:
|
||||
"""The scale-up arm of :func:`ep_joiner_of`."""
|
||||
return cfg.ep_join_mode == "scale"
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Any, Dict
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
context_parallel_attn_dp_size,
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
use_mla_backend,
|
||||
@@ -85,13 +86,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
logger.warning(
|
||||
"Context parallel feature is still under experiment. It has only been verified on Hopper platform."
|
||||
)
|
||||
# DSA CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
attn_dp_size = context_parallel_attn_dp_size(
|
||||
cfg, "DSA context parallelism"
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
overrides["moe_dense_tp_size"] = 1
|
||||
@@ -166,13 +163,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"MLA prefill context parallel is still experimental. "
|
||||
"Verified on Hopper with the fa3 backend."
|
||||
)
|
||||
# MLA CP runs data-parallel groups as attention DP.
|
||||
assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), (
|
||||
f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: "
|
||||
"data-parallel replicas combined with attention data parallelism "
|
||||
"are not supported."
|
||||
)
|
||||
attn_dp_size = cfg.attn_dp_size * cfg.dp_size
|
||||
attn_dp_size = context_parallel_attn_dp_size(cfg, "MLA context parallelism")
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
# TODO(kpham-sgl) Supports moe_dense_tp_size != 1.
|
||||
|
||||
@@ -39,22 +39,23 @@ def handle_deprecated_dp_attention(server_args: Any):
|
||||
|
||||
Runs before any handler reads the DP layout. The old spelling counted
|
||||
attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and
|
||||
`dp_size` counts replicas again.
|
||||
`dp_size` counts replicas again. Next to `--attn-dp-size` the flag adds
|
||||
nothing: a serialized config reports it set whenever attention DP runs
|
||||
(`ServerArgs.resolved_dict`), so reading one back resolves the same layout.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.enable_dp_attention:
|
||||
if cfg.attn_dp_size != 1:
|
||||
raise ValueError(
|
||||
"--enable-dp-attention is the deprecated spelling of "
|
||||
"--attn-dp-size; pass --attn-dp-size alone."
|
||||
)
|
||||
if not cfg.enable_dp_attention:
|
||||
return
|
||||
if cfg.attn_dp_size == 1:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_deprecated_dp_attention",
|
||||
attn_dp_size=cfg.dp_size,
|
||||
dp_size=1,
|
||||
enable_dp_attention=False,
|
||||
)
|
||||
declare_resolution(
|
||||
server_args, "_handle_deprecated_dp_attention", enable_dp_attention=False
|
||||
)
|
||||
|
||||
|
||||
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
|
||||
|
||||
@@ -158,8 +158,6 @@ class BaseKVSender(ABC):
|
||||
mgr: BaseKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
): ...
|
||||
|
||||
|
||||
@@ -1766,8 +1766,6 @@ class CommonKVSender(BaseKVSender):
|
||||
mgr: CommonKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
self.kv_mgr = mgr
|
||||
|
||||
@@ -31,7 +31,6 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.srt.configs.mamba_utils import Mamba2CacheParams
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
@@ -382,7 +381,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
scheduler: Scheduler,
|
||||
transfer_queue: DecodeTransferQueue,
|
||||
tree_cache: BasePrefixCache,
|
||||
gloo_group: ProcessGroup,
|
||||
gpu_id: int,
|
||||
bootstrap_port: int,
|
||||
max_total_num_tokens: int,
|
||||
@@ -402,7 +400,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.tree_cache = tree_cache
|
||||
self.host_pool = None
|
||||
self.host_reserved_tokens = 0
|
||||
self.gloo_group = gloo_group
|
||||
self.gloo_group = parallel.attn_tp_group.cpu_group
|
||||
# Destinations visible to prefill but not yet on the transfer queue.
|
||||
self._num_published_destinations = 0
|
||||
self.tp_rank = parallel.tp_rank
|
||||
@@ -2379,14 +2377,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
gloo_group: ProcessGroup,
|
||||
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
|
||||
metadata_buffers: MetadataBuffers,
|
||||
scheduler: Scheduler,
|
||||
tree_cache: BasePrefixCache,
|
||||
):
|
||||
self.queue: List[DecodeRequest] = []
|
||||
self.gloo_group = gloo_group
|
||||
self.gloo_group = get_parallel().attn_tp_group.cpu_group
|
||||
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.metadata_buffers = metadata_buffers
|
||||
|
||||
@@ -25,10 +25,7 @@ from fastapi import FastAPI
|
||||
from fastapi.responses import ORJSONResponse, Response
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
GroupCoordinator,
|
||||
get_mooncake_transfer_engine,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.io_struct import (
|
||||
EncoderDispatchErrorReq,
|
||||
@@ -1870,9 +1867,6 @@ class MMReceiverBase(ABC):
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
):
|
||||
@@ -1892,10 +1886,13 @@ class MMReceiverBase(ABC):
|
||||
)
|
||||
self.recv_timeout = envs.SGLANG_ENCODER_RECV_TIMEOUT.get()
|
||||
self.host = get_local_ip_auto(get_serving().host)
|
||||
self.pp_rank = pp_rank
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_group = tp_group
|
||||
# A scheduler-side receiver works across its TP group; the tokenizer
|
||||
# side has no placement.
|
||||
parallel = get_parallel()
|
||||
self.pp_rank = parallel.pp_rank if scheduler is not None else None
|
||||
self.tp_rank = parallel.tp_rank if scheduler is not None else None
|
||||
self.tp_size = parallel.tp_size
|
||||
self.tp_group = parallel.tp_group if scheduler is not None else None
|
||||
self.nnodes = get_parallel().nnodes
|
||||
self.hostname = get_local_ip_auto()
|
||||
self.waiting_list: List[WaitingMMRequestBase] = []
|
||||
@@ -1908,7 +1905,7 @@ class MMReceiverBase(ABC):
|
||||
and scheduler is not None
|
||||
):
|
||||
self.registration_runner = _ReceiveRegistrationRunner(
|
||||
f"encoder-receive-registration-{tp_rank}"
|
||||
f"encoder-receive-registration-{self.tp_rank}"
|
||||
)
|
||||
(
|
||||
self.scheduler_embedding_port,
|
||||
@@ -2674,9 +2671,6 @@ class MMReceiverHTTP(MMReceiverBase):
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
):
|
||||
@@ -2684,9 +2678,6 @@ class MMReceiverHTTP(MMReceiverBase):
|
||||
server_args,
|
||||
dtype=dtype,
|
||||
hf_config=hf_config,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=tp_group,
|
||||
scheduler=scheduler,
|
||||
encode_urls=encode_urls,
|
||||
)
|
||||
@@ -2813,9 +2804,6 @@ class MMReceiverGrpc(MMReceiverBase):
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
):
|
||||
@@ -2830,9 +2818,6 @@ class MMReceiverGrpc(MMReceiverBase):
|
||||
server_args,
|
||||
dtype=dtype,
|
||||
hf_config=hf_config,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=tp_group,
|
||||
scheduler=scheduler,
|
||||
encode_urls=encode_urls,
|
||||
)
|
||||
@@ -2946,9 +2931,6 @@ def create_mm_receiver(
|
||||
server_args: ServerArgs,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
hf_config: Optional[PretrainedConfig] = None,
|
||||
pp_rank: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_group: Optional[GroupCoordinator] = None,
|
||||
scheduler: Optional["Scheduler"] = None,
|
||||
transport_mode: Optional[str] = None,
|
||||
encode_urls: Optional[List[str]] = None,
|
||||
@@ -2967,9 +2949,6 @@ def create_mm_receiver(
|
||||
server_args,
|
||||
dtype=dtype,
|
||||
hf_config=hf_config,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=tp_group,
|
||||
scheduler=scheduler,
|
||||
encode_urls=encode_urls,
|
||||
)
|
||||
|
||||
@@ -43,8 +43,6 @@ class FakeKVSender(BaseKVSender):
|
||||
mgr: BaseKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
self.kv_mgr = mgr
|
||||
|
||||
@@ -2813,16 +2813,12 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
mgr: MooncakeKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
dest_tp_ranks,
|
||||
pp_rank,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.init_time = time.time()
|
||||
|
||||
@@ -1785,16 +1785,12 @@ class MoriKVSender(CommonKVSender):
|
||||
mgr: MoriKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
dest_tp_ranks,
|
||||
pp_rank,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.conclude_state: Optional[KVPoll] = None
|
||||
|
||||
@@ -3133,16 +3133,12 @@ class NixlKVSender(CommonKVSender):
|
||||
mgr: NixlKVManager,
|
||||
bootstrap_addr: str,
|
||||
bootstrap_room: int,
|
||||
dest_tp_ranks: List[int],
|
||||
pp_rank: int,
|
||||
req_has_disagg_prefill_dp_rank: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
mgr,
|
||||
bootstrap_addr,
|
||||
bootstrap_room,
|
||||
dest_tp_ranks,
|
||||
pp_rank,
|
||||
req_has_disagg_prefill_dp_rank,
|
||||
)
|
||||
self.init_time = time.time()
|
||||
|
||||
@@ -382,14 +382,10 @@ class PrefillBootstrapQueue:
|
||||
)
|
||||
kv_sender_class = get_kv_class(backend, KVClassType.SENDER)
|
||||
|
||||
dest_tp_ranks = [self.tp_rank]
|
||||
|
||||
req.disagg_kv_sender = kv_sender_class(
|
||||
mgr=self.kv_manager,
|
||||
bootstrap_addr=f"{req.bootstrap_host}:{self.bootstrap_port}",
|
||||
bootstrap_room=req.bootstrap_room,
|
||||
dest_tp_ranks=dest_tp_ranks,
|
||||
pp_rank=self.pp_rank,
|
||||
req_has_disagg_prefill_dp_rank=req.disagg_prefill_dp_rank is not None,
|
||||
)
|
||||
self._process_req(req)
|
||||
|
||||
@@ -83,6 +83,7 @@ from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from sglang.srt.distributed import utils as dist_utils
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -1099,6 +1100,7 @@ class VisionAttention(nn.Module):
|
||||
] = None,
|
||||
use_data_parallel: bool = False,
|
||||
use_dp_attention_reduce: bool = False,
|
||||
allow_tp_reduce_mismatch: bool = False,
|
||||
aux_stream: Optional[torch.cuda.Stream] = None,
|
||||
workspace_buffer: Optional[torch.Tensor] = None,
|
||||
use_sink: bool = False,
|
||||
@@ -1208,6 +1210,17 @@ class VisionAttention(nn.Module):
|
||||
tp_size=self.tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
if not allow_tp_reduce_mismatch:
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=self.tp_size,
|
||||
reduces_over_attn_tp=use_dp_attention_reduce,
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=self.dummy_dim,
|
||||
output_size=embed_dim,
|
||||
|
||||
@@ -455,6 +455,79 @@ def is_dp_attention_enabled() -> bool:
|
||||
return get_flags().dp.enabled
|
||||
|
||||
|
||||
def get_dp_tp_group() -> GroupCoordinator:
|
||||
"""The TP ranks that run one DP rank's batch: the attention-TP group with
|
||||
attention DP, otherwise the whole TP group.
|
||||
|
||||
Attention-CP peers of a DP rank are members only in the second case.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
|
||||
|
||||
|
||||
def multimodal_encoder_runs_here() -> bool:
|
||||
"""Whether a multimodal tower built on this worker is also forwarded here.
|
||||
|
||||
A tower is built unconditionally by most models, but an encoder-disaggregated
|
||||
language instance forwards it on the encoder instance, a language-model-only
|
||||
instance rejects multimodal requests, and a PD decode instance embeds nothing
|
||||
(`general_mm_embed_routine` skips decode and target-verify forwards).
|
||||
|
||||
Adaptive dispatch is the exception on the first of those: it keeps the
|
||||
requests it does not send to the encoder, and forwards the tower for them.
|
||||
"""
|
||||
from sglang.srt.runtime_context import get_disagg
|
||||
|
||||
disagg = get_disagg()
|
||||
offloaded_to_an_encoder_instance = (
|
||||
disagg.language_only and not disagg.enable_adaptive_dispatch_to_encoder
|
||||
)
|
||||
return not (
|
||||
offloaded_to_an_encoder_instance
|
||||
or disagg.language_model_only
|
||||
or disagg.disaggregation_mode == "decode"
|
||||
)
|
||||
|
||||
|
||||
def reject_attn_tp_shard_with_tp_reduce(
|
||||
layer: str,
|
||||
*,
|
||||
shard_tp_size: int,
|
||||
reduces_over_attn_tp: bool,
|
||||
multimodal_encoder: bool = False,
|
||||
hint: str = "",
|
||||
) -> None:
|
||||
"""Reject a layer that shards over attention TP but all-reduces over the TP group.
|
||||
|
||||
The two groups differ only when attention DP or attention CP makes attention
|
||||
TP narrower than TP. There the all-reduce mixes ranks that hold other
|
||||
requests or replicated inputs, so the output is wrong; a one-rank shard does
|
||||
not reduce at all.
|
||||
"""
|
||||
tp_size = get_parallel().tp_size
|
||||
if reduces_over_attn_tp or not 1 < shard_tp_size < tp_size:
|
||||
return
|
||||
if multimodal_encoder and not multimodal_encoder_runs_here():
|
||||
return
|
||||
# Name only the widths that are actually narrowing the group here, so the
|
||||
# remedy is one the operator can apply.
|
||||
parallel = get_parallel()
|
||||
narrowed_by = []
|
||||
if parallel.attn_dp_size > 1:
|
||||
narrowed_by.append(f"--attn-dp-size {parallel.attn_dp_size}")
|
||||
if parallel.attn_cp_size > 1:
|
||||
narrowed_by.append(f"--attn-cp-size {parallel.attn_cp_size}")
|
||||
remedy = " and ".join(f"drop {flag}" for flag in narrowed_by) or (
|
||||
"widen the attention TP group"
|
||||
)
|
||||
raise ValueError(
|
||||
f"{layer} shards over the attention TP group ({shard_tp_size} ranks) "
|
||||
f"but all-reduces over the full TP group ({tp_size} ranks), so it does "
|
||||
f"not support {' with '.join(narrowed_by) or 'this layout'} yet. "
|
||||
f"Use --attn-dp-size equal to --tp-size, or {remedy}{hint}."
|
||||
)
|
||||
|
||||
|
||||
def is_allocation_symmetric() -> bool:
|
||||
return not is_dp_attention_enabled() or is_dp_max_padding()
|
||||
|
||||
|
||||
@@ -464,6 +464,7 @@ class FusedMoE(torch.nn.Module):
|
||||
is_gated=is_gated,
|
||||
routing_method_type=routing_method_type,
|
||||
gate_up_interleaved=gate_up_interleaved,
|
||||
layer=self,
|
||||
)
|
||||
|
||||
self.quant_method = quant_method
|
||||
|
||||
@@ -68,6 +68,35 @@ class MoeRunnerConfig:
|
||||
# False preserves backend defaults, including their existing FP32 paths.
|
||||
silu_mul_keep_fp32: bool = False
|
||||
|
||||
# The MoE placement a fused func needs is the layer's own, not the
|
||||
# deployment's: DWDP's `FusedMoE.bind_full_expert_weights` collapses a
|
||||
# layer's EP view after load. Read it off the layer so there is one
|
||||
# address to keep correct.
|
||||
@property
|
||||
def moe_tp_size(self) -> int:
|
||||
return self._layer_placement("moe_tp_size")
|
||||
|
||||
@property
|
||||
def moe_tp_rank(self) -> int:
|
||||
return self._layer_placement("moe_tp_rank")
|
||||
|
||||
@property
|
||||
def moe_ep_size(self) -> int:
|
||||
return self._layer_placement("moe_ep_size")
|
||||
|
||||
@property
|
||||
def moe_ep_rank(self) -> int:
|
||||
return self._layer_placement("moe_ep_rank")
|
||||
|
||||
def _layer_placement(self, name: str) -> int:
|
||||
if self.layer is None:
|
||||
raise AttributeError(
|
||||
f"{name} is the layer's MoE placement, but this runner config "
|
||||
"has no layer. A layer that runs fused MoE funcs states itself "
|
||||
"on its runner config."
|
||||
)
|
||||
return getattr(self.layer, name)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunnerInput(ABC):
|
||||
|
||||
@@ -58,10 +58,6 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
|
||||
swiglu_alpha: Optional[torch.Tensor] = None
|
||||
swiglu_beta: Optional[torch.Tensor] = None
|
||||
swiglu_limit: Optional[torch.Tensor] = None
|
||||
moe_tp_size: int = 1
|
||||
moe_tp_rank: int = 0
|
||||
moe_ep_size: int = 1
|
||||
moe_ep_rank: int = 0
|
||||
apply_routed_scaling_factor: bool = True
|
||||
|
||||
|
||||
@@ -144,12 +140,6 @@ class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo):
|
||||
# SwigluStep variant.
|
||||
use_swiglu_step: bool = False
|
||||
|
||||
# TP/EP topology (forwarded to the FlashInfer kernel)
|
||||
moe_tp_size: int = 1
|
||||
moe_tp_rank: int = 0
|
||||
moe_ep_size: int = 1
|
||||
moe_ep_rank: int = 0
|
||||
|
||||
# GPT-OSS pads its input hidden dim up to the (pre-padded) loaded weight
|
||||
# width and trims the output back. DSv4 leaves this as ``None`` (no pad).
|
||||
padded_hidden: Optional[int] = None
|
||||
@@ -318,10 +308,10 @@ def _run_flashinfer_cutlass(
|
||||
swiglu_alpha=quant_info.swiglu_alpha,
|
||||
swiglu_beta=quant_info.swiglu_beta,
|
||||
swiglu_limit=quant_info.swiglu_limit,
|
||||
ep_size=quant_info.moe_ep_size,
|
||||
ep_rank=quant_info.moe_ep_rank,
|
||||
tp_size=quant_info.moe_tp_size,
|
||||
tp_rank=quant_info.moe_tp_rank,
|
||||
ep_size=runner_config.moe_ep_size,
|
||||
ep_rank=runner_config.moe_ep_rank,
|
||||
tp_size=runner_config.moe_tp_size,
|
||||
tp_rank=runner_config.moe_tp_rank,
|
||||
tune_max_num_tokens=next_power_of_2(x.shape[0]),
|
||||
activation_type=_activation_type(runner_config),
|
||||
enable_alltoall=enable_alltoall,
|
||||
@@ -525,10 +515,10 @@ def _fused_experts_flashinfer_mxfp4_cutlass(
|
||||
swiglu_alpha=quant_info.swiglu_alpha,
|
||||
swiglu_beta=quant_info.swiglu_beta,
|
||||
swiglu_limit=quant_info.swiglu_limit,
|
||||
tp_size=quant_info.moe_tp_size,
|
||||
tp_rank=quant_info.moe_tp_rank,
|
||||
ep_size=quant_info.moe_ep_size,
|
||||
ep_rank=quant_info.moe_ep_rank,
|
||||
tp_size=runner_config.moe_tp_size,
|
||||
tp_rank=runner_config.moe_tp_rank,
|
||||
ep_size=runner_config.moe_ep_size,
|
||||
ep_rank=runner_config.moe_ep_rank,
|
||||
use_w4_group_scaling=not use_mxfp8_act_scaling,
|
||||
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
|
||||
activation_type=(
|
||||
|
||||
@@ -80,7 +80,6 @@ class HpcOpsMoeQuantInfo(MoeQuantInfo):
|
||||
w2_weight: torch.Tensor
|
||||
block_quant: bool
|
||||
global_num_experts: int
|
||||
moe_ep_rank: int
|
||||
# Blockwise path
|
||||
w13_weight_scale_inv: Optional[torch.Tensor] = None
|
||||
w2_weight_scale_inv: Optional[torch.Tensor] = None
|
||||
@@ -183,7 +182,7 @@ def fused_experts_none_to_hpc_ops(
|
||||
quant_info.w2_weight_scale_inv,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
quant_info.moe_ep_rank,
|
||||
runner_config.moe_ep_rank,
|
||||
quant_info.global_num_experts,
|
||||
)
|
||||
else:
|
||||
@@ -198,7 +197,7 @@ def fused_experts_none_to_hpc_ops(
|
||||
act_and_mul_scale,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
quant_info.moe_ep_rank,
|
||||
runner_config.moe_ep_rank,
|
||||
quant_info.global_num_experts,
|
||||
)
|
||||
|
||||
|
||||
+1
-2
@@ -71,7 +71,6 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
|
||||
assert not config.actorder, (
|
||||
"Actorder is not supported by flashinfer_trtllm backend"
|
||||
)
|
||||
self.moe_ep_rank = get_parallel().moe_ep_rank
|
||||
|
||||
if self.quant_config.quant_format != CompressionFormat.pack_quantized.value:
|
||||
raise ValueError(
|
||||
@@ -348,7 +347,7 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
|
||||
n_group=topk_config.num_expert_group,
|
||||
topk_group=topk_config.topk_group,
|
||||
intermediate_size=self.moe_runner_config.intermediate_size_per_partition,
|
||||
local_expert_offset=self.moe_ep_rank * local_num_experts,
|
||||
local_expert_offset=self.moe_runner_config.moe_ep_rank * local_num_experts,
|
||||
local_num_experts=local_num_experts,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_method_type=routing_method_type,
|
||||
|
||||
-4
@@ -368,10 +368,6 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=False,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -2701,7 +2701,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_weight=layer.w2_weight,
|
||||
block_quant=True,
|
||||
global_num_experts=int(layer.num_experts),
|
||||
moe_ep_rank=int(layer.moe_ep_rank),
|
||||
w13_weight_scale_inv=layer.hpc_ops_w13_weight_scale,
|
||||
w2_weight_scale_inv=layer.hpc_ops_w2_weight_scale,
|
||||
block_shape=self.quant_config.weight_block_size,
|
||||
@@ -2712,7 +2711,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_weight=layer.w2_weight,
|
||||
block_quant=False,
|
||||
global_num_experts=int(layer.num_experts),
|
||||
moe_ep_rank=int(layer.moe_ep_rank),
|
||||
gate_up_alphas=layer.hpc_ops_gate_up_alphas,
|
||||
down_alphas=layer.hpc_ops_down_alphas,
|
||||
w13_input_scale=layer.w13_input_scale,
|
||||
|
||||
@@ -1426,10 +1426,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
@@ -3253,10 +3249,6 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=False,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -1482,10 +1482,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=layer.swiglu_alpha,
|
||||
swiglu_beta=layer.swiglu_beta,
|
||||
swiglu_limit=layer.swiglu_limit,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
padded_hidden=self._padded_hidden,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
@@ -1529,10 +1525,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_alpha=layer.swiglu_alpha,
|
||||
swiglu_beta=layer.swiglu_beta,
|
||||
swiglu_limit=layer.swiglu_limit,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
padded_hidden=self._padded_hidden,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -87,6 +87,9 @@ class Mxfp4FlashinferCutlassMoEMethod:
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend
|
||||
|
||||
self.moe_runner_config = moe_runner_config
|
||||
# The fused func reads the layer's MoE placement off the config.
|
||||
if moe_runner_config.layer is None:
|
||||
moe_runner_config.layer = layer
|
||||
|
||||
E = layer.num_local_experts
|
||||
device = layer.w13_weight.device
|
||||
@@ -249,10 +252,6 @@ class Mxfp4FlashinferCutlassMoEMethod:
|
||||
swiglu_beta=None,
|
||||
swiglu_limit=self._swiglu_limit_tensor,
|
||||
use_swiglu_step=self._use_swiglu_step,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
padded_hidden=None,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -1105,10 +1105,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
moe_tp_rank=layer.moe_tp_rank,
|
||||
apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk,
|
||||
)
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
@@ -1149,9 +1149,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
has_active_lora=has_active_lora,
|
||||
experts_shared_outer_loras=self.experts_shared_outer_loras,
|
||||
cg_buffers=cg_buffers,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
hidden_size=getattr(self.base_layer, "hidden_size", 0),
|
||||
lora_use_virtual_experts=self.lora_use_virtual_experts,
|
||||
)
|
||||
|
||||
|
||||
@@ -75,8 +75,6 @@ class LoRAManager:
|
||||
dtype: torch.dtype,
|
||||
server_args: ServerArgs,
|
||||
lora_backend: str = "triton",
|
||||
tp_size: int = 1,
|
||||
tp_rank: int = 0,
|
||||
max_lora_rank: Optional[int] = None,
|
||||
target_modules: Optional[Iterable[str]] = None,
|
||||
lora_paths: Optional[List[LoRARef]] = None,
|
||||
@@ -90,8 +88,6 @@ class LoRAManager:
|
||||
self.load_config: LoadConfig = load_config
|
||||
self.dtype: torch.dtype = dtype
|
||||
self.device: torch.device = next(self.base_model.parameters()).device
|
||||
self.tp_size: int = tp_size
|
||||
self.tp_rank: int = tp_rank
|
||||
# Attention projections shard on the attn-TP group; extracted once
|
||||
# here (parallel groups are frozen after init_torch_distributed).
|
||||
self.attn_tp_size: int = get_parallel().attn_tp_size
|
||||
@@ -1004,9 +1000,6 @@ class LoRAManager:
|
||||
base_hf_config=self.base_hf_config,
|
||||
max_loras_per_batch=self.max_loras_per_batch,
|
||||
dtype=self.dtype,
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=self.tp_rank,
|
||||
attn_tp_size=self.attn_tp_size,
|
||||
max_lora_rank=self.max_lora_rank,
|
||||
target_modules=self.target_modules,
|
||||
base_model=self.base_model,
|
||||
|
||||
@@ -184,10 +184,6 @@ class LoRAInfo:
|
||||
experts_shared_outer_loras: bool = False
|
||||
cg_buffers: dict | None = None
|
||||
|
||||
fully_sharded: bool = False
|
||||
tp_size: int = 1
|
||||
tp_rank: int = 0
|
||||
hidden_size: int = 0
|
||||
lora_use_virtual_experts: bool = False
|
||||
|
||||
|
||||
@@ -391,7 +387,6 @@ def _add_lora_gate_up_delta(
|
||||
expand_num_warps=4,
|
||||
expand_num_stages=2,
|
||||
expand_split_k=1,
|
||||
fully_sharded=lora_info.fully_sharded,
|
||||
)
|
||||
|
||||
|
||||
@@ -424,12 +419,6 @@ def _add_lora_down_delta(
|
||||
if lora_info.experts_shared_outer_loras and not lora_info.lora_use_virtual_experts:
|
||||
down_lora_b = down_lora_b.expand(-1, lora_info.num_experts, -1, -1)
|
||||
|
||||
if lora_info.fully_sharded and lora_info.tp_size > 1:
|
||||
shard_size = lora_info.hidden_size // lora_info.tp_size
|
||||
offset = shard_size * lora_info.tp_rank
|
||||
else:
|
||||
offset = 0
|
||||
|
||||
if lora_info.lora_use_virtual_experts:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
output=intermediate_cache,
|
||||
@@ -474,8 +463,6 @@ def _add_lora_down_delta(
|
||||
expand_num_stages=2,
|
||||
expand_split_k=1,
|
||||
mul_routed_weight=True,
|
||||
fully_sharded=lora_info.fully_sharded,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -138,9 +138,6 @@ class LoRAMemoryPool:
|
||||
base_hf_config: AutoConfig,
|
||||
max_loras_per_batch: int,
|
||||
dtype: torch.dtype,
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
attn_tp_size: int,
|
||||
max_lora_rank: int,
|
||||
target_modules: Set[str],
|
||||
base_model: torch.nn.Module,
|
||||
@@ -154,8 +151,9 @@ class LoRAMemoryPool:
|
||||
self.num_layer: int = base_hf_config.num_hidden_layers
|
||||
self.max_loras_per_batch: int = max_loras_per_batch
|
||||
self.dtype: torch.dtype = dtype
|
||||
self.tp_size: int = tp_size
|
||||
self.tp_rank: int = tp_rank
|
||||
parallel = get_parallel()
|
||||
self.tp_size: int = parallel.tp_size
|
||||
self.tp_rank: int = parallel.tp_rank
|
||||
self.lora_added_tokens_size: int = lora_added_tokens_size
|
||||
self.max_lora_rank: int = max_lora_rank
|
||||
self.target_modules: Set[str] = target_modules
|
||||
@@ -193,7 +191,7 @@ class LoRAMemoryPool:
|
||||
# under attention DP is `attn_tp_size = tp_size // attn_dp_size`.
|
||||
# The corresponding LoRA wrappers slice weights by the base layer's
|
||||
# attn_tp-local rank, so the buffer shapes must match that shard.
|
||||
self.attn_tp_size: int = attn_tp_size
|
||||
self.attn_tp_size: int = parallel.attn_tp_size
|
||||
|
||||
# Initialize eviction policy
|
||||
self.eviction_policy = get_eviction_policy(eviction_policy)
|
||||
@@ -227,7 +225,7 @@ class LoRAMemoryPool:
|
||||
# Cache lm_head shard_indices from the base model so that buffer
|
||||
# allocation uses the same sharding as the base ParallelLMHead layer.
|
||||
self.lm_head_shard_indices = None
|
||||
if "lm_head" in target_modules and tp_size > 1:
|
||||
if "lm_head" in target_modules and self.tp_size > 1:
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
for _, module in base_model.named_modules():
|
||||
|
||||
@@ -705,17 +705,15 @@ class HiCacheController:
|
||||
if storage_backend_extra_config is None:
|
||||
storage_backend_extra_config = {}
|
||||
|
||||
parallel = get_parallel()
|
||||
if is_dp_attention_enabled():
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.tp_size = get_parallel().attn_tp_size
|
||||
self.dp_rank = get_parallel().attn_dp_rank
|
||||
tp_rank = parallel.attn_tp_rank
|
||||
tp_size = parallel.attn_tp_size
|
||||
dp_rank = parallel.attn_dp_rank
|
||||
else:
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.dp_rank = 0
|
||||
|
||||
self.pp_rank = get_parallel().pp_rank
|
||||
self.pp_size = get_parallel().pp_size
|
||||
tp_rank = parallel.tp_rank
|
||||
tp_size = parallel.tp_size
|
||||
dp_rank = 0
|
||||
|
||||
# Currently, NPUMLATokenToKVPool is the subclass of MLATokenToKVPool.
|
||||
# DeepSeekV4TokenToKVPool has compressed MLA-style rank-replicated cache
|
||||
@@ -732,23 +730,23 @@ class HiCacheController:
|
||||
should_split_heads = False
|
||||
|
||||
if tp_lcm_size:
|
||||
assert tp_lcm_size % self.tp_size == 0, (
|
||||
assert tp_lcm_size % tp_size == 0, (
|
||||
"tp_lcm_size must be divisible by tp_size."
|
||||
)
|
||||
should_split_heads = (
|
||||
not is_rank_replicated
|
||||
and self.mem_pool_host.layout == "page_head"
|
||||
and tp_lcm_size > self.tp_size
|
||||
and tp_lcm_size > tp_size
|
||||
)
|
||||
|
||||
attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
|
||||
model_name = storage_model_name(model_name, self.storage_host_pool)
|
||||
|
||||
return HiCacheStorageConfig(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
pp_rank=self.pp_rank,
|
||||
pp_size=self.pp_size,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
pp_rank=parallel.pp_rank,
|
||||
pp_size=parallel.pp_size,
|
||||
attn_cp_rank=attn_cp_rank,
|
||||
attn_cp_size=attn_cp_size,
|
||||
# TODO(hzh): Rename is_mla_model to is_rank_replicated.
|
||||
@@ -758,7 +756,7 @@ class HiCacheController:
|
||||
model_name=model_name,
|
||||
tp_lcm_size=tp_lcm_size,
|
||||
should_split_heads=should_split_heads,
|
||||
dp_rank=self.dp_rank,
|
||||
dp_rank=dp_rank,
|
||||
extra_config=storage_backend_extra_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -76,12 +76,9 @@ class _NegotiateOutput(NamedTuple):
|
||||
class PrefillDelayer:
|
||||
def __init__(
|
||||
self,
|
||||
cpu_group,
|
||||
max_delay_passes: int,
|
||||
token_usage_low_watermark: Optional[float],
|
||||
metrics_collector: Optional["SchedulerMetricsCollector"] = None,
|
||||
device: Optional["torch.device"] = "cpu",
|
||||
device_group=None,
|
||||
debug_log_enabled: bool = True,
|
||||
):
|
||||
self._max_delay_passes = max_delay_passes
|
||||
@@ -117,14 +114,12 @@ class PrefillDelayer:
|
||||
get_schedule().disable_overlap_schedule
|
||||
or envs.SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH.get()
|
||||
)
|
||||
tp_group = parallel.tp_group
|
||||
if use_nccl:
|
||||
assert device_group is not None, (
|
||||
"device_group is required when using NCCL for PrefillDelayer all-gather"
|
||||
)
|
||||
self._gather_group = device_group
|
||||
self._gather_device = device
|
||||
self._gather_group = tp_group.device_group
|
||||
self._gather_device = tp_group.device
|
||||
else:
|
||||
self._gather_group = cpu_group
|
||||
self._gather_group = tp_group.cpu_group
|
||||
self._gather_device = "cpu"
|
||||
|
||||
# Fields packed per rank into the all-gather tensor: prefillable,
|
||||
|
||||
@@ -109,6 +109,7 @@ from sglang.srt.dllm.mixin.scheduler import SchedulerDllmMixin
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.moe import initialize_moe_config
|
||||
from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config
|
||||
from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config
|
||||
@@ -568,9 +569,6 @@ class Scheduler(
|
||||
tp_worker=self.tp_worker,
|
||||
page_size=self.page_size,
|
||||
spec_algorithm=self.spec_algorithm,
|
||||
attn_tp_cpu_group=self.attn_tp_cpu_group,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
attn_cp_cpu_group=self.attn_cp_cpu_group,
|
||||
enable_metrics=get_observability().enable_metrics,
|
||||
enable_kv_cache_events=bool(
|
||||
get_observability().kv_events_config
|
||||
@@ -578,8 +576,6 @@ class Scheduler(
|
||||
and get_parallel().attn_tp_rank == 0
|
||||
and get_parallel().attn_cp_rank == 0
|
||||
),
|
||||
tp_group=self.tp_group,
|
||||
pp_group=self.pp_group,
|
||||
enable_hierarchical_cache=self.enable_hierarchical_cache,
|
||||
hicache_draft_plan=(
|
||||
self.draft_worker.hicache_draft_plan
|
||||
@@ -626,11 +622,7 @@ class Scheduler(
|
||||
self.decode_offload_manager = DecodeKVCacheOffloadManager(
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
tp_group=(
|
||||
self.attn_tp_cpu_group
|
||||
if self.attn_dp_enabled
|
||||
else self.tp_cpu_group
|
||||
),
|
||||
tp_group=self.dp_tp_cpu_group,
|
||||
tree_cache=self.tree_cache,
|
||||
)
|
||||
else:
|
||||
@@ -747,16 +739,16 @@ class Scheduler(
|
||||
):
|
||||
return
|
||||
|
||||
parallel = get_parallel()
|
||||
tp_group = parallel.tp_group
|
||||
rank = (
|
||||
get_parallel().dp_rank
|
||||
if get_parallel().dp_rank is not None
|
||||
else self.tp_group.rank_in_group
|
||||
parallel.dp_rank if parallel.dp_rank is not None else tp_group.rank_in_group
|
||||
)
|
||||
logger.info("HCCL DP prewarm start: rank=%s", rank)
|
||||
_prewarm_hccl_group(
|
||||
device=self.tp_group.device,
|
||||
group=self.tp_group.device_group,
|
||||
device_module=self.tp_group.device_module,
|
||||
device=tp_group.device,
|
||||
group=tp_group.device_group,
|
||||
device_module=tp_group.device_module,
|
||||
)
|
||||
logger.info("HCCL DP prewarm done: rank=%s", rank)
|
||||
|
||||
@@ -1213,11 +1205,9 @@ class Scheduler(
|
||||
self.world_group = get_parallel().world_group
|
||||
|
||||
# NOTE: dp_tp_* are request/data-plane coordination groups (not tensor collectives).
|
||||
# When DP attention is enabled, scope to the attention-TP group; otherwise use
|
||||
# the base TP group. Entry rank is the local rank 0 in that group.
|
||||
# Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA
|
||||
# stream/device coupling (#11910).
|
||||
self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group
|
||||
# Entry rank is the local rank 0 in that group. Use the CPU (gloo) group to
|
||||
# broadcast VLM Python objects and avoid CUDA stream/device coupling (#11910).
|
||||
self.dp_tp_group = get_dp_tp_group()
|
||||
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
|
||||
|
||||
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
|
||||
@@ -1408,8 +1398,6 @@ class Scheduler(
|
||||
)
|
||||
else:
|
||||
self.prefill_delayer = PrefillDelayer(
|
||||
cpu_group=self.tp_cpu_group,
|
||||
device_group=self.tp_group.device_group,
|
||||
metrics_collector=(
|
||||
self.metrics_collector
|
||||
if self.metrics_reporter.enable_metrics
|
||||
@@ -1417,7 +1405,6 @@ class Scheduler(
|
||||
),
|
||||
max_delay_passes=get_schedule().prefill_delayer_max_delay_passes,
|
||||
token_usage_low_watermark=get_schedule().prefill_delayer_token_usage_low_watermark,
|
||||
device=self.tp_group.device,
|
||||
debug_log_enabled=get_parallel().attn_tp_rank == 0,
|
||||
)
|
||||
|
||||
@@ -1536,7 +1523,6 @@ class Scheduler(
|
||||
|
||||
# The decode requests polling kv cache
|
||||
self.disagg_decode_transfer_queue = DecodeTransferQueue(
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
|
||||
metadata_buffers=self.disagg_metadata_buffers,
|
||||
scheduler=self,
|
||||
@@ -1553,7 +1539,6 @@ class Scheduler(
|
||||
scheduler=self,
|
||||
transfer_queue=self.disagg_decode_transfer_queue,
|
||||
tree_cache=self.tree_cache,
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
gpu_id=get_device().gpu_id,
|
||||
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
@@ -1613,9 +1598,6 @@ class Scheduler(
|
||||
self.server_args,
|
||||
dtype=self.model_config.dtype,
|
||||
hf_config=self.model_config.hf_config,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
tp_group=self.tp_group,
|
||||
scheduler=self,
|
||||
)
|
||||
|
||||
@@ -2242,7 +2224,6 @@ class Scheduler(
|
||||
|
||||
def init_profiler(self) -> None:
|
||||
self.profiler_manager = SchedulerProfilerManager(
|
||||
dp_tp_cpu_group=self.dp_tp_cpu_group,
|
||||
get_forward_ct=lambda: self.forward_ct,
|
||||
)
|
||||
|
||||
@@ -2250,7 +2231,6 @@ class Scheduler(
|
||||
self.weight_updater = SchedulerWeightUpdaterManager(
|
||||
tp_worker=self.tp_worker,
|
||||
draft_worker=self.draft_worker,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
memory_saver_adapter=self.memory_saver_adapter,
|
||||
flush_cache=self.flush_cache,
|
||||
is_fully_idle=self.is_fully_idle,
|
||||
@@ -2333,11 +2313,6 @@ class Scheduler(
|
||||
recv_skipper=self.recv_skipper,
|
||||
input_blocker=self.input_blocker,
|
||||
mm_receiver=self.mm_receiver,
|
||||
tp_group=self.tp_group,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
attn_tp_cpu_group=self.attn_tp_cpu_group,
|
||||
attn_cp_cpu_group=self.attn_cp_cpu_group,
|
||||
world_group=self.world_group,
|
||||
server_args=self.server_args,
|
||||
model_config=self.model_config,
|
||||
max_recv_per_poll=self.max_recv_per_poll,
|
||||
|
||||
@@ -4,7 +4,7 @@ import gc
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
@@ -17,6 +17,7 @@ from typing import (
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.managers.io_struct import ProfileReq, ProfileReqOutput, ProfileReqType
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.step_span_utils import set_detailed_annotations_enabled
|
||||
@@ -51,10 +52,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class SchedulerProfilerManager:
|
||||
dp_tp_cpu_group: Any
|
||||
get_forward_ct: Callable[[], int]
|
||||
# Taken at construction: a batch can stop profiling while a scoped
|
||||
# override has moved the TP group.
|
||||
dp_tp_cpu_group: Any = field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.dp_tp_cpu_group = get_dp_tp_group().cpu_group
|
||||
if envs.SGLANG_PROFILE_V2.get():
|
||||
self._profile_manager = ProfileManager(
|
||||
cpu_group=self.dp_tp_cpu_group,
|
||||
|
||||
@@ -63,11 +63,6 @@ class SchedulerRequestReceiver:
|
||||
recv_skipper: Any
|
||||
input_blocker: Any
|
||||
mm_receiver: Any
|
||||
tp_group: Any
|
||||
tp_cpu_group: Any
|
||||
attn_tp_cpu_group: Any
|
||||
attn_cp_cpu_group: Any
|
||||
world_group: Any
|
||||
server_args: ServerArgs
|
||||
model_config: ModelConfig
|
||||
max_recv_per_poll: int
|
||||
@@ -157,7 +152,7 @@ class SchedulerRequestReceiver:
|
||||
recv_reqs = point_to_point_pyobj(
|
||||
[],
|
||||
get_parallel().pp_rank * get_parallel().tp_size + dp_offset,
|
||||
self.world_group.cpu_group,
|
||||
get_parallel().world_group.cpu_group,
|
||||
(get_parallel().pp_rank - 1) * get_parallel().tp_size + dp_offset,
|
||||
get_parallel().pp_rank * get_parallel().tp_size + dp_offset,
|
||||
)
|
||||
@@ -195,22 +190,24 @@ class SchedulerRequestReceiver:
|
||||
if _local_ctrl:
|
||||
control_reqs = attn_cp_tp_broadcast_pyobj(control_reqs)
|
||||
elif get_parallel().tp_size != 1:
|
||||
tp_group = get_parallel().tp_group
|
||||
control_reqs = broadcast_pyobj(
|
||||
control_reqs,
|
||||
self.tp_group.rank,
|
||||
self.tp_cpu_group,
|
||||
src=self.tp_group.ranks[0],
|
||||
tp_group.rank,
|
||||
tp_group.cpu_group,
|
||||
src=tp_group.ranks[0],
|
||||
)
|
||||
recv_reqs = work_reqs + control_reqs
|
||||
else:
|
||||
if recv_reqs is not None:
|
||||
recv_reqs = [*recv_reqs, *local_reqs]
|
||||
if get_parallel().tp_size != 1:
|
||||
tp_group = get_parallel().tp_group
|
||||
recv_reqs = broadcast_pyobj(
|
||||
recv_reqs,
|
||||
self.tp_group.rank,
|
||||
self.tp_cpu_group,
|
||||
src=self.tp_group.ranks[0],
|
||||
tp_group.rank,
|
||||
tp_group.cpu_group,
|
||||
src=tp_group.ranks[0],
|
||||
)
|
||||
return recv_reqs
|
||||
|
||||
@@ -268,11 +265,11 @@ class SchedulerRequestReceiver:
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_dp_enabled:
|
||||
if parallel.attn_tp_size > 1:
|
||||
barrier(group=self.attn_tp_cpu_group)
|
||||
barrier(group=parallel.attn_tp_group.cpu_group)
|
||||
if parallel.attn_cp_size > 1:
|
||||
barrier(group=self.attn_cp_cpu_group)
|
||||
barrier(group=parallel.attn_cp_group.cpu_group)
|
||||
elif parallel.tp_size > 1:
|
||||
barrier(group=self.tp_cpu_group)
|
||||
barrier(group=parallel.tp_group.cpu_group)
|
||||
|
||||
# 2. materialize independently so one bad VLM request does not stop the loop
|
||||
failed = torch.zeros(len(tokenized_reqs), dtype=torch.int32)
|
||||
@@ -292,11 +289,15 @@ class SchedulerRequestReceiver:
|
||||
# 3. all ranks reject the same requests before entering model collectives
|
||||
if parallel.attn_dp_enabled:
|
||||
if parallel.attn_tp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group)
|
||||
all_reduce(
|
||||
failed, op=ReduceOp.MAX, group=parallel.attn_tp_group.cpu_group
|
||||
)
|
||||
if parallel.attn_cp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.attn_cp_cpu_group)
|
||||
all_reduce(
|
||||
failed, op=ReduceOp.MAX, group=parallel.attn_cp_group.cpu_group
|
||||
)
|
||||
elif parallel.tp_size > 1:
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=self.tp_cpu_group)
|
||||
all_reduce(failed, op=ReduceOp.MAX, group=parallel.tp_group.cpu_group)
|
||||
|
||||
error = MMInputsProcessError(
|
||||
"Failed to materialize shared-memory multimodal features on a scheduler rank."
|
||||
|
||||
@@ -44,7 +44,7 @@ from sglang.srt.managers.io_struct import (
|
||||
UpdateWeightsFromTensorReqInput,
|
||||
UpdateWeightsFromTensorReqOutput,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.runtime_context import get_model, get_parallel
|
||||
from sglang.srt.utils.weight_checker import overall_checksum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -88,7 +88,6 @@ class _WeightUpdateSession(msgspec.Struct, frozen=True):
|
||||
class SchedulerWeightUpdaterManager:
|
||||
tp_worker: Any
|
||||
draft_worker: Any
|
||||
tp_cpu_group: Any
|
||||
memory_saver_adapter: Any
|
||||
flush_cache: Callable[..., bool]
|
||||
is_fully_idle: Callable[..., bool]
|
||||
@@ -98,6 +97,10 @@ class SchedulerWeightUpdaterManager:
|
||||
stashed_model_static_state: Any = None
|
||||
# replicated on every TP rank, so a rejected call returns on all ranks before any barrier
|
||||
_session: Optional[_WeightUpdateSession] = None
|
||||
tp_cpu_group: Any = field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.tp_cpu_group = get_parallel().tp_group.cpu_group
|
||||
|
||||
@contextmanager
|
||||
def _observe_weight_load(self, source: str) -> Iterator[None]:
|
||||
|
||||
@@ -865,7 +865,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
def get_prefetch_submission(self, rid: str) -> Optional[PrefetchSubmission]:
|
||||
if self.pp_prefetch_command_group is None:
|
||||
return None
|
||||
if self.pp_rank != 0:
|
||||
if self.storage_config.pp_rank != 0:
|
||||
return PrefetchSubmission()
|
||||
with self.pp_prefetch_state_lock:
|
||||
state = self.pp_prefetch_states.get(rid)
|
||||
@@ -898,7 +898,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
)
|
||||
|
||||
if self.pp_rank != 0:
|
||||
if self.storage_config.pp_rank != 0:
|
||||
raise RuntimeError("Only PP0 can submit a PP prefetch ticket.")
|
||||
|
||||
rid = handle.rid
|
||||
@@ -928,7 +928,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
storage_hit_count = len(ticket.prefetch_key.token_ids)
|
||||
try:
|
||||
for pp_rank in range(self.tp_rank, self.pp_size, self.tp_size):
|
||||
config = self.storage_config
|
||||
for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size):
|
||||
_, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank)
|
||||
storage_hit_count = min(storage_hit_count, rank_hit_count)
|
||||
except Exception:
|
||||
@@ -1096,7 +1097,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
assert group is not None
|
||||
rank = torch.distributed.get_rank()
|
||||
source = torch.distributed.get_process_group_ranks(group)[0]
|
||||
is_source = self.pp_rank == 0
|
||||
is_source = self.storage_config.pp_rank == 0
|
||||
|
||||
while True:
|
||||
objects = []
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -58,10 +59,7 @@ from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
|
||||
from sglang.srt.utils import ceil_align, is_hip
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.managers.tp_worker import BaseTpWorker
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
|
||||
@@ -266,13 +264,8 @@ def build_kv_cache(
|
||||
tp_worker: BaseTpWorker,
|
||||
page_size: int,
|
||||
spec_algorithm: SpeculativeAlgorithm,
|
||||
attn_tp_cpu_group: ProcessGroup,
|
||||
tp_cpu_group: ProcessGroup,
|
||||
attn_cp_cpu_group: ProcessGroup,
|
||||
enable_metrics: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
tp_group: GroupCoordinator,
|
||||
pp_group: GroupCoordinator,
|
||||
enable_hierarchical_cache: bool,
|
||||
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
|
||||
) -> KVCacheBuildResult:
|
||||
@@ -358,12 +351,10 @@ def build_kv_cache(
|
||||
else token_to_kv_pool_allocator.page_size
|
||||
),
|
||||
is_eagle=spec_algorithm.is_eagle(),
|
||||
tp_cache_group=(
|
||||
attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group
|
||||
),
|
||||
attn_cp_cache_group=attn_cp_cpu_group,
|
||||
attn_tp_cache_group=attn_tp_cpu_group,
|
||||
pp_cache_group=pp_group.cpu_group,
|
||||
tp_cache_group=get_dp_tp_group().cpu_group,
|
||||
attn_cp_cache_group=parallel.attn_cp_group.cpu_group,
|
||||
attn_tp_cache_group=parallel.attn_tp_group.cpu_group,
|
||||
pp_cache_group=parallel.pp_group.cpu_group,
|
||||
eviction_policy=get_memory().radix_eviction_policy,
|
||||
eviction_policy_config=get_memory().radix_eviction_policy_config,
|
||||
enable_metrics=enable_metrics,
|
||||
@@ -394,7 +385,7 @@ def build_kv_cache(
|
||||
model_config=model_config,
|
||||
tp_size=parallel.tp_size,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_group=tp_group,
|
||||
tp_group=parallel.tp_group,
|
||||
)
|
||||
with auto_size_hicache(
|
||||
params,
|
||||
|
||||
@@ -123,6 +123,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
|
||||
extra_config, *_ = HybridCacheController.parse_storage_backend_extra_config(
|
||||
get_memory().hicache_storage_backend_extra_config
|
||||
)
|
||||
# TODO: unlike `HiCacheController._generate_storage_config`, this sets no
|
||||
# `dp_rank`. Aligning the storage-config producers changes persisted
|
||||
# storage keys, so it waits for a storage-key format change.
|
||||
storage_config = HiCacheStorageConfig(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
|
||||
@@ -295,6 +295,11 @@ class UMBPDirectLinker(UnifiedCacheLinker):
|
||||
)
|
||||
extra_config["dram_page_size"] = dram_page_size
|
||||
|
||||
# TODO: `tp_rank` is the rank in the cache TP group but `tp_size` is the
|
||||
# full TP width, and no `dp_rank` is set, unlike
|
||||
# `HiCacheController._generate_storage_config`. Aligning the
|
||||
# storage-config producers changes persisted storage keys, so it waits
|
||||
# for a storage-key format change.
|
||||
storage_config = HiCacheStorageConfig(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=get_parallel().tp_size,
|
||||
|
||||
@@ -276,13 +276,14 @@ class StorageAttachment:
|
||||
"""
|
||||
cache = self._cache
|
||||
controller = cache.cache_controller
|
||||
config = controller.storage_config
|
||||
attn_cp_rank, attn_cp_size = controller.get_attn_cp_rank_and_size()
|
||||
labels = {
|
||||
"storage_backend": storage_backend,
|
||||
"tp_rank": controller.tp_rank,
|
||||
"dp_rank": controller.dp_rank,
|
||||
"pp_rank": controller.pp_rank,
|
||||
"pp_size": controller.pp_size,
|
||||
"tp_rank": config.tp_rank,
|
||||
"dp_rank": config.dp_rank,
|
||||
"pp_rank": config.pp_rank,
|
||||
"pp_size": config.pp_size,
|
||||
"attn_cp_rank": attn_cp_rank,
|
||||
"attn_cp_size": attn_cp_size,
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ from sglang.srt.layers.cp.utils import (
|
||||
is_cp_active,
|
||||
is_mla_cp_enabled,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.sampler import create_sampler
|
||||
from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers
|
||||
@@ -963,11 +964,7 @@ class ModelRunner:
|
||||
top_k=hisparse_top_k,
|
||||
device_buffer_size=hisparse_cfg.device_buffer_size,
|
||||
device=self.device,
|
||||
tp_group=(
|
||||
get_parallel().attn_tp_group.cpu_group
|
||||
if get_parallel().attn_dp_enabled
|
||||
else get_parallel().tp_group.cpu_group
|
||||
),
|
||||
tp_group=get_dp_tp_group().cpu_group,
|
||||
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
|
||||
swap_in_block_size=hisparse_cfg.swap_in_block_size,
|
||||
shared_index_layers=resolve_shared_index_layers(
|
||||
@@ -1368,8 +1365,6 @@ class ModelRunner:
|
||||
dtype=self.dtype,
|
||||
server_args=self.server_args,
|
||||
lora_backend=get_lora().lora_backend,
|
||||
tp_size=get_parallel().tp_size,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
max_lora_rank=get_lora().max_lora_rank,
|
||||
target_modules=get_lora().lora_target_modules,
|
||||
lora_paths=get_lora().lora_paths,
|
||||
|
||||
@@ -12,6 +12,7 @@ from transformers import CLIPConfig, CLIPTextConfig, CLIPVisionConfig
|
||||
|
||||
from sglang.srt.layers.activation import QuickGELU, get_act_fn
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -200,6 +201,15 @@ class CLIPAttention(nn.Module):
|
||||
tp_rank=parallel.attn_tp_rank,
|
||||
tp_size=parallel.attn_tp_size,
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=parallel.attn_tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
multimodal_encoder=True,
|
||||
)
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=config.hidden_size,
|
||||
|
||||
@@ -29,9 +29,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
|
||||
@@ -209,6 +209,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin):
|
||||
gemm1_alpha=None,
|
||||
gemm1_clamp_limit=None,
|
||||
is_gated=True,
|
||||
layer=self,
|
||||
)
|
||||
|
||||
FusedMoELoadingMixin.__init__(
|
||||
|
||||
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
@@ -313,6 +314,14 @@ class IQuestQ1Attention(nn.Module):
|
||||
tp_size=attn_tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# The decoder boundary reduces partial attention outputs. Only a
|
||||
# standalone attention that owns its TP reduction needs this guard.
|
||||
if reduce_results:
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=attn_tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
|
||||
@@ -59,6 +59,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -357,6 +358,16 @@ class MLP2(nn.Module):
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
elif use_tensor_parallel:
|
||||
# TODO: these layers shard over attention TP but reduce over the full
|
||||
# TP group; reduce over the attention-TP group so attention DP and
|
||||
# attention CP narrower than TP can run them.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.fc0 = ColumnParallelLinear(
|
||||
dims[0],
|
||||
dims[1],
|
||||
|
||||
@@ -514,6 +514,12 @@ class AudioEncoderAttention(nn.Module):
|
||||
window_size=window_size,
|
||||
customized_position_embedding_applier=_audio_rope_applier,
|
||||
prefix="attn",
|
||||
# TODO: the audio encoder does not follow --mm-enable-dp-encoder and
|
||||
# reduces over the full TP group, so its output is wrong under
|
||||
# attention DP narrower than TP. Rejecting that would reject the
|
||||
# documented MiMo-V2.5 `--tp 8 --attn-dp-size 2` launch; fix the
|
||||
# reduce group or run the encoder data-parallel instead.
|
||||
allow_tp_reduce_mismatch=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -17,7 +17,10 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -83,6 +86,16 @@ class MiniMaxVLMultiModalProjector(nn.Module):
|
||||
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
|
||||
self.linear_1 = ColumnParallelLinear(
|
||||
vision_hidden_size,
|
||||
@@ -138,6 +151,16 @@ class MiniMaxVLPatchMerger(nn.Module):
|
||||
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
|
||||
self.linear_1 = ColumnParallelLinear(
|
||||
text_hidden_size * spatial_merge_size**2,
|
||||
@@ -260,6 +283,16 @@ class CLIPEncoderLayer(nn.Module):
|
||||
self.use_data_parallel = use_data_parallel
|
||||
tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
|
||||
self.self_attn = VisionAttention(
|
||||
embed_dim=config.hidden_size,
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -282,6 +283,12 @@ class PhiMoEAttention(nn.Module):
|
||||
tp_size=attn_tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__, shard_tp_size=attn_tp_size, reduces_over_attn_tp=False
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
hidden_size,
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
@@ -148,6 +149,16 @@ class Qwen3_VisionMLP(nn.Module):
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=self.tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.linear_fc1 = ColumnParallelLinear(
|
||||
in_features,
|
||||
hidden_features,
|
||||
@@ -317,6 +328,16 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module):
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group without attention DP; reduce over the attention-TP group so
|
||||
# attention CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=self.tp_size,
|
||||
reduces_over_attn_tp=is_dp_attention_enabled(),
|
||||
multimodal_encoder=True,
|
||||
hint=", or --mm-enable-dp-encoder where the model supports it",
|
||||
)
|
||||
self.linear_fc1 = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.padded_context_dim,
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
reject_attn_tp_shard_with_tp_reduce,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
@@ -545,6 +546,15 @@ class Step3VisionMLP(nn.Module):
|
||||
prefix=add_prefix("gate_proj", prefix),
|
||||
)
|
||||
self.act = ACT2FN[hidden_act] # quick_gelu
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
# CP narrower than TP can run it.
|
||||
reject_attn_tp_shard_with_tp_reduce(
|
||||
type(self).__name__,
|
||||
shard_tp_size=attn_tp_size,
|
||||
reduces_over_attn_tp=False,
|
||||
multimodal_encoder=True,
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
intermediate_size,
|
||||
dim,
|
||||
|
||||
@@ -263,12 +263,23 @@ class ServerArgs:
|
||||
return getattr(self, "_launch_command", None)
|
||||
|
||||
def resolved_dict(self) -> dict[str, Any]:
|
||||
"""Serialize resolved field values, expanding nested records and excluding bookkeeping."""
|
||||
"""Serialize resolved field values, expanding nested records and excluding bookkeeping.
|
||||
|
||||
return {
|
||||
One exception: the deprecated `enable_dp_attention` reports whether an
|
||||
attention-DP width was configured, for clients that still read it from
|
||||
`/server_info`. Resolution leaves the field false. It reports the width
|
||||
rather than `attn_dp_enabled`, whose elastic scale-joiner arm is true at
|
||||
width one: a dump carrying that would fold the joiner's replicas into
|
||||
attention-DP groups when it is read back.
|
||||
"""
|
||||
|
||||
resolved = {
|
||||
field.name: _plain(resolution_result(self, field.name))
|
||||
for field in record_fields(type(self))
|
||||
}
|
||||
# TODO: drop together with `--enable-dp-attention` after 2026-12-31.
|
||||
resolved["enable_dp_attention"] = resolving_view(self).attn_dp_size > 1
|
||||
return resolved
|
||||
|
||||
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
@@ -351,7 +362,7 @@ class ServerArgs:
|
||||
help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.",
|
||||
)
|
||||
# `enable_dp_attention` is `no_cli=True` too; resolution turns it into
|
||||
# `attn_dp_size`.
|
||||
# `attn_dp_size`. TODO: remove the flag and the field after 2026-12-31.
|
||||
parser.add_argument(
|
||||
"--enable-dp-attention",
|
||||
action=DeprecatedStoreTrueAction,
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
|
||||
)
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
@@ -402,11 +403,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._full_embed_gpu: Optional[torch.Tensor] = None
|
||||
# Under dp attention, peer DP ranks run different (idle) paths, so
|
||||
# spec broadcasts must stay within the attn-TP group.
|
||||
self._tp_sync = SpecTpSync(
|
||||
get_parallel().attn_tp_group
|
||||
if get_parallel().attn_dp_enabled
|
||||
else get_parallel().tp_group
|
||||
)
|
||||
self._tp_sync = SpecTpSync(get_dp_tp_group())
|
||||
|
||||
# Under dp attention, the draft worker runs on the per-DP attn-TP
|
||||
# group, independent of idle peer DP ranks; it is built and run under
|
||||
|
||||
@@ -8,6 +8,7 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
)
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -233,9 +234,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
self._mask_token_id = runtime_config.mask_token_id
|
||||
|
||||
parallel = get_parallel()
|
||||
self._tp_sync = SpecTpSync(
|
||||
parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group
|
||||
)
|
||||
self._tp_sync = SpecTpSync(get_dp_tp_group())
|
||||
self._draft_graph_group = (
|
||||
parallel.attn_tp_group
|
||||
if self._draft_dp_context_enabled
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.lora.lora_manager import LoRAManager
|
||||
from sglang.srt.lora.lora_registry import LoRARef
|
||||
from sglang.srt.runtime_context import get_parallel, get_spec
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -45,8 +45,6 @@ def init_uno_lora_manager(
|
||||
dtype=model_runner.dtype,
|
||||
server_args=model_runner.server_args,
|
||||
lora_backend="uno_cublas", # fast path
|
||||
tp_size=get_parallel().tp_size,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
# Infer these from the one trained adapter.
|
||||
max_lora_rank=None,
|
||||
target_modules=None,
|
||||
|
||||
Reference in New Issue
Block a user