From 10edb05e72afb79ecfdf7dbe7603f90ba05ad3ee Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:59:32 -0700 Subject: [PATCH] [Refactor] Let the remaining placement consumers read the parallel context (#42348) --- .../advanced_features/dp_dpa_smg_guide.mdx | 6 +- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 13 +- .../sglang/srt/arg_groups/fields/parallel.py | 2 +- .../srt/arg_groups/model_override_base.py | 21 +++ .../arg_groups/model_overrides/deepseek_v2.py | 17 +- python/sglang/srt/arg_groups/parallel_hook.py | 17 +- python/sglang/srt/disaggregation/base/conn.py | 2 - .../sglang/srt/disaggregation/common/conn.py | 2 - python/sglang/srt/disaggregation/decode.py | 7 +- .../srt/disaggregation/encoder/receiver.py | 39 +---- python/sglang/srt/disaggregation/fake/conn.py | 2 - .../srt/disaggregation/mooncake/conn.py | 4 - python/sglang/srt/disaggregation/mori/conn.py | 4 - python/sglang/srt/disaggregation/nixl/conn.py | 4 - python/sglang/srt/disaggregation/prefill.py | 4 - python/sglang/srt/layers/attention/vision.py | 13 ++ python/sglang/srt/layers/dp_attention.py | 73 ++++++++ .../srt/layers/moe/fused_moe_triton/layer.py | 1 + .../sglang/srt/layers/moe/moe_runner/base.py | 29 +++ .../moe/moe_runner/flashinfer_cutlass.py | 26 +-- .../srt/layers/moe/moe_runner/hpc_ops.py | 5 +- .../compressed_tensors_w4a4_mxint4_moe.py | 3 +- .../compressed_tensors_w4a4_nvfp4_moe.py | 4 - python/sglang/srt/layers/quantization/fp8.py | 2 - .../srt/layers/quantization/modelopt_quant.py | 8 - .../sglang/srt/layers/quantization/mxfp4.py | 8 - .../mxfp4_flashinfer_cutlass_moe.py | 7 +- .../sglang/srt/layers/quantization/unquant.py | 4 - python/sglang/srt/lora/layers.py | 3 - python/sglang/srt/lora/lora_manager.py | 7 - python/sglang/srt/lora/lora_moe_runners.py | 13 -- python/sglang/srt/lora/mem_pool.py | 12 +- .../sglang/srt/managers/cache_controller.py | 30 ++-- python/sglang/srt/managers/prefill_delayer.py | 13 +- python/sglang/srt/managers/scheduler.py | 47 ++--- .../scheduler_components/profiler_manager.py | 8 +- .../scheduler_components/request_receiver.py | 37 ++-- .../scheduler_components/weight_updater.py | 7 +- .../hybrid_cache/hybrid_cache_controller.py | 9 +- .../sglang/srt/mem_cache/kv_cache_builder.py | 21 +-- .../mooncake_store/mooncake_direct_linker.py | 3 + .../storage/umbp/umbp_direct_linker.py | 5 + .../unified_cache/storage_attachment.py | 9 +- .../sglang/srt/model_executor/model_runner.py | 9 +- python/sglang/srt/models/clip.py | 10 ++ python/sglang/srt/models/exaone_moe.py | 4 +- .../srt/models/inkling_common/dense_mlp.py | 1 + python/sglang/srt/models/iquest_q1.py | 9 + python/sglang/srt/models/kimi_vl_moonvit.py | 11 ++ python/sglang/srt/models/mimo_audio.py | 6 + python/sglang/srt/models/minimax_vl_common.py | 35 +++- python/sglang/srt/models/phimoe.py | 7 + python/sglang/srt/models/qwen3_vl.py | 21 +++ python/sglang/srt/models/step3_vl.py | 10 ++ python/sglang/srt/server_args.py | 17 +- .../srt/speculative/dflash_worker_v2.py | 7 +- .../dspark_components/dspark_worker_v2.py | 5 +- python/sglang/srt/speculative/uno_lora.py | 4 +- test/registered/moe/test_hpc_ops_moe.py | 7 +- .../unit/disaggregation/test_dcp_pack.py | 2 +- .../disaggregation/test_fake_kv_sender.py | 2 - .../mlx/test_metal_profiler.py | 6 +- .../moe/test_moe_runner_config_placement.py | 147 ++++++++++++++++ .../quantization/test_mxfp4_sm120_cutlass.py | 2 + .../quantization/test_mxfp4_sm90_cutlass.py | 23 +-- .../layers/test_attn_tp_reduce_rejection.py | 165 ++++++++++++++++++ .../unit/lora/test_mem_pool_ep_unit.py | 20 ++- .../managers/test_mm_shm_error_consensus.py | 26 +-- .../unit/managers/test_pp_cp_rank_offsets.py | 18 +- .../managers/test_prefill_delayer_gather.py | 6 +- .../test_scheduler_weight_version_tracking.py | 10 +- .../unit/mem_cache/test_pp_prefetch_ticket.py | 38 ++-- .../unit/server_args/test_attn_dp_size.py | 58 +++++- 73 files changed, 875 insertions(+), 362 deletions(-) create mode 100644 test/registered/unit/layers/moe/test_moe_runner_config_placement.py create mode 100644 test/registered/unit/layers/test_attn_tp_reduce_rejection.py diff --git a/docs/docs/advanced_features/dp_dpa_smg_guide.mdx b/docs/docs/advanced_features/dp_dpa_smg_guide.mdx index 896b533d76..16471bfdc6 100644 --- a/docs/docs/advanced_features/dp_dpa_smg_guide.mdx +++ b/docs/docs/advanced_features/dp_dpa_smg_guide.mdx @@ -120,12 +120,14 @@ python -m sglang.launch_server \ **Important**: `--attn-dp-size` must be greater than 1 for DPA to work. When `attn_dp_size == 1` (default), DPA is disabled. The constraint `tp_size % attn_dp_size == 0` must also be satisfied. `--dp-size` only sets the number of data-parallel replicas of the model; combining replicas (`--dp-size` > 1) with DPA is not supported. -> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead. +> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead. The flag will be removed after 2026-12-31. +> +> Under DeepSeek context parallelism (DSA, MLA, and DeepSeek-V4 prefill CP), `--dp-size N` alone still runs N attention-DP groups, and logs a deprecation warning; it will be rejected after 2026-12-31. Use `--attn-dp-size N` there too. **Elastic EP scale joiners.** A rank that joins an existing DPA deployment through elastic EP scale-up runs DPA even when it is launched with `--attn-dp-size 1`: its own attention DP group is one rank wide. **DP rank count.** The number of DP ranks (one scheduler each) is `dp_size × attn_dp_size`; one of the two is always 1. -- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Its deprecated `enable_dp_attention` field is always `false`; read `attn_dp_size` instead. +- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Until the flag is removed, its deprecated `enable_dp_attention` field still reports whether DPA runs; new clients should read `attn_dp_size`. - Wire formats that predate `--attn-dp-size` keep the key `dp_size` for the DP rank count: KV-event descriptors, the PD prefill server info, and load snapshots. ### TP LMHead with all-to-all diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index afd88cd583..09a1321d74 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -3,7 +3,10 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING -from sglang.srt.arg_groups.model_override_base import attention_backends_of +from sglang.srt.arg_groups.model_override_base import ( + attention_backends_of, + context_parallel_attn_dp_size, +) from sglang.srt.arg_groups.overrides import ( _deepseek_v4_kv_cache_dtype, declare_resolution, @@ -157,13 +160,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: f"DeepSeekV4 prefill CP on HIP does not support {feature} yet." ) - # DeepSeek-V4 CP runs data-parallel groups as attention DP. - assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( - f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " - "data-parallel replicas combined with attention data parallelism " - "are not supported." - ) - attn_dp_size = cfg.attn_dp_size * cfg.dp_size + attn_dp_size = context_parallel_attn_dp_size(cfg, "DeepSeek-V4 context parallelism") declare_resolution( server_args, "validate_deepseek_v4_cp", diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index edf20e3d41..cca9bc325f 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -171,7 +171,7 @@ class Parallel(msgspec.Struct): "Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.", ] = False # Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention` - # resolves to `attn_dp_size = N`, `dp_size = 1`. + # resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31. enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False enable_dp_attention_local_control_broadcast: A[ bool, diff --git a/python/sglang/srt/arg_groups/model_override_base.py b/python/sglang/srt/arg_groups/model_override_base.py index 5cc545a89f..a8f86f3111 100644 --- a/python/sglang/srt/arg_groups/model_override_base.py +++ b/python/sglang/srt/arg_groups/model_override_base.py @@ -226,6 +226,27 @@ def ep_joiner_of(cfg: Any) -> bool: return cfg.ep_join_mode in ("scale", "recover") +def context_parallel_attn_dp_size(cfg: Any, layout: str) -> int: + """The attention-DP width a DeepSeek context-parallel layout runs. + + These layouts run data-parallel groups as attention DP, so `--dp-size N` + alone still means N attention-DP groups there. That reading is deprecated. + """ + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + if cfg.dp_size > 1: + # TODO: reject `--dp-size > 1` here after 2026-12-31. + logger.warning( + f"--dp-size {cfg.dp_size} under {layout} runs {cfg.dp_size} attention " + "data-parallel groups. This is deprecated and will be rejected after " + f"2026-12-31; use --attn-dp-size {cfg.dp_size} instead." + ) + return cfg.attn_dp_size * cfg.dp_size + + def ep_scale_joiner_of(cfg: Any) -> bool: """The scale-up arm of :func:`ep_joiner_of`.""" return cfg.ep_join_mode == "scale" diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 87b19a5c8d..54689473fd 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -5,6 +5,7 @@ from typing import Any, Dict from sglang.srt.arg_groups.model_override_base import ( _register_for, + context_parallel_attn_dp_size, is_attention_backend_not_set, resolving_view, use_mla_backend, @@ -85,13 +86,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: logger.warning( "Context parallel feature is still under experiment. It has only been verified on Hopper platform." ) - # DSA CP runs data-parallel groups as attention DP. - assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( - f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " - "data-parallel replicas combined with attention data parallelism " - "are not supported." + attn_dp_size = context_parallel_attn_dp_size( + cfg, "DSA context parallelism" ) - attn_dp_size = cfg.attn_dp_size * cfg.dp_size overrides["attn_dp_size"] = attn_dp_size overrides["dp_size"] = 1 overrides["moe_dense_tp_size"] = 1 @@ -166,13 +163,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: "MLA prefill context parallel is still experimental. " "Verified on Hopper with the fa3 backend." ) - # MLA CP runs data-parallel groups as attention DP. - assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( - f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " - "data-parallel replicas combined with attention data parallelism " - "are not supported." - ) - attn_dp_size = cfg.attn_dp_size * cfg.dp_size + attn_dp_size = context_parallel_attn_dp_size(cfg, "MLA context parallelism") overrides["attn_dp_size"] = attn_dp_size overrides["dp_size"] = 1 # TODO(kpham-sgl) Supports moe_dense_tp_size != 1. diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index a760fcef59..49f176de3a 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -39,22 +39,23 @@ def handle_deprecated_dp_attention(server_args: Any): Runs before any handler reads the DP layout. The old spelling counted attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and - `dp_size` counts replicas again. + `dp_size` counts replicas again. Next to `--attn-dp-size` the flag adds + nothing: a serialized config reports it set whenever attention DP runs + (`ServerArgs.resolved_dict`), so reading one back resolves the same layout. """ cfg = resolving_view(server_args) - if cfg.enable_dp_attention: - if cfg.attn_dp_size != 1: - raise ValueError( - "--enable-dp-attention is the deprecated spelling of " - "--attn-dp-size; pass --attn-dp-size alone." - ) + if not cfg.enable_dp_attention: + return + if cfg.attn_dp_size == 1: declare_resolution( server_args, "_handle_deprecated_dp_attention", attn_dp_size=cfg.dp_size, dp_size=1, - enable_dp_attention=False, ) + declare_resolution( + server_args, "_handle_deprecated_dp_attention", enable_dp_attention=False + ) def _boundary_parallelism_overrides(cfg, model_type: str) -> dict: diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index 88b263d208..a3272299d5 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -158,8 +158,6 @@ class BaseKVSender(ABC): mgr: BaseKVManager, bootstrap_addr: str, bootstrap_room: int, - dest_tp_ranks: List[int], - pp_rank: int, req_has_disagg_prefill_dp_rank: bool = False, ): ... diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index e4ba54615b..45359599ca 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -1766,8 +1766,6 @@ class CommonKVSender(BaseKVSender): mgr: CommonKVManager, bootstrap_addr: str, bootstrap_room: int, - dest_tp_ranks: List[int], - pp_rank: int, req_has_disagg_prefill_dp_rank: bool = False, ): self.kv_mgr = mgr diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index d5c52a4258..22c1e6e1bc 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -31,7 +31,6 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple import numpy as np import torch -from torch.distributed import ProcessGroup from sglang.srt.configs.mamba_utils import Mamba2CacheParams from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE @@ -382,7 +381,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): scheduler: Scheduler, transfer_queue: DecodeTransferQueue, tree_cache: BasePrefixCache, - gloo_group: ProcessGroup, gpu_id: int, bootstrap_port: int, max_total_num_tokens: int, @@ -402,7 +400,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.tree_cache = tree_cache self.host_pool = None self.host_reserved_tokens = 0 - self.gloo_group = gloo_group + self.gloo_group = parallel.attn_tp_group.cpu_group # Destinations visible to prefill but not yet on the transfer queue. self._num_published_destinations = 0 self.tp_rank = parallel.tp_rank @@ -2379,14 +2377,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): def __init__( self, - gloo_group: ProcessGroup, req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator, metadata_buffers: MetadataBuffers, scheduler: Scheduler, tree_cache: BasePrefixCache, ): self.queue: List[DecodeRequest] = [] - self.gloo_group = gloo_group + self.gloo_group = get_parallel().attn_tp_group.cpu_group self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator self.tp_rank = get_parallel().tp_rank self.metadata_buffers = metadata_buffers diff --git a/python/sglang/srt/disaggregation/encoder/receiver.py b/python/sglang/srt/disaggregation/encoder/receiver.py index 7c1cb6258b..4fa9e1149d 100644 --- a/python/sglang/srt/disaggregation/encoder/receiver.py +++ b/python/sglang/srt/disaggregation/encoder/receiver.py @@ -25,10 +25,7 @@ from fastapi import FastAPI from fastapi.responses import ORJSONResponse, Response from transformers import PretrainedConfig -from sglang.srt.distributed.parallel_state import ( - GroupCoordinator, - get_mooncake_transfer_engine, -) +from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine from sglang.srt.environ import envs from sglang.srt.managers.io_struct import ( EncoderDispatchErrorReq, @@ -1870,9 +1867,6 @@ class MMReceiverBase(ABC): server_args: ServerArgs, dtype: Optional[torch.dtype] = None, hf_config: Optional[PretrainedConfig] = None, - pp_rank: Optional[int] = None, - tp_rank: Optional[int] = None, - tp_group: Optional[GroupCoordinator] = None, scheduler: Optional["Scheduler"] = None, encode_urls: Optional[List[str]] = None, ): @@ -1892,10 +1886,13 @@ class MMReceiverBase(ABC): ) self.recv_timeout = envs.SGLANG_ENCODER_RECV_TIMEOUT.get() self.host = get_local_ip_auto(get_serving().host) - self.pp_rank = pp_rank - self.tp_rank = tp_rank - self.tp_size = get_parallel().tp_size - self.tp_group = tp_group + # A scheduler-side receiver works across its TP group; the tokenizer + # side has no placement. + parallel = get_parallel() + self.pp_rank = parallel.pp_rank if scheduler is not None else None + self.tp_rank = parallel.tp_rank if scheduler is not None else None + self.tp_size = parallel.tp_size + self.tp_group = parallel.tp_group if scheduler is not None else None self.nnodes = get_parallel().nnodes self.hostname = get_local_ip_auto() self.waiting_list: List[WaitingMMRequestBase] = [] @@ -1908,7 +1905,7 @@ class MMReceiverBase(ABC): and scheduler is not None ): self.registration_runner = _ReceiveRegistrationRunner( - f"encoder-receive-registration-{tp_rank}" + f"encoder-receive-registration-{self.tp_rank}" ) ( self.scheduler_embedding_port, @@ -2674,9 +2671,6 @@ class MMReceiverHTTP(MMReceiverBase): server_args: ServerArgs, dtype: Optional[torch.dtype] = None, hf_config: Optional[PretrainedConfig] = None, - pp_rank: Optional[int] = None, - tp_rank: Optional[int] = None, - tp_group: Optional[GroupCoordinator] = None, scheduler: Optional["Scheduler"] = None, encode_urls: Optional[List[str]] = None, ): @@ -2684,9 +2678,6 @@ class MMReceiverHTTP(MMReceiverBase): server_args, dtype=dtype, hf_config=hf_config, - pp_rank=pp_rank, - tp_rank=tp_rank, - tp_group=tp_group, scheduler=scheduler, encode_urls=encode_urls, ) @@ -2813,9 +2804,6 @@ class MMReceiverGrpc(MMReceiverBase): server_args: ServerArgs, dtype: Optional[torch.dtype] = None, hf_config: Optional[PretrainedConfig] = None, - pp_rank: Optional[int] = None, - tp_rank: Optional[int] = None, - tp_group: Optional[GroupCoordinator] = None, scheduler: Optional["Scheduler"] = None, encode_urls: Optional[List[str]] = None, ): @@ -2830,9 +2818,6 @@ class MMReceiverGrpc(MMReceiverBase): server_args, dtype=dtype, hf_config=hf_config, - pp_rank=pp_rank, - tp_rank=tp_rank, - tp_group=tp_group, scheduler=scheduler, encode_urls=encode_urls, ) @@ -2946,9 +2931,6 @@ def create_mm_receiver( server_args: ServerArgs, dtype: Optional[torch.dtype] = None, hf_config: Optional[PretrainedConfig] = None, - pp_rank: Optional[int] = None, - tp_rank: Optional[int] = None, - tp_group: Optional[GroupCoordinator] = None, scheduler: Optional["Scheduler"] = None, transport_mode: Optional[str] = None, encode_urls: Optional[List[str]] = None, @@ -2967,9 +2949,6 @@ def create_mm_receiver( server_args, dtype=dtype, hf_config=hf_config, - pp_rank=pp_rank, - tp_rank=tp_rank, - tp_group=tp_group, scheduler=scheduler, encode_urls=encode_urls, ) diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py index a701cb062f..63d5e1c107 100644 --- a/python/sglang/srt/disaggregation/fake/conn.py +++ b/python/sglang/srt/disaggregation/fake/conn.py @@ -43,8 +43,6 @@ class FakeKVSender(BaseKVSender): mgr: BaseKVManager, bootstrap_addr: str, bootstrap_room: int, - dest_tp_ranks: List[int], - pp_rank: int, req_has_disagg_prefill_dp_rank: bool = False, ): self.kv_mgr = mgr diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 394cc49c3e..a4b5c01b3b 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2813,16 +2813,12 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender): mgr: MooncakeKVManager, bootstrap_addr: str, bootstrap_room: int, - dest_tp_ranks: List[int], - pp_rank: int, req_has_disagg_prefill_dp_rank: bool = False, ): super().__init__( mgr, bootstrap_addr, bootstrap_room, - dest_tp_ranks, - pp_rank, req_has_disagg_prefill_dp_rank, ) self.init_time = time.time() diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 370f38012e..95b891069a 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -1785,16 +1785,12 @@ class MoriKVSender(CommonKVSender): mgr: MoriKVManager, bootstrap_addr: str, bootstrap_room: int, - dest_tp_ranks: List[int], - pp_rank: int, req_has_disagg_prefill_dp_rank: bool = False, ): super().__init__( mgr, bootstrap_addr, bootstrap_room, - dest_tp_ranks, - pp_rank, req_has_disagg_prefill_dp_rank, ) self.conclude_state: Optional[KVPoll] = None diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 7e42c2039f..1c31bea92b 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -3133,16 +3133,12 @@ class NixlKVSender(CommonKVSender): mgr: NixlKVManager, bootstrap_addr: str, bootstrap_room: int, - dest_tp_ranks: List[int], - pp_rank: int, req_has_disagg_prefill_dp_rank: bool = False, ): super().__init__( mgr, bootstrap_addr, bootstrap_room, - dest_tp_ranks, - pp_rank, req_has_disagg_prefill_dp_rank, ) self.init_time = time.time() diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index e50dad09db..9d89cb4a39 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -382,14 +382,10 @@ class PrefillBootstrapQueue: ) kv_sender_class = get_kv_class(backend, KVClassType.SENDER) - dest_tp_ranks = [self.tp_rank] - req.disagg_kv_sender = kv_sender_class( mgr=self.kv_manager, bootstrap_addr=f"{req.bootstrap_host}:{self.bootstrap_port}", bootstrap_room=req.bootstrap_room, - dest_tp_ranks=dest_tp_ranks, - pp_rank=self.pp_rank, req_has_disagg_prefill_dp_rank=req.disagg_prefill_dp_rank is not None, ) self._process_req(req) diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index b18f597525..ceeaea811d 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -83,6 +83,7 @@ from sglang.srt.distributed import ( tensor_model_parallel_all_gather, ) from sglang.srt.distributed import utils as dist_utils +from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -1099,6 +1100,7 @@ class VisionAttention(nn.Module): ] = None, use_data_parallel: bool = False, use_dp_attention_reduce: bool = False, + allow_tp_reduce_mismatch: bool = False, aux_stream: Optional[torch.cuda.Stream] = None, workspace_buffer: Optional[torch.Tensor] = None, use_sink: bool = False, @@ -1208,6 +1210,17 @@ class VisionAttention(nn.Module): tp_size=self.tp_size, prefix=add_prefix("qkv_proj", prefix), ) + # TODO: this layer shards over attention TP but reduces over the full TP + # group; reduce over the attention-TP group so attention DP and attention + # CP narrower than TP can run it. + if not allow_tp_reduce_mismatch: + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=self.tp_size, + reduces_over_attn_tp=use_dp_attention_reduce, + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.proj = RowParallelLinear( input_size=self.dummy_dim, output_size=embed_dim, diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 521f0d964a..1942e0e9eb 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -455,6 +455,79 @@ def is_dp_attention_enabled() -> bool: return get_flags().dp.enabled +def get_dp_tp_group() -> GroupCoordinator: + """The TP ranks that run one DP rank's batch: the attention-TP group with + attention DP, otherwise the whole TP group. + + Attention-CP peers of a DP rank are members only in the second case. + """ + parallel = get_parallel() + return parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group + + +def multimodal_encoder_runs_here() -> bool: + """Whether a multimodal tower built on this worker is also forwarded here. + + A tower is built unconditionally by most models, but an encoder-disaggregated + language instance forwards it on the encoder instance, a language-model-only + instance rejects multimodal requests, and a PD decode instance embeds nothing + (`general_mm_embed_routine` skips decode and target-verify forwards). + + Adaptive dispatch is the exception on the first of those: it keeps the + requests it does not send to the encoder, and forwards the tower for them. + """ + from sglang.srt.runtime_context import get_disagg + + disagg = get_disagg() + offloaded_to_an_encoder_instance = ( + disagg.language_only and not disagg.enable_adaptive_dispatch_to_encoder + ) + return not ( + offloaded_to_an_encoder_instance + or disagg.language_model_only + or disagg.disaggregation_mode == "decode" + ) + + +def reject_attn_tp_shard_with_tp_reduce( + layer: str, + *, + shard_tp_size: int, + reduces_over_attn_tp: bool, + multimodal_encoder: bool = False, + hint: str = "", +) -> None: + """Reject a layer that shards over attention TP but all-reduces over the TP group. + + The two groups differ only when attention DP or attention CP makes attention + TP narrower than TP. There the all-reduce mixes ranks that hold other + requests or replicated inputs, so the output is wrong; a one-rank shard does + not reduce at all. + """ + tp_size = get_parallel().tp_size + if reduces_over_attn_tp or not 1 < shard_tp_size < tp_size: + return + if multimodal_encoder and not multimodal_encoder_runs_here(): + return + # Name only the widths that are actually narrowing the group here, so the + # remedy is one the operator can apply. + parallel = get_parallel() + narrowed_by = [] + if parallel.attn_dp_size > 1: + narrowed_by.append(f"--attn-dp-size {parallel.attn_dp_size}") + if parallel.attn_cp_size > 1: + narrowed_by.append(f"--attn-cp-size {parallel.attn_cp_size}") + remedy = " and ".join(f"drop {flag}" for flag in narrowed_by) or ( + "widen the attention TP group" + ) + raise ValueError( + f"{layer} shards over the attention TP group ({shard_tp_size} ranks) " + f"but all-reduces over the full TP group ({tp_size} ranks), so it does " + f"not support {' with '.join(narrowed_by) or 'this layout'} yet. " + f"Use --attn-dp-size equal to --tp-size, or {remedy}{hint}." + ) + + def is_allocation_symmetric() -> bool: return not is_dp_attention_enabled() or is_dp_max_padding() diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 71b9227044..7ed93a9942 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -464,6 +464,7 @@ class FusedMoE(torch.nn.Module): is_gated=is_gated, routing_method_type=routing_method_type, gate_up_interleaved=gate_up_interleaved, + layer=self, ) self.quant_method = quant_method diff --git a/python/sglang/srt/layers/moe/moe_runner/base.py b/python/sglang/srt/layers/moe/moe_runner/base.py index 51dc53c7ff..77adb35301 100644 --- a/python/sglang/srt/layers/moe/moe_runner/base.py +++ b/python/sglang/srt/layers/moe/moe_runner/base.py @@ -68,6 +68,35 @@ class MoeRunnerConfig: # False preserves backend defaults, including their existing FP32 paths. silu_mul_keep_fp32: bool = False + # The MoE placement a fused func needs is the layer's own, not the + # deployment's: DWDP's `FusedMoE.bind_full_expert_weights` collapses a + # layer's EP view after load. Read it off the layer so there is one + # address to keep correct. + @property + def moe_tp_size(self) -> int: + return self._layer_placement("moe_tp_size") + + @property + def moe_tp_rank(self) -> int: + return self._layer_placement("moe_tp_rank") + + @property + def moe_ep_size(self) -> int: + return self._layer_placement("moe_ep_size") + + @property + def moe_ep_rank(self) -> int: + return self._layer_placement("moe_ep_rank") + + def _layer_placement(self, name: str) -> int: + if self.layer is None: + raise AttributeError( + f"{name} is the layer's MoE placement, but this runner config " + "has no layer. A layer that runs fused MoE funcs states itself " + "on its runner config." + ) + return getattr(self.layer, name) + @dataclass class RunnerInput(ABC): diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py index 8e21077460..1db31e0738 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py @@ -58,10 +58,6 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo): swiglu_alpha: Optional[torch.Tensor] = None swiglu_beta: Optional[torch.Tensor] = None swiglu_limit: Optional[torch.Tensor] = None - moe_tp_size: int = 1 - moe_tp_rank: int = 0 - moe_ep_size: int = 1 - moe_ep_rank: int = 0 apply_routed_scaling_factor: bool = True @@ -144,12 +140,6 @@ class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo): # SwigluStep variant. use_swiglu_step: bool = False - # TP/EP topology (forwarded to the FlashInfer kernel) - moe_tp_size: int = 1 - moe_tp_rank: int = 0 - moe_ep_size: int = 1 - moe_ep_rank: int = 0 - # GPT-OSS pads its input hidden dim up to the (pre-padded) loaded weight # width and trims the output back. DSv4 leaves this as ``None`` (no pad). padded_hidden: Optional[int] = None @@ -318,10 +308,10 @@ def _run_flashinfer_cutlass( swiglu_alpha=quant_info.swiglu_alpha, swiglu_beta=quant_info.swiglu_beta, swiglu_limit=quant_info.swiglu_limit, - ep_size=quant_info.moe_ep_size, - ep_rank=quant_info.moe_ep_rank, - tp_size=quant_info.moe_tp_size, - tp_rank=quant_info.moe_tp_rank, + ep_size=runner_config.moe_ep_size, + ep_rank=runner_config.moe_ep_rank, + tp_size=runner_config.moe_tp_size, + tp_rank=runner_config.moe_tp_rank, tune_max_num_tokens=next_power_of_2(x.shape[0]), activation_type=_activation_type(runner_config), enable_alltoall=enable_alltoall, @@ -525,10 +515,10 @@ def _fused_experts_flashinfer_mxfp4_cutlass( swiglu_alpha=quant_info.swiglu_alpha, swiglu_beta=quant_info.swiglu_beta, swiglu_limit=quant_info.swiglu_limit, - tp_size=quant_info.moe_tp_size, - tp_rank=quant_info.moe_tp_rank, - ep_size=quant_info.moe_ep_size, - ep_rank=quant_info.moe_ep_rank, + tp_size=runner_config.moe_tp_size, + tp_rank=runner_config.moe_tp_rank, + ep_size=runner_config.moe_ep_size, + ep_rank=runner_config.moe_ep_rank, use_w4_group_scaling=not use_mxfp8_act_scaling, use_mxfp8_act_scaling=use_mxfp8_act_scaling, activation_type=( diff --git a/python/sglang/srt/layers/moe/moe_runner/hpc_ops.py b/python/sglang/srt/layers/moe/moe_runner/hpc_ops.py index 6cfad99d4a..ae6df5b56f 100644 --- a/python/sglang/srt/layers/moe/moe_runner/hpc_ops.py +++ b/python/sglang/srt/layers/moe/moe_runner/hpc_ops.py @@ -80,7 +80,6 @@ class HpcOpsMoeQuantInfo(MoeQuantInfo): w2_weight: torch.Tensor block_quant: bool global_num_experts: int - moe_ep_rank: int # Blockwise path w13_weight_scale_inv: Optional[torch.Tensor] = None w2_weight_scale_inv: Optional[torch.Tensor] = None @@ -183,7 +182,7 @@ def fused_experts_none_to_hpc_ops( quant_info.w2_weight_scale_inv, topk_ids, topk_weights, - quant_info.moe_ep_rank, + runner_config.moe_ep_rank, quant_info.global_num_experts, ) else: @@ -198,7 +197,7 @@ def fused_experts_none_to_hpc_ops( act_and_mul_scale, topk_ids, topk_weights, - quant_info.moe_ep_rank, + runner_config.moe_ep_rank, quant_info.global_num_experts, ) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py index c29e01efec..d24b3b2223 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py @@ -71,7 +71,6 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme): assert not config.actorder, ( "Actorder is not supported by flashinfer_trtllm backend" ) - self.moe_ep_rank = get_parallel().moe_ep_rank if self.quant_config.quant_format != CompressionFormat.pack_quantized.value: raise ValueError( @@ -348,7 +347,7 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme): n_group=topk_config.num_expert_group, topk_group=topk_config.topk_group, intermediate_size=self.moe_runner_config.intermediate_size_per_partition, - local_expert_offset=self.moe_ep_rank * local_num_experts, + local_expert_offset=self.moe_runner_config.moe_ep_rank * local_num_experts, local_num_experts=local_num_experts, routed_scaling_factor=routed_scaling_factor, routing_method_type=routing_method_type, diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py index 8a3e12b3b3..596fbf5488 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py @@ -368,10 +368,6 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme): swiglu_alpha=swiglu_alpha, swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, apply_routed_scaling_factor=False, ) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 66dc62c18a..3d62fd5e1e 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -2701,7 +2701,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): w2_weight=layer.w2_weight, block_quant=True, global_num_experts=int(layer.num_experts), - moe_ep_rank=int(layer.moe_ep_rank), w13_weight_scale_inv=layer.hpc_ops_w13_weight_scale, w2_weight_scale_inv=layer.hpc_ops_w2_weight_scale, block_shape=self.quant_config.weight_block_size, @@ -2712,7 +2711,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): w2_weight=layer.w2_weight, block_quant=False, global_num_experts=int(layer.num_experts), - moe_ep_rank=int(layer.moe_ep_rank), gate_up_alphas=layer.hpc_ops_gate_up_alphas, down_alphas=layer.hpc_ops_down_alphas, w13_input_scale=layer.w13_input_scale, diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index c154a142c1..9ab02023f8 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -1426,10 +1426,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): swiglu_alpha=swiglu_alpha, swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk, ) return self.runner.run(dispatch_output, quant_info) @@ -3253,10 +3249,6 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): swiglu_alpha=swiglu_alpha, swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, apply_routed_scaling_factor=False, ) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py index 1d25a81feb..bd79467318 100644 --- a/python/sglang/srt/layers/quantization/mxfp4.py +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -1482,10 +1482,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): swiglu_alpha=layer.swiglu_alpha, swiglu_beta=layer.swiglu_beta, swiglu_limit=layer.swiglu_limit, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, padded_hidden=self._padded_hidden, ) return self.runner.run(dispatch_output, quant_info) @@ -1529,10 +1525,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): swiglu_alpha=layer.swiglu_alpha, swiglu_beta=layer.swiglu_beta, swiglu_limit=layer.swiglu_limit, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, padded_hidden=self._padded_hidden, ) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py index 62c9aa9e04..b8e6ff0810 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py @@ -87,6 +87,9 @@ class Mxfp4FlashinferCutlassMoEMethod: from sglang.srt.layers.moe.utils import MoeRunnerBackend self.moe_runner_config = moe_runner_config + # The fused func reads the layer's MoE placement off the config. + if moe_runner_config.layer is None: + moe_runner_config.layer = layer E = layer.num_local_experts device = layer.w13_weight.device @@ -249,10 +252,6 @@ class Mxfp4FlashinferCutlassMoEMethod: swiglu_beta=None, swiglu_limit=self._swiglu_limit_tensor, use_swiglu_step=self._use_swiglu_step, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, padded_hidden=None, ) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index 8edd7e7240..50f54c3b74 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -1105,10 +1105,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): swiglu_alpha=swiglu_alpha, swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, - moe_ep_size=layer.moe_ep_size, - moe_ep_rank=layer.moe_ep_rank, - moe_tp_size=layer.moe_tp_size, - moe_tp_rank=layer.moe_tp_rank, apply_routed_scaling_factor=not layer.should_fuse_routed_scaling_factor_in_topk, ) return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/lora/layers.py b/python/sglang/srt/lora/layers.py index 6ce0205d4c..9a314e580a 100644 --- a/python/sglang/srt/lora/layers.py +++ b/python/sglang/srt/lora/layers.py @@ -1149,9 +1149,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): has_active_lora=has_active_lora, experts_shared_outer_loras=self.experts_shared_outer_loras, cg_buffers=cg_buffers, - tp_size=self.tp_size, - tp_rank=self.tp_rank, - hidden_size=getattr(self.base_layer, "hidden_size", 0), lora_use_virtual_experts=self.lora_use_virtual_experts, ) diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index 4ef61510f9..6142089f95 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -75,8 +75,6 @@ class LoRAManager: dtype: torch.dtype, server_args: ServerArgs, lora_backend: str = "triton", - tp_size: int = 1, - tp_rank: int = 0, max_lora_rank: Optional[int] = None, target_modules: Optional[Iterable[str]] = None, lora_paths: Optional[List[LoRARef]] = None, @@ -90,8 +88,6 @@ class LoRAManager: self.load_config: LoadConfig = load_config self.dtype: torch.dtype = dtype self.device: torch.device = next(self.base_model.parameters()).device - self.tp_size: int = tp_size - self.tp_rank: int = tp_rank # Attention projections shard on the attn-TP group; extracted once # here (parallel groups are frozen after init_torch_distributed). self.attn_tp_size: int = get_parallel().attn_tp_size @@ -1004,9 +1000,6 @@ class LoRAManager: base_hf_config=self.base_hf_config, max_loras_per_batch=self.max_loras_per_batch, dtype=self.dtype, - tp_size=self.tp_size, - tp_rank=self.tp_rank, - attn_tp_size=self.attn_tp_size, max_lora_rank=self.max_lora_rank, target_modules=self.target_modules, base_model=self.base_model, diff --git a/python/sglang/srt/lora/lora_moe_runners.py b/python/sglang/srt/lora/lora_moe_runners.py index b630b2db79..adddaf10d2 100644 --- a/python/sglang/srt/lora/lora_moe_runners.py +++ b/python/sglang/srt/lora/lora_moe_runners.py @@ -184,10 +184,6 @@ class LoRAInfo: experts_shared_outer_loras: bool = False cg_buffers: dict | None = None - fully_sharded: bool = False - tp_size: int = 1 - tp_rank: int = 0 - hidden_size: int = 0 lora_use_virtual_experts: bool = False @@ -391,7 +387,6 @@ def _add_lora_gate_up_delta( expand_num_warps=4, expand_num_stages=2, expand_split_k=1, - fully_sharded=lora_info.fully_sharded, ) @@ -424,12 +419,6 @@ def _add_lora_down_delta( if lora_info.experts_shared_outer_loras and not lora_info.lora_use_virtual_experts: down_lora_b = down_lora_b.expand(-1, lora_info.num_experts, -1, -1) - if lora_info.fully_sharded and lora_info.tp_size > 1: - shard_size = lora_info.hidden_size // lora_info.tp_size - offset = shard_size * lora_info.tp_rank - else: - offset = 0 - if lora_info.lora_use_virtual_experts: merged_experts_fused_moe_lora_add( output=intermediate_cache, @@ -474,8 +463,6 @@ def _add_lora_down_delta( expand_num_stages=2, expand_split_k=1, mul_routed_weight=True, - fully_sharded=lora_info.fully_sharded, - offset=offset, ) diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py index 7a3e820005..4c52e2fd78 100644 --- a/python/sglang/srt/lora/mem_pool.py +++ b/python/sglang/srt/lora/mem_pool.py @@ -138,9 +138,6 @@ class LoRAMemoryPool: base_hf_config: AutoConfig, max_loras_per_batch: int, dtype: torch.dtype, - tp_size: int, - tp_rank: int, - attn_tp_size: int, max_lora_rank: int, target_modules: Set[str], base_model: torch.nn.Module, @@ -154,8 +151,9 @@ class LoRAMemoryPool: self.num_layer: int = base_hf_config.num_hidden_layers self.max_loras_per_batch: int = max_loras_per_batch self.dtype: torch.dtype = dtype - self.tp_size: int = tp_size - self.tp_rank: int = tp_rank + parallel = get_parallel() + self.tp_size: int = parallel.tp_size + self.tp_rank: int = parallel.tp_rank self.lora_added_tokens_size: int = lora_added_tokens_size self.max_lora_rank: int = max_lora_rank self.target_modules: Set[str] = target_modules @@ -193,7 +191,7 @@ class LoRAMemoryPool: # under attention DP is `attn_tp_size = tp_size // attn_dp_size`. # The corresponding LoRA wrappers slice weights by the base layer's # attn_tp-local rank, so the buffer shapes must match that shard. - self.attn_tp_size: int = attn_tp_size + self.attn_tp_size: int = parallel.attn_tp_size # Initialize eviction policy self.eviction_policy = get_eviction_policy(eviction_policy) @@ -227,7 +225,7 @@ class LoRAMemoryPool: # Cache lm_head shard_indices from the base model so that buffer # allocation uses the same sharding as the base ParallelLMHead layer. self.lm_head_shard_indices = None - if "lm_head" in target_modules and tp_size > 1: + if "lm_head" in target_modules and self.tp_size > 1: from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead for _, module in base_model.named_modules(): diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index b92260af2c..8261cd60fd 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -705,17 +705,15 @@ class HiCacheController: if storage_backend_extra_config is None: storage_backend_extra_config = {} + parallel = get_parallel() if is_dp_attention_enabled(): - self.tp_rank = get_parallel().attn_tp_rank - self.tp_size = get_parallel().attn_tp_size - self.dp_rank = get_parallel().attn_dp_rank + tp_rank = parallel.attn_tp_rank + tp_size = parallel.attn_tp_size + dp_rank = parallel.attn_dp_rank else: - self.tp_rank = get_parallel().tp_rank - self.tp_size = get_parallel().tp_size - self.dp_rank = 0 - - self.pp_rank = get_parallel().pp_rank - self.pp_size = get_parallel().pp_size + tp_rank = parallel.tp_rank + tp_size = parallel.tp_size + dp_rank = 0 # Currently, NPUMLATokenToKVPool is the subclass of MLATokenToKVPool. # DeepSeekV4TokenToKVPool has compressed MLA-style rank-replicated cache @@ -732,23 +730,23 @@ class HiCacheController: should_split_heads = False if tp_lcm_size: - assert tp_lcm_size % self.tp_size == 0, ( + assert tp_lcm_size % tp_size == 0, ( "tp_lcm_size must be divisible by tp_size." ) should_split_heads = ( not is_rank_replicated and self.mem_pool_host.layout == "page_head" - and tp_lcm_size > self.tp_size + and tp_lcm_size > tp_size ) attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size() model_name = storage_model_name(model_name, self.storage_host_pool) return HiCacheStorageConfig( - tp_rank=self.tp_rank, - tp_size=self.tp_size, - pp_rank=self.pp_rank, - pp_size=self.pp_size, + tp_rank=tp_rank, + tp_size=tp_size, + pp_rank=parallel.pp_rank, + pp_size=parallel.pp_size, attn_cp_rank=attn_cp_rank, attn_cp_size=attn_cp_size, # TODO(hzh): Rename is_mla_model to is_rank_replicated. @@ -758,7 +756,7 @@ class HiCacheController: model_name=model_name, tp_lcm_size=tp_lcm_size, should_split_heads=should_split_heads, - dp_rank=self.dp_rank, + dp_rank=dp_rank, extra_config=storage_backend_extra_config, ) diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index f7d990574d..c150e9d90a 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -76,12 +76,9 @@ class _NegotiateOutput(NamedTuple): class PrefillDelayer: def __init__( self, - cpu_group, max_delay_passes: int, token_usage_low_watermark: Optional[float], metrics_collector: Optional["SchedulerMetricsCollector"] = None, - device: Optional["torch.device"] = "cpu", - device_group=None, debug_log_enabled: bool = True, ): self._max_delay_passes = max_delay_passes @@ -117,14 +114,12 @@ class PrefillDelayer: get_schedule().disable_overlap_schedule or envs.SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH.get() ) + tp_group = parallel.tp_group if use_nccl: - assert device_group is not None, ( - "device_group is required when using NCCL for PrefillDelayer all-gather" - ) - self._gather_group = device_group - self._gather_device = device + self._gather_group = tp_group.device_group + self._gather_device = tp_group.device else: - self._gather_group = cpu_group + self._gather_group = tp_group.cpu_group self._gather_device = "cpu" # Fields packed per rank into the all-gather tensor: prefillable, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 2fb58d381c..b6a3aa45c9 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -109,6 +109,7 @@ from sglang.srt.dllm.mixin.scheduler import SchedulerDllmMixin from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.hardware_backend.mlx.runtime import use_mlx +from sglang.srt.layers.dp_attention import get_dp_tp_group from sglang.srt.layers.moe import initialize_moe_config from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config @@ -568,9 +569,6 @@ class Scheduler( tp_worker=self.tp_worker, page_size=self.page_size, spec_algorithm=self.spec_algorithm, - attn_tp_cpu_group=self.attn_tp_cpu_group, - tp_cpu_group=self.tp_cpu_group, - attn_cp_cpu_group=self.attn_cp_cpu_group, enable_metrics=get_observability().enable_metrics, enable_kv_cache_events=bool( get_observability().kv_events_config @@ -578,8 +576,6 @@ class Scheduler( and get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0 ), - tp_group=self.tp_group, - pp_group=self.pp_group, enable_hierarchical_cache=self.enable_hierarchical_cache, hicache_draft_plan=( self.draft_worker.hicache_draft_plan @@ -626,11 +622,7 @@ class Scheduler( self.decode_offload_manager = DecodeKVCacheOffloadManager( req_to_token_pool=self.req_to_token_pool, token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - tp_group=( - self.attn_tp_cpu_group - if self.attn_dp_enabled - else self.tp_cpu_group - ), + tp_group=self.dp_tp_cpu_group, tree_cache=self.tree_cache, ) else: @@ -747,16 +739,16 @@ class Scheduler( ): return + parallel = get_parallel() + tp_group = parallel.tp_group rank = ( - get_parallel().dp_rank - if get_parallel().dp_rank is not None - else self.tp_group.rank_in_group + parallel.dp_rank if parallel.dp_rank is not None else tp_group.rank_in_group ) logger.info("HCCL DP prewarm start: rank=%s", rank) _prewarm_hccl_group( - device=self.tp_group.device, - group=self.tp_group.device_group, - device_module=self.tp_group.device_module, + device=tp_group.device, + group=tp_group.device_group, + device_module=tp_group.device_module, ) logger.info("HCCL DP prewarm done: rank=%s", rank) @@ -1213,11 +1205,9 @@ class Scheduler( self.world_group = get_parallel().world_group # NOTE: dp_tp_* are request/data-plane coordination groups (not tensor collectives). - # When DP attention is enabled, scope to the attention-TP group; otherwise use - # the base TP group. Entry rank is the local rank 0 in that group. - # Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA - # stream/device coupling (#11910). - self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group + # Entry rank is the local rank 0 in that group. Use the CPU (gloo) group to + # broadcast VLM Python objects and avoid CUDA stream/device coupling (#11910). + self.dp_tp_group = get_dp_tp_group() self.dp_tp_cpu_group = self.dp_tp_group.cpu_group self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func() @@ -1408,8 +1398,6 @@ class Scheduler( ) else: self.prefill_delayer = PrefillDelayer( - cpu_group=self.tp_cpu_group, - device_group=self.tp_group.device_group, metrics_collector=( self.metrics_collector if self.metrics_reporter.enable_metrics @@ -1417,7 +1405,6 @@ class Scheduler( ), max_delay_passes=get_schedule().prefill_delayer_max_delay_passes, token_usage_low_watermark=get_schedule().prefill_delayer_token_usage_low_watermark, - device=self.tp_group.device, debug_log_enabled=get_parallel().attn_tp_rank == 0, ) @@ -1536,7 +1523,6 @@ class Scheduler( # The decode requests polling kv cache self.disagg_decode_transfer_queue = DecodeTransferQueue( - gloo_group=self.attn_tp_cpu_group, req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator, metadata_buffers=self.disagg_metadata_buffers, scheduler=self, @@ -1553,7 +1539,6 @@ class Scheduler( scheduler=self, transfer_queue=self.disagg_decode_transfer_queue, tree_cache=self.tree_cache, - gloo_group=self.attn_tp_cpu_group, gpu_id=get_device().gpu_id, bootstrap_port=get_disagg().disaggregation_bootstrap_port, max_total_num_tokens=self.max_total_num_tokens, @@ -1613,9 +1598,6 @@ class Scheduler( self.server_args, dtype=self.model_config.dtype, hf_config=self.model_config.hf_config, - pp_rank=get_parallel().pp_rank, - tp_rank=get_parallel().tp_rank, - tp_group=self.tp_group, scheduler=self, ) @@ -2242,7 +2224,6 @@ class Scheduler( def init_profiler(self) -> None: self.profiler_manager = SchedulerProfilerManager( - dp_tp_cpu_group=self.dp_tp_cpu_group, get_forward_ct=lambda: self.forward_ct, ) @@ -2250,7 +2231,6 @@ class Scheduler( self.weight_updater = SchedulerWeightUpdaterManager( tp_worker=self.tp_worker, draft_worker=self.draft_worker, - tp_cpu_group=self.tp_cpu_group, memory_saver_adapter=self.memory_saver_adapter, flush_cache=self.flush_cache, is_fully_idle=self.is_fully_idle, @@ -2333,11 +2313,6 @@ class Scheduler( recv_skipper=self.recv_skipper, input_blocker=self.input_blocker, mm_receiver=self.mm_receiver, - tp_group=self.tp_group, - tp_cpu_group=self.tp_cpu_group, - attn_tp_cpu_group=self.attn_tp_cpu_group, - attn_cp_cpu_group=self.attn_cp_cpu_group, - world_group=self.world_group, server_args=self.server_args, model_config=self.model_config, max_recv_per_poll=self.max_recv_per_poll, diff --git a/python/sglang/srt/managers/scheduler_components/profiler_manager.py b/python/sglang/srt/managers/scheduler_components/profiler_manager.py index 27814e104f..debaa72fda 100644 --- a/python/sglang/srt/managers/scheduler_components/profiler_manager.py +++ b/python/sglang/srt/managers/scheduler_components/profiler_manager.py @@ -4,7 +4,7 @@ import gc import logging import os import time -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import ( TYPE_CHECKING, @@ -17,6 +17,7 @@ from typing import ( import torch from sglang.srt.environ import envs +from sglang.srt.layers.dp_attention import get_dp_tp_group from sglang.srt.managers.io_struct import ProfileReq, ProfileReqOutput, ProfileReqType from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.step_span_utils import set_detailed_annotations_enabled @@ -51,10 +52,13 @@ logger = logging.getLogger(__name__) @dataclass(kw_only=True) class SchedulerProfilerManager: - dp_tp_cpu_group: Any get_forward_ct: Callable[[], int] + # Taken at construction: a batch can stop profiling while a scoped + # override has moved the TP group. + dp_tp_cpu_group: Any = field(init=False) def __post_init__(self) -> None: + self.dp_tp_cpu_group = get_dp_tp_group().cpu_group if envs.SGLANG_PROFILE_V2.get(): self._profile_manager = ProfileManager( cpu_group=self.dp_tp_cpu_group, diff --git a/python/sglang/srt/managers/scheduler_components/request_receiver.py b/python/sglang/srt/managers/scheduler_components/request_receiver.py index f188b547d8..cb2b76cb6c 100644 --- a/python/sglang/srt/managers/scheduler_components/request_receiver.py +++ b/python/sglang/srt/managers/scheduler_components/request_receiver.py @@ -63,11 +63,6 @@ class SchedulerRequestReceiver: recv_skipper: Any input_blocker: Any mm_receiver: Any - tp_group: Any - tp_cpu_group: Any - attn_tp_cpu_group: Any - attn_cp_cpu_group: Any - world_group: Any server_args: ServerArgs model_config: ModelConfig max_recv_per_poll: int @@ -157,7 +152,7 @@ class SchedulerRequestReceiver: recv_reqs = point_to_point_pyobj( [], get_parallel().pp_rank * get_parallel().tp_size + dp_offset, - self.world_group.cpu_group, + get_parallel().world_group.cpu_group, (get_parallel().pp_rank - 1) * get_parallel().tp_size + dp_offset, get_parallel().pp_rank * get_parallel().tp_size + dp_offset, ) @@ -195,22 +190,24 @@ class SchedulerRequestReceiver: if _local_ctrl: control_reqs = attn_cp_tp_broadcast_pyobj(control_reqs) elif get_parallel().tp_size != 1: + tp_group = get_parallel().tp_group control_reqs = broadcast_pyobj( control_reqs, - self.tp_group.rank, - self.tp_cpu_group, - src=self.tp_group.ranks[0], + tp_group.rank, + tp_group.cpu_group, + src=tp_group.ranks[0], ) recv_reqs = work_reqs + control_reqs else: if recv_reqs is not None: recv_reqs = [*recv_reqs, *local_reqs] if get_parallel().tp_size != 1: + tp_group = get_parallel().tp_group recv_reqs = broadcast_pyobj( recv_reqs, - self.tp_group.rank, - self.tp_cpu_group, - src=self.tp_group.ranks[0], + tp_group.rank, + tp_group.cpu_group, + src=tp_group.ranks[0], ) return recv_reqs @@ -268,11 +265,11 @@ class SchedulerRequestReceiver: parallel = get_parallel() if parallel.attn_dp_enabled: if parallel.attn_tp_size > 1: - barrier(group=self.attn_tp_cpu_group) + barrier(group=parallel.attn_tp_group.cpu_group) if parallel.attn_cp_size > 1: - barrier(group=self.attn_cp_cpu_group) + barrier(group=parallel.attn_cp_group.cpu_group) elif parallel.tp_size > 1: - barrier(group=self.tp_cpu_group) + barrier(group=parallel.tp_group.cpu_group) # 2. materialize independently so one bad VLM request does not stop the loop failed = torch.zeros(len(tokenized_reqs), dtype=torch.int32) @@ -292,11 +289,15 @@ class SchedulerRequestReceiver: # 3. all ranks reject the same requests before entering model collectives if parallel.attn_dp_enabled: if parallel.attn_tp_size > 1: - all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group) + all_reduce( + failed, op=ReduceOp.MAX, group=parallel.attn_tp_group.cpu_group + ) if parallel.attn_cp_size > 1: - all_reduce(failed, op=ReduceOp.MAX, group=self.attn_cp_cpu_group) + all_reduce( + failed, op=ReduceOp.MAX, group=parallel.attn_cp_group.cpu_group + ) elif parallel.tp_size > 1: - all_reduce(failed, op=ReduceOp.MAX, group=self.tp_cpu_group) + all_reduce(failed, op=ReduceOp.MAX, group=parallel.tp_group.cpu_group) error = MMInputsProcessError( "Failed to materialize shared-memory multimodal features on a scheduler rank." diff --git a/python/sglang/srt/managers/scheduler_components/weight_updater.py b/python/sglang/srt/managers/scheduler_components/weight_updater.py index d9a1173732..bfcf845480 100644 --- a/python/sglang/srt/managers/scheduler_components/weight_updater.py +++ b/python/sglang/srt/managers/scheduler_components/weight_updater.py @@ -44,7 +44,7 @@ from sglang.srt.managers.io_struct import ( UpdateWeightsFromTensorReqInput, UpdateWeightsFromTensorReqOutput, ) -from sglang.srt.runtime_context import get_model +from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.utils.weight_checker import overall_checksum logger = logging.getLogger(__name__) @@ -88,7 +88,6 @@ class _WeightUpdateSession(msgspec.Struct, frozen=True): class SchedulerWeightUpdaterManager: tp_worker: Any draft_worker: Any - tp_cpu_group: Any memory_saver_adapter: Any flush_cache: Callable[..., bool] is_fully_idle: Callable[..., bool] @@ -98,6 +97,10 @@ class SchedulerWeightUpdaterManager: stashed_model_static_state: Any = None # replicated on every TP rank, so a rejected call returns on all ranks before any barrier _session: Optional[_WeightUpdateSession] = None + tp_cpu_group: Any = field(init=False) + + def __post_init__(self) -> None: + self.tp_cpu_group = get_parallel().tp_group.cpu_group @contextmanager def _observe_weight_load(self, source: str) -> Iterator[None]: diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index f2556a12a6..bed036f0de 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -865,7 +865,7 @@ class HybridCacheController(BaseHiCacheController): def get_prefetch_submission(self, rid: str) -> Optional[PrefetchSubmission]: if self.pp_prefetch_command_group is None: return None - if self.pp_rank != 0: + if self.storage_config.pp_rank != 0: return PrefetchSubmission() with self.pp_prefetch_state_lock: state = self.pp_prefetch_states.get(rid) @@ -898,7 +898,7 @@ class HybridCacheController(BaseHiCacheController): ) ) - if self.pp_rank != 0: + if self.storage_config.pp_rank != 0: raise RuntimeError("Only PP0 can submit a PP prefetch ticket.") rid = handle.rid @@ -928,7 +928,8 @@ class HybridCacheController(BaseHiCacheController): storage_hit_count = len(ticket.prefetch_key.token_ids) try: - for pp_rank in range(self.tp_rank, self.pp_size, self.tp_size): + config = self.storage_config + for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size): _, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank) storage_hit_count = min(storage_hit_count, rank_hit_count) except Exception: @@ -1096,7 +1097,7 @@ class HybridCacheController(BaseHiCacheController): assert group is not None rank = torch.distributed.get_rank() source = torch.distributed.get_process_group_ranks(group)[0] - is_source = self.pp_rank == 0 + is_source = self.storage_config.pp_rank == 0 while True: objects = [] diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index b03769326a..1442a6ab24 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +from sglang.srt.layers.dp_attention import get_dp_tp_group from sglang.srt.runtime_context import get_exec logger = logging.getLogger(__name__) @@ -58,10 +59,7 @@ from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode from sglang.srt.utils import ceil_align, is_hip if TYPE_CHECKING: - from torch.distributed import ProcessGroup - from sglang.srt.configs.model_config import ModelConfig - from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.managers.tp_worker import BaseTpWorker from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan @@ -266,13 +264,8 @@ def build_kv_cache( tp_worker: BaseTpWorker, page_size: int, spec_algorithm: SpeculativeAlgorithm, - attn_tp_cpu_group: ProcessGroup, - tp_cpu_group: ProcessGroup, - attn_cp_cpu_group: ProcessGroup, enable_metrics: bool, enable_kv_cache_events: bool, - tp_group: GroupCoordinator, - pp_group: GroupCoordinator, enable_hierarchical_cache: bool, hicache_draft_plan: Optional[HiCacheDraftPlan] = None, ) -> KVCacheBuildResult: @@ -358,12 +351,10 @@ def build_kv_cache( else token_to_kv_pool_allocator.page_size ), is_eagle=spec_algorithm.is_eagle(), - tp_cache_group=( - attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group - ), - attn_cp_cache_group=attn_cp_cpu_group, - attn_tp_cache_group=attn_tp_cpu_group, - pp_cache_group=pp_group.cpu_group, + tp_cache_group=get_dp_tp_group().cpu_group, + attn_cp_cache_group=parallel.attn_cp_group.cpu_group, + attn_tp_cache_group=parallel.attn_tp_group.cpu_group, + pp_cache_group=parallel.pp_group.cpu_group, eviction_policy=get_memory().radix_eviction_policy, eviction_policy_config=get_memory().radix_eviction_policy_config, enable_metrics=enable_metrics, @@ -394,7 +385,7 @@ def build_kv_cache( model_config=model_config, tp_size=parallel.tp_size, tp_rank=parallel.tp_rank, - tp_group=tp_group, + tp_group=parallel.tp_group, ) with auto_size_hicache( params, diff --git a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py index 09d6b45bf6..f162d5928c 100644 --- a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py @@ -123,6 +123,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker): extra_config, *_ = HybridCacheController.parse_storage_backend_extra_config( get_memory().hicache_storage_backend_extra_config ) + # TODO: unlike `HiCacheController._generate_storage_config`, this sets no + # `dp_rank`. Aligning the storage-config producers changes persisted + # storage keys, so it waits for a storage-key format change. storage_config = HiCacheStorageConfig( tp_rank=tp_rank, tp_size=tp_size, diff --git a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py index 542835eb2f..9e00ec2d37 100644 --- a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py @@ -295,6 +295,11 @@ class UMBPDirectLinker(UnifiedCacheLinker): ) extra_config["dram_page_size"] = dram_page_size + # TODO: `tp_rank` is the rank in the cache TP group but `tp_size` is the + # full TP width, and no `dp_rank` is set, unlike + # `HiCacheController._generate_storage_config`. Aligning the + # storage-config producers changes persisted storage keys, so it waits + # for a storage-key format change. storage_config = HiCacheStorageConfig( tp_rank=tp_rank, tp_size=get_parallel().tp_size, diff --git a/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py b/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py index 82ee8bd21e..ee264d3253 100644 --- a/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py +++ b/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py @@ -276,13 +276,14 @@ class StorageAttachment: """ cache = self._cache controller = cache.cache_controller + config = controller.storage_config attn_cp_rank, attn_cp_size = controller.get_attn_cp_rank_and_size() labels = { "storage_backend": storage_backend, - "tp_rank": controller.tp_rank, - "dp_rank": controller.dp_rank, - "pp_rank": controller.pp_rank, - "pp_size": controller.pp_size, + "tp_rank": config.tp_rank, + "dp_rank": config.dp_rank, + "pp_rank": config.pp_rank, + "pp_size": config.pp_size, "attn_cp_rank": attn_cp_rank, "attn_cp_size": attn_cp_size, } diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 5700480688..614a26ad2f 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -76,6 +76,7 @@ from sglang.srt.layers.cp.utils import ( is_cp_active, is_mla_cp_enabled, ) +from sglang.srt.layers.dp_attention import get_dp_tp_group from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.sampler import create_sampler from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers @@ -963,11 +964,7 @@ class ModelRunner: top_k=hisparse_top_k, device_buffer_size=hisparse_cfg.device_buffer_size, device=self.device, - tp_group=( - get_parallel().attn_tp_group.cpu_group - if get_parallel().attn_dp_enabled - else get_parallel().tp_group.cpu_group - ), + tp_group=get_dp_tp_group().cpu_group, host_to_device_ratio=hisparse_cfg.host_to_device_ratio, swap_in_block_size=hisparse_cfg.swap_in_block_size, shared_index_layers=resolve_shared_index_layers( @@ -1368,8 +1365,6 @@ class ModelRunner: dtype=self.dtype, server_args=self.server_args, lora_backend=get_lora().lora_backend, - tp_size=get_parallel().tp_size, - tp_rank=get_parallel().tp_rank, max_lora_rank=get_lora().max_lora_rank, target_modules=get_lora().lora_target_modules, lora_paths=get_lora().lora_paths, diff --git a/python/sglang/srt/models/clip.py b/python/sglang/srt/models/clip.py index 55c71fd64b..5addc3725a 100644 --- a/python/sglang/srt/models/clip.py +++ b/python/sglang/srt/models/clip.py @@ -12,6 +12,7 @@ from transformers import CLIPConfig, CLIPTextConfig, CLIPVisionConfig from sglang.srt.layers.activation import QuickGELU, get_act_fn from sglang.srt.layers.conv import Conv2dLayer +from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce from sglang.srt.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -200,6 +201,15 @@ class CLIPAttention(nn.Module): tp_rank=parallel.attn_tp_rank, tp_size=parallel.attn_tp_size, ) + # TODO: this layer shards over attention TP but reduces over the full TP + # group; reduce over the attention-TP group so attention DP and attention + # CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=parallel.attn_tp_size, + reduces_over_attn_tp=False, + multimodal_encoder=True, + ) self.proj = RowParallelLinear( input_size=config.hidden_size, output_size=config.hidden_size, diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 606bbd6919..912904d484 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -29,9 +29,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList -from sglang.srt.layers.dp_attention import ( - is_dp_attention_enabled, -) +from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, diff --git a/python/sglang/srt/models/inkling_common/dense_mlp.py b/python/sglang/srt/models/inkling_common/dense_mlp.py index 35748c8e53..8dd19a0813 100644 --- a/python/sglang/srt/models/inkling_common/dense_mlp.py +++ b/python/sglang/srt/models/inkling_common/dense_mlp.py @@ -209,6 +209,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin): gemm1_alpha=None, gemm1_clamp_limit=None, is_gated=True, + layer=self, ) FusedMoELoadingMixin.__init__( diff --git a/python/sglang/srt/models/iquest_q1.py b/python/sglang/srt/models/iquest_q1.py index 0d591c6577..ad3cc08e59 100644 --- a/python/sglang/srt/models/iquest_q1.py +++ b/python/sglang/srt/models/iquest_q1.py @@ -9,6 +9,7 @@ from transformers import PretrainedConfig from triton.language.extra import libdevice from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, @@ -313,6 +314,14 @@ class IQuestQ1Attention(nn.Module): tp_size=attn_tp_size, prefix=add_prefix("qkv_proj", prefix), ) + # The decoder boundary reduces partial attention outputs. Only a + # standalone attention that owns its TP reduction needs this guard. + if reduce_results: + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=attn_tp_size, + reduces_over_attn_tp=False, + ) self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, config.hidden_size, diff --git a/python/sglang/srt/models/kimi_vl_moonvit.py b/python/sglang/srt/models/kimi_vl_moonvit.py index e166b382d7..b90a213322 100644 --- a/python/sglang/srt/models/kimi_vl_moonvit.py +++ b/python/sglang/srt/models/kimi_vl_moonvit.py @@ -59,6 +59,7 @@ from sglang.srt.layers.attention.vision import ( prepare_vision_attention_metadata, ) from sglang.srt.layers.conv import Conv2dLayer +from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce from sglang.srt.layers.linear import ( ColumnParallelLinear, ReplicatedLinear, @@ -357,6 +358,16 @@ class MLP2(nn.Module): prefix=add_prefix("fc1", prefix), ) elif use_tensor_parallel: + # TODO: these layers shard over attention TP but reduce over the full + # TP group; reduce over the attention-TP group so attention DP and + # attention CP narrower than TP can run them. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=tp_size, + reduces_over_attn_tp=False, + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.fc0 = ColumnParallelLinear( dims[0], dims[1], diff --git a/python/sglang/srt/models/mimo_audio.py b/python/sglang/srt/models/mimo_audio.py index c266e060fc..8c0e9c8734 100644 --- a/python/sglang/srt/models/mimo_audio.py +++ b/python/sglang/srt/models/mimo_audio.py @@ -514,6 +514,12 @@ class AudioEncoderAttention(nn.Module): window_size=window_size, customized_position_embedding_applier=_audio_rope_applier, prefix="attn", + # TODO: the audio encoder does not follow --mm-enable-dp-encoder and + # reduces over the full TP group, so its output is wrong under + # attention DP narrower than TP. Rejecting that would reject the + # documented MiMo-V2.5 `--tp 8 --attn-dp-size 2` launch; fix the + # reduce group or run the encoder data-parallel instead. + allow_tp_reduce_mismatch=True, ) def forward( diff --git a/python/sglang/srt/models/minimax_vl_common.py b/python/sglang/srt/models/minimax_vl_common.py index 333076efdd..853cf14519 100644 --- a/python/sglang/srt/models/minimax_vl_common.py +++ b/python/sglang/srt/models/minimax_vl_common.py @@ -17,7 +17,10 @@ from sglang.srt.layers.attention.vision import ( VisionAttentionMetadata, prepare_vision_attention_metadata, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, + reject_attn_tp_shard_with_tp_reduce, +) from sglang.srt.layers.linear import ( ColumnParallelLinear, RowParallelLinear, @@ -83,6 +86,16 @@ class MiniMaxVLMultiModalProjector(nn.Module): tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank + # TODO: this layer shards over attention TP but reduces over the full TP + # group without attention DP; reduce over the attention-TP group so + # attention CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=tp_size, + reduces_over_attn_tp=is_dp_attention_enabled(), + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.linear_1 = ColumnParallelLinear( vision_hidden_size, @@ -138,6 +151,16 @@ class MiniMaxVLPatchMerger(nn.Module): tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank + # TODO: this layer shards over attention TP but reduces over the full TP + # group without attention DP; reduce over the attention-TP group so + # attention CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=tp_size, + reduces_over_attn_tp=is_dp_attention_enabled(), + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.linear_1 = ColumnParallelLinear( text_hidden_size * spatial_merge_size**2, @@ -260,6 +283,16 @@ class CLIPEncoderLayer(nn.Module): self.use_data_parallel = use_data_parallel tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size tp_rank = 0 if use_data_parallel else get_parallel().attn_tp_rank + # TODO: this layer shards over attention TP but reduces over the full TP + # group without attention DP; reduce over the attention-TP group so + # attention CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=tp_size, + reduces_over_attn_tp=is_dp_attention_enabled(), + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.self_attn = VisionAttention( embed_dim=config.hidden_size, diff --git a/python/sglang/srt/models/phimoe.py b/python/sglang/srt/models/phimoe.py index f5cd3a0011..b0ef96b7f7 100644 --- a/python/sglang/srt/models/phimoe.py +++ b/python/sglang/srt/models/phimoe.py @@ -4,6 +4,7 @@ import torch from torch import nn from transformers.configuration_utils import PretrainedConfig +from sglang.srt.layers.dp_attention import reject_attn_tp_shard_with_tp_reduce from sglang.srt.layers.linear import ( QKVParallelLinear, ReplicatedLinear, @@ -282,6 +283,12 @@ class PhiMoEAttention(nn.Module): tp_size=attn_tp_size, prefix=add_prefix("qkv_proj", prefix), ) + # TODO: this layer shards over attention TP but reduces over the full TP + # group; reduce over the attention-TP group so attention DP and attention + # CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, shard_tp_size=attn_tp_size, reduces_over_attn_tp=False + ) self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, hidden_size, diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 9c447ec7fa..6c38dc9b9c 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -41,6 +41,7 @@ from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, + reject_attn_tp_shard_with_tp_reduce, ) from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear @@ -148,6 +149,16 @@ class Qwen3_VisionMLP(nn.Module): tp_size=tp_size, tp_rank=tp_rank, ) + # TODO: this layer shards over attention TP but reduces over the full TP + # group without attention DP; reduce over the attention-TP group so + # attention CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=self.tp_size, + reduces_over_attn_tp=is_dp_attention_enabled(), + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.linear_fc1 = ColumnParallelLinear( in_features, hidden_features, @@ -317,6 +328,16 @@ class Qwen3VLMoeVisionPatchMerger(nn.Module): tp_size=tp_size, tp_rank=tp_rank, ) + # TODO: this layer shards over attention TP but reduces over the full TP + # group without attention DP; reduce over the attention-TP group so + # attention CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=self.tp_size, + reduces_over_attn_tp=is_dp_attention_enabled(), + multimodal_encoder=True, + hint=", or --mm-enable-dp-encoder where the model supports it", + ) self.linear_fc1 = ColumnParallelLinear( self.hidden_size, self.padded_context_dim, diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index e752313e16..f8626c39bf 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -22,6 +22,7 @@ from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.conv import Conv2dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, + reject_attn_tp_shard_with_tp_reduce, ) from sglang.srt.layers.layer_boundary import ( declare_attn, @@ -545,6 +546,15 @@ class Step3VisionMLP(nn.Module): prefix=add_prefix("gate_proj", prefix), ) self.act = ACT2FN[hidden_act] # quick_gelu + # TODO: this layer shards over attention TP but reduces over the full TP + # group; reduce over the attention-TP group so attention DP and attention + # CP narrower than TP can run it. + reject_attn_tp_shard_with_tp_reduce( + type(self).__name__, + shard_tp_size=attn_tp_size, + reduces_over_attn_tp=False, + multimodal_encoder=True, + ) self.fc2 = RowParallelLinear( intermediate_size, dim, diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index bcd2371510..86d90360d2 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -263,12 +263,23 @@ class ServerArgs: return getattr(self, "_launch_command", None) def resolved_dict(self) -> dict[str, Any]: - """Serialize resolved field values, expanding nested records and excluding bookkeeping.""" + """Serialize resolved field values, expanding nested records and excluding bookkeeping. - return { + One exception: the deprecated `enable_dp_attention` reports whether an + attention-DP width was configured, for clients that still read it from + `/server_info`. Resolution leaves the field false. It reports the width + rather than `attn_dp_enabled`, whose elastic scale-joiner arm is true at + width one: a dump carrying that would fold the joiner's replicas into + attention-DP groups when it is read back. + """ + + resolved = { field.name: _plain(resolution_result(self, field.name)) for field in record_fields(type(self)) } + # TODO: drop together with `--enable-dp-attention` after 2026-12-31. + resolved["enable_dp_attention"] = resolving_view(self).attn_dp_size > 1 + return resolved LANGUAGE_MODEL_ONLY_ARCHITECTURES = ( "MuseGlimmerForConditionalGeneration", @@ -351,7 +362,7 @@ class ServerArgs: help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.", ) # `enable_dp_attention` is `no_cli=True` too; resolution turns it into - # `attn_dp_size`. + # `attn_dp_size`. TODO: remove the flag and the field after 2026-12-31. parser.add_argument( "--enable-dp-attention", action=DeprecatedStoreTrueAction, diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 0a8cd2dff8..f48870c1c7 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -19,6 +19,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import ( ) from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.environ import envs +from sglang.srt.layers.dp_attention import get_dp_tp_group from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method from sglang.srt.layers.logprob_processor import compute_spec_logprobs from sglang.srt.lora.layers import unwrap_lora_layer @@ -402,11 +403,7 @@ class DFlashWorkerV2(BaseSpecWorker): self._full_embed_gpu: Optional[torch.Tensor] = None # Under dp attention, peer DP ranks run different (idle) paths, so # spec broadcasts must stay within the attn-TP group. - self._tp_sync = SpecTpSync( - get_parallel().attn_tp_group - if get_parallel().attn_dp_enabled - else get_parallel().tp_group - ) + self._tp_sync = SpecTpSync(get_dp_tp_group()) # Under dp attention, the draft worker runs on the per-DP attn-TP # group, independent of idle peer DP ranks; it is built and run under diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 99d8327d47..490c5798b6 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -8,6 +8,7 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( ) from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.environ import envs +from sglang.srt.layers.dp_attention import get_dp_tp_group from sglang.srt.layers.logprob_processor import compute_spec_logprobs from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.managers.schedule_batch import ScheduleBatch @@ -233,9 +234,7 @@ class DSparkWorkerV2(BaseSpecWorker): self._mask_token_id = runtime_config.mask_token_id parallel = get_parallel() - self._tp_sync = SpecTpSync( - parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group - ) + self._tp_sync = SpecTpSync(get_dp_tp_group()) self._draft_graph_group = ( parallel.attn_tp_group if self._draft_dp_context_enabled diff --git a/python/sglang/srt/speculative/uno_lora.py b/python/sglang/srt/speculative/uno_lora.py index 132c53186c..95db269fe6 100644 --- a/python/sglang/srt/speculative/uno_lora.py +++ b/python/sglang/srt/speculative/uno_lora.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING from sglang.srt.lora.lora_manager import LoRAManager from sglang.srt.lora.lora_registry import LoRARef -from sglang.srt.runtime_context import get_parallel, get_spec +from sglang.srt.runtime_context import get_spec if TYPE_CHECKING: from sglang.srt.model_executor.model_runner import ModelRunner @@ -45,8 +45,6 @@ def init_uno_lora_manager( dtype=model_runner.dtype, server_args=model_runner.server_args, lora_backend="uno_cublas", # fast path - tp_size=get_parallel().tp_size, - tp_rank=get_parallel().tp_rank, # Infer these from the one trained adapter. max_lora_rank=None, target_modules=None, diff --git a/test/registered/moe/test_hpc_ops_moe.py b/test/registered/moe/test_hpc_ops_moe.py index 40af9d0097..40b16bc0e5 100644 --- a/test/registered/moe/test_hpc_ops_moe.py +++ b/test/registered/moe/test_hpc_ops_moe.py @@ -7,6 +7,7 @@ is not installed or the GPU is not SM90 (the kernels ship sm90a only). """ import unittest +from types import SimpleNamespace import torch @@ -91,6 +92,10 @@ class TestHpcOpsMoeBlockwise(CustomTestCase): activation="silu", is_gated=True, inplace=False, + # The fused func reads the layer's MoE placement off the config. + layer=SimpleNamespace( + moe_tp_size=1, moe_tp_rank=0, moe_ep_size=1, moe_ep_rank=0 + ), ) def _run_hpc_ops(self, x, w13, w2, w13_scale, w2_scale, topk_weights, topk_ids): @@ -104,7 +109,6 @@ class TestHpcOpsMoeBlockwise(CustomTestCase): w2_weight=w2, block_quant=True, global_num_experts=E, - moe_ep_rank=0, w13_weight_scale_inv=pad_hpc_ops_block_scale(w13_scale), w2_weight_scale_inv=pad_hpc_ops_block_scale(w2_scale), block_shape=[128, 128], @@ -175,7 +179,6 @@ class TestHpcOpsMoeBlockwise(CustomTestCase): w2_weight=w2, block_quant=False, global_num_experts=E, - moe_ep_rank=0, gate_up_alphas=w13_scale * a1_scale, down_alphas=w2_scale * a2_scale, w13_input_scale=a1_scale, diff --git a/test/registered/unit/disaggregation/test_dcp_pack.py b/test/registered/unit/disaggregation/test_dcp_pack.py index 3919c7e52a..01e2e78f9f 100644 --- a/test/registered/unit/disaggregation/test_dcp_pack.py +++ b/test/registered/unit/disaggregation/test_dcp_pack.py @@ -244,7 +244,7 @@ class TestDcpCachedPrefixSend(CustomTestCase): } mgr.add_transfer_request = Mock() with get_context().override_server_args(dp_size=1): - sender = NixlKVSender(mgr, "unused", 1, [0], 0) + sender = NixlKVSender(mgr, "unused", 1) sender.init((total - prefix + page_size - 1) // page_size, 3) req = SimpleNamespace( rid="cached-prefix", diff --git a/test/registered/unit/disaggregation/test_fake_kv_sender.py b/test/registered/unit/disaggregation/test_fake_kv_sender.py index dcaa599911..71704c3d04 100644 --- a/test/registered/unit/disaggregation/test_fake_kv_sender.py +++ b/test/registered/unit/disaggregation/test_fake_kv_sender.py @@ -33,8 +33,6 @@ class TestFakeKVSender(unittest.TestCase): mgr=self.mgr if mgr is None else mgr, bootstrap_addr="fake_addr:1234", bootstrap_room=42, - dest_tp_ranks=[0], - pp_rank=0, ) def _expire_deadline(self, sender: FakeKVSender, by: float = 1.0) -> None: diff --git a/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py b/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py index 128a7fa46d..35ddcd45dd 100644 --- a/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py +++ b/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py @@ -18,6 +18,7 @@ import platform import tempfile import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch from sglang.srt.runtime_context import get_parallel @@ -164,7 +165,10 @@ class TestSchedulerProfilerManagerMPS(unittest.TestCase): SchedulerProfilerManager, ) - mgr = SchedulerProfilerManager(dp_tp_cpu_group=None, get_forward_ct=lambda: 0) + with get_parallel().override( + tp_group=SimpleNamespace(cpu_group=None), attn_dp_enabled=False + ): + mgr = SchedulerProfilerManager(get_forward_ct=lambda: 0) mgr._init_profile(output_dir, None, None, None, None, None, False, "test") return mgr diff --git a/test/registered/unit/layers/moe/test_moe_runner_config_placement.py b/test/registered/unit/layers/moe/test_moe_runner_config_placement.py new file mode 100644 index 0000000000..1384ac833e --- /dev/null +++ b/test/registered/unit/layers/moe/test_moe_runner_config_placement.py @@ -0,0 +1,147 @@ +"""A MoE layer owns its placement, and the fused funcs read it through the +layer's `MoeRunnerConfig`.""" + +import sys +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.moe_runner import flashinfer_cutlass +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + FlashInferCutlassMoeQuantInfo, +) +from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.runtime_context import get_context, get_flags, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="stage-b-test-cpu-intel") + + +def _layer(monkeypatch) -> FusedMoE: + method = UnquantizedFusedMoEMethod() + monkeypatch.setattr(method, "create_weights", lambda **kwargs: None) + monkeypatch.setattr(method, "create_moe_runner", lambda layer, config: None) + monkeypatch.setattr( + fused_moe_layer_module, + "create_moe_dispatcher", + lambda config, quant_method: SimpleNamespace(), + ) + with ( + get_context().override_server_args(model_path="dummy"), + get_flags().moe.override( + runner_backend=MoeRunnerBackend.AUTO, + a2a_backend=MoeA2ABackend.NONE, + ), + get_parallel().override( + moe_ep_size=2, + moe_ep_rank=1, + moe_tp_size=2, + moe_tp_rank=1, + tp_size=4, + tp_rank=3, + attn_tp_size=4, + attn_tp_rank=3, + ), + ): + return FusedMoE( + num_experts=4, + hidden_size=4, + intermediate_size=8, + layer_id=0, + quant_method=method, + ) + + +def _placement(obj): + return (obj.moe_tp_size, obj.moe_tp_rank, obj.moe_ep_size, obj.moe_ep_rank) + + +def test_a_config_without_a_layer_has_no_placement() -> None: + config = MoeRunnerConfig() + for name in ("moe_tp_size", "moe_tp_rank", "moe_ep_size", "moe_ep_rank"): + with pytest.raises(AttributeError, match="has no layer"): + getattr(config, name) + + +def test_the_config_reads_the_layer_placement(monkeypatch) -> None: + layer = _layer(monkeypatch) + assert _placement(layer) == (2, 1, 2, 1) + assert _placement(layer.moe_runner_config) == _placement(layer) + + +def test_a_full_expert_bind_collapses_ep_on_the_config(monkeypatch) -> None: + layer = _layer(monkeypatch) + layer.bind_full_expert_weights({}) + assert _placement(layer) == (2, 1, 1, 0) + assert _placement(layer.moe_runner_config) == (2, 1, 1, 0) + assert layer.moe_runner_config.num_local_experts == 4 + + +def test_the_kernel_gets_the_bound_placement(monkeypatch) -> None: + """The reason the placement lives on the layer: after a full-expert bind + the kernel must be told this rank owns every expert.""" + layer = _layer(monkeypatch) + before = _kernel_placement(monkeypatch, layer.moe_runner_config) + assert before == (2, 1, 2, 1) + + layer.bind_full_expert_weights({}) + + # The layer kept its TP shard and now owns every expert. + assert _kernel_placement(monkeypatch, layer.moe_runner_config) == (2, 1, 1, 0) + + +def _kernel_placement(monkeypatch, runner_config): + """Run the CUTLASS fused func against a fake kernel and report the + (tp_size, tp_rank, ep_size, ep_rank) it was handed.""" + calls = [] + + def fake_kernel(**kwargs): + calls.append(kwargs) + return [kwargs["output"]] + + monkeypatch.setattr( + flashinfer_cutlass, + "_flashinfer_cutlass_fused_moe", + lambda: (fake_kernel, None), + ) + monkeypatch.setattr(flashinfer_cutlass, "_activation_type", lambda config: None) + + hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16) + flashinfer_cutlass._run_flashinfer_cutlass( + dispatch_output=SimpleNamespace( + hidden_states=hidden_states, + hidden_states_scale=None, + topk_output=SimpleNamespace( + topk_weights=torch.ones(2, 1), + topk_ids=torch.zeros(2, 1, dtype=torch.int64), + ), + ), + quant_info=FlashInferCutlassMoeQuantInfo( + quant_type="bf16", w13_weight=torch.zeros(1), w2_weight=torch.zeros(1) + ), + runner_config=runner_config, + output=torch.empty_like(hidden_states), + ) + (call,) = calls + return (call["tp_size"], call["tp_rank"], call["ep_size"], call["ep_rank"]) + + +def test_cutlass_passes_the_runner_config_placement(monkeypatch) -> None: + placement = _kernel_placement( + monkeypatch, + MoeRunnerConfig( + layer=SimpleNamespace( + moe_tp_size=2, moe_tp_rank=1, moe_ep_size=4, moe_ep_rank=3 + ) + ), + ) + assert placement == (2, 1, 4, 3) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py b/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py index e358e2411a..0b14a1a714 100644 --- a/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py +++ b/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py @@ -342,6 +342,8 @@ def test_gpt_oss_sm120_padding_layout_and_kernel(monkeypatch, stated_tp_group): gemm1_alpha=1.702, gemm1_clamp_limit=7.0, ) + # The fused func reads the layer's MoE placement off the config. + config.layer = layer method.moe_runner_config = config method.runner = MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, config) method._process_weights_for_sm120_cutlass(layer) diff --git a/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py b/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py index 62696f0112..83e7b792ba 100644 --- a/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py +++ b/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py @@ -173,7 +173,7 @@ def _round_up(x, base): return ((x + base - 1) // base) * base -def _build_method(num_experts, hidden, inter, *, use_humming=False): +def _build_method(num_experts, hidden, inter, *, layer=None, use_humming=False): from sglang.srt.layers.quantization.mxfp4 import Mxfp4MoEMethod method = Mxfp4MoEMethod.__new__(Mxfp4MoEMethod) @@ -188,16 +188,17 @@ def _build_method(num_experts, hidden, inter, *, use_humming=False): method._padded_hidden = _round_up(hidden, 128) method._padded_intermediate = _round_up(inter, 128) method.use_flashinfer = True - method.runner = _build_flashinfer_mxfp4_runner(num_experts, hidden, inter) + method.runner = _build_flashinfer_mxfp4_runner(num_experts, hidden, inter, layer) return method -def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter): +def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter, layer=None): """Construct a real MoeRunner bound to the flashinfer_mxfp4 fused func. Bypasses ``create_moe_runner`` (which needs a live server arg context) and wires the runner with a minimal MoeRunnerConfig sufficient for the - cutlass SM90 fused func, which only reads dispatch_output / quant_info. + cutlass SM90 fused func. The func reads the layer's MoE placement off the + config, so the layer states itself here the way `FusedMoE` does. """ import sglang.srt.layers.moe.moe_runner.flashinfer_cutlass # noqa: F401 from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig @@ -212,6 +213,7 @@ def _build_flashinfer_mxfp4_runner(num_experts, hidden, inter): top_k=None, activation="silu", is_gated=True, + layer=layer, ) return MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, cfg) @@ -299,7 +301,7 @@ def test_process_weights_matches_direct_interleave(num_experts, hidden, inter): layer = _build_mock_layer( num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b ) - method = _build_method(num_experts, hidden, inter) + method = _build_method(num_experts, hidden, inter, layer=layer) method._process_weights_for_sm90_cutlass(layer) N_pad = _round_up(inter, 128) @@ -375,7 +377,7 @@ def test_apply_sm90_cutlass_matches_flashinfer_direct( layer = _build_mock_layer( num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b ) - method = _build_method(num_experts, hidden, inter) + method = _build_method(num_experts, hidden, inter, layer=layer) method._process_weights_for_sm90_cutlass(layer) out_sglang = method._apply_sm90_cutlass( @@ -444,7 +446,7 @@ def test_process_weights_humming_matches_flashinfer_direct(): layer = _build_mock_layer( num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b ) - method = _build_method(num_experts, hidden, inter, use_humming=True) + method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True) method._process_weights_for_sm90_cutlass(layer) # GPT-OSS loads pair-wise [gate, up]; FlashInfer consumes halved [up; gate]. @@ -484,7 +486,7 @@ def test_humming_padding_preserves_per_expert_residual(): layer = _build_mock_layer( num_experts, hidden, inter, w13, w2, w13_s, w2_s, w13_b, w2_b ) - method = _build_method(num_experts, hidden, inter, use_humming=True) + method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True) method._process_weights_for_sm90_cutlass(layer) assert torch.equal(layer.w13_humming_residual_scale, expected_w13_residual * 64.0) @@ -572,7 +574,7 @@ def test_humming_range_ignores_prerounded_hidden_tail(): layer = _build_mock_layer( E, hidden_rounded, inter, w13, w2, w13_s, w2_s, w13_b, w2_b ) - method = _build_method(E, hidden_rounded, inter, use_humming=True) + method = _build_method(E, hidden_rounded, inter, layer=layer, use_humming=True) # What create_weights records from layer.hidden_size_unpadded. method._unpadded_hidden = hidden_real method._process_weights_for_sm90_cutlass(layer) @@ -627,7 +629,7 @@ def test_apply_sm90_humming_matches_flashinfer_direct( ) layer.moe_ep_size = ep_size layer.moe_ep_rank = ep_rank - method = _build_method(num_experts, hidden, inter, use_humming=True) + method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True) method._process_weights_for_sm90_cutlass(layer) out_sglang = method._apply_sm90_cutlass( layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) @@ -784,6 +786,7 @@ def test_dsv4_apply_matches_flashinfer_direct( layer.moe_tp_rank = 0 layer.moe_ep_size = 1 layer.moe_ep_rank = 0 + method.runner.config.layer = layer method.process_weights_after_loading(layer) diff --git a/test/registered/unit/layers/test_attn_tp_reduce_rejection.py b/test/registered/unit/layers/test_attn_tp_reduce_rejection.py new file mode 100644 index 0000000000..5cf3dda305 --- /dev/null +++ b/test/registered/unit/layers/test_attn_tp_reduce_rejection.py @@ -0,0 +1,165 @@ +"""Layers that shard over attention TP but all-reduce over the full TP group +refuse the layouts where the two groups differ.""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.layers.dp_attention import ( + initialize_dp_attention_flags, + reject_attn_tp_shard_with_tp_reduce, +) +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + + +class TestAttnTpReduceRejection(CustomTestCase): + def publish(self, **fields): + reset_context() + self.addCleanup(reset_context) + server_args = ServerArgs(model_path="dummy", device="cpu", **fields) + publish(server_args, role="test", ranks=SpawnRanks(world_rank=0)) + initialize_dp_attention_flags(server_args) + + def test_only_a_narrower_wider_than_one_shard_with_a_tp_reduce_is_refused(self): + self.publish(tp_size=4) + with get_parallel().override(tp_size=4): + with self.assertRaisesRegex(ValueError, "Layer shards over the attention"): + reject_attn_tp_shard_with_tp_reduce( + "Layer", shard_tp_size=2, reduces_over_attn_tp=False + ) + for shard_tp_size, reduces_over_attn_tp in ( + (2, True), + (1, False), + (4, False), + ): + with self.subTest(shard=shard_tp_size, reduces=reduces_over_attn_tp): + reject_attn_tp_shard_with_tp_reduce( + "Layer", + shard_tp_size=shard_tp_size, + reduces_over_attn_tp=reduces_over_attn_tp, + ) + + def test_an_attention_that_always_reduces_over_tp(self): + from sglang.srt.models.clip import CLIPAttention + + config = SimpleNamespace( + hidden_size=16, num_attention_heads=4, attention_dropout=0.0 + ) + self.publish(tp_size=4, attn_dp_size=2) + with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"): + CLIPAttention(config) + # A one-rank attention-TP shard does not reduce at all. + self.publish(tp_size=4, attn_dp_size=4) + CLIPAttention(config) + self.publish(tp_size=4) + CLIPAttention(config) + + def test_a_layer_that_reduces_over_attention_tp_only_with_attention_dp(self): + from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP + + def build(): + return Qwen3_VisionMLP(16, 32, bias=False, hidden_act="relu") + + self.publish(tp_size=4, attn_cp_size=2) + with self.assertRaisesRegex(ValueError, "Qwen3_VisionMLP shards over"): + build() + for fields in ( + dict(tp_size=4, attn_dp_size=2), + dict(tp_size=8, attn_dp_size=2, attn_cp_size=2), + dict(tp_size=4), + ): + with self.subTest(**fields): + self.publish(**fields) + build() + # The data-parallel encoder runs one-rank shards. + self.publish(tp_size=4, attn_cp_size=2) + Qwen3_VisionMLP(16, 32, bias=False, hidden_act="relu", use_data_parallel=True) + + def test_a_worker_that_never_forwards_its_tower(self): + """Most models build their multimodal tower unconditionally. Where the + tower is not forwarded, the layout it would need must not be refused.""" + from sglang.srt.models.clip import CLIPAttention + + config = SimpleNamespace( + hidden_size=16, num_attention_heads=4, attention_dropout=0.0 + ) + self.publish(tp_size=4, attn_dp_size=2) + with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"): + CLIPAttention(config) + for fields in ( + # The encoder instance forwards the tower instead. + dict(language_only=True), + # Adaptive dispatch is off, so nothing is kept back for local work. + dict(language_only=True, enable_adaptive_dispatch_to_encoder=False), + # Multimodal requests are rejected outright. + dict(language_model_only=True), + # A decode instance embeds nothing. + dict(disaggregation_mode="decode"), + ): + with self.subTest(**fields): + self.publish(tp_size=4, attn_dp_size=2, **fields) + CLIPAttention(config) + + def test_a_tower_kept_for_local_work_is_still_checked(self): + """Adaptive dispatch sends only some requests to the encoder instance + and forwards the tower for the rest, so the layout still has to hold.""" + from sglang.srt.models.clip import CLIPAttention + + config = SimpleNamespace( + hidden_size=16, num_attention_heads=4, attention_dropout=0.0 + ) + self.publish( + tp_size=4, + attn_dp_size=2, + language_only=True, + enable_adaptive_dispatch_to_encoder=True, + ) + with self.assertRaisesRegex(ValueError, "CLIPAttention shards over"): + CLIPAttention(config) + + def test_the_message_names_the_width_that_narrowed_the_group(self): + self.publish(tp_size=4, attn_dp_size=2) + with self.assertRaisesRegex(ValueError, r"drop --attn-dp-size 2"): + reject_attn_tp_shard_with_tp_reduce( + "Layer", shard_tp_size=2, reduces_over_attn_tp=False + ) + self.publish(tp_size=4, attn_cp_size=2) + with self.assertRaisesRegex(ValueError, r"drop --attn-cp-size 2"): + reject_attn_tp_shard_with_tp_reduce( + "Layer", shard_tp_size=2, reduces_over_attn_tp=False + ) + + def test_a_text_layer_is_checked_on_every_worker(self): + """The exemption belongs to towers that do not run, not to the worker: + a text attention still reduces on a language-only instance.""" + from sglang.srt.models.phimoe import PhiMoEAttention + + for fields in ( + {}, + dict(language_only=True), + dict(language_model_only=True), + dict(disaggregation_mode="decode"), + ): + with self.subTest(**fields): + self.publish(tp_size=4, attn_dp_size=2, **fields) + with self.assertRaisesRegex(ValueError, "PhiMoEAttention shards over"): + PhiMoEAttention(hidden_size=16, num_heads=4, num_kv_heads=4) + + def test_a_replicated_branch_is_not_gated(self): + """MoonViT's tensor-parallel MLP is refused, while its ModelSlim branch + builds replicated layers that neither shard nor reduce.""" + from sglang.srt.models.kimi_vl_moonvit import MLP2 + + self.publish(tp_size=4, attn_dp_size=2) + with self.assertRaisesRegex(ValueError, "MLP2 shards over"): + MLP2(dims=[16, 32, 16], activation=None, use_tensor_parallel=True) + # Without tensor parallelism the shard is one rank wide. + MLP2(dims=[16, 32, 16], activation=None) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/lora/test_mem_pool_ep_unit.py b/test/registered/unit/lora/test_mem_pool_ep_unit.py index 2848a80648..c36cf338ff 100644 --- a/test/registered/unit/lora/test_mem_pool_ep_unit.py +++ b/test/registered/unit/lora/test_mem_pool_ep_unit.py @@ -68,6 +68,8 @@ with mock.patch.dict( _moe_runner_keeps_global_expert_ids, ) +from sglang.srt.runtime_context import get_parallel + class _IdentityMoeSlices: def slice_moe_lora_a_weights(self, weights, _rank, _target): @@ -982,6 +984,21 @@ class TestPoolInitPicksUpEpContext(unittest.TestCase): return_value=keeps_global, ), mock.patch.object(LoRAMemoryPool, "init_buffers", lambda self, _m: None), + # The pool reads its TP placement from the parallel context; state + # a whole layout so the topology check sees a consistent one. + get_parallel().override( + tp_size=tp_size, + tp_rank=tp_rank, + attn_tp_size=tp_size, + attn_tp_rank=tp_rank, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=tp_size, + moe_ep_size=1, + moe_dp_size=1, + ), ): hf_cfg = types.SimpleNamespace( num_hidden_layers=1, @@ -995,9 +1012,6 @@ class TestPoolInitPicksUpEpContext(unittest.TestCase): base_hf_config=hf_cfg, max_loras_per_batch=1, dtype=torch.bfloat16, - tp_size=tp_size, - tp_rank=tp_rank, - attn_tp_size=tp_size, max_lora_rank=8, target_modules={"qkv_proj"}, base_model=base_model, diff --git a/test/registered/unit/managers/test_mm_shm_error_consensus.py b/test/registered/unit/managers/test_mm_shm_error_consensus.py index 26e647043c..d234f96ffb 100644 --- a/test/registered/unit/managers/test_mm_shm_error_consensus.py +++ b/test/registered/unit/managers/test_mm_shm_error_consensus.py @@ -92,18 +92,12 @@ def _request(feature, rid: str = "vlm-request") -> TokenizedEmbeddingReqInput: def _receiver() -> SchedulerRequestReceiver: - group = SimpleNamespace(rank=0, ranks=[0], cpu_group=object()) return SchedulerRequestReceiver( recv_from_tokenizer=None, recv_from_rpc=None, recv_skipper=None, input_blocker=None, mm_receiver=None, - tp_group=group, - tp_cpu_group=group, - attn_tp_cpu_group=group, - attn_cp_cpu_group=group, - world_group=group, server_args=SimpleNamespace(), model_config=SimpleNamespace(is_multimodal=True), max_recv_per_poll=-1, @@ -121,9 +115,12 @@ def _run_consensus_rank(rank: int, world_size: int, init_file: str) -> None: ) try: req = _request(_failed_pointer() if rank == 1 else _successful_pointer()) - parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=world_size) + parallel = SimpleNamespace( + attn_dp_enabled=False, + tp_size=world_size, + tp_group=SimpleNamespace(cpu_group=torch.distributed.group.WORLD), + ) receiver = _receiver() - object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD) with ( patch( "sglang.srt.managers.mm_utils._get_is_default_transport", @@ -152,7 +149,6 @@ def _run_image_receiver(rank, init_file, pipe): ) try: receiver = _receiver() - object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD) torch.distributed.barrier() torch.distributed.all_reduce(torch.zeros(1)) initial_fds = len(os.listdir("/proc/self/fd")) @@ -169,7 +165,11 @@ def _run_image_receiver(rank, init_file, pipe): ), patch( "sglang.srt.managers.scheduler_components.request_receiver.get_parallel", - return_value=SimpleNamespace(attn_dp_enabled=False, tp_size=2), + return_value=SimpleNamespace( + attn_dp_enabled=False, + tp_size=2, + tp_group=SimpleNamespace(cpu_group=torch.distributed.group.WORLD), + ), ), ): for base in [30, 90]: @@ -396,7 +396,11 @@ class TestShmRequestFailureConsensus(unittest.TestCase): def test_peer_failure_rejects_the_local_request(self): req = _request(torch.zeros(1)) - parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=2) + parallel = SimpleNamespace( + attn_dp_enabled=False, + tp_size=2, + tp_group=SimpleNamespace(cpu_group=object()), + ) def inject_peer_failure(mask, **kwargs): mask.fill_(1) diff --git a/test/registered/unit/managers/test_pp_cp_rank_offsets.py b/test/registered/unit/managers/test_pp_cp_rank_offsets.py index b837ce0fac..56ae036b0d 100644 --- a/test/registered/unit/managers/test_pp_cp_rank_offsets.py +++ b/test/registered/unit/managers/test_pp_cp_rank_offsets.py @@ -22,6 +22,7 @@ from sglang.srt.managers.scheduler_pp_mixin import ( # noqa: E402 _pp_exchange_outputs_before_forward, ) from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 +from sglang.srt.runtime_context import get_parallel # noqa: E402 register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -46,21 +47,12 @@ def _fake_group() -> SimpleNamespace: def _make_receiver() -> SchedulerRequestReceiver: - tp_group = _fake_group() - attn_tp_group = _fake_group() - attn_cp_group = _fake_group() - world_group = _fake_group() return SchedulerRequestReceiver( recv_from_tokenizer=None, recv_from_rpc=None, recv_skipper=None, input_blocker=None, mm_receiver=None, - tp_group=tp_group, - tp_cpu_group=tp_group, - attn_tp_cpu_group=attn_tp_group, - attn_cp_cpu_group=attn_cp_group, - world_group=world_group, server_args=SimpleNamespace( enable_dp_attention_local_control_broadcast=False, ), @@ -120,6 +112,7 @@ class TestRequestReceiverBroadcast(unittest.TestCase): attn_tp_size=1, attn_cp_size=1, tp_size=32, + tp_group=_fake_group(), ) with ( @@ -150,15 +143,16 @@ class TestRequestReceiverBroadcast(unittest.TestCase): self.assertEqual(result, [control_req]) broadcast.assert_called_once_with( [control_req], - receiver.tp_group.rank, - receiver.tp_cpu_group, - src=receiver.tp_group.ranks[0], + parallel.tp_group.rank, + parallel.tp_group.cpu_group, + src=parallel.tp_group.ranks[0], ) class TestPPCPRankOffsets(unittest.TestCase): def test_request_receiver_uses_cp_size_for_pp_recv_rank(self): enter_scope(self, _published_topology()) + enter_scope(self, get_parallel().override(world_group=_fake_group())) calls = [] def fake_point_to_point_pyobj(data, rank, group, src, dst, **kwargs): diff --git a/test/registered/unit/managers/test_prefill_delayer_gather.py b/test/registered/unit/managers/test_prefill_delayer_gather.py index 9c5177716d..b2b586876d 100644 --- a/test/registered/unit/managers/test_prefill_delayer_gather.py +++ b/test/registered/unit/managers/test_prefill_delayer_gather.py @@ -1,4 +1,5 @@ import unittest +from types import SimpleNamespace from unittest.mock import patch import torch @@ -41,9 +42,12 @@ class TestPrefillDelayerGather(CustomTestCase): "sglang.srt.managers.prefill_delayer.all_gather_single", side_effect=_all_gather_over_tp, ), + get_parallel().override( + tp_group=SimpleNamespace(cpu_group=None, device_group=None) + ), ): delayer = PrefillDelayer( - cpu_group=None, max_delay_passes=4, token_usage_low_watermark=None + max_delay_passes=4, token_usage_low_watermark=None ) info = delayer._gather_info( local_prefillable=True, diff --git a/test/registered/unit/managers/test_scheduler_weight_version_tracking.py b/test/registered/unit/managers/test_scheduler_weight_version_tracking.py index 8eb58463dd..f94c780807 100644 --- a/test/registered/unit/managers/test_scheduler_weight_version_tracking.py +++ b/test/registered/unit/managers/test_scheduler_weight_version_tracking.py @@ -11,6 +11,7 @@ from sglang.srt.managers.scheduler_components.weight_updater import ( SchedulerWeightUpdaterManager, _WeightUpdateSession, ) +from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -140,8 +141,8 @@ class _WeightUpdaterManagerTestBase(CustomTestCase): self.addCleanup(patcher.stop) self.recorded = [] - def _manager(self, target, draft=None, *, session=True): - manager = SchedulerWeightUpdaterManager( + def _build_manager(self, target, draft): + return SchedulerWeightUpdaterManager( tp_worker=SimpleNamespace( model_runner=target, weight_update_runners=lambda: [("target", target)], @@ -152,7 +153,6 @@ class _WeightUpdaterManagerTestBase(CustomTestCase): if draft is None else SimpleNamespace(weight_update_runners=lambda: [("draft", draft)]) ), - tp_cpu_group=None, memory_saver_adapter=None, flush_cache=lambda **kwargs: True, is_fully_idle=lambda **kwargs: True, @@ -162,6 +162,10 @@ class _WeightUpdaterManagerTestBase(CustomTestCase): ) ), ) + + def _manager(self, target, draft=None, *, session=True): + with get_parallel().override(tp_group=SimpleNamespace(cpu_group=None)): + manager = self._build_manager(target, draft) if session: # update_weights_from_* require an open session manager._session = _WeightUpdateSession(selector="all") diff --git a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py index 1f46b17464..4bd1669d63 100644 --- a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py +++ b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py @@ -16,7 +16,11 @@ from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestOutcome, ) from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline -from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer +from sglang.srt.mem_cache.hicache_storage import ( + HiCacheStorageConfig, + PoolName, + PoolTransfer, +) from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( HybridCacheController, PPPrefetchDecision, @@ -34,8 +38,18 @@ class TestPPPrefetchTicket(unittest.TestCase): def setUp(self): self.c = c = HybridCacheController.__new__(HybridCacheController) c.page_size = c.prefetch_threshold = 4 - c.pp_rank = c.tp_rank = 0 - c.pp_size, c.tp_size = 4, 2 + c.storage_config = HiCacheStorageConfig( + tp_rank=0, + tp_size=2, + pp_rank=0, + pp_size=4, + attn_cp_rank=0, + attn_cp_size=1, + is_mla_model=False, + enable_storage_metrics=False, + is_page_first_layout=False, + model_name=None, + ) c.pp_group, c.pp_prefetch_command_group = "pp", "command" c.pp_prefetch_command_thread = None c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"] @@ -61,7 +75,9 @@ class TestPPPrefetchTicket(unittest.TestCase): patcher.start() self.addCleanup(patcher.stop) patcher = patch.object( - torch.distributed, "get_rank", side_effect=lambda: c.pp_rank * 2 + c.tp_rank + torch.distributed, + "get_rank", + side_effect=lambda: c.storage_config.pp_rank * 2 + c.storage_config.tp_rank, ) patcher.start() self.addCleanup(patcher.stop) @@ -98,7 +114,7 @@ class TestPPPrefetchTicket(unittest.TestCase): self.c.pp_prefetch_command_queue.put(None) def broadcast(objects, *args, **kwargs): - return [next(commands)] if self.c.pp_rank else objects + return [next(commands)] if self.c.storage_config.pp_rank else objects with patch(f"{HybridCacheController.__module__}.broadcast_pyobj", broadcast): self.c.pp_prefetch_command_thread_func() @@ -179,7 +195,7 @@ class TestPPPrefetchTicket(unittest.TestCase): handle = CacheRequestHandle("hit", 3) ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket)) ticket.last_hash = None - c.pp_rank = 1 + c.storage_config.pp_rank = 1 c.pp_prefetch_states.clear() cache.bind_prefetch_ticket("hit") self.assertFalse(cache.check_prefetch_progress(handle)) @@ -216,7 +232,7 @@ class TestPPPrefetchTicket(unittest.TestCase): ticket = c.pp_prefetch_states.pop("hit").ticket following = pickle.loads(pickle.dumps(ticket)) following.handle = CacheRequestHandle("next", 0) - c.pp_rank = 1 + c.storage_config.pp_rank = 1 kv = torch.arange(8) c.mem_pool_host.alloc.side_effect = [ kv, @@ -257,7 +273,7 @@ class TestPPPrefetchTicket(unittest.TestCase): c, cache = self.c, self.cache self.submit(pools=[PoolTransfer(PoolName.SWA, host_indices=torch.arange(4))]) ticket = c.pp_prefetch_states.pop("hit").ticket - c.pp_rank = 1 + c.storage_config.pp_rank = 1 cache.bind_prefetch_ticket("hit") cache.finish(ticket.handle, CacheRequestOutcome.ABORT) cache.finish(ticket.handle, CacheRequestOutcome.ABORT) @@ -290,7 +306,7 @@ class TestPPPrefetchTicket(unittest.TestCase): ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket)) for rank in (0, 1): with self.subTest(rank=rank): - c.pp_rank = rank + c.storage_config.pp_rank = rank if rank: c.pp_prefetch_states.clear() c.mem_pool_host.alloc.reset_mock() @@ -358,7 +374,7 @@ class TestPPPrefetchTicket(unittest.TestCase): def test_idle_source_broadcasts_empty_then_processes_ticket_and_stop(self): c = self.c - c.tp_rank = 1 # The source is a global rank, not pp_rank=0. + c.storage_config.tp_rank = 1 # The source is a global rank, not pp_rank=0. operation = self.submit().operation c.pp_prefetch_command_queue.put(None) get = c.pp_prefetch_command_queue.get @@ -397,7 +413,7 @@ class TestPPPrefetchTicket(unittest.TestCase): c = self.c self.submit() ticket = c.pp_prefetch_states.pop("hit").ticket - c.pp_rank = 1 + c.storage_config.pp_rank = 1 with ( patch( f"{HybridCacheController.__module__}.broadcast_pyobj", diff --git a/test/registered/unit/server_args/test_attn_dp_size.py b/test/registered/unit/server_args/test_attn_dp_size.py index 6a70050f6e..d3e9a8ab65 100644 --- a/test/registered/unit/server_args/test_attn_dp_size.py +++ b/test/registered/unit/server_args/test_attn_dp_size.py @@ -89,8 +89,13 @@ class TestAttnDpSize(CustomTestCase): "replicas combined with attention data parallelism", ), ( - {"tp_size": 4, "attn_dp_size": 2, "enable_dp_attention": True}, - "pass --attn-dp-size alone", + { + "tp_size": 4, + "dp_size": 2, + "attn_dp_size": 2, + "enable_dp_attention": True, + }, + "replicas combined with attention data parallelism", ), ({"tp_size": 2, "attn_dp_size": 0}, "must be positive"), ): @@ -98,6 +103,55 @@ class TestAttnDpSize(CustomTestCase): with self.assertRaisesRegex(ValueError, message): self.resolve(**fields) + def test_the_deprecated_flag_next_to_attn_dp_size_is_redundant(self): + both = self.resolve(tp_size=4, attn_dp_size=2, enable_dp_attention=True) + new = self.resolve(tp_size=4, attn_dp_size=2) + self.assertEqual(self.layout(both), (1, 2, False)) + self.assertEqual(both.resolved_dict(), new.resolved_dict()) + + def test_the_readback_reports_the_deprecated_field_as_attention_dp(self): + for fields, attention_dp in ( + ({"tp_size": 4, "attn_dp_size": 2}, True), + ({"tp_size": 4, "dp_size": 2, "enable_dp_attention": True}, True), + ({"tp_size": 4, "dp_size": 2}, False), + ({"tp_size": 4}, False), + ): + with self.subTest(**fields): + server_args = self.resolve(**fields) + self.assertFalse(resolution_result(server_args, "enable_dp_attention")) + readback = server_args.resolved_dict() + self.assertIs(readback["enable_dp_attention"], attention_dp) + # A read-back config resolves to the same layout. + again = self.resolve( + **{ + name: readback[name] + for name in ( + "tp_size", + "dp_size", + "attn_dp_size", + "enable_dp_attention", + ) + } + ) + self.assertEqual(self.layout(again), self.layout(server_args)) + + def test_the_readback_reports_the_width_not_the_joiner_arm(self): + """`attn_dp_enabled` is also true for an elastic scale joiner at width + one. The readback reports the configured width instead, so a joiner's + replicas are not folded into attention-DP groups when it is read back.""" + from sglang.srt.arg_groups.overrides import resolving_view + from sglang.srt.runtime_context import attn_dp_enabled_of + + joiner = ServerArgs( + model_path="dummy", tp_size=2, dp_size=2, ep_join_mode="scale" + ) + joiner.resolve_once() + self.assertTrue(attn_dp_enabled_of(resolving_view(joiner))) + readback = joiner.resolved_dict() + self.assertFalse(readback["enable_dp_attention"]) + self.assertEqual(readback["dp_size"], 2) + self.assertEqual(readback["attn_dp_size"], 1) + def test_both_cli_spellings_parse(self): parser = argparse.ArgumentParser() ServerArgs.add_cli_args(parser)