[Refactor] Read parallel placement in consumers (#40638)

This commit is contained in:
Cheng Wan
2026-09-23 14:57:25 -07:00
committed by GitHub
parent f2a1366584
commit 79fec592a8
43 changed files with 242 additions and 424 deletions
+1 -12
View File
@@ -95,7 +95,6 @@ from sglang.srt.runtime_context import (
SpawnRanks,
get_device,
get_model,
get_parallel,
get_schedule,
publish,
spawn_world_rank,
@@ -543,10 +542,6 @@ def _maybe_prepare_mlp_sync_batch(batch: ScheduleBatch, model_runner):
prepare_mlp_sync_batch_raw(
batch,
model_runner=model_runner,
dp_size=get_parallel().dp_size,
attn_tp_size=get_parallel().attn_tp_size,
attn_cp_size=model_runner.attn_cp_size,
tp_group=model_runner.tp_group,
get_idle_batch=None,
disable_cuda_graph=cuda_graph_fully_disabled(),
require_mlp_tp_gather=require_mlp_tp_gather(),
@@ -908,13 +903,7 @@ def latency_test(
initialize_fp4_gemm_config()
if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
parallel = get_parallel()
set_gpu_proc_affinity(
parallel.pp_size,
parallel.tp_size,
parallel.nnodes,
tp_rank,
)
set_gpu_proc_affinity(tp_rank)
# Configure the logger
configure_logger(server_args, prefix=f" TP{tp_rank}")
+7 -11
View File
@@ -365,16 +365,13 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
transfer_queue: DecodeTransferQueue,
tree_cache: BasePrefixCache,
gloo_group: ProcessGroup,
tp_rank: int,
tp_size: int,
dp_size: int,
gpu_id: int,
bootstrap_port: int,
max_total_num_tokens: int,
pp_rank: int,
num_reserved_decode_tokens: int,
transfer_backend: TransferBackend,
):
parallel = get_parallel()
self.req_to_token_pool = req_to_token_pool
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
self.token_to_kv_pool = token_to_kv_pool_allocator.get_kvcache()
@@ -388,14 +385,14 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.gloo_group = gloo_group
# Destinations visible to prefill but not yet on the transfer queue.
self._num_published_destinations = 0
self.tp_rank = tp_rank
self.tp_size = tp_size
self.dp_size = dp_size
self.tp_rank = parallel.tp_rank
self.tp_size = parallel.tp_size
self.dp_size = parallel.dp_size
self.gpu_id = gpu_id
self.bootstrap_port = bootstrap_port
self.max_total_num_tokens = max_total_num_tokens
self.pp_rank = pp_rank
self.pp_size = get_parallel().pp_size
self.pp_rank = parallel.pp_rank
self.pp_size = parallel.pp_size
self.num_reserved_decode_tokens = num_reserved_decode_tokens
self.transfer_backend = transfer_backend
# Queue for requests pending pre-allocation
@@ -2200,7 +2197,6 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self,
gloo_group: ProcessGroup,
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
tp_rank: int,
metadata_buffers: MetadataBuffers,
scheduler: Scheduler,
tree_cache: BasePrefixCache,
@@ -2208,7 +2204,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self.queue: List[DecodeRequest] = []
self.gloo_group = gloo_group
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
self.tp_rank = tp_rank
self.tp_rank = get_parallel().tp_rank
self.metadata_buffers = metadata_buffers
self.scheduler = scheduler
self.tree_cache = tree_cache
@@ -436,11 +436,11 @@ class EncoderPreprocessor:
vr,
video_config,
*,
tp_rank: int,
tp_size: int,
video_processor_kwargs: dict,
precomputed_indices: Optional[List[int]] = None,
):
parallel = get_parallel()
tp_rank, tp_size = parallel.attn_tp_rank, parallel.attn_tp_size
video_config = video_config or {}
video_fps = vr.avg_fps
duration = len(vr) / video_fps if video_fps else 0
@@ -596,8 +596,6 @@ class EncoderPreprocessor:
result = await self._dp_sharded_decode_single_video(
video_items[0],
video_configs[0],
tp_rank=parallel.attn_tp_rank,
tp_size=tp_size,
video_processor_kwargs=video_processor_kwargs,
precomputed_indices=sampled,
)
@@ -670,11 +670,8 @@ class MMEncoder:
get_mm().mm_global_cache_backend,
)
self.mm_global_cache = EmbeddingCacheController(
rank,
get_parallel().tp_size,
embedding_store=embedding_store,
hidden_dims=self._embedding_dims,
tp_group=get_parallel().tp_group.cpu_group,
all_rank_get=False,
dtype=self._embedding_dtype,
)
+5 -8
View File
@@ -164,27 +164,24 @@ class PrefillBootstrapQueue:
draft_token_to_kv_pool: Optional[KVCache],
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
metadata_buffers: MetadataBuffers,
tp_rank: int,
tp_size: int,
gpu_id: int,
bootstrap_port: int,
gloo_group: ProcessGroup,
max_total_num_tokens: int,
scheduler: Scheduler,
scheduler_stage_metrics: SchedulerStageMetricsRecorder,
pp_rank: int,
pp_size: int,
transfer_backend: TransferBackend,
):
parallel = get_parallel()
self.token_to_kv_pool = token_to_kv_pool
self.draft_token_to_kv_pool = draft_token_to_kv_pool
self.is_mla_backend = is_mla_backend(token_to_kv_pool)
self.metadata_buffers = metadata_buffers
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
self.tp_rank = tp_rank
self.tp_size = tp_size
self.pp_rank = pp_rank
self.pp_size = pp_size
self.tp_rank = parallel.tp_rank
self.tp_size = parallel.tp_size
self.pp_rank = parallel.pp_rank
self.pp_size = parallel.pp_size
self.gpu_id = gpu_id
self.bootstrap_port = bootstrap_port
self.queue: List[Req] = []
+8 -13
View File
@@ -142,8 +142,6 @@ def init_parallel_runtime(
parallel = get_parallel()
if device == "cpu":
_init_cpu_threads_env(
tp_size=parallel.tp_size,
tp_rank=parallel.tp_rank,
local_omp_cpuid=local_omp_cpuid,
dist_init_method=dist_init_method,
)
@@ -160,11 +158,7 @@ def init_parallel_runtime(
if get_exec().comm.pre_warm_nccl and (
parallel.tp_size > 1 or parallel.pp_size > 1 or parallel.moe_ep_size > 1
):
_prewarm_nccl(
tp_size=parallel.tp_size,
pp_size=parallel.pp_size,
moe_ep_size=parallel.moe_ep_size,
)
_prewarm_nccl()
# CUDA graph capture enables the PyNCCL communicator for TP LM-head
# all-to-all. Exercise that exact send/recv path before measuring
@@ -262,17 +256,16 @@ def _set_shm_master_env(dist_init_method: Optional[str]) -> None:
def _init_cpu_threads_env(
*,
tp_size: int,
tp_rank: int,
local_omp_cpuid: Optional[List[int]],
dist_init_method: Optional[str] = None,
) -> None:
if _is_cpu_amx_available or _is_cpu_arm64:
parallel = get_parallel()
# Bind OpenMP threads to CPU cores
torch.ops.sgl_kernel.init_cpu_threads_env(local_omp_cpuid)
# Set local size to hint SGLang to use shared memory based AllReduce
os.environ["LOCAL_SIZE"] = str(tp_size)
os.environ["LOCAL_SIZE"] = str(parallel.tp_size)
# shm.cpp names its /dev/shm segments from MASTER_ADDR/MASTER_PORT.
# Feed each engine's unique dist_init_method (tcp://host:port) into
@@ -280,7 +273,7 @@ def _init_cpu_threads_env(
# don't collide.
_set_shm_master_env(dist_init_method)
torch.ops.sgl_kernel.initialize(tp_size, tp_rank)
torch.ops.sgl_kernel.initialize(parallel.tp_size, parallel.tp_rank)
else:
logger.warning(
@@ -338,7 +331,8 @@ def init_layer_runtime(*, model_config: ModelConfig) -> None:
initialize_layernorm_sp(model_config=model_config)
def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None:
def _prewarm_nccl() -> None:
parallel = get_parallel()
warmup_start = time.perf_counter()
tp_group_handle = get_tp_group().device_group
@@ -350,7 +344,8 @@ def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None:
warmup_elapsed = time.perf_counter() - warmup_start
logger.info(
f"NCCL/RCCL/HCCL warmup completed in {warmup_elapsed:.3f}s "
f"(tp_size={tp_size}, pp_size={pp_size}, ep_size={moe_ep_size})"
f"(tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, "
f"ep_size={parallel.moe_ep_size})"
)
@@ -30,19 +30,18 @@ class ExpertBackupClient:
self,
*,
model_config,
moe_ep_size: int,
moe_ep_rank: int,
get_model: Callable[[], Any],
):
context = zmq.Context(2)
self.engine_num = get_parallel().nnodes
self.engine_rank = get_parallel().node_rank
parallel = get_parallel()
self.engine_num = parallel.nnodes
self.engine_rank = parallel.node_rank
self.recv_list = [None] * self.engine_num
self.ready_sockets = [None] * self.engine_num
self._get_model = get_model
self.moe_ep_size = moe_ep_size
self.moe_ep_size = parallel.moe_ep_size
self.model_config = model_config
self.moe_ep_rank = moe_ep_rank
self.moe_ep_rank = parallel.moe_ep_rank
self.dram_map_list = [None] * self.engine_num
self.session_id_list = [None] * self.engine_num
self.transfer_engine = None
+12 -60
View File
@@ -730,15 +730,8 @@ class Engine(EngineScoreMixin, EngineBase):
"--dist-init-addr so all nodes rendezvous at the same endpoint."
)
tp_size = get_parallel().tp_size
pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = (
_calculate_rank_ranges(
get_parallel().nnodes,
get_parallel().pp_size,
tp_size,
get_parallel().node_rank,
)
_calculate_rank_ranges(get_parallel().node_rank)
)
# Build the distributed init method (multi-node uses the user-provided
@@ -902,12 +895,7 @@ class Engine(EngineScoreMixin, EngineBase):
scheduler_pipe_readers = []
pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = (
_calculate_rank_ranges(
get_parallel().nnodes,
get_parallel().pp_size,
get_parallel().tp_size,
get_parallel().node_rank,
)
_calculate_rank_ranges(get_parallel().node_rank)
)
for pp_rank in pp_rank_range:
@@ -918,9 +906,6 @@ class Engine(EngineScoreMixin, EngineBase):
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
+ (tp_rank % tp_size_per_node) * get_device().gpu_id_step
)
attn_cp_rank, moe_dp_rank, moe_ep_rank = _compute_parallelism_ranks(
tp_rank
)
with maybe_reindex_device_id(gpu_id) as gpu_id:
proc = mp.Process(
@@ -930,9 +915,6 @@ class Engine(EngineScoreMixin, EngineBase):
port_args,
gpu_id,
tp_rank,
attn_cp_rank,
moe_dp_rank,
moe_ep_rank,
pp_rank,
None,
writer,
@@ -1925,15 +1907,13 @@ def _wait_for_scheduler_ready(
return scheduler_infos
def _calculate_rank_ranges(
nnodes: int, pp_size: int, tp_size: int, node_rank: int
) -> Tuple[range, range, int, int]:
def _calculate_rank_ranges(node_rank: int) -> Tuple[range, range, int, int]:
"""Calculate pp_rank_range and tp_rank_range for a given node.
`node_rank` stays an argument because the Ray launchers size every node
from the driver, not just their own.
Args:
nnodes: Total number of nodes.
pp_size: Pipeline parallel size.
tp_size: Tensor parallel size.
node_rank: The rank of the node to compute ranges for.
Returns:
@@ -1943,15 +1923,17 @@ def _calculate_rank_ranges(
- pp_size_per_node: number of PP ranks per node.
- tp_size_per_node: number of TP ranks per node.
"""
pp_size_per_node = max(pp_size // nnodes, 1)
nnodes_per_pp_rank = max(nnodes // pp_size, 1)
parallel = get_parallel()
nnodes = parallel.nnodes
pp_size_per_node = max(parallel.pp_size // nnodes, 1)
nnodes_per_pp_rank = max(nnodes // parallel.pp_size, 1)
pp_rank_range = range(
pp_size_per_node * (node_rank // nnodes_per_pp_rank),
pp_size_per_node * (node_rank // nnodes_per_pp_rank + 1),
)
nnodes_per_tp_group = nnodes_per_pp_rank
tp_size_per_node = tp_size // nnodes_per_tp_group
tp_size_per_node = parallel.tp_size // nnodes_per_tp_group
tp_rank_range = range(
tp_size_per_node * (node_rank % nnodes_per_tp_group),
tp_size_per_node * (node_rank % nnodes_per_tp_group + 1),
@@ -1963,12 +1945,7 @@ def _calculate_rank_ranges(
def node_hosts_rust_server() -> bool:
"""Whether this node contains a Rust listener rank, assuming Rust mode."""
parallel = get_parallel()
pp_rank_range, tp_rank_range, _, _ = _calculate_rank_ranges(
parallel.nnodes,
parallel.pp_size,
parallel.tp_size,
parallel.node_rank,
)
pp_rank_range, tp_rank_range, _, _ = _calculate_rank_ranges(parallel.node_rank)
if 0 not in pp_rank_range:
return False
@@ -1983,28 +1960,3 @@ def node_hosts_rust_server() -> bool:
if rank_within_dp_group == 0:
return True
return False
def _compute_parallelism_ranks(tp_rank: int) -> Tuple[int, int, int]:
"""Compute attention-CP, MoE-DP, and MoE-EP ranks for a TP rank.
Called while the launcher is deciding what to spawn, so the sizes are the
configured ones -- the groups this is laying out do not exist yet.
"""
attn_dp_size = get_parallel().dp_size if get_parallel().enable_dp_attention else 1
tp_size = get_parallel().tp_size
attn_cp_size = get_parallel().attn_cp_size
moe_dp_size = get_parallel().moe_dp_size
# Parallelism hierarchy (outermost to innermost):
# - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost)
# - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost)
attn_tp_size = tp_size // attn_dp_size // attn_cp_size
attn_cp_rank = (tp_rank // attn_tp_size) % attn_cp_size
moe_dp_rank = tp_rank // (tp_size // moe_dp_size)
moe_ep_rank = (
tp_rank
% (tp_size // moe_dp_size)
// (tp_size // moe_dp_size // get_parallel().ep_size)
)
return attn_cp_rank, moe_dp_rank, moe_ep_rank
+11 -16
View File
@@ -41,16 +41,16 @@ def _accelerator_name() -> Optional[str]:
return get_device_name()
@lru_cache(maxsize=1)
def _num_accelerators_per_dp_rank(
tp_size: int,
pp_size: int,
dp_size: int,
enable_dp_attention: bool,
) -> int:
num_accelerators = tp_size * pp_size
if enable_dp_attention:
num_accelerators //= dp_size
def _num_accelerators_per_dp_rank() -> int:
"""Accelerators behind one DP rank.
Not cached: the arguments were the cache key, so an ``lru_cache`` here
would freeze the first answer past a post-publish override.
"""
parallel = get_parallel()
num_accelerators = parallel.tp_size * parallel.pp_size
if parallel.enable_dp_attention:
num_accelerators //= parallel.dp_size
return num_accelerators
@@ -145,11 +145,6 @@ async def get_loads(
"timestamp": datetime.now(timezone.utc).isoformat(),
"version": __version__,
"accelerator": _accelerator_name(),
"num_accelerators": _num_accelerators_per_dp_rank(
get_parallel().tp_size,
get_parallel().pp_size,
get_parallel().dp_size,
get_parallel().enable_dp_attention,
),
"num_accelerators": _num_accelerators_per_dp_rank(),
"loads": loads,
}
+1 -3
View File
@@ -157,7 +157,6 @@ class EPLBManager:
model=self._get_model(),
new_expert_location_metadata=expert_location_metadata,
update_layer_ids=chunk_layer_ids,
nnodes=get_parallel().nnodes,
tp_rank=(
self._elastic_global_rank()
if is_post_scale_rebalance
@@ -302,7 +301,6 @@ def update_expert_location_with_recovery(
model: nn.Module,
new_expert_location_metadata: ExpertLocationMetadata,
update_layer_ids: List[int],
nnodes: int,
tp_rank: int,
use_flat_topology: bool = False,
expert_backup_client,
@@ -314,7 +312,7 @@ def update_expert_location_with_recovery(
model.routed_experts_weights_of_layer,
new_expert_location_metadata,
update_layer_ids=update_layer_ids,
nnodes=nnodes,
nnodes=get_parallel().nnodes,
rank=tp_rank,
use_flat_topology=use_flat_topology,
)
+3 -4
View File
@@ -1089,9 +1089,7 @@ class LogitsProcessor(nn.Module):
get_parallel().tp_group.all_to_all_single(
all_to_all_output.view(-1), logits.view(-1)
)
return _reassemble_tp_lm_head_all_to_all_output(
all_to_all_output, get_parallel().tp_size
)
return _reassemble_tp_lm_head_all_to_all_output(all_to_all_output)
def _scatter_dp_attn_logits(
self,
@@ -1252,7 +1250,7 @@ class LogitsProcessor(nn.Module):
def _reassemble_tp_lm_head_all_to_all_output(
all_to_all_output: torch.Tensor, tp_size: int
all_to_all_output: torch.Tensor,
) -> torch.Tensor:
"""Convert source-major all-to-all output to row-major full-vocab logits.
@@ -1261,6 +1259,7 @@ def _reassemble_tp_lm_head_all_to_all_output(
along dim 0, while the sampler expects the vocab shards concatenated along
dim 1.
"""
tp_size = get_parallel().tp_size
assert all_to_all_output.shape[0] % tp_size == 0
local_rows = all_to_all_output.shape[0] // tp_size
vocab_shard = all_to_all_output.shape[1]
@@ -467,7 +467,7 @@ class DataParallelController:
)
else:
# Other nodes: Receive worker ports from node 0
return self._receive_ports_as_client(endpoint, get_parallel().node_rank)
return self._receive_ports_as_client(endpoint)
def _broadcast_ports_as_server(
self, endpoint: str, expected_clients: int, worker_ports: list[int]
@@ -522,9 +522,10 @@ class DataParallelController:
sock_send(rep_socket, wrap_as_pickle(worker_ports))
logger.debug(f"Sent worker ports to node {client_rank}")
def _receive_ports_as_client(self, endpoint: str, node_rank: int) -> list[int]:
def _receive_ports_as_client(self, endpoint: str) -> list[int]:
"""Receive worker ports from the server node."""
logger.debug("Connecting to node 0 to receive worker ports")
node_rank = get_parallel().node_rank
req_socket = get_zmq_socket(self.context, zmq.REQ, endpoint, False)
req_socket.setsockopt(zmq.RCVTIMEO, 600 * 1000) # 10 minute timeout
@@ -567,9 +568,7 @@ class DataParallelController:
primary_endpoint = NetworkAddress(
primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA
).to_tcp()
all_ports = self._receive_ports_as_client(
primary_endpoint, get_parallel().node_rank
)
all_ports = self._receive_ports_as_client(primary_endpoint)
offset = self._joiner_slot_offset(server_args)
local_tp_span = self._joiner_local_tp_span(server_args)
broadcasted_ports = all_ports[offset : offset + local_tp_span]
@@ -636,8 +635,6 @@ class DataParallelController:
tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group + 1),
)
attn_cp_rank = 0
moe_dp_rank = 0
for pp_rank in pp_rank_range:
for tp_rank in tp_rank_range:
rank_port_args = port_args
@@ -679,22 +676,7 @@ class DataParallelController:
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
+ (tp_rank % tp_size_per_node) * get_device().gpu_id_step
)
attn_dp_size = (
get_parallel().dp_size if get_parallel().enable_dp_attention else 1
)
# Parallelism hierarchy (outermost to innermost):
# - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost)
# - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost)
attn_tp_size = (
get_parallel().tp_size
// attn_dp_size
// get_parallel().attn_cp_size
)
attn_cp_rank = (tp_rank // attn_tp_size) % get_parallel().attn_cp_size
moe_dp_rank = tp_rank // (
get_parallel().tp_size // get_parallel().moe_dp_size
)
# Derive the child's EP rank for its display label.
moe_ep_rank = (
tp_rank
% (get_parallel().tp_size // get_parallel().moe_dp_size)
@@ -719,9 +701,6 @@ class DataParallelController:
rank_port_args,
gpu_id,
tp_rank,
attn_cp_rank,
moe_dp_rank,
moe_ep_rank,
pp_rank,
dp_rank,
writer,
@@ -47,6 +47,7 @@ from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool, ReqToTokenPool
from sglang.srt.mem_cache.memory_pool_host import DeepSeekV4PagedHostPool
from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.runtime_context import get_parallel
device_module = get_device_module()
@@ -72,7 +73,6 @@ class HiSparseTokenStats(NamedTuple):
def resolve_shared_index_layers(
*,
hf_text_config,
pp_size: int,
is_speculative: bool,
) -> Optional[List[bool]]:
"""Per-layer "reuses the previous layer's top-k index" pattern, or None.
@@ -91,7 +91,7 @@ def resolve_shared_index_layers(
pattern = [dsa_layer_skips_topk(hf_text_config, i) for i in range(num_layers)]
if not any(pattern):
return None
if pp_size != 1 or is_speculative:
if get_parallel().pp_size != 1 or is_speculative:
logger.warning(
"HiSparse shared-index prefetch is unsupported under pipeline "
"parallelism / speculative decoding; falling back to synchronous "
@@ -75,8 +75,6 @@ class _NegotiateOutput(NamedTuple):
class PrefillDelayer:
def __init__(
self,
dp_size: int,
attn_tp_size: int,
cpu_group,
max_delay_passes: int,
token_usage_low_watermark: Optional[float],
@@ -106,9 +104,10 @@ class PrefillDelayer:
f"max_delay_ms={self._max_delay_ms} "
f"queue_trigger_enabled={self._queue_trigger_enabled}"
)
self.dp_size = dp_size
self.enable_dp_attention = get_parallel().enable_dp_attention
dp_size_dim = dp_size if self.enable_dp_attention else 1
parallel = get_parallel()
self.dp_size = parallel.dp_size
self.enable_dp_attention = parallel.enable_dp_attention
dp_size_dim = self.dp_size if self.enable_dp_attention else 1
# Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the
# env flag is on (or overlap scheduling is disabled), ride the NCCL
@@ -131,7 +130,7 @@ class PrefillDelayer:
# token_watermark_force_allow, running_batch, max_prefill_bs,
# waiting_queue_len.
self._global_info_buffer = torch.empty(
(dp_size_dim, attn_tp_size, 5),
(dp_size_dim, parallel.attn_tp_size, 5),
dtype=torch.int64,
device=self._gather_device,
)
+22 -77
View File
@@ -449,9 +449,6 @@ class Scheduler(
self,
server_args: ServerArgs,
port_args: PortArgs,
tp_rank: int,
pp_rank: int,
dp_rank: Optional[int],
):
# NOTE: KEEP THE FOLLOWING CODE STYLE for this function:
# Keep __init__ as an orchestrator: sequence init_* and maybe_init_* calls
@@ -526,7 +523,7 @@ class Scheduler(
bootstrap.init_layer_runtime(model_config=self.model_config)
# Init metrics stats
self.init_metrics_collector(tp_rank, pp_rank, dp_rank)
self.init_metrics_collector()
# Init inter-process communication
self.init_ipc_channels(port_args)
@@ -637,7 +634,7 @@ class Scheduler(
# Init diffusion LLM
self.init_diffusion_llm()
self.init_metrics_reporter(tp_rank, pp_rank, dp_rank)
self.init_metrics_reporter()
self.scheduler_stage_metrics = self.metrics_reporter.scheduler_stage_metrics
# Init schedule policy and new token estimation
@@ -772,14 +769,9 @@ class Scheduler(
else None
)
def init_metrics_collector(
self, tp_rank: int, pp_rank: int, dp_rank: Optional[int]
) -> None:
def init_metrics_collector(self) -> None:
self.metrics_collector_context = SchedulerMetricsCollector.init_new(
server_args=self.server_args,
tp_rank=tp_rank,
pp_rank=pp_rank,
dp_rank=dp_rank,
enable_priority_scheduling=self.enable_priority_scheduling,
enable_lora=self.enable_lora,
enable_hierarchical_cache=self.enable_hierarchical_cache,
@@ -1340,9 +1332,6 @@ class Scheduler(
max_prefill_tokens=self.max_prefill_tokens,
page_size=self.page_size,
device=self.device,
pp_group=self.pp_group,
world_group=self.world_group,
pp_rank=get_parallel().pp_rank,
)
if sizer.profile_and_fit():
self.dynamic_chunk_sizer = sizer
@@ -1369,15 +1358,10 @@ class Scheduler(
if is_extend:
self._prefill_decode_interval_remaining = self.prefill_decode_interval
def init_metrics_reporter(
self, tp_rank: int, pp_rank: int, dp_rank: Optional[int]
) -> None:
def init_metrics_reporter(self) -> None:
# Override point for deployments that need a specialized reporter.
self.metrics_reporter = SchedulerMetricsReporter(
scheduler=self,
tp_rank=tp_rank,
pp_rank=pp_rank,
dp_rank=dp_rank,
metrics_collector_context=self.metrics_collector_context,
metrics_collector=self.metrics_collector,
)
@@ -1404,8 +1388,6 @@ class Scheduler(
)
else:
self.prefill_delayer = PrefillDelayer(
dp_size=get_parallel().dp_size,
attn_tp_size=get_parallel().attn_tp_size,
cpu_group=self.tp_cpu_group,
device_group=self.tp_group.device_group,
metrics_collector=(
@@ -1536,7 +1518,6 @@ class Scheduler(
self.disagg_decode_transfer_queue = DecodeTransferQueue(
gloo_group=self.attn_tp_cpu_group,
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
tp_rank=get_parallel().tp_rank,
metadata_buffers=self.disagg_metadata_buffers,
scheduler=self,
tree_cache=self.tree_cache,
@@ -1553,13 +1534,9 @@ class Scheduler(
transfer_queue=self.disagg_decode_transfer_queue,
tree_cache=self.tree_cache,
gloo_group=self.attn_tp_cpu_group,
tp_rank=get_parallel().tp_rank,
tp_size=get_parallel().tp_size,
dp_size=get_parallel().dp_size,
gpu_id=get_device().gpu_id,
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
max_total_num_tokens=self.max_total_num_tokens,
pp_rank=get_parallel().pp_rank,
num_reserved_decode_tokens=get_disagg().num_reserved_decode_tokens,
transfer_backend=self.transfer_backend,
)
@@ -1585,16 +1562,12 @@ class Scheduler(
draft_token_to_kv_pool=draft_token_to_kv_pool,
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
metadata_buffers=self.disagg_metadata_buffers,
tp_rank=get_parallel().tp_rank,
tp_size=get_parallel().tp_size,
gpu_id=get_device().gpu_id,
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
gloo_group=self.attn_tp_cpu_group,
max_total_num_tokens=self.max_total_num_tokens,
scheduler=self,
scheduler_stage_metrics=self.scheduler_stage_metrics,
pp_rank=get_parallel().pp_rank,
pp_size=get_parallel().pp_size,
transfer_backend=self.transfer_backend,
)
# The prefill requests that are in the middle of kv sending
@@ -2337,7 +2310,6 @@ class Scheduler(
)
self.dp_attn_adapter = SchedulerDPAttnAdapter(
model_runner=target_worker.model_runner,
tp_group=self.tp_group,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
tree_cache=self.tree_cache,
@@ -2398,10 +2370,6 @@ class Scheduler(
def init_kv_events_publisher(self) -> None:
self.kv_events_publisher = SchedulerKvEventsPublisher(
kv_events_config=get_observability().kv_events_config,
attn_tp_rank=get_parallel().attn_tp_rank,
attn_cp_rank=get_parallel().attn_cp_rank,
attn_dp_rank=get_parallel().attn_dp_rank,
dp_rank=get_parallel().dp_rank,
tree_cache=self.tree_cache,
send_metrics_from_scheduler=self.ipc_channels.send_metrics_from_scheduler,
max_running_requests=self.max_running_requests,
@@ -5954,42 +5922,37 @@ def resolve_spawn_dp_rank(dp_rank: Optional[int]) -> Optional[int]:
def configure_scheduler_process(
server_args: ServerArgs,
gpu_id: int,
tp_rank: int,
attn_cp_rank: int,
moe_dp_rank: int,
moe_ep_rank: int,
pp_rank: int,
dp_rank: Optional[int],
display_tp_rank: Optional[int] = None,
display_dp_rank: Optional[int] = None,
display_moe_ep_rank: Optional[int] = None,
) -> None:
"""Configure scheduler worker logging and process title.
display_* ranks are cosmetic; runtime ranks stay local. `dp_rank` arrives
already resolved -- see `resolve_spawn_dp_rank`.
Runs after `publish`, so every rank it labels the process with comes from
the context. display_* ranks are cosmetic; runtime ranks stay local.
"""
kill_itself_when_parent_died()
# Generate the logger prefix
shown_dp = display_dp_rank if display_dp_rank is not None else dp_rank
shown_tp = display_tp_rank if display_tp_rank is not None else tp_rank
parallel = get_parallel()
shown_dp = display_dp_rank if display_dp_rank is not None else parallel.dp_rank
shown_tp = display_tp_rank if display_tp_rank is not None else parallel.tp_rank
shown_moe_ep = (
display_moe_ep_rank if display_moe_ep_rank is not None else moe_ep_rank
display_moe_ep_rank if display_moe_ep_rank is not None else parallel.moe_ep_rank
)
prefix = ""
if shown_dp is not None:
prefix += f" DP{shown_dp}"
if get_parallel().pp_size > 1:
prefix += f" PP{pp_rank}"
if get_parallel().attn_cp_size > 1:
prefix += f" ATTN_CP{attn_cp_rank}"
if get_parallel().moe_dp_size > 1:
prefix += f" MOE_DP{moe_dp_rank}"
if get_parallel().tp_size > 1:
if parallel.pp_size > 1:
prefix += f" PP{parallel.pp_rank}"
if parallel.attn_cp_size > 1:
prefix += f" ATTN_CP{parallel.attn_cp_rank}"
if parallel.moe_dp_size > 1:
prefix += f" MOE_DP{parallel.moe_dp_rank}"
if parallel.tp_size > 1:
prefix += f" TP{shown_tp}"
if get_parallel().ep_size > 1:
if parallel.ep_size > 1:
prefix += f" EP{shown_moe_ep}"
# Config the process
@@ -6002,12 +5965,7 @@ def configure_scheduler_process(
# Set cpu affinity to this gpu process
if envs.SGLANG_SET_CPU_AFFINITY.get():
set_gpu_proc_affinity(
get_parallel().pp_size,
get_parallel().tp_size,
get_parallel().nnodes,
gpu_id,
)
set_gpu_proc_affinity(gpu_id)
if not envs.SGLANG_NUMA_BIND_V2.get():
numa_node = get_numa_node_if_available(server_args, gpu_id)
if numa_node is not None:
@@ -6019,9 +5977,6 @@ def run_scheduler_process(
port_args: PortArgs,
gpu_id: int,
tp_rank: int,
attn_cp_rank: int,
moe_dp_rank: int,
moe_ep_rank: int,
pp_rank: int,
dp_rank: Optional[int],
pipe_writer,
@@ -6032,6 +5987,8 @@ def run_scheduler_process(
# Load plugins so hooks can override Scheduler and its dependencies.
load_plugins()
dp_rank = resolve_spawn_dp_rank(dp_rank)
# Placement is needed before publish(): TP/PP select a WORLD rank;
# ordinary DP replicas have separate WORLDs and need an explicit DP rank.
publish(
server_args,
role="scheduler",
@@ -6044,12 +6001,6 @@ def run_scheduler_process(
configure_scheduler_process(
server_args,
gpu_id,
tp_rank,
attn_cp_rank,
moe_dp_rank,
moe_ep_rank,
pp_rank,
dp_rank,
display_tp_rank=display_tp_rank,
display_dp_rank=display_dp_rank,
display_moe_ep_rank=display_moe_ep_rank,
@@ -6073,13 +6024,7 @@ def run_scheduler_process(
# Create a scheduler and run the event loop
scheduler = None
try:
scheduler = Scheduler(
server_args,
port_args,
tp_rank,
pp_rank,
dp_rank,
)
scheduler = Scheduler(server_args, port_args)
# Send initialization info back to the parent process
pipe_writer.send(scheduler.get_init_info())
@@ -38,7 +38,6 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils.common import require_mlp_tp_gather
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -363,10 +362,6 @@ def _local_prefill_cuda_graph_vote(
def prepare_mlp_sync_batch_raw(
local_batch: ScheduleBatch,
model_runner: ModelRunner,
dp_size: int,
attn_tp_size: int,
attn_cp_size: int,
tp_group: GroupCoordinator,
get_idle_batch: Callable[[], ScheduleBatch],
disable_cuda_graph: bool,
require_mlp_tp_gather: bool,
@@ -374,6 +369,10 @@ def prepare_mlp_sync_batch_raw(
offload_tags: set[str],
dwdp: bool = False,
):
parallel = get_parallel()
dp_size = parallel.dp_size
attn_tp_size = parallel.attn_tp_size
tp_group = parallel.tp_group
# Check if other DP workers have running batches
if (
local_batch is None
@@ -432,9 +431,7 @@ def prepare_mlp_sync_batch_raw(
tbo_preparer = TboDPAttentionPreparer()
use_world_group = world_dp_gather_enabled()
if use_world_group:
from sglang.srt.runtime_context import get_parallel
world = get_parallel().world_group
world = parallel.world_group
group = torch.distributed.group.WORLD
device = world.device
elif len(offload_tags) == 0 and (
@@ -460,7 +457,7 @@ def prepare_mlp_sync_batch_raw(
mlp_sync_info = MLPSyncBatchInfo(
dp_size=dp_size,
tp_size=attn_tp_size,
cp_size=attn_cp_size,
cp_size=parallel.attn_cp_size,
num_tokens=num_tokens,
num_tokens_for_logprob=num_tokens_for_logprob,
can_run_decode_cuda_graph=can_run_decode_cuda_graph,
@@ -529,7 +526,6 @@ def prepare_mlp_sync_batch_raw(
@dataclass(kw_only=True, slots=True, frozen=True)
class SchedulerDPAttnAdapter:
model_runner: ModelRunner
tp_group: GroupCoordinator
req_to_token_pool: ReqToTokenPool
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator
tree_cache: BasePrefixCache
@@ -543,10 +539,6 @@ class SchedulerDPAttnAdapter:
return prepare_mlp_sync_batch_raw(
local_batch,
model_runner=self.model_runner,
dp_size=get_parallel().dp_size,
attn_tp_size=get_parallel().attn_tp_size,
attn_cp_size=get_parallel().attn_cp_size,
tp_group=self.tp_group,
get_idle_batch=self.get_idle_batch,
disable_cuda_graph=cuda_graph_fully_disabled(),
require_mlp_tp_gather=require_mlp_tp_gather(),
@@ -25,7 +25,6 @@ from sglang.srt.utils.common import get_device_module
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
@@ -51,9 +50,6 @@ class DynamicChunkSizer:
max_prefill_tokens: int,
page_size: int,
device: str,
pp_group: GroupCoordinator,
world_group: GroupCoordinator,
pp_rank: int,
):
self.model_runner = model_runner
self.model_config = model_config
@@ -65,9 +61,7 @@ class DynamicChunkSizer:
self.max_prefill_tokens = max_prefill_tokens
self.page_size = page_size
self.device = device
self.pp_group = pp_group
self.world_group = world_group
self.pp_rank = pp_rank
self.pp_rank = get_parallel().pp_rank
self.predictor = ChunkSizePredictor()
def profile_and_fit(self) -> bool:
@@ -75,7 +69,8 @@ class DynamicChunkSizer:
returns whether the predictor is ready."""
samples: Optional[Tuple[List[int], List[float]]] = None
if self.pp_group.is_first_rank:
parallel = get_parallel()
if parallel.pp_group.is_first_rank:
try:
samples = self._profile_prefill_latency()
except Exception as e:
@@ -86,8 +81,9 @@ class DynamicChunkSizer:
# The samples are global, so one broadcast from global rank 0 (a PP0 rank)
# reaches every stage and attention rank; a failure travels as None.
world_group = parallel.world_group
samples = broadcast_pyobj(
[samples], self.world_group.rank, self.world_group.cpu_group, src=0
[samples], world_group.rank, world_group.cpu_group, src=0
)[0]
if samples is None:
@@ -45,10 +45,6 @@ hook_custom_types(KvMetrics)
@dataclass(kw_only=True, slots=True)
class SchedulerKvEventsPublisher:
kv_events_config: Optional[str]
attn_tp_rank: int
attn_cp_rank: int
attn_dp_rank: int
dp_rank: Optional[int]
tree_cache: BasePrefixCache
send_metrics_from_scheduler: Optional[zmq.Socket]
max_running_requests: int
@@ -203,9 +203,6 @@ class _ForwardOccupancyLogWindow(msgspec.Struct):
@dataclass(kw_only=True)
class SchedulerMetricsReporter:
scheduler: Scheduler
tp_rank: int
pp_rank: int
dp_rank: Optional[int]
metrics_collector_context: SchedulerMetricsCollectorContext
metrics_collector: Optional[SchedulerMetricsCollector]
num_retracted_reqs: int = 0
@@ -222,7 +219,7 @@ class SchedulerMetricsReporter:
self.enable_kv_cache_events = (
self.metrics_collector_context.enable_kv_cache_events
)
self._init_metrics(self.tp_rank, self.pp_rank, self.dp_rank)
self._init_metrics()
self._install_device_timer_on_runners()
# Keep log history after the existing async result copy so reporting does
# not synchronize the model stream once per generated token.
@@ -244,12 +241,7 @@ class SchedulerMetricsReporter:
self.last_gen_throughput = 0.0
return self.last_gen_throughput
def _init_metrics(
self,
tp_rank: int,
pp_rank: int,
dp_rank: Optional[int],
):
def _init_metrics(self):
# Basic stats
self.forward_ct_decode = 0
self.num_generated_tokens = 0
@@ -18,6 +18,7 @@ except ImportError:
from sglang.srt.managers.schedule_batch import Modality
from sglang.srt.mem_cache.embedding_store import EmbeddingStore
from sglang.srt.runtime_context import get_parallel
logger = logging.getLogger(__name__)
@@ -344,20 +345,18 @@ class EmbeddingInsertOperation:
class EmbeddingCacheController:
def __init__(
self,
tp_rank,
tp_size,
embedding_store: EmbeddingStore,
max_pool_size_gb=4.0,
hidden_dims: dict = None,
tp_group=None,
all_rank_get=False,
enable_eviction: bool = True,
max_eviction_batch: int = 100,
dtype: torch.dtype = torch.float32,
):
self.tp_world_size = tp_size
self.tp_group = tp_group
self.tp_rank = tp_rank
parallel = get_parallel()
self.tp_world_size = parallel.tp_size
self.tp_group = parallel.tp_group.cpu_group
self.tp_rank = parallel.tp_rank
self.all_rank_get = all_rank_get
self.hidden_dims = hidden_dims or {}
# Pool dtype must match the model's embedding dtype so that pool views,
@@ -35,6 +35,7 @@ from sglang.srt.model_executor.input_buffers import (
INDEX_SEMANTIC_BUFFERS,
share_input_buffer,
)
from sglang.srt.runtime_context import get_parallel
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -1011,7 +1012,6 @@ def build_eager_registry(
is_encoder_decoder: bool = False,
encoder_len_fill_value: int = 0,
encoder_lens_dtype: torch.dtype = torch.int32,
dp_size: int = 1,
) -> CudaGraphBufferRegistry:
"""One fixed-max input registry for the ``EagerRunner``, serving BOTH eager
decode and eager prefill.
@@ -1042,7 +1042,7 @@ def build_eager_registry(
register_global_num_tokens=False,
require_gathered_buffer=False,
require_mlp_tp_gather=False,
dp_size=dp_size,
dp_size=get_parallel().dp_size,
share_pool=True,
source=None,
)
@@ -437,7 +437,7 @@ class ModelRunner:
self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None
# CPU offload
set_offloader(create_offloader(dp_rank=get_parallel().dp_rank))
set_offloader(create_offloader())
self._weight_checker = WeightChecker(get_model=lambda: self.model)
@@ -582,7 +582,6 @@ class ModelRunner:
def init_remote_instance_weight_transporter(self):
self.remote_instance_weight_transporter = RemoteInstanceWeightTransporter(
get_model=lambda: self.model,
tp_rank=get_parallel().tp_rank,
gpu_id=self.gpu_id,
)
@@ -653,12 +652,7 @@ class ModelRunner:
self.init_token_oracle()
self.sampler = create_sampler()
self.load_model()
prepare_moe_topk(
model=self.model,
model_config=self.model_config,
moe_ep_size=get_parallel().moe_ep_size,
moe_ep_rank=get_parallel().moe_ep_rank,
)
prepare_moe_topk(model=self.model, model_config=self.model_config)
self.maybe_init_dwdp()
@@ -754,8 +748,6 @@ class ModelRunner:
self.expert_backup_client = (
ExpertBackupClient(
model_config=self.model_config,
moe_ep_size=get_parallel().moe_ep_size,
moe_ep_rank=get_parallel().moe_ep_rank,
get_model=lambda: self.model,
)
if (
@@ -797,16 +789,12 @@ class ModelRunner:
def get_pp_proxy_topk_size(self) -> Optional[int]:
return misc_utils.resolve_pp_proxy_topk_size(
model_config=self.model_config,
pp_size=get_parallel().pp_size,
pp_rank=get_parallel().pp_rank,
start_layer=self.layer_info.start_layer,
)
def get_pp_proxy_residual_num_blocks(self) -> Optional[int]:
return misc_utils.resolve_pp_proxy_residual_num_blocks(
model_config=self.model_config,
pp_size=get_parallel().pp_size,
pp_rank=get_parallel().pp_rank,
start_layer=self.layer_info.start_layer,
)
@@ -974,7 +962,6 @@ class ModelRunner:
swap_in_block_size=hisparse_cfg.swap_in_block_size,
shared_index_layers=resolve_shared_index_layers(
hf_text_config=self.model_config.hf_text_config,
pp_size=get_parallel().pp_size,
is_speculative=self.spec_algorithm.is_speculative(),
),
)
@@ -12,6 +12,7 @@ from sglang.srt.runtime_context import (
attention_backends,
get_context,
get_observability,
get_parallel,
get_schedule,
)
from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS
@@ -66,12 +67,13 @@ def create_msprobe_debugger() -> Optional[Any]:
def resolve_pp_proxy_topk_size(
*, model_config: ModelConfig, pp_size: int, pp_rank: int, start_layer: int
*, model_config: ModelConfig, start_layer: int
) -> Optional[int]:
hf_config = model_config.hf_text_config
parallel = get_parallel()
if (
pp_size <= 1
or pp_rank == 0
parallel.pp_size <= 1
or parallel.pp_rank == 0
or not is_deepseek_dsa(hf_config)
or not dsa_layer_skips_topk(hf_config, start_layer)
):
@@ -80,10 +82,15 @@ def resolve_pp_proxy_topk_size(
def resolve_pp_proxy_residual_num_blocks(
*, model_config: ModelConfig, pp_size: int, pp_rank: int, start_layer: int
*, model_config: ModelConfig, start_layer: int
) -> Optional[int]:
"""Return the inherited Kimi K3 attention-residual bank width."""
if pp_size <= 1 or pp_rank == 0 or not is_kimi_k3(model_config.hf_config):
parallel = get_parallel()
if (
parallel.pp_size <= 1
or parallel.pp_rank == 0
or not is_kimi_k3(model_config.hf_config)
):
return None
block_size = getattr(model_config.hf_text_config, "attn_res_block_size", None)
@@ -13,7 +13,7 @@ from sglang.srt.eplb.lplb_solver import (
)
from sglang.srt.layers.moe.hash_topk import HashTopK
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.runtime_context import get_exec
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0
if TYPE_CHECKING:
@@ -24,13 +24,8 @@ logger = logging.getLogger(__name__)
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
def prepare_moe_topk(
*,
model,
model_config: ModelConfig,
moe_ep_size: int,
moe_ep_rank: int,
) -> None:
def prepare_moe_topk(*, model, model_config: ModelConfig) -> None:
parallel = get_parallel()
balancer_cls = None
num_prepared = 0
num_routed_experts = None
@@ -61,8 +56,8 @@ def prepare_moe_topk(
routed_scaling_factor = module.routed_scaling_factor
module.waterfill_balancer = balancer_cls(
num_routed_experts=num_physical_routed_experts,
world_size=moe_ep_size,
rank=moe_ep_rank,
world_size=parallel.moe_ep_size,
rank=parallel.moe_ep_rank,
layer_id=module.layer_id,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
@@ -1,7 +1,7 @@
from __future__ import annotations
import logging
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any, Callable, Optional
import torch
@@ -24,7 +24,8 @@ logger = logging.getLogger(__name__)
@dataclass(slots=True, kw_only=True)
class RemoteInstanceWeightTransporter:
get_model: Callable[[], torch.nn.Module]
tp_rank: int
# Registration may run after the runner's construction scope has exited.
tp_rank: int = field(init=False, default_factory=lambda: get_parallel().tp_rank)
gpu_id: int
engine: Optional[Any] = None
session_id: str = ""
@@ -76,8 +76,6 @@ def _allocate_decode_buffers(
hidden_size: int,
vocab_size: int,
dtype: torch.dtype,
dp_size: int,
pp_size: int,
is_encoder_decoder: bool,
require_mlp_tp_gather: bool,
seq_len_fill_value: int,
@@ -92,6 +90,7 @@ def _allocate_decode_buffers(
allocate_logits_buffer: bool = True,
) -> SimpleNamespace:
"""Allocate the FB-shared decode buffers."""
parallel = get_parallel()
with torch.device(device):
input_ids = torch.zeros((max_num_token,), dtype=torch.int64)
input_embeds = torch.zeros((max_num_token, hidden_size), dtype=dtype)
@@ -127,7 +126,7 @@ def _allocate_decode_buffers(
torch.zeros((max_bs,), dtype=torch.bool) if enable_mamba_track else None
)
if pp_size > 1:
if parallel.pp_size > 1:
# mHC (e.g. DSV4) flattens residual into hidden_states (size = hc_hidden_size).
is_mhc = hc_hidden_size is not None
hs = hc_hidden_size if is_mhc else hidden_size
@@ -163,9 +162,9 @@ def _allocate_decode_buffers(
encoder_lens = None
if require_mlp_tp_gather:
global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32)
global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32)
global_num_tokens_for_logprob_gpu = torch.zeros(
(dp_size,), dtype=torch.int32
(parallel.dp_size,), dtype=torch.int32
)
else:
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
@@ -367,8 +366,6 @@ class BaseRunner(ABC):
hidden_size=mr.model_config.hidden_size,
vocab_size=mr.model_config.vocab_size,
dtype=mr.model_config.dtype,
dp_size=get_parallel().dp_size,
pp_size=get_parallel().pp_size,
is_encoder_decoder=mr.model_config.is_encoder_decoder,
require_mlp_tp_gather=require_mlp_tp_gather(),
seq_len_fill_value=mr.attn_backend.get_cuda_graph_seq_len_fill_value(),
@@ -849,7 +849,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
export_cuda_graph_capture_trace(
prof_context,
runner_name=type(self).__name__,
tp_rank=get_parallel().tp_rank,
)
def capture_prepare(
@@ -59,7 +59,6 @@ from sglang.srt.model_executor.runner_utils import (
)
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
get_spec,
max_prefill_buffer_tokens,
max_speculative_num_draft_tokens,
@@ -150,7 +149,6 @@ class EagerRunner(BaseRunner):
encoder_lens_dtype=(
torch.int64 if torch.device(mr.device).type == "cpu" else torch.int32
),
dp_size=get_parallel().dp_size,
)
# Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up).
if run_warmup:
-1
View File
@@ -117,7 +117,6 @@ class GraniteMoeMoE(nn.Module):
layer_id: int,
params_dtype: Optional[torch.dtype] = None,
quant_config: Optional[QuantizationConfig] = None,
tp_size: Optional[int] = None,
prefix: str = "",
):
super().__init__()
@@ -66,7 +66,6 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
layer_id=layer_idx,
quant_config=quant_config,
tp_size=get_parallel().tp_size,
prefix=f"{prefix}.block_sparse_moe",
)
@@ -252,7 +251,6 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
layer_id=layer_idx,
quant_config=quant_config,
tp_size=get_parallel().tp_size,
prefix=f"{prefix}.block_sparse_moe",
)
+6 -12
View File
@@ -713,12 +713,12 @@ def _build_inkling_shared_experts(
intermediate_size: int,
layer_id: int,
prefix: str,
moe_tp_rank: int,
moe_tp_size: int,
quant_config: QuantizationConfig | None = None,
) -> nn.Module | None:
if n_shared_experts <= 0:
return None
# The shared dense MLP is reconstructed by the full-TP all-reduce in forward.
parallel = get_parallel()
if shared_expert_sink:
shared_prefix = add_prefix("shared_experts", prefix)
shared_sink_serves_fp4 = InklingBatchDenseMLP._resolve_fp4_strategy(
@@ -729,8 +729,6 @@ def _build_inkling_shared_experts(
shared_sink_serves_fp4=shared_sink_serves_fp4,
)
if use_fused_shared:
# moe_tp_rank/moe_tp_size are derived internally; kept as args only
# for the legacy and non-sink InklingDenseMLP paths below.
return InklingSharedFusedMoE(
n_shared_experts=n_shared_experts,
hidden_size=hidden_size,
@@ -748,8 +746,8 @@ def _build_inkling_shared_experts(
prefix=shared_prefix,
quant_config=quant_config,
inference_moe_w13_interleaved=inference_moe_w13_interleaved,
tp_rank=moe_tp_rank,
tp_size=moe_tp_size,
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
tp_group=get_parallel().tp_group,
)
return InklingBatchDenseMLP(
@@ -767,8 +765,8 @@ def _build_inkling_shared_experts(
layer_id=layer_id,
prefix=add_prefix("shared_experts", prefix),
quant_config=quant_config,
tp_rank=moe_tp_rank,
tp_size=moe_tp_size,
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
tp_group=get_parallel().tp_group,
)
@@ -870,10 +868,6 @@ class InklingMoE(nn.Module):
intermediate_size=self.intermediate_dim,
layer_id=layer_id,
prefix=prefix,
# Shared expert is a replicated dense MLP: shard over the full tp group, not
# moe_tp (the single full-tp all_reduce in forward() reconstructs it).
moe_tp_rank=get_parallel().tp_rank,
moe_tp_size=get_parallel().tp_size,
quant_config=self.quant_config,
)
if isinstance(self.shared_experts, InklingSharedFusedMoE):
+3 -3
View File
@@ -646,10 +646,9 @@ class TransformersBase(nn.Module):
# Pipeline parallel
self.pipeline_parallel()
# Module replacement (Linear → TP, RMSNorm → fused, MoE overridden by MoEMixin)
tp_size = get_parallel().tp_size
self.recursive_replace()
# Attention instances
self.attention_instances = self._create_attention_instances(tp_size)
self.attention_instances = self._create_attention_instances()
# Vocab embeddings
self.replace_vocab_embed_class(self.model)
@@ -902,7 +901,8 @@ class TransformersBase(nn.Module):
self._register_missing_prefix(maybe_prefix("model", name))
# -- Attention instances ------------------------------------------------
def _create_attention_instances(self, tp_size: int) -> dict[int, RadixAttention]:
def _create_attention_instances(self) -> dict[int, RadixAttention]:
tp_size = get_parallel().tp_size
num_heads = self.text_config.num_attention_heads
num_kv_heads = getattr(self.text_config, "num_key_value_heads", num_heads)
hidden_size = self.text_config.hidden_size
@@ -1103,9 +1103,6 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
cls,
*,
server_args: ServerArgs,
tp_rank: int,
pp_rank: int,
dp_rank: Optional[int],
enable_priority_scheduling: bool,
enable_lora: bool,
enable_hierarchical_cache: bool,
@@ -1134,14 +1131,14 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
labels = {
"model_name": get_serving().served_model_name,
"engine_type": engine_type,
"tp_rank": tp_rank,
"pp_rank": pp_rank,
"tp_rank": parallel.tp_rank,
"pp_rank": parallel.pp_rank,
"moe_ep_rank": parallel.moe_ep_rank,
}
if enable_priority_scheduling:
labels["priority"] = ""
if dp_rank is not None:
labels["dp_rank"] = dp_rank
if parallel.dp_rank is not None:
labels["dp_rank"] = parallel.dp_rank
if get_observability().extra_metric_labels:
labels.update(get_observability().extra_metric_labels)
scheduler_collector_cls = resolve_collector_class(
@@ -146,10 +146,7 @@ class RayDataParallelController(DataParallelController):
for node_idx in range(nnodes):
bundle_idx = self.bundle_for_node[node_idx]
pp_range, tp_range, pp_per_node, tp_per_node = _calculate_rank_ranges(
nnodes,
get_parallel().pp_size,
get_parallel().tp_size,
node_rank=node_idx,
node_rank=node_idx
)
for pp_rank in pp_range:
for tp_rank in tp_range:
+1 -12
View File
@@ -29,7 +29,6 @@ from sglang.srt.entrypoints.engine import (
Engine,
SchedulerInitResult,
_calculate_rank_ranges,
_compute_parallelism_ranks,
)
from sglang.srt.environ import envs
from sglang.srt.ray.scheduler_actor import SchedulerActor
@@ -204,8 +203,6 @@ def _create_scheduler_actor(
rank0_node_ip: IP of rank-0's node, used for NCCL rendezvous.
dist_init_addr: Distributed init address (tcp://rank0_node_ip:nccl_port).
"""
attn_cp_rank, moe_dp_rank, moe_ep_rank = _compute_parallelism_ranks(tp_rank)
return SchedulerActor.options(
num_cpus=0,
num_gpus=1,
@@ -223,9 +220,6 @@ def _create_scheduler_actor(
port_args=port_args,
gpu_id=gpu_id,
tp_rank=tp_rank,
attn_cp_rank=attn_cp_rank,
moe_dp_rank=moe_dp_rank,
moe_ep_rank=moe_ep_rank,
pp_rank=pp_rank,
dp_rank=dp_rank,
dist_init_addr=dist_init_addr,
@@ -340,12 +334,7 @@ class RayEngine(Engine):
for node_idx in range(nnodes):
bundle_idx = bundle_for_node[node_idx]
pp_range, tp_range, pp_per_node, tp_per_node = (
_calculate_rank_ranges(
nnodes,
get_parallel().pp_size,
get_parallel().tp_size,
node_rank=node_idx,
)
_calculate_rank_ranges(node_rank=node_idx)
)
for pp_rank in pp_range:
for tp_rank in tp_range:
+1 -16
View File
@@ -41,9 +41,6 @@ class SchedulerActor:
port_args: PortArgs,
gpu_id: int,
tp_rank: int,
attn_cp_rank: int,
moe_dp_rank: int,
moe_ep_rank: int,
pp_rank: int,
dp_rank: Optional[int],
dist_init_addr: Optional[str] = None,
@@ -98,16 +95,7 @@ class SchedulerActor:
)
# Configure worker (logging, process title, etc.)
configure_scheduler_process(
server_args,
actual_gpu_id,
tp_rank,
attn_cp_rank,
moe_dp_rank,
moe_ep_rank,
pp_rank,
dp_rank,
)
configure_scheduler_process(server_args, actual_gpu_id)
# Ray actors can't use the numactl subprocess-wrapping approach
# (SGLANG_NUMA_BIND_V2's normal path), so bind in-process via libnuma.
@@ -125,9 +113,6 @@ class SchedulerActor:
self.scheduler = Scheduler(
server_args=server_args,
port_args=port_args,
tp_rank=tp_rank,
pp_rank=pp_rank,
dp_rank=dp_rank,
)
self._tp_rank = tp_rank
+4 -8
View File
@@ -2962,18 +2962,14 @@ def direct_register_custom_op(
raise error
def set_gpu_proc_affinity(
pp_size: int,
tp_size: int,
nnodes: int,
gpu_id: int,
):
def set_gpu_proc_affinity(gpu_id: int):
# current process
pid = os.getpid()
p = psutil.Process(pid)
nnodes_per_tp_group = max(nnodes // pp_size, 1)
tp_size_per_node = tp_size // nnodes_per_tp_group
parallel = get_parallel()
nnodes_per_tp_group = max(parallel.nnodes // parallel.pp_size, 1)
tp_size_per_node = parallel.tp_size // nnodes_per_tp_group
# total physical cores
total_pcores = psutil.cpu_count(logical=False)
+3 -5
View File
@@ -65,7 +65,7 @@ def set_offloader(instance: BaseOffloader):
_instance = instance
def create_offloader(dp_rank: int):
def create_offloader():
if get_exec().offload.cpu_offload_gb > 0:
return OffloaderV1(
cpu_offload_max_bytes=int(get_exec().offload.cpu_offload_gb * 1024**3)
@@ -79,8 +79,6 @@ def create_offloader(dp_rank: int):
num_in_group=get_exec().offload.offload_num_in_group,
prefetch_step=get_exec().offload.offload_prefetch_step,
mode=get_exec().offload.offload_mode,
dp_rank=dp_rank,
dp_size=get_parallel().dp_size,
)
return NoopOffloader()
@@ -210,9 +208,9 @@ class OffloaderV2(BaseOffloader):
num_in_group: int,
prefetch_step: int,
mode: str,
dp_rank: int,
dp_size: int,
):
parallel = get_parallel()
dp_rank, dp_size = parallel.dp_rank, parallel.dp_size
self.group_size = group_size
self.num_in_group = num_in_group
self.prefetch_step = prefetch_step
+2 -1
View File
@@ -48,7 +48,7 @@ def graph_capture_profile_dir() -> str:
)
def export_cuda_graph_capture_trace(prof_context, *, runner_name: str, tp_rank: int):
def export_cuda_graph_capture_trace(prof_context, *, runner_name: str):
"""Persist a CUDA-graph capture profiler trace (chrome trace) to disk.
Opt-in via ``SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE`` (no-op otherwise). The
@@ -60,6 +60,7 @@ def export_cuda_graph_capture_trace(prof_context, *, runner_name: str, tp_rank:
"""
if not envs.SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE.get():
return
tp_rank = get_parallel().tp_rank
output_dir = graph_capture_profile_dir()
os.makedirs(output_dir, exist_ok=True)
path = os.path.join(