[Refactor] Read parallel placement in consumers (#40638)
This commit is contained in:
@@ -95,7 +95,6 @@ from sglang.srt.runtime_context import (
|
||||
SpawnRanks,
|
||||
get_device,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
publish,
|
||||
spawn_world_rank,
|
||||
@@ -543,10 +542,6 @@ def _maybe_prepare_mlp_sync_batch(batch: ScheduleBatch, model_runner):
|
||||
prepare_mlp_sync_batch_raw(
|
||||
batch,
|
||||
model_runner=model_runner,
|
||||
dp_size=get_parallel().dp_size,
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
attn_cp_size=model_runner.attn_cp_size,
|
||||
tp_group=model_runner.tp_group,
|
||||
get_idle_batch=None,
|
||||
disable_cuda_graph=cuda_graph_fully_disabled(),
|
||||
require_mlp_tp_gather=require_mlp_tp_gather(),
|
||||
@@ -908,13 +903,7 @@ def latency_test(
|
||||
initialize_fp4_gemm_config()
|
||||
|
||||
if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
|
||||
parallel = get_parallel()
|
||||
set_gpu_proc_affinity(
|
||||
parallel.pp_size,
|
||||
parallel.tp_size,
|
||||
parallel.nnodes,
|
||||
tp_rank,
|
||||
)
|
||||
set_gpu_proc_affinity(tp_rank)
|
||||
|
||||
# Configure the logger
|
||||
configure_logger(server_args, prefix=f" TP{tp_rank}")
|
||||
|
||||
@@ -365,16 +365,13 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
transfer_queue: DecodeTransferQueue,
|
||||
tree_cache: BasePrefixCache,
|
||||
gloo_group: ProcessGroup,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
dp_size: int,
|
||||
gpu_id: int,
|
||||
bootstrap_port: int,
|
||||
max_total_num_tokens: int,
|
||||
pp_rank: int,
|
||||
num_reserved_decode_tokens: int,
|
||||
transfer_backend: TransferBackend,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
self.req_to_token_pool = req_to_token_pool
|
||||
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
|
||||
self.token_to_kv_pool = token_to_kv_pool_allocator.get_kvcache()
|
||||
@@ -388,14 +385,14 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.gloo_group = gloo_group
|
||||
# Destinations visible to prefill but not yet on the transfer queue.
|
||||
self._num_published_destinations = 0
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = tp_size
|
||||
self.dp_size = dp_size
|
||||
self.tp_rank = parallel.tp_rank
|
||||
self.tp_size = parallel.tp_size
|
||||
self.dp_size = parallel.dp_size
|
||||
self.gpu_id = gpu_id
|
||||
self.bootstrap_port = bootstrap_port
|
||||
self.max_total_num_tokens = max_total_num_tokens
|
||||
self.pp_rank = pp_rank
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.pp_rank = parallel.pp_rank
|
||||
self.pp_size = parallel.pp_size
|
||||
self.num_reserved_decode_tokens = num_reserved_decode_tokens
|
||||
self.transfer_backend = transfer_backend
|
||||
# Queue for requests pending pre-allocation
|
||||
@@ -2200,7 +2197,6 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self,
|
||||
gloo_group: ProcessGroup,
|
||||
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
|
||||
tp_rank: int,
|
||||
metadata_buffers: MetadataBuffers,
|
||||
scheduler: Scheduler,
|
||||
tree_cache: BasePrefixCache,
|
||||
@@ -2208,7 +2204,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self.queue: List[DecodeRequest] = []
|
||||
self.gloo_group = gloo_group
|
||||
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.metadata_buffers = metadata_buffers
|
||||
self.scheduler = scheduler
|
||||
self.tree_cache = tree_cache
|
||||
|
||||
@@ -436,11 +436,11 @@ class EncoderPreprocessor:
|
||||
vr,
|
||||
video_config,
|
||||
*,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
video_processor_kwargs: dict,
|
||||
precomputed_indices: Optional[List[int]] = None,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
tp_rank, tp_size = parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
video_config = video_config or {}
|
||||
video_fps = vr.avg_fps
|
||||
duration = len(vr) / video_fps if video_fps else 0
|
||||
@@ -596,8 +596,6 @@ class EncoderPreprocessor:
|
||||
result = await self._dp_sharded_decode_single_video(
|
||||
video_items[0],
|
||||
video_configs[0],
|
||||
tp_rank=parallel.attn_tp_rank,
|
||||
tp_size=tp_size,
|
||||
video_processor_kwargs=video_processor_kwargs,
|
||||
precomputed_indices=sampled,
|
||||
)
|
||||
|
||||
@@ -670,11 +670,8 @@ class MMEncoder:
|
||||
get_mm().mm_global_cache_backend,
|
||||
)
|
||||
self.mm_global_cache = EmbeddingCacheController(
|
||||
rank,
|
||||
get_parallel().tp_size,
|
||||
embedding_store=embedding_store,
|
||||
hidden_dims=self._embedding_dims,
|
||||
tp_group=get_parallel().tp_group.cpu_group,
|
||||
all_rank_get=False,
|
||||
dtype=self._embedding_dtype,
|
||||
)
|
||||
|
||||
@@ -164,27 +164,24 @@ class PrefillBootstrapQueue:
|
||||
draft_token_to_kv_pool: Optional[KVCache],
|
||||
req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator,
|
||||
metadata_buffers: MetadataBuffers,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
gpu_id: int,
|
||||
bootstrap_port: int,
|
||||
gloo_group: ProcessGroup,
|
||||
max_total_num_tokens: int,
|
||||
scheduler: Scheduler,
|
||||
scheduler_stage_metrics: SchedulerStageMetricsRecorder,
|
||||
pp_rank: int,
|
||||
pp_size: int,
|
||||
transfer_backend: TransferBackend,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
self.token_to_kv_pool = token_to_kv_pool
|
||||
self.draft_token_to_kv_pool = draft_token_to_kv_pool
|
||||
self.is_mla_backend = is_mla_backend(token_to_kv_pool)
|
||||
self.metadata_buffers = metadata_buffers
|
||||
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = tp_size
|
||||
self.pp_rank = pp_rank
|
||||
self.pp_size = pp_size
|
||||
self.tp_rank = parallel.tp_rank
|
||||
self.tp_size = parallel.tp_size
|
||||
self.pp_rank = parallel.pp_rank
|
||||
self.pp_size = parallel.pp_size
|
||||
self.gpu_id = gpu_id
|
||||
self.bootstrap_port = bootstrap_port
|
||||
self.queue: List[Req] = []
|
||||
|
||||
@@ -142,8 +142,6 @@ def init_parallel_runtime(
|
||||
parallel = get_parallel()
|
||||
if device == "cpu":
|
||||
_init_cpu_threads_env(
|
||||
tp_size=parallel.tp_size,
|
||||
tp_rank=parallel.tp_rank,
|
||||
local_omp_cpuid=local_omp_cpuid,
|
||||
dist_init_method=dist_init_method,
|
||||
)
|
||||
@@ -160,11 +158,7 @@ def init_parallel_runtime(
|
||||
if get_exec().comm.pre_warm_nccl and (
|
||||
parallel.tp_size > 1 or parallel.pp_size > 1 or parallel.moe_ep_size > 1
|
||||
):
|
||||
_prewarm_nccl(
|
||||
tp_size=parallel.tp_size,
|
||||
pp_size=parallel.pp_size,
|
||||
moe_ep_size=parallel.moe_ep_size,
|
||||
)
|
||||
_prewarm_nccl()
|
||||
|
||||
# CUDA graph capture enables the PyNCCL communicator for TP LM-head
|
||||
# all-to-all. Exercise that exact send/recv path before measuring
|
||||
@@ -262,17 +256,16 @@ def _set_shm_master_env(dist_init_method: Optional[str]) -> None:
|
||||
|
||||
def _init_cpu_threads_env(
|
||||
*,
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
local_omp_cpuid: Optional[List[int]],
|
||||
dist_init_method: Optional[str] = None,
|
||||
) -> None:
|
||||
if _is_cpu_amx_available or _is_cpu_arm64:
|
||||
parallel = get_parallel()
|
||||
# Bind OpenMP threads to CPU cores
|
||||
torch.ops.sgl_kernel.init_cpu_threads_env(local_omp_cpuid)
|
||||
|
||||
# Set local size to hint SGLang to use shared memory based AllReduce
|
||||
os.environ["LOCAL_SIZE"] = str(tp_size)
|
||||
os.environ["LOCAL_SIZE"] = str(parallel.tp_size)
|
||||
|
||||
# shm.cpp names its /dev/shm segments from MASTER_ADDR/MASTER_PORT.
|
||||
# Feed each engine's unique dist_init_method (tcp://host:port) into
|
||||
@@ -280,7 +273,7 @@ def _init_cpu_threads_env(
|
||||
# don't collide.
|
||||
_set_shm_master_env(dist_init_method)
|
||||
|
||||
torch.ops.sgl_kernel.initialize(tp_size, tp_rank)
|
||||
torch.ops.sgl_kernel.initialize(parallel.tp_size, parallel.tp_rank)
|
||||
|
||||
else:
|
||||
logger.warning(
|
||||
@@ -338,7 +331,8 @@ def init_layer_runtime(*, model_config: ModelConfig) -> None:
|
||||
initialize_layernorm_sp(model_config=model_config)
|
||||
|
||||
|
||||
def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None:
|
||||
def _prewarm_nccl() -> None:
|
||||
parallel = get_parallel()
|
||||
warmup_start = time.perf_counter()
|
||||
tp_group_handle = get_tp_group().device_group
|
||||
|
||||
@@ -350,7 +344,8 @@ def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None:
|
||||
warmup_elapsed = time.perf_counter() - warmup_start
|
||||
logger.info(
|
||||
f"NCCL/RCCL/HCCL warmup completed in {warmup_elapsed:.3f}s "
|
||||
f"(tp_size={tp_size}, pp_size={pp_size}, ep_size={moe_ep_size})"
|
||||
f"(tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, "
|
||||
f"ep_size={parallel.moe_ep_size})"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -30,19 +30,18 @@ class ExpertBackupClient:
|
||||
self,
|
||||
*,
|
||||
model_config,
|
||||
moe_ep_size: int,
|
||||
moe_ep_rank: int,
|
||||
get_model: Callable[[], Any],
|
||||
):
|
||||
context = zmq.Context(2)
|
||||
self.engine_num = get_parallel().nnodes
|
||||
self.engine_rank = get_parallel().node_rank
|
||||
parallel = get_parallel()
|
||||
self.engine_num = parallel.nnodes
|
||||
self.engine_rank = parallel.node_rank
|
||||
self.recv_list = [None] * self.engine_num
|
||||
self.ready_sockets = [None] * self.engine_num
|
||||
self._get_model = get_model
|
||||
self.moe_ep_size = moe_ep_size
|
||||
self.moe_ep_size = parallel.moe_ep_size
|
||||
self.model_config = model_config
|
||||
self.moe_ep_rank = moe_ep_rank
|
||||
self.moe_ep_rank = parallel.moe_ep_rank
|
||||
self.dram_map_list = [None] * self.engine_num
|
||||
self.session_id_list = [None] * self.engine_num
|
||||
self.transfer_engine = None
|
||||
|
||||
@@ -730,15 +730,8 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
"--dist-init-addr so all nodes rendezvous at the same endpoint."
|
||||
)
|
||||
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = (
|
||||
_calculate_rank_ranges(
|
||||
get_parallel().nnodes,
|
||||
get_parallel().pp_size,
|
||||
tp_size,
|
||||
get_parallel().node_rank,
|
||||
)
|
||||
_calculate_rank_ranges(get_parallel().node_rank)
|
||||
)
|
||||
|
||||
# Build the distributed init method (multi-node uses the user-provided
|
||||
@@ -902,12 +895,7 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
scheduler_pipe_readers = []
|
||||
|
||||
pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = (
|
||||
_calculate_rank_ranges(
|
||||
get_parallel().nnodes,
|
||||
get_parallel().pp_size,
|
||||
get_parallel().tp_size,
|
||||
get_parallel().node_rank,
|
||||
)
|
||||
_calculate_rank_ranges(get_parallel().node_rank)
|
||||
)
|
||||
|
||||
for pp_rank in pp_rank_range:
|
||||
@@ -918,9 +906,6 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
|
||||
+ (tp_rank % tp_size_per_node) * get_device().gpu_id_step
|
||||
)
|
||||
attn_cp_rank, moe_dp_rank, moe_ep_rank = _compute_parallelism_ranks(
|
||||
tp_rank
|
||||
)
|
||||
|
||||
with maybe_reindex_device_id(gpu_id) as gpu_id:
|
||||
proc = mp.Process(
|
||||
@@ -930,9 +915,6 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
port_args,
|
||||
gpu_id,
|
||||
tp_rank,
|
||||
attn_cp_rank,
|
||||
moe_dp_rank,
|
||||
moe_ep_rank,
|
||||
pp_rank,
|
||||
None,
|
||||
writer,
|
||||
@@ -1925,15 +1907,13 @@ def _wait_for_scheduler_ready(
|
||||
return scheduler_infos
|
||||
|
||||
|
||||
def _calculate_rank_ranges(
|
||||
nnodes: int, pp_size: int, tp_size: int, node_rank: int
|
||||
) -> Tuple[range, range, int, int]:
|
||||
def _calculate_rank_ranges(node_rank: int) -> Tuple[range, range, int, int]:
|
||||
"""Calculate pp_rank_range and tp_rank_range for a given node.
|
||||
|
||||
`node_rank` stays an argument because the Ray launchers size every node
|
||||
from the driver, not just their own.
|
||||
|
||||
Args:
|
||||
nnodes: Total number of nodes.
|
||||
pp_size: Pipeline parallel size.
|
||||
tp_size: Tensor parallel size.
|
||||
node_rank: The rank of the node to compute ranges for.
|
||||
|
||||
Returns:
|
||||
@@ -1943,15 +1923,17 @@ def _calculate_rank_ranges(
|
||||
- pp_size_per_node: number of PP ranks per node.
|
||||
- tp_size_per_node: number of TP ranks per node.
|
||||
"""
|
||||
pp_size_per_node = max(pp_size // nnodes, 1)
|
||||
nnodes_per_pp_rank = max(nnodes // pp_size, 1)
|
||||
parallel = get_parallel()
|
||||
nnodes = parallel.nnodes
|
||||
pp_size_per_node = max(parallel.pp_size // nnodes, 1)
|
||||
nnodes_per_pp_rank = max(nnodes // parallel.pp_size, 1)
|
||||
pp_rank_range = range(
|
||||
pp_size_per_node * (node_rank // nnodes_per_pp_rank),
|
||||
pp_size_per_node * (node_rank // nnodes_per_pp_rank + 1),
|
||||
)
|
||||
|
||||
nnodes_per_tp_group = nnodes_per_pp_rank
|
||||
tp_size_per_node = tp_size // nnodes_per_tp_group
|
||||
tp_size_per_node = parallel.tp_size // nnodes_per_tp_group
|
||||
tp_rank_range = range(
|
||||
tp_size_per_node * (node_rank % nnodes_per_tp_group),
|
||||
tp_size_per_node * (node_rank % nnodes_per_tp_group + 1),
|
||||
@@ -1963,12 +1945,7 @@ def _calculate_rank_ranges(
|
||||
def node_hosts_rust_server() -> bool:
|
||||
"""Whether this node contains a Rust listener rank, assuming Rust mode."""
|
||||
parallel = get_parallel()
|
||||
pp_rank_range, tp_rank_range, _, _ = _calculate_rank_ranges(
|
||||
parallel.nnodes,
|
||||
parallel.pp_size,
|
||||
parallel.tp_size,
|
||||
parallel.node_rank,
|
||||
)
|
||||
pp_rank_range, tp_rank_range, _, _ = _calculate_rank_ranges(parallel.node_rank)
|
||||
if 0 not in pp_rank_range:
|
||||
return False
|
||||
|
||||
@@ -1983,28 +1960,3 @@ def node_hosts_rust_server() -> bool:
|
||||
if rank_within_dp_group == 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _compute_parallelism_ranks(tp_rank: int) -> Tuple[int, int, int]:
|
||||
"""Compute attention-CP, MoE-DP, and MoE-EP ranks for a TP rank.
|
||||
|
||||
Called while the launcher is deciding what to spawn, so the sizes are the
|
||||
configured ones -- the groups this is laying out do not exist yet.
|
||||
"""
|
||||
attn_dp_size = get_parallel().dp_size if get_parallel().enable_dp_attention else 1
|
||||
tp_size = get_parallel().tp_size
|
||||
attn_cp_size = get_parallel().attn_cp_size
|
||||
moe_dp_size = get_parallel().moe_dp_size
|
||||
|
||||
# Parallelism hierarchy (outermost to innermost):
|
||||
# - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost)
|
||||
# - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost)
|
||||
attn_tp_size = tp_size // attn_dp_size // attn_cp_size
|
||||
attn_cp_rank = (tp_rank // attn_tp_size) % attn_cp_size
|
||||
moe_dp_rank = tp_rank // (tp_size // moe_dp_size)
|
||||
moe_ep_rank = (
|
||||
tp_rank
|
||||
% (tp_size // moe_dp_size)
|
||||
// (tp_size // moe_dp_size // get_parallel().ep_size)
|
||||
)
|
||||
return attn_cp_rank, moe_dp_rank, moe_ep_rank
|
||||
|
||||
@@ -41,16 +41,16 @@ def _accelerator_name() -> Optional[str]:
|
||||
return get_device_name()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _num_accelerators_per_dp_rank(
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
dp_size: int,
|
||||
enable_dp_attention: bool,
|
||||
) -> int:
|
||||
num_accelerators = tp_size * pp_size
|
||||
if enable_dp_attention:
|
||||
num_accelerators //= dp_size
|
||||
def _num_accelerators_per_dp_rank() -> int:
|
||||
"""Accelerators behind one DP rank.
|
||||
|
||||
Not cached: the arguments were the cache key, so an ``lru_cache`` here
|
||||
would freeze the first answer past a post-publish override.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
num_accelerators = parallel.tp_size * parallel.pp_size
|
||||
if parallel.enable_dp_attention:
|
||||
num_accelerators //= parallel.dp_size
|
||||
return num_accelerators
|
||||
|
||||
|
||||
@@ -145,11 +145,6 @@ async def get_loads(
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"version": __version__,
|
||||
"accelerator": _accelerator_name(),
|
||||
"num_accelerators": _num_accelerators_per_dp_rank(
|
||||
get_parallel().tp_size,
|
||||
get_parallel().pp_size,
|
||||
get_parallel().dp_size,
|
||||
get_parallel().enable_dp_attention,
|
||||
),
|
||||
"num_accelerators": _num_accelerators_per_dp_rank(),
|
||||
"loads": loads,
|
||||
}
|
||||
|
||||
@@ -157,7 +157,6 @@ class EPLBManager:
|
||||
model=self._get_model(),
|
||||
new_expert_location_metadata=expert_location_metadata,
|
||||
update_layer_ids=chunk_layer_ids,
|
||||
nnodes=get_parallel().nnodes,
|
||||
tp_rank=(
|
||||
self._elastic_global_rank()
|
||||
if is_post_scale_rebalance
|
||||
@@ -302,7 +301,6 @@ def update_expert_location_with_recovery(
|
||||
model: nn.Module,
|
||||
new_expert_location_metadata: ExpertLocationMetadata,
|
||||
update_layer_ids: List[int],
|
||||
nnodes: int,
|
||||
tp_rank: int,
|
||||
use_flat_topology: bool = False,
|
||||
expert_backup_client,
|
||||
@@ -314,7 +312,7 @@ def update_expert_location_with_recovery(
|
||||
model.routed_experts_weights_of_layer,
|
||||
new_expert_location_metadata,
|
||||
update_layer_ids=update_layer_ids,
|
||||
nnodes=nnodes,
|
||||
nnodes=get_parallel().nnodes,
|
||||
rank=tp_rank,
|
||||
use_flat_topology=use_flat_topology,
|
||||
)
|
||||
|
||||
@@ -1089,9 +1089,7 @@ class LogitsProcessor(nn.Module):
|
||||
get_parallel().tp_group.all_to_all_single(
|
||||
all_to_all_output.view(-1), logits.view(-1)
|
||||
)
|
||||
return _reassemble_tp_lm_head_all_to_all_output(
|
||||
all_to_all_output, get_parallel().tp_size
|
||||
)
|
||||
return _reassemble_tp_lm_head_all_to_all_output(all_to_all_output)
|
||||
|
||||
def _scatter_dp_attn_logits(
|
||||
self,
|
||||
@@ -1252,7 +1250,7 @@ class LogitsProcessor(nn.Module):
|
||||
|
||||
|
||||
def _reassemble_tp_lm_head_all_to_all_output(
|
||||
all_to_all_output: torch.Tensor, tp_size: int
|
||||
all_to_all_output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Convert source-major all-to-all output to row-major full-vocab logits.
|
||||
|
||||
@@ -1261,6 +1259,7 @@ def _reassemble_tp_lm_head_all_to_all_output(
|
||||
along dim 0, while the sampler expects the vocab shards concatenated along
|
||||
dim 1.
|
||||
"""
|
||||
tp_size = get_parallel().tp_size
|
||||
assert all_to_all_output.shape[0] % tp_size == 0
|
||||
local_rows = all_to_all_output.shape[0] // tp_size
|
||||
vocab_shard = all_to_all_output.shape[1]
|
||||
|
||||
@@ -467,7 +467,7 @@ class DataParallelController:
|
||||
)
|
||||
else:
|
||||
# Other nodes: Receive worker ports from node 0
|
||||
return self._receive_ports_as_client(endpoint, get_parallel().node_rank)
|
||||
return self._receive_ports_as_client(endpoint)
|
||||
|
||||
def _broadcast_ports_as_server(
|
||||
self, endpoint: str, expected_clients: int, worker_ports: list[int]
|
||||
@@ -522,9 +522,10 @@ class DataParallelController:
|
||||
sock_send(rep_socket, wrap_as_pickle(worker_ports))
|
||||
logger.debug(f"Sent worker ports to node {client_rank}")
|
||||
|
||||
def _receive_ports_as_client(self, endpoint: str, node_rank: int) -> list[int]:
|
||||
def _receive_ports_as_client(self, endpoint: str) -> list[int]:
|
||||
"""Receive worker ports from the server node."""
|
||||
logger.debug("Connecting to node 0 to receive worker ports")
|
||||
node_rank = get_parallel().node_rank
|
||||
|
||||
req_socket = get_zmq_socket(self.context, zmq.REQ, endpoint, False)
|
||||
req_socket.setsockopt(zmq.RCVTIMEO, 600 * 1000) # 10 minute timeout
|
||||
@@ -567,9 +568,7 @@ class DataParallelController:
|
||||
primary_endpoint = NetworkAddress(
|
||||
primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA
|
||||
).to_tcp()
|
||||
all_ports = self._receive_ports_as_client(
|
||||
primary_endpoint, get_parallel().node_rank
|
||||
)
|
||||
all_ports = self._receive_ports_as_client(primary_endpoint)
|
||||
offset = self._joiner_slot_offset(server_args)
|
||||
local_tp_span = self._joiner_local_tp_span(server_args)
|
||||
broadcasted_ports = all_ports[offset : offset + local_tp_span]
|
||||
@@ -636,8 +635,6 @@ class DataParallelController:
|
||||
tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group + 1),
|
||||
)
|
||||
|
||||
attn_cp_rank = 0
|
||||
moe_dp_rank = 0
|
||||
for pp_rank in pp_rank_range:
|
||||
for tp_rank in tp_rank_range:
|
||||
rank_port_args = port_args
|
||||
@@ -679,22 +676,7 @@ class DataParallelController:
|
||||
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
|
||||
+ (tp_rank % tp_size_per_node) * get_device().gpu_id_step
|
||||
)
|
||||
attn_dp_size = (
|
||||
get_parallel().dp_size if get_parallel().enable_dp_attention else 1
|
||||
)
|
||||
|
||||
# Parallelism hierarchy (outermost to innermost):
|
||||
# - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost)
|
||||
# - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost)
|
||||
attn_tp_size = (
|
||||
get_parallel().tp_size
|
||||
// attn_dp_size
|
||||
// get_parallel().attn_cp_size
|
||||
)
|
||||
attn_cp_rank = (tp_rank // attn_tp_size) % get_parallel().attn_cp_size
|
||||
moe_dp_rank = tp_rank // (
|
||||
get_parallel().tp_size // get_parallel().moe_dp_size
|
||||
)
|
||||
# Derive the child's EP rank for its display label.
|
||||
moe_ep_rank = (
|
||||
tp_rank
|
||||
% (get_parallel().tp_size // get_parallel().moe_dp_size)
|
||||
@@ -719,9 +701,6 @@ class DataParallelController:
|
||||
rank_port_args,
|
||||
gpu_id,
|
||||
tp_rank,
|
||||
attn_cp_rank,
|
||||
moe_dp_rank,
|
||||
moe_ep_rank,
|
||||
pp_rank,
|
||||
dp_rank,
|
||||
writer,
|
||||
|
||||
@@ -47,6 +47,7 @@ from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.memory_pool_host import DeepSeekV4PagedHostPool
|
||||
from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
device_module = get_device_module()
|
||||
|
||||
@@ -72,7 +73,6 @@ class HiSparseTokenStats(NamedTuple):
|
||||
def resolve_shared_index_layers(
|
||||
*,
|
||||
hf_text_config,
|
||||
pp_size: int,
|
||||
is_speculative: bool,
|
||||
) -> Optional[List[bool]]:
|
||||
"""Per-layer "reuses the previous layer's top-k index" pattern, or None.
|
||||
@@ -91,7 +91,7 @@ def resolve_shared_index_layers(
|
||||
pattern = [dsa_layer_skips_topk(hf_text_config, i) for i in range(num_layers)]
|
||||
if not any(pattern):
|
||||
return None
|
||||
if pp_size != 1 or is_speculative:
|
||||
if get_parallel().pp_size != 1 or is_speculative:
|
||||
logger.warning(
|
||||
"HiSparse shared-index prefetch is unsupported under pipeline "
|
||||
"parallelism / speculative decoding; falling back to synchronous "
|
||||
|
||||
@@ -75,8 +75,6 @@ class _NegotiateOutput(NamedTuple):
|
||||
class PrefillDelayer:
|
||||
def __init__(
|
||||
self,
|
||||
dp_size: int,
|
||||
attn_tp_size: int,
|
||||
cpu_group,
|
||||
max_delay_passes: int,
|
||||
token_usage_low_watermark: Optional[float],
|
||||
@@ -106,9 +104,10 @@ class PrefillDelayer:
|
||||
f"max_delay_ms={self._max_delay_ms} "
|
||||
f"queue_trigger_enabled={self._queue_trigger_enabled}"
|
||||
)
|
||||
self.dp_size = dp_size
|
||||
self.enable_dp_attention = get_parallel().enable_dp_attention
|
||||
dp_size_dim = dp_size if self.enable_dp_attention else 1
|
||||
parallel = get_parallel()
|
||||
self.dp_size = parallel.dp_size
|
||||
self.enable_dp_attention = parallel.enable_dp_attention
|
||||
dp_size_dim = self.dp_size if self.enable_dp_attention else 1
|
||||
|
||||
# Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the
|
||||
# env flag is on (or overlap scheduling is disabled), ride the NCCL
|
||||
@@ -131,7 +130,7 @@ class PrefillDelayer:
|
||||
# token_watermark_force_allow, running_batch, max_prefill_bs,
|
||||
# waiting_queue_len.
|
||||
self._global_info_buffer = torch.empty(
|
||||
(dp_size_dim, attn_tp_size, 5),
|
||||
(dp_size_dim, parallel.attn_tp_size, 5),
|
||||
dtype=torch.int64,
|
||||
device=self._gather_device,
|
||||
)
|
||||
|
||||
@@ -449,9 +449,6 @@ class Scheduler(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
port_args: PortArgs,
|
||||
tp_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
):
|
||||
# NOTE: KEEP THE FOLLOWING CODE STYLE for this function:
|
||||
# Keep __init__ as an orchestrator: sequence init_* and maybe_init_* calls
|
||||
@@ -526,7 +523,7 @@ class Scheduler(
|
||||
bootstrap.init_layer_runtime(model_config=self.model_config)
|
||||
|
||||
# Init metrics stats
|
||||
self.init_metrics_collector(tp_rank, pp_rank, dp_rank)
|
||||
self.init_metrics_collector()
|
||||
|
||||
# Init inter-process communication
|
||||
self.init_ipc_channels(port_args)
|
||||
@@ -637,7 +634,7 @@ class Scheduler(
|
||||
# Init diffusion LLM
|
||||
self.init_diffusion_llm()
|
||||
|
||||
self.init_metrics_reporter(tp_rank, pp_rank, dp_rank)
|
||||
self.init_metrics_reporter()
|
||||
self.scheduler_stage_metrics = self.metrics_reporter.scheduler_stage_metrics
|
||||
|
||||
# Init schedule policy and new token estimation
|
||||
@@ -772,14 +769,9 @@ class Scheduler(
|
||||
else None
|
||||
)
|
||||
|
||||
def init_metrics_collector(
|
||||
self, tp_rank: int, pp_rank: int, dp_rank: Optional[int]
|
||||
) -> None:
|
||||
def init_metrics_collector(self) -> None:
|
||||
self.metrics_collector_context = SchedulerMetricsCollector.init_new(
|
||||
server_args=self.server_args,
|
||||
tp_rank=tp_rank,
|
||||
pp_rank=pp_rank,
|
||||
dp_rank=dp_rank,
|
||||
enable_priority_scheduling=self.enable_priority_scheduling,
|
||||
enable_lora=self.enable_lora,
|
||||
enable_hierarchical_cache=self.enable_hierarchical_cache,
|
||||
@@ -1340,9 +1332,6 @@ class Scheduler(
|
||||
max_prefill_tokens=self.max_prefill_tokens,
|
||||
page_size=self.page_size,
|
||||
device=self.device,
|
||||
pp_group=self.pp_group,
|
||||
world_group=self.world_group,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
)
|
||||
if sizer.profile_and_fit():
|
||||
self.dynamic_chunk_sizer = sizer
|
||||
@@ -1369,15 +1358,10 @@ class Scheduler(
|
||||
if is_extend:
|
||||
self._prefill_decode_interval_remaining = self.prefill_decode_interval
|
||||
|
||||
def init_metrics_reporter(
|
||||
self, tp_rank: int, pp_rank: int, dp_rank: Optional[int]
|
||||
) -> None:
|
||||
def init_metrics_reporter(self) -> None:
|
||||
# Override point for deployments that need a specialized reporter.
|
||||
self.metrics_reporter = SchedulerMetricsReporter(
|
||||
scheduler=self,
|
||||
tp_rank=tp_rank,
|
||||
pp_rank=pp_rank,
|
||||
dp_rank=dp_rank,
|
||||
metrics_collector_context=self.metrics_collector_context,
|
||||
metrics_collector=self.metrics_collector,
|
||||
)
|
||||
@@ -1404,8 +1388,6 @@ class Scheduler(
|
||||
)
|
||||
else:
|
||||
self.prefill_delayer = PrefillDelayer(
|
||||
dp_size=get_parallel().dp_size,
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
cpu_group=self.tp_cpu_group,
|
||||
device_group=self.tp_group.device_group,
|
||||
metrics_collector=(
|
||||
@@ -1536,7 +1518,6 @@ class Scheduler(
|
||||
self.disagg_decode_transfer_queue = DecodeTransferQueue(
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
metadata_buffers=self.disagg_metadata_buffers,
|
||||
scheduler=self,
|
||||
tree_cache=self.tree_cache,
|
||||
@@ -1553,13 +1534,9 @@ class Scheduler(
|
||||
transfer_queue=self.disagg_decode_transfer_queue,
|
||||
tree_cache=self.tree_cache,
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
tp_size=get_parallel().tp_size,
|
||||
dp_size=get_parallel().dp_size,
|
||||
gpu_id=get_device().gpu_id,
|
||||
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
num_reserved_decode_tokens=get_disagg().num_reserved_decode_tokens,
|
||||
transfer_backend=self.transfer_backend,
|
||||
)
|
||||
@@ -1585,16 +1562,12 @@ class Scheduler(
|
||||
draft_token_to_kv_pool=draft_token_to_kv_pool,
|
||||
req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
|
||||
metadata_buffers=self.disagg_metadata_buffers,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
tp_size=get_parallel().tp_size,
|
||||
gpu_id=get_device().gpu_id,
|
||||
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
scheduler=self,
|
||||
scheduler_stage_metrics=self.scheduler_stage_metrics,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
pp_size=get_parallel().pp_size,
|
||||
transfer_backend=self.transfer_backend,
|
||||
)
|
||||
# The prefill requests that are in the middle of kv sending
|
||||
@@ -2337,7 +2310,6 @@ class Scheduler(
|
||||
)
|
||||
self.dp_attn_adapter = SchedulerDPAttnAdapter(
|
||||
model_runner=target_worker.model_runner,
|
||||
tp_group=self.tp_group,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
tree_cache=self.tree_cache,
|
||||
@@ -2398,10 +2370,6 @@ class Scheduler(
|
||||
def init_kv_events_publisher(self) -> None:
|
||||
self.kv_events_publisher = SchedulerKvEventsPublisher(
|
||||
kv_events_config=get_observability().kv_events_config,
|
||||
attn_tp_rank=get_parallel().attn_tp_rank,
|
||||
attn_cp_rank=get_parallel().attn_cp_rank,
|
||||
attn_dp_rank=get_parallel().attn_dp_rank,
|
||||
dp_rank=get_parallel().dp_rank,
|
||||
tree_cache=self.tree_cache,
|
||||
send_metrics_from_scheduler=self.ipc_channels.send_metrics_from_scheduler,
|
||||
max_running_requests=self.max_running_requests,
|
||||
@@ -5954,42 +5922,37 @@ def resolve_spawn_dp_rank(dp_rank: Optional[int]) -> Optional[int]:
|
||||
def configure_scheduler_process(
|
||||
server_args: ServerArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
attn_cp_rank: int,
|
||||
moe_dp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
display_tp_rank: Optional[int] = None,
|
||||
display_dp_rank: Optional[int] = None,
|
||||
display_moe_ep_rank: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Configure scheduler worker logging and process title.
|
||||
|
||||
display_* ranks are cosmetic; runtime ranks stay local. `dp_rank` arrives
|
||||
already resolved -- see `resolve_spawn_dp_rank`.
|
||||
Runs after `publish`, so every rank it labels the process with comes from
|
||||
the context. display_* ranks are cosmetic; runtime ranks stay local.
|
||||
"""
|
||||
kill_itself_when_parent_died()
|
||||
|
||||
# Generate the logger prefix
|
||||
shown_dp = display_dp_rank if display_dp_rank is not None else dp_rank
|
||||
shown_tp = display_tp_rank if display_tp_rank is not None else tp_rank
|
||||
parallel = get_parallel()
|
||||
shown_dp = display_dp_rank if display_dp_rank is not None else parallel.dp_rank
|
||||
shown_tp = display_tp_rank if display_tp_rank is not None else parallel.tp_rank
|
||||
shown_moe_ep = (
|
||||
display_moe_ep_rank if display_moe_ep_rank is not None else moe_ep_rank
|
||||
display_moe_ep_rank if display_moe_ep_rank is not None else parallel.moe_ep_rank
|
||||
)
|
||||
|
||||
prefix = ""
|
||||
if shown_dp is not None:
|
||||
prefix += f" DP{shown_dp}"
|
||||
if get_parallel().pp_size > 1:
|
||||
prefix += f" PP{pp_rank}"
|
||||
if get_parallel().attn_cp_size > 1:
|
||||
prefix += f" ATTN_CP{attn_cp_rank}"
|
||||
if get_parallel().moe_dp_size > 1:
|
||||
prefix += f" MOE_DP{moe_dp_rank}"
|
||||
if get_parallel().tp_size > 1:
|
||||
if parallel.pp_size > 1:
|
||||
prefix += f" PP{parallel.pp_rank}"
|
||||
if parallel.attn_cp_size > 1:
|
||||
prefix += f" ATTN_CP{parallel.attn_cp_rank}"
|
||||
if parallel.moe_dp_size > 1:
|
||||
prefix += f" MOE_DP{parallel.moe_dp_rank}"
|
||||
if parallel.tp_size > 1:
|
||||
prefix += f" TP{shown_tp}"
|
||||
if get_parallel().ep_size > 1:
|
||||
if parallel.ep_size > 1:
|
||||
prefix += f" EP{shown_moe_ep}"
|
||||
|
||||
# Config the process
|
||||
@@ -6002,12 +5965,7 @@ def configure_scheduler_process(
|
||||
|
||||
# Set cpu affinity to this gpu process
|
||||
if envs.SGLANG_SET_CPU_AFFINITY.get():
|
||||
set_gpu_proc_affinity(
|
||||
get_parallel().pp_size,
|
||||
get_parallel().tp_size,
|
||||
get_parallel().nnodes,
|
||||
gpu_id,
|
||||
)
|
||||
set_gpu_proc_affinity(gpu_id)
|
||||
if not envs.SGLANG_NUMA_BIND_V2.get():
|
||||
numa_node = get_numa_node_if_available(server_args, gpu_id)
|
||||
if numa_node is not None:
|
||||
@@ -6019,9 +5977,6 @@ def run_scheduler_process(
|
||||
port_args: PortArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
attn_cp_rank: int,
|
||||
moe_dp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
pipe_writer,
|
||||
@@ -6032,6 +5987,8 @@ def run_scheduler_process(
|
||||
# Load plugins so hooks can override Scheduler and its dependencies.
|
||||
load_plugins()
|
||||
dp_rank = resolve_spawn_dp_rank(dp_rank)
|
||||
# Placement is needed before publish(): TP/PP select a WORLD rank;
|
||||
# ordinary DP replicas have separate WORLDs and need an explicit DP rank.
|
||||
publish(
|
||||
server_args,
|
||||
role="scheduler",
|
||||
@@ -6044,12 +6001,6 @@ def run_scheduler_process(
|
||||
configure_scheduler_process(
|
||||
server_args,
|
||||
gpu_id,
|
||||
tp_rank,
|
||||
attn_cp_rank,
|
||||
moe_dp_rank,
|
||||
moe_ep_rank,
|
||||
pp_rank,
|
||||
dp_rank,
|
||||
display_tp_rank=display_tp_rank,
|
||||
display_dp_rank=display_dp_rank,
|
||||
display_moe_ep_rank=display_moe_ep_rank,
|
||||
@@ -6073,13 +6024,7 @@ def run_scheduler_process(
|
||||
# Create a scheduler and run the event loop
|
||||
scheduler = None
|
||||
try:
|
||||
scheduler = Scheduler(
|
||||
server_args,
|
||||
port_args,
|
||||
tp_rank,
|
||||
pp_rank,
|
||||
dp_rank,
|
||||
)
|
||||
scheduler = Scheduler(server_args, port_args)
|
||||
|
||||
# Send initialization info back to the parent process
|
||||
pipe_writer.send(scheduler.get_init_info())
|
||||
|
||||
@@ -38,7 +38,6 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils.common import require_mlp_tp_gather
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
|
||||
@@ -363,10 +362,6 @@ def _local_prefill_cuda_graph_vote(
|
||||
def prepare_mlp_sync_batch_raw(
|
||||
local_batch: ScheduleBatch,
|
||||
model_runner: ModelRunner,
|
||||
dp_size: int,
|
||||
attn_tp_size: int,
|
||||
attn_cp_size: int,
|
||||
tp_group: GroupCoordinator,
|
||||
get_idle_batch: Callable[[], ScheduleBatch],
|
||||
disable_cuda_graph: bool,
|
||||
require_mlp_tp_gather: bool,
|
||||
@@ -374,6 +369,10 @@ def prepare_mlp_sync_batch_raw(
|
||||
offload_tags: set[str],
|
||||
dwdp: bool = False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
dp_size = parallel.dp_size
|
||||
attn_tp_size = parallel.attn_tp_size
|
||||
tp_group = parallel.tp_group
|
||||
# Check if other DP workers have running batches
|
||||
if (
|
||||
local_batch is None
|
||||
@@ -432,9 +431,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
tbo_preparer = TboDPAttentionPreparer()
|
||||
use_world_group = world_dp_gather_enabled()
|
||||
if use_world_group:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
world = get_parallel().world_group
|
||||
world = parallel.world_group
|
||||
group = torch.distributed.group.WORLD
|
||||
device = world.device
|
||||
elif len(offload_tags) == 0 and (
|
||||
@@ -460,7 +457,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
mlp_sync_info = MLPSyncBatchInfo(
|
||||
dp_size=dp_size,
|
||||
tp_size=attn_tp_size,
|
||||
cp_size=attn_cp_size,
|
||||
cp_size=parallel.attn_cp_size,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_for_logprob=num_tokens_for_logprob,
|
||||
can_run_decode_cuda_graph=can_run_decode_cuda_graph,
|
||||
@@ -529,7 +526,6 @@ def prepare_mlp_sync_batch_raw(
|
||||
@dataclass(kw_only=True, slots=True, frozen=True)
|
||||
class SchedulerDPAttnAdapter:
|
||||
model_runner: ModelRunner
|
||||
tp_group: GroupCoordinator
|
||||
req_to_token_pool: ReqToTokenPool
|
||||
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator
|
||||
tree_cache: BasePrefixCache
|
||||
@@ -543,10 +539,6 @@ class SchedulerDPAttnAdapter:
|
||||
return prepare_mlp_sync_batch_raw(
|
||||
local_batch,
|
||||
model_runner=self.model_runner,
|
||||
dp_size=get_parallel().dp_size,
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
attn_cp_size=get_parallel().attn_cp_size,
|
||||
tp_group=self.tp_group,
|
||||
get_idle_batch=self.get_idle_batch,
|
||||
disable_cuda_graph=cuda_graph_fully_disabled(),
|
||||
require_mlp_tp_gather=require_mlp_tp_gather(),
|
||||
|
||||
@@ -25,7 +25,6 @@ from sglang.srt.utils.common import get_device_module
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
@@ -51,9 +50,6 @@ class DynamicChunkSizer:
|
||||
max_prefill_tokens: int,
|
||||
page_size: int,
|
||||
device: str,
|
||||
pp_group: GroupCoordinator,
|
||||
world_group: GroupCoordinator,
|
||||
pp_rank: int,
|
||||
):
|
||||
self.model_runner = model_runner
|
||||
self.model_config = model_config
|
||||
@@ -65,9 +61,7 @@ class DynamicChunkSizer:
|
||||
self.max_prefill_tokens = max_prefill_tokens
|
||||
self.page_size = page_size
|
||||
self.device = device
|
||||
self.pp_group = pp_group
|
||||
self.world_group = world_group
|
||||
self.pp_rank = pp_rank
|
||||
self.pp_rank = get_parallel().pp_rank
|
||||
self.predictor = ChunkSizePredictor()
|
||||
|
||||
def profile_and_fit(self) -> bool:
|
||||
@@ -75,7 +69,8 @@ class DynamicChunkSizer:
|
||||
returns whether the predictor is ready."""
|
||||
samples: Optional[Tuple[List[int], List[float]]] = None
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
parallel = get_parallel()
|
||||
if parallel.pp_group.is_first_rank:
|
||||
try:
|
||||
samples = self._profile_prefill_latency()
|
||||
except Exception as e:
|
||||
@@ -86,8 +81,9 @@ class DynamicChunkSizer:
|
||||
|
||||
# The samples are global, so one broadcast from global rank 0 (a PP0 rank)
|
||||
# reaches every stage and attention rank; a failure travels as None.
|
||||
world_group = parallel.world_group
|
||||
samples = broadcast_pyobj(
|
||||
[samples], self.world_group.rank, self.world_group.cpu_group, src=0
|
||||
[samples], world_group.rank, world_group.cpu_group, src=0
|
||||
)[0]
|
||||
|
||||
if samples is None:
|
||||
|
||||
@@ -45,10 +45,6 @@ hook_custom_types(KvMetrics)
|
||||
@dataclass(kw_only=True, slots=True)
|
||||
class SchedulerKvEventsPublisher:
|
||||
kv_events_config: Optional[str]
|
||||
attn_tp_rank: int
|
||||
attn_cp_rank: int
|
||||
attn_dp_rank: int
|
||||
dp_rank: Optional[int]
|
||||
tree_cache: BasePrefixCache
|
||||
send_metrics_from_scheduler: Optional[zmq.Socket]
|
||||
max_running_requests: int
|
||||
|
||||
@@ -203,9 +203,6 @@ class _ForwardOccupancyLogWindow(msgspec.Struct):
|
||||
@dataclass(kw_only=True)
|
||||
class SchedulerMetricsReporter:
|
||||
scheduler: Scheduler
|
||||
tp_rank: int
|
||||
pp_rank: int
|
||||
dp_rank: Optional[int]
|
||||
metrics_collector_context: SchedulerMetricsCollectorContext
|
||||
metrics_collector: Optional[SchedulerMetricsCollector]
|
||||
num_retracted_reqs: int = 0
|
||||
@@ -222,7 +219,7 @@ class SchedulerMetricsReporter:
|
||||
self.enable_kv_cache_events = (
|
||||
self.metrics_collector_context.enable_kv_cache_events
|
||||
)
|
||||
self._init_metrics(self.tp_rank, self.pp_rank, self.dp_rank)
|
||||
self._init_metrics()
|
||||
self._install_device_timer_on_runners()
|
||||
# Keep log history after the existing async result copy so reporting does
|
||||
# not synchronize the model stream once per generated token.
|
||||
@@ -244,12 +241,7 @@ class SchedulerMetricsReporter:
|
||||
self.last_gen_throughput = 0.0
|
||||
return self.last_gen_throughput
|
||||
|
||||
def _init_metrics(
|
||||
self,
|
||||
tp_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
):
|
||||
def _init_metrics(self):
|
||||
# Basic stats
|
||||
self.forward_ct_decode = 0
|
||||
self.num_generated_tokens = 0
|
||||
|
||||
@@ -18,6 +18,7 @@ except ImportError:
|
||||
|
||||
from sglang.srt.managers.schedule_batch import Modality
|
||||
from sglang.srt.mem_cache.embedding_store import EmbeddingStore
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -344,20 +345,18 @@ class EmbeddingInsertOperation:
|
||||
class EmbeddingCacheController:
|
||||
def __init__(
|
||||
self,
|
||||
tp_rank,
|
||||
tp_size,
|
||||
embedding_store: EmbeddingStore,
|
||||
max_pool_size_gb=4.0,
|
||||
hidden_dims: dict = None,
|
||||
tp_group=None,
|
||||
all_rank_get=False,
|
||||
enable_eviction: bool = True,
|
||||
max_eviction_batch: int = 100,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
):
|
||||
self.tp_world_size = tp_size
|
||||
self.tp_group = tp_group
|
||||
self.tp_rank = tp_rank
|
||||
parallel = get_parallel()
|
||||
self.tp_world_size = parallel.tp_size
|
||||
self.tp_group = parallel.tp_group.cpu_group
|
||||
self.tp_rank = parallel.tp_rank
|
||||
self.all_rank_get = all_rank_get
|
||||
self.hidden_dims = hidden_dims or {}
|
||||
# Pool dtype must match the model's embedding dtype so that pool views,
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.model_executor.input_buffers import (
|
||||
INDEX_SEMANTIC_BUFFERS,
|
||||
share_input_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -1011,7 +1012,6 @@ def build_eager_registry(
|
||||
is_encoder_decoder: bool = False,
|
||||
encoder_len_fill_value: int = 0,
|
||||
encoder_lens_dtype: torch.dtype = torch.int32,
|
||||
dp_size: int = 1,
|
||||
) -> CudaGraphBufferRegistry:
|
||||
"""One fixed-max input registry for the ``EagerRunner``, serving BOTH eager
|
||||
decode and eager prefill.
|
||||
@@ -1042,7 +1042,7 @@ def build_eager_registry(
|
||||
register_global_num_tokens=False,
|
||||
require_gathered_buffer=False,
|
||||
require_mlp_tp_gather=False,
|
||||
dp_size=dp_size,
|
||||
dp_size=get_parallel().dp_size,
|
||||
share_pool=True,
|
||||
source=None,
|
||||
)
|
||||
|
||||
@@ -437,7 +437,7 @@ class ModelRunner:
|
||||
self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None
|
||||
|
||||
# CPU offload
|
||||
set_offloader(create_offloader(dp_rank=get_parallel().dp_rank))
|
||||
set_offloader(create_offloader())
|
||||
|
||||
self._weight_checker = WeightChecker(get_model=lambda: self.model)
|
||||
|
||||
@@ -582,7 +582,6 @@ class ModelRunner:
|
||||
def init_remote_instance_weight_transporter(self):
|
||||
self.remote_instance_weight_transporter = RemoteInstanceWeightTransporter(
|
||||
get_model=lambda: self.model,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
gpu_id=self.gpu_id,
|
||||
)
|
||||
|
||||
@@ -653,12 +652,7 @@ class ModelRunner:
|
||||
self.init_token_oracle()
|
||||
self.sampler = create_sampler()
|
||||
self.load_model()
|
||||
prepare_moe_topk(
|
||||
model=self.model,
|
||||
model_config=self.model_config,
|
||||
moe_ep_size=get_parallel().moe_ep_size,
|
||||
moe_ep_rank=get_parallel().moe_ep_rank,
|
||||
)
|
||||
prepare_moe_topk(model=self.model, model_config=self.model_config)
|
||||
|
||||
self.maybe_init_dwdp()
|
||||
|
||||
@@ -754,8 +748,6 @@ class ModelRunner:
|
||||
self.expert_backup_client = (
|
||||
ExpertBackupClient(
|
||||
model_config=self.model_config,
|
||||
moe_ep_size=get_parallel().moe_ep_size,
|
||||
moe_ep_rank=get_parallel().moe_ep_rank,
|
||||
get_model=lambda: self.model,
|
||||
)
|
||||
if (
|
||||
@@ -797,16 +789,12 @@ class ModelRunner:
|
||||
def get_pp_proxy_topk_size(self) -> Optional[int]:
|
||||
return misc_utils.resolve_pp_proxy_topk_size(
|
||||
model_config=self.model_config,
|
||||
pp_size=get_parallel().pp_size,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
)
|
||||
|
||||
def get_pp_proxy_residual_num_blocks(self) -> Optional[int]:
|
||||
return misc_utils.resolve_pp_proxy_residual_num_blocks(
|
||||
model_config=self.model_config,
|
||||
pp_size=get_parallel().pp_size,
|
||||
pp_rank=get_parallel().pp_rank,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
)
|
||||
|
||||
@@ -974,7 +962,6 @@ class ModelRunner:
|
||||
swap_in_block_size=hisparse_cfg.swap_in_block_size,
|
||||
shared_index_layers=resolve_shared_index_layers(
|
||||
hf_text_config=self.model_config.hf_text_config,
|
||||
pp_size=get_parallel().pp_size,
|
||||
is_speculative=self.spec_algorithm.is_speculative(),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.runtime_context import (
|
||||
attention_backends,
|
||||
get_context,
|
||||
get_observability,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS
|
||||
@@ -66,12 +67,13 @@ def create_msprobe_debugger() -> Optional[Any]:
|
||||
|
||||
|
||||
def resolve_pp_proxy_topk_size(
|
||||
*, model_config: ModelConfig, pp_size: int, pp_rank: int, start_layer: int
|
||||
*, model_config: ModelConfig, start_layer: int
|
||||
) -> Optional[int]:
|
||||
hf_config = model_config.hf_text_config
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
pp_size <= 1
|
||||
or pp_rank == 0
|
||||
parallel.pp_size <= 1
|
||||
or parallel.pp_rank == 0
|
||||
or not is_deepseek_dsa(hf_config)
|
||||
or not dsa_layer_skips_topk(hf_config, start_layer)
|
||||
):
|
||||
@@ -80,10 +82,15 @@ def resolve_pp_proxy_topk_size(
|
||||
|
||||
|
||||
def resolve_pp_proxy_residual_num_blocks(
|
||||
*, model_config: ModelConfig, pp_size: int, pp_rank: int, start_layer: int
|
||||
*, model_config: ModelConfig, start_layer: int
|
||||
) -> Optional[int]:
|
||||
"""Return the inherited Kimi K3 attention-residual bank width."""
|
||||
if pp_size <= 1 or pp_rank == 0 or not is_kimi_k3(model_config.hf_config):
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.pp_size <= 1
|
||||
or parallel.pp_rank == 0
|
||||
or not is_kimi_k3(model_config.hf_config)
|
||||
):
|
||||
return None
|
||||
|
||||
block_size = getattr(model_config.hf_text_config, "attn_res_block_size", None)
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.srt.eplb.lplb_solver import (
|
||||
)
|
||||
from sglang.srt.layers.moe.hash_topk import HashTopK
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -24,13 +24,8 @@ logger = logging.getLogger(__name__)
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
|
||||
|
||||
def prepare_moe_topk(
|
||||
*,
|
||||
model,
|
||||
model_config: ModelConfig,
|
||||
moe_ep_size: int,
|
||||
moe_ep_rank: int,
|
||||
) -> None:
|
||||
def prepare_moe_topk(*, model, model_config: ModelConfig) -> None:
|
||||
parallel = get_parallel()
|
||||
balancer_cls = None
|
||||
num_prepared = 0
|
||||
num_routed_experts = None
|
||||
@@ -61,8 +56,8 @@ def prepare_moe_topk(
|
||||
routed_scaling_factor = module.routed_scaling_factor
|
||||
module.waterfill_balancer = balancer_cls(
|
||||
num_routed_experts=num_physical_routed_experts,
|
||||
world_size=moe_ep_size,
|
||||
rank=moe_ep_rank,
|
||||
world_size=parallel.moe_ep_size,
|
||||
rank=parallel.moe_ep_rank,
|
||||
layer_id=module.layer_id,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
|
||||
+3
-2
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import torch
|
||||
@@ -24,7 +24,8 @@ logger = logging.getLogger(__name__)
|
||||
@dataclass(slots=True, kw_only=True)
|
||||
class RemoteInstanceWeightTransporter:
|
||||
get_model: Callable[[], torch.nn.Module]
|
||||
tp_rank: int
|
||||
# Registration may run after the runner's construction scope has exited.
|
||||
tp_rank: int = field(init=False, default_factory=lambda: get_parallel().tp_rank)
|
||||
gpu_id: int
|
||||
engine: Optional[Any] = None
|
||||
session_id: str = ""
|
||||
|
||||
@@ -76,8 +76,6 @@ def _allocate_decode_buffers(
|
||||
hidden_size: int,
|
||||
vocab_size: int,
|
||||
dtype: torch.dtype,
|
||||
dp_size: int,
|
||||
pp_size: int,
|
||||
is_encoder_decoder: bool,
|
||||
require_mlp_tp_gather: bool,
|
||||
seq_len_fill_value: int,
|
||||
@@ -92,6 +90,7 @@ def _allocate_decode_buffers(
|
||||
allocate_logits_buffer: bool = True,
|
||||
) -> SimpleNamespace:
|
||||
"""Allocate the FB-shared decode buffers."""
|
||||
parallel = get_parallel()
|
||||
with torch.device(device):
|
||||
input_ids = torch.zeros((max_num_token,), dtype=torch.int64)
|
||||
input_embeds = torch.zeros((max_num_token, hidden_size), dtype=dtype)
|
||||
@@ -127,7 +126,7 @@ def _allocate_decode_buffers(
|
||||
torch.zeros((max_bs,), dtype=torch.bool) if enable_mamba_track else None
|
||||
)
|
||||
|
||||
if pp_size > 1:
|
||||
if parallel.pp_size > 1:
|
||||
# mHC (e.g. DSV4) flattens residual into hidden_states (size = hc_hidden_size).
|
||||
is_mhc = hc_hidden_size is not None
|
||||
hs = hc_hidden_size if is_mhc else hidden_size
|
||||
@@ -163,9 +162,9 @@ def _allocate_decode_buffers(
|
||||
encoder_lens = None
|
||||
|
||||
if require_mlp_tp_gather:
|
||||
global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32)
|
||||
global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32)
|
||||
global_num_tokens_for_logprob_gpu = torch.zeros(
|
||||
(dp_size,), dtype=torch.int32
|
||||
(parallel.dp_size,), dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32)
|
||||
@@ -367,8 +366,6 @@ class BaseRunner(ABC):
|
||||
hidden_size=mr.model_config.hidden_size,
|
||||
vocab_size=mr.model_config.vocab_size,
|
||||
dtype=mr.model_config.dtype,
|
||||
dp_size=get_parallel().dp_size,
|
||||
pp_size=get_parallel().pp_size,
|
||||
is_encoder_decoder=mr.model_config.is_encoder_decoder,
|
||||
require_mlp_tp_gather=require_mlp_tp_gather(),
|
||||
seq_len_fill_value=mr.attn_backend.get_cuda_graph_seq_len_fill_value(),
|
||||
|
||||
@@ -849,7 +849,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
export_cuda_graph_capture_trace(
|
||||
prof_context,
|
||||
runner_name=type(self).__name__,
|
||||
tp_rank=get_parallel().tp_rank,
|
||||
)
|
||||
|
||||
def capture_prepare(
|
||||
|
||||
@@ -59,7 +59,6 @@ from sglang.srt.model_executor.runner_utils import (
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
max_prefill_buffer_tokens,
|
||||
max_speculative_num_draft_tokens,
|
||||
@@ -150,7 +149,6 @@ class EagerRunner(BaseRunner):
|
||||
encoder_lens_dtype=(
|
||||
torch.int64 if torch.device(mr.device).type == "cpu" else torch.int32
|
||||
),
|
||||
dp_size=get_parallel().dp_size,
|
||||
)
|
||||
# Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up).
|
||||
if run_warmup:
|
||||
|
||||
@@ -117,7 +117,6 @@ class GraniteMoeMoE(nn.Module):
|
||||
layer_id: int,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -66,7 +66,6 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
layer_id=layer_idx,
|
||||
quant_config=quant_config,
|
||||
tp_size=get_parallel().tp_size,
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
)
|
||||
|
||||
@@ -252,7 +251,6 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
layer_id=layer_idx,
|
||||
quant_config=quant_config,
|
||||
tp_size=get_parallel().tp_size,
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
)
|
||||
|
||||
|
||||
@@ -713,12 +713,12 @@ def _build_inkling_shared_experts(
|
||||
intermediate_size: int,
|
||||
layer_id: int,
|
||||
prefix: str,
|
||||
moe_tp_rank: int,
|
||||
moe_tp_size: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> nn.Module | None:
|
||||
if n_shared_experts <= 0:
|
||||
return None
|
||||
# The shared dense MLP is reconstructed by the full-TP all-reduce in forward.
|
||||
parallel = get_parallel()
|
||||
if shared_expert_sink:
|
||||
shared_prefix = add_prefix("shared_experts", prefix)
|
||||
shared_sink_serves_fp4 = InklingBatchDenseMLP._resolve_fp4_strategy(
|
||||
@@ -729,8 +729,6 @@ def _build_inkling_shared_experts(
|
||||
shared_sink_serves_fp4=shared_sink_serves_fp4,
|
||||
)
|
||||
if use_fused_shared:
|
||||
# moe_tp_rank/moe_tp_size are derived internally; kept as args only
|
||||
# for the legacy and non-sink InklingDenseMLP paths below.
|
||||
return InklingSharedFusedMoE(
|
||||
n_shared_experts=n_shared_experts,
|
||||
hidden_size=hidden_size,
|
||||
@@ -748,8 +746,8 @@ def _build_inkling_shared_experts(
|
||||
prefix=shared_prefix,
|
||||
quant_config=quant_config,
|
||||
inference_moe_w13_interleaved=inference_moe_w13_interleaved,
|
||||
tp_rank=moe_tp_rank,
|
||||
tp_size=moe_tp_size,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
tp_group=get_parallel().tp_group,
|
||||
)
|
||||
return InklingBatchDenseMLP(
|
||||
@@ -767,8 +765,8 @@ def _build_inkling_shared_experts(
|
||||
layer_id=layer_id,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
quant_config=quant_config,
|
||||
tp_rank=moe_tp_rank,
|
||||
tp_size=moe_tp_size,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
tp_group=get_parallel().tp_group,
|
||||
)
|
||||
|
||||
@@ -870,10 +868,6 @@ class InklingMoE(nn.Module):
|
||||
intermediate_size=self.intermediate_dim,
|
||||
layer_id=layer_id,
|
||||
prefix=prefix,
|
||||
# Shared expert is a replicated dense MLP: shard over the full tp group, not
|
||||
# moe_tp (the single full-tp all_reduce in forward() reconstructs it).
|
||||
moe_tp_rank=get_parallel().tp_rank,
|
||||
moe_tp_size=get_parallel().tp_size,
|
||||
quant_config=self.quant_config,
|
||||
)
|
||||
if isinstance(self.shared_experts, InklingSharedFusedMoE):
|
||||
|
||||
@@ -646,10 +646,9 @@ class TransformersBase(nn.Module):
|
||||
# Pipeline parallel
|
||||
self.pipeline_parallel()
|
||||
# Module replacement (Linear → TP, RMSNorm → fused, MoE overridden by MoEMixin)
|
||||
tp_size = get_parallel().tp_size
|
||||
self.recursive_replace()
|
||||
# Attention instances
|
||||
self.attention_instances = self._create_attention_instances(tp_size)
|
||||
self.attention_instances = self._create_attention_instances()
|
||||
# Vocab embeddings
|
||||
self.replace_vocab_embed_class(self.model)
|
||||
|
||||
@@ -902,7 +901,8 @@ class TransformersBase(nn.Module):
|
||||
self._register_missing_prefix(maybe_prefix("model", name))
|
||||
|
||||
# -- Attention instances ------------------------------------------------
|
||||
def _create_attention_instances(self, tp_size: int) -> dict[int, RadixAttention]:
|
||||
def _create_attention_instances(self) -> dict[int, RadixAttention]:
|
||||
tp_size = get_parallel().tp_size
|
||||
num_heads = self.text_config.num_attention_heads
|
||||
num_kv_heads = getattr(self.text_config, "num_key_value_heads", num_heads)
|
||||
hidden_size = self.text_config.hidden_size
|
||||
|
||||
@@ -1103,9 +1103,6 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
cls,
|
||||
*,
|
||||
server_args: ServerArgs,
|
||||
tp_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
enable_priority_scheduling: bool,
|
||||
enable_lora: bool,
|
||||
enable_hierarchical_cache: bool,
|
||||
@@ -1134,14 +1131,14 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
labels = {
|
||||
"model_name": get_serving().served_model_name,
|
||||
"engine_type": engine_type,
|
||||
"tp_rank": tp_rank,
|
||||
"pp_rank": pp_rank,
|
||||
"tp_rank": parallel.tp_rank,
|
||||
"pp_rank": parallel.pp_rank,
|
||||
"moe_ep_rank": parallel.moe_ep_rank,
|
||||
}
|
||||
if enable_priority_scheduling:
|
||||
labels["priority"] = ""
|
||||
if dp_rank is not None:
|
||||
labels["dp_rank"] = dp_rank
|
||||
if parallel.dp_rank is not None:
|
||||
labels["dp_rank"] = parallel.dp_rank
|
||||
if get_observability().extra_metric_labels:
|
||||
labels.update(get_observability().extra_metric_labels)
|
||||
scheduler_collector_cls = resolve_collector_class(
|
||||
|
||||
@@ -146,10 +146,7 @@ class RayDataParallelController(DataParallelController):
|
||||
for node_idx in range(nnodes):
|
||||
bundle_idx = self.bundle_for_node[node_idx]
|
||||
pp_range, tp_range, pp_per_node, tp_per_node = _calculate_rank_ranges(
|
||||
nnodes,
|
||||
get_parallel().pp_size,
|
||||
get_parallel().tp_size,
|
||||
node_rank=node_idx,
|
||||
node_rank=node_idx
|
||||
)
|
||||
for pp_rank in pp_range:
|
||||
for tp_rank in tp_range:
|
||||
|
||||
@@ -29,7 +29,6 @@ from sglang.srt.entrypoints.engine import (
|
||||
Engine,
|
||||
SchedulerInitResult,
|
||||
_calculate_rank_ranges,
|
||||
_compute_parallelism_ranks,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.ray.scheduler_actor import SchedulerActor
|
||||
@@ -204,8 +203,6 @@ def _create_scheduler_actor(
|
||||
rank0_node_ip: IP of rank-0's node, used for NCCL rendezvous.
|
||||
dist_init_addr: Distributed init address (tcp://rank0_node_ip:nccl_port).
|
||||
"""
|
||||
attn_cp_rank, moe_dp_rank, moe_ep_rank = _compute_parallelism_ranks(tp_rank)
|
||||
|
||||
return SchedulerActor.options(
|
||||
num_cpus=0,
|
||||
num_gpus=1,
|
||||
@@ -223,9 +220,6 @@ def _create_scheduler_actor(
|
||||
port_args=port_args,
|
||||
gpu_id=gpu_id,
|
||||
tp_rank=tp_rank,
|
||||
attn_cp_rank=attn_cp_rank,
|
||||
moe_dp_rank=moe_dp_rank,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
pp_rank=pp_rank,
|
||||
dp_rank=dp_rank,
|
||||
dist_init_addr=dist_init_addr,
|
||||
@@ -340,12 +334,7 @@ class RayEngine(Engine):
|
||||
for node_idx in range(nnodes):
|
||||
bundle_idx = bundle_for_node[node_idx]
|
||||
pp_range, tp_range, pp_per_node, tp_per_node = (
|
||||
_calculate_rank_ranges(
|
||||
nnodes,
|
||||
get_parallel().pp_size,
|
||||
get_parallel().tp_size,
|
||||
node_rank=node_idx,
|
||||
)
|
||||
_calculate_rank_ranges(node_rank=node_idx)
|
||||
)
|
||||
for pp_rank in pp_range:
|
||||
for tp_rank in tp_range:
|
||||
|
||||
@@ -41,9 +41,6 @@ class SchedulerActor:
|
||||
port_args: PortArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
attn_cp_rank: int,
|
||||
moe_dp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
dist_init_addr: Optional[str] = None,
|
||||
@@ -98,16 +95,7 @@ class SchedulerActor:
|
||||
)
|
||||
|
||||
# Configure worker (logging, process title, etc.)
|
||||
configure_scheduler_process(
|
||||
server_args,
|
||||
actual_gpu_id,
|
||||
tp_rank,
|
||||
attn_cp_rank,
|
||||
moe_dp_rank,
|
||||
moe_ep_rank,
|
||||
pp_rank,
|
||||
dp_rank,
|
||||
)
|
||||
configure_scheduler_process(server_args, actual_gpu_id)
|
||||
|
||||
# Ray actors can't use the numactl subprocess-wrapping approach
|
||||
# (SGLANG_NUMA_BIND_V2's normal path), so bind in-process via libnuma.
|
||||
@@ -125,9 +113,6 @@ class SchedulerActor:
|
||||
self.scheduler = Scheduler(
|
||||
server_args=server_args,
|
||||
port_args=port_args,
|
||||
tp_rank=tp_rank,
|
||||
pp_rank=pp_rank,
|
||||
dp_rank=dp_rank,
|
||||
)
|
||||
|
||||
self._tp_rank = tp_rank
|
||||
|
||||
@@ -2962,18 +2962,14 @@ def direct_register_custom_op(
|
||||
raise error
|
||||
|
||||
|
||||
def set_gpu_proc_affinity(
|
||||
pp_size: int,
|
||||
tp_size: int,
|
||||
nnodes: int,
|
||||
gpu_id: int,
|
||||
):
|
||||
def set_gpu_proc_affinity(gpu_id: int):
|
||||
# current process
|
||||
pid = os.getpid()
|
||||
p = psutil.Process(pid)
|
||||
|
||||
nnodes_per_tp_group = max(nnodes // pp_size, 1)
|
||||
tp_size_per_node = tp_size // nnodes_per_tp_group
|
||||
parallel = get_parallel()
|
||||
nnodes_per_tp_group = max(parallel.nnodes // parallel.pp_size, 1)
|
||||
tp_size_per_node = parallel.tp_size // nnodes_per_tp_group
|
||||
|
||||
# total physical cores
|
||||
total_pcores = psutil.cpu_count(logical=False)
|
||||
|
||||
@@ -65,7 +65,7 @@ def set_offloader(instance: BaseOffloader):
|
||||
_instance = instance
|
||||
|
||||
|
||||
def create_offloader(dp_rank: int):
|
||||
def create_offloader():
|
||||
if get_exec().offload.cpu_offload_gb > 0:
|
||||
return OffloaderV1(
|
||||
cpu_offload_max_bytes=int(get_exec().offload.cpu_offload_gb * 1024**3)
|
||||
@@ -79,8 +79,6 @@ def create_offloader(dp_rank: int):
|
||||
num_in_group=get_exec().offload.offload_num_in_group,
|
||||
prefetch_step=get_exec().offload.offload_prefetch_step,
|
||||
mode=get_exec().offload.offload_mode,
|
||||
dp_rank=dp_rank,
|
||||
dp_size=get_parallel().dp_size,
|
||||
)
|
||||
return NoopOffloader()
|
||||
|
||||
@@ -210,9 +208,9 @@ class OffloaderV2(BaseOffloader):
|
||||
num_in_group: int,
|
||||
prefetch_step: int,
|
||||
mode: str,
|
||||
dp_rank: int,
|
||||
dp_size: int,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
dp_rank, dp_size = parallel.dp_rank, parallel.dp_size
|
||||
self.group_size = group_size
|
||||
self.num_in_group = num_in_group
|
||||
self.prefetch_step = prefetch_step
|
||||
|
||||
@@ -48,7 +48,7 @@ def graph_capture_profile_dir() -> str:
|
||||
)
|
||||
|
||||
|
||||
def export_cuda_graph_capture_trace(prof_context, *, runner_name: str, tp_rank: int):
|
||||
def export_cuda_graph_capture_trace(prof_context, *, runner_name: str):
|
||||
"""Persist a CUDA-graph capture profiler trace (chrome trace) to disk.
|
||||
|
||||
Opt-in via ``SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE`` (no-op otherwise). The
|
||||
@@ -60,6 +60,7 @@ def export_cuda_graph_capture_trace(prof_context, *, runner_name: str, tp_rank:
|
||||
"""
|
||||
if not envs.SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE.get():
|
||||
return
|
||||
tp_rank = get_parallel().tp_rank
|
||||
output_dir = graph_capture_profile_dir()
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
path = os.path.join(
|
||||
|
||||
Reference in New Issue
Block a user