diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index 35921dae64..7b95a6243a 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -95,7 +95,6 @@ from sglang.srt.runtime_context import ( SpawnRanks, get_device, get_model, - get_parallel, get_schedule, publish, spawn_world_rank, @@ -543,10 +542,6 @@ def _maybe_prepare_mlp_sync_batch(batch: ScheduleBatch, model_runner): prepare_mlp_sync_batch_raw( batch, model_runner=model_runner, - dp_size=get_parallel().dp_size, - attn_tp_size=get_parallel().attn_tp_size, - attn_cp_size=model_runner.attn_cp_size, - tp_group=model_runner.tp_group, get_idle_batch=None, disable_cuda_graph=cuda_graph_fully_disabled(), require_mlp_tp_gather=require_mlp_tp_gather(), @@ -908,13 +903,7 @@ def latency_test( initialize_fp4_gemm_config() if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"): - parallel = get_parallel() - set_gpu_proc_affinity( - parallel.pp_size, - parallel.tp_size, - parallel.nnodes, - tp_rank, - ) + set_gpu_proc_affinity(tp_rank) # Configure the logger configure_logger(server_args, prefix=f" TP{tp_rank}") diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index ecb39d3e76..1f6dfabcc2 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -365,16 +365,13 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): transfer_queue: DecodeTransferQueue, tree_cache: BasePrefixCache, gloo_group: ProcessGroup, - tp_rank: int, - tp_size: int, - dp_size: int, gpu_id: int, bootstrap_port: int, max_total_num_tokens: int, - pp_rank: int, num_reserved_decode_tokens: int, transfer_backend: TransferBackend, ): + parallel = get_parallel() self.req_to_token_pool = req_to_token_pool self.token_to_kv_pool_allocator = token_to_kv_pool_allocator self.token_to_kv_pool = token_to_kv_pool_allocator.get_kvcache() @@ -388,14 +385,14 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.gloo_group = gloo_group # Destinations visible to prefill but not yet on the transfer queue. self._num_published_destinations = 0 - self.tp_rank = tp_rank - self.tp_size = tp_size - self.dp_size = dp_size + self.tp_rank = parallel.tp_rank + self.tp_size = parallel.tp_size + self.dp_size = parallel.dp_size self.gpu_id = gpu_id self.bootstrap_port = bootstrap_port self.max_total_num_tokens = max_total_num_tokens - self.pp_rank = pp_rank - self.pp_size = get_parallel().pp_size + self.pp_rank = parallel.pp_rank + self.pp_size = parallel.pp_size self.num_reserved_decode_tokens = num_reserved_decode_tokens self.transfer_backend = transfer_backend # Queue for requests pending pre-allocation @@ -2200,7 +2197,6 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self, gloo_group: ProcessGroup, req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator, - tp_rank: int, metadata_buffers: MetadataBuffers, scheduler: Scheduler, tree_cache: BasePrefixCache, @@ -2208,7 +2204,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self.queue: List[DecodeRequest] = [] self.gloo_group = gloo_group self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator - self.tp_rank = tp_rank + self.tp_rank = get_parallel().tp_rank self.metadata_buffers = metadata_buffers self.scheduler = scheduler self.tree_cache = tree_cache diff --git a/python/sglang/srt/disaggregation/encoder/preprocessor.py b/python/sglang/srt/disaggregation/encoder/preprocessor.py index 2fca0d7981..c0344903b5 100644 --- a/python/sglang/srt/disaggregation/encoder/preprocessor.py +++ b/python/sglang/srt/disaggregation/encoder/preprocessor.py @@ -436,11 +436,11 @@ class EncoderPreprocessor: vr, video_config, *, - tp_rank: int, - tp_size: int, video_processor_kwargs: dict, precomputed_indices: Optional[List[int]] = None, ): + parallel = get_parallel() + tp_rank, tp_size = parallel.attn_tp_rank, parallel.attn_tp_size video_config = video_config or {} video_fps = vr.avg_fps duration = len(vr) / video_fps if video_fps else 0 @@ -596,8 +596,6 @@ class EncoderPreprocessor: result = await self._dp_sharded_decode_single_video( video_items[0], video_configs[0], - tp_rank=parallel.attn_tp_rank, - tp_size=tp_size, video_processor_kwargs=video_processor_kwargs, precomputed_indices=sampled, ) diff --git a/python/sglang/srt/disaggregation/encoder/server.py b/python/sglang/srt/disaggregation/encoder/server.py index e13e254da5..02e557552f 100644 --- a/python/sglang/srt/disaggregation/encoder/server.py +++ b/python/sglang/srt/disaggregation/encoder/server.py @@ -670,11 +670,8 @@ class MMEncoder: get_mm().mm_global_cache_backend, ) self.mm_global_cache = EmbeddingCacheController( - rank, - get_parallel().tp_size, embedding_store=embedding_store, hidden_dims=self._embedding_dims, - tp_group=get_parallel().tp_group.cpu_group, all_rank_get=False, dtype=self._embedding_dtype, ) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index a2623adcff..4019bda079 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -164,27 +164,24 @@ class PrefillBootstrapQueue: draft_token_to_kv_pool: Optional[KVCache], req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator, metadata_buffers: MetadataBuffers, - tp_rank: int, - tp_size: int, gpu_id: int, bootstrap_port: int, gloo_group: ProcessGroup, max_total_num_tokens: int, scheduler: Scheduler, scheduler_stage_metrics: SchedulerStageMetricsRecorder, - pp_rank: int, - pp_size: int, transfer_backend: TransferBackend, ): + parallel = get_parallel() self.token_to_kv_pool = token_to_kv_pool self.draft_token_to_kv_pool = draft_token_to_kv_pool self.is_mla_backend = is_mla_backend(token_to_kv_pool) self.metadata_buffers = metadata_buffers self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator - self.tp_rank = tp_rank - self.tp_size = tp_size - self.pp_rank = pp_rank - self.pp_size = pp_size + self.tp_rank = parallel.tp_rank + self.tp_size = parallel.tp_size + self.pp_rank = parallel.pp_rank + self.pp_size = parallel.pp_size self.gpu_id = gpu_id self.bootstrap_port = bootstrap_port self.queue: List[Req] = [] diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index 6f82a92f3e..6630962729 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -142,8 +142,6 @@ def init_parallel_runtime( parallel = get_parallel() if device == "cpu": _init_cpu_threads_env( - tp_size=parallel.tp_size, - tp_rank=parallel.tp_rank, local_omp_cpuid=local_omp_cpuid, dist_init_method=dist_init_method, ) @@ -160,11 +158,7 @@ def init_parallel_runtime( if get_exec().comm.pre_warm_nccl and ( parallel.tp_size > 1 or parallel.pp_size > 1 or parallel.moe_ep_size > 1 ): - _prewarm_nccl( - tp_size=parallel.tp_size, - pp_size=parallel.pp_size, - moe_ep_size=parallel.moe_ep_size, - ) + _prewarm_nccl() # CUDA graph capture enables the PyNCCL communicator for TP LM-head # all-to-all. Exercise that exact send/recv path before measuring @@ -262,17 +256,16 @@ def _set_shm_master_env(dist_init_method: Optional[str]) -> None: def _init_cpu_threads_env( *, - tp_size: int, - tp_rank: int, local_omp_cpuid: Optional[List[int]], dist_init_method: Optional[str] = None, ) -> None: if _is_cpu_amx_available or _is_cpu_arm64: + parallel = get_parallel() # Bind OpenMP threads to CPU cores torch.ops.sgl_kernel.init_cpu_threads_env(local_omp_cpuid) # Set local size to hint SGLang to use shared memory based AllReduce - os.environ["LOCAL_SIZE"] = str(tp_size) + os.environ["LOCAL_SIZE"] = str(parallel.tp_size) # shm.cpp names its /dev/shm segments from MASTER_ADDR/MASTER_PORT. # Feed each engine's unique dist_init_method (tcp://host:port) into @@ -280,7 +273,7 @@ def _init_cpu_threads_env( # don't collide. _set_shm_master_env(dist_init_method) - torch.ops.sgl_kernel.initialize(tp_size, tp_rank) + torch.ops.sgl_kernel.initialize(parallel.tp_size, parallel.tp_rank) else: logger.warning( @@ -338,7 +331,8 @@ def init_layer_runtime(*, model_config: ModelConfig) -> None: initialize_layernorm_sp(model_config=model_config) -def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None: +def _prewarm_nccl() -> None: + parallel = get_parallel() warmup_start = time.perf_counter() tp_group_handle = get_tp_group().device_group @@ -350,7 +344,8 @@ def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None: warmup_elapsed = time.perf_counter() - warmup_start logger.info( f"NCCL/RCCL/HCCL warmup completed in {warmup_elapsed:.3f}s " - f"(tp_size={tp_size}, pp_size={pp_size}, ep_size={moe_ep_size})" + f"(tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, " + f"ep_size={parallel.moe_ep_size})" ) diff --git a/python/sglang/srt/elastic_ep/expert_backup_client.py b/python/sglang/srt/elastic_ep/expert_backup_client.py index ad4c555722..b4855a20c0 100644 --- a/python/sglang/srt/elastic_ep/expert_backup_client.py +++ b/python/sglang/srt/elastic_ep/expert_backup_client.py @@ -30,19 +30,18 @@ class ExpertBackupClient: self, *, model_config, - moe_ep_size: int, - moe_ep_rank: int, get_model: Callable[[], Any], ): context = zmq.Context(2) - self.engine_num = get_parallel().nnodes - self.engine_rank = get_parallel().node_rank + parallel = get_parallel() + self.engine_num = parallel.nnodes + self.engine_rank = parallel.node_rank self.recv_list = [None] * self.engine_num self.ready_sockets = [None] * self.engine_num self._get_model = get_model - self.moe_ep_size = moe_ep_size + self.moe_ep_size = parallel.moe_ep_size self.model_config = model_config - self.moe_ep_rank = moe_ep_rank + self.moe_ep_rank = parallel.moe_ep_rank self.dram_map_list = [None] * self.engine_num self.session_id_list = [None] * self.engine_num self.transfer_engine = None diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index aa267992c6..9b2454f9a8 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -730,15 +730,8 @@ class Engine(EngineScoreMixin, EngineBase): "--dist-init-addr so all nodes rendezvous at the same endpoint." ) - tp_size = get_parallel().tp_size - pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = ( - _calculate_rank_ranges( - get_parallel().nnodes, - get_parallel().pp_size, - tp_size, - get_parallel().node_rank, - ) + _calculate_rank_ranges(get_parallel().node_rank) ) # Build the distributed init method (multi-node uses the user-provided @@ -902,12 +895,7 @@ class Engine(EngineScoreMixin, EngineBase): scheduler_pipe_readers = [] pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = ( - _calculate_rank_ranges( - get_parallel().nnodes, - get_parallel().pp_size, - get_parallel().tp_size, - get_parallel().node_rank, - ) + _calculate_rank_ranges(get_parallel().node_rank) ) for pp_rank in pp_rank_range: @@ -918,9 +906,6 @@ class Engine(EngineScoreMixin, EngineBase): + ((pp_rank % pp_size_per_node) * tp_size_per_node) + (tp_rank % tp_size_per_node) * get_device().gpu_id_step ) - attn_cp_rank, moe_dp_rank, moe_ep_rank = _compute_parallelism_ranks( - tp_rank - ) with maybe_reindex_device_id(gpu_id) as gpu_id: proc = mp.Process( @@ -930,9 +915,6 @@ class Engine(EngineScoreMixin, EngineBase): port_args, gpu_id, tp_rank, - attn_cp_rank, - moe_dp_rank, - moe_ep_rank, pp_rank, None, writer, @@ -1925,15 +1907,13 @@ def _wait_for_scheduler_ready( return scheduler_infos -def _calculate_rank_ranges( - nnodes: int, pp_size: int, tp_size: int, node_rank: int -) -> Tuple[range, range, int, int]: +def _calculate_rank_ranges(node_rank: int) -> Tuple[range, range, int, int]: """Calculate pp_rank_range and tp_rank_range for a given node. + `node_rank` stays an argument because the Ray launchers size every node + from the driver, not just their own. + Args: - nnodes: Total number of nodes. - pp_size: Pipeline parallel size. - tp_size: Tensor parallel size. node_rank: The rank of the node to compute ranges for. Returns: @@ -1943,15 +1923,17 @@ def _calculate_rank_ranges( - pp_size_per_node: number of PP ranks per node. - tp_size_per_node: number of TP ranks per node. """ - pp_size_per_node = max(pp_size // nnodes, 1) - nnodes_per_pp_rank = max(nnodes // pp_size, 1) + parallel = get_parallel() + nnodes = parallel.nnodes + pp_size_per_node = max(parallel.pp_size // nnodes, 1) + nnodes_per_pp_rank = max(nnodes // parallel.pp_size, 1) pp_rank_range = range( pp_size_per_node * (node_rank // nnodes_per_pp_rank), pp_size_per_node * (node_rank // nnodes_per_pp_rank + 1), ) nnodes_per_tp_group = nnodes_per_pp_rank - tp_size_per_node = tp_size // nnodes_per_tp_group + tp_size_per_node = parallel.tp_size // nnodes_per_tp_group tp_rank_range = range( tp_size_per_node * (node_rank % nnodes_per_tp_group), tp_size_per_node * (node_rank % nnodes_per_tp_group + 1), @@ -1963,12 +1945,7 @@ def _calculate_rank_ranges( def node_hosts_rust_server() -> bool: """Whether this node contains a Rust listener rank, assuming Rust mode.""" parallel = get_parallel() - pp_rank_range, tp_rank_range, _, _ = _calculate_rank_ranges( - parallel.nnodes, - parallel.pp_size, - parallel.tp_size, - parallel.node_rank, - ) + pp_rank_range, tp_rank_range, _, _ = _calculate_rank_ranges(parallel.node_rank) if 0 not in pp_rank_range: return False @@ -1983,28 +1960,3 @@ def node_hosts_rust_server() -> bool: if rank_within_dp_group == 0: return True return False - - -def _compute_parallelism_ranks(tp_rank: int) -> Tuple[int, int, int]: - """Compute attention-CP, MoE-DP, and MoE-EP ranks for a TP rank. - - Called while the launcher is deciding what to spawn, so the sizes are the - configured ones -- the groups this is laying out do not exist yet. - """ - attn_dp_size = get_parallel().dp_size if get_parallel().enable_dp_attention else 1 - tp_size = get_parallel().tp_size - attn_cp_size = get_parallel().attn_cp_size - moe_dp_size = get_parallel().moe_dp_size - - # Parallelism hierarchy (outermost to innermost): - # - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost) - # - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost) - attn_tp_size = tp_size // attn_dp_size // attn_cp_size - attn_cp_rank = (tp_rank // attn_tp_size) % attn_cp_size - moe_dp_rank = tp_rank // (tp_size // moe_dp_size) - moe_ep_rank = ( - tp_rank - % (tp_size // moe_dp_size) - // (tp_size // moe_dp_size // get_parallel().ep_size) - ) - return attn_cp_rank, moe_dp_rank, moe_ep_rank diff --git a/python/sglang/srt/entrypoints/v1_loads.py b/python/sglang/srt/entrypoints/v1_loads.py index e3a6c27d29..1d466e4320 100644 --- a/python/sglang/srt/entrypoints/v1_loads.py +++ b/python/sglang/srt/entrypoints/v1_loads.py @@ -41,16 +41,16 @@ def _accelerator_name() -> Optional[str]: return get_device_name() -@lru_cache(maxsize=1) -def _num_accelerators_per_dp_rank( - tp_size: int, - pp_size: int, - dp_size: int, - enable_dp_attention: bool, -) -> int: - num_accelerators = tp_size * pp_size - if enable_dp_attention: - num_accelerators //= dp_size +def _num_accelerators_per_dp_rank() -> int: + """Accelerators behind one DP rank. + + Not cached: the arguments were the cache key, so an ``lru_cache`` here + would freeze the first answer past a post-publish override. + """ + parallel = get_parallel() + num_accelerators = parallel.tp_size * parallel.pp_size + if parallel.enable_dp_attention: + num_accelerators //= parallel.dp_size return num_accelerators @@ -145,11 +145,6 @@ async def get_loads( "timestamp": datetime.now(timezone.utc).isoformat(), "version": __version__, "accelerator": _accelerator_name(), - "num_accelerators": _num_accelerators_per_dp_rank( - get_parallel().tp_size, - get_parallel().pp_size, - get_parallel().dp_size, - get_parallel().enable_dp_attention, - ), + "num_accelerators": _num_accelerators_per_dp_rank(), "loads": loads, } diff --git a/python/sglang/srt/eplb/eplb_manager.py b/python/sglang/srt/eplb/eplb_manager.py index 6f59f3e400..0388dbee15 100644 --- a/python/sglang/srt/eplb/eplb_manager.py +++ b/python/sglang/srt/eplb/eplb_manager.py @@ -157,7 +157,6 @@ class EPLBManager: model=self._get_model(), new_expert_location_metadata=expert_location_metadata, update_layer_ids=chunk_layer_ids, - nnodes=get_parallel().nnodes, tp_rank=( self._elastic_global_rank() if is_post_scale_rebalance @@ -302,7 +301,6 @@ def update_expert_location_with_recovery( model: nn.Module, new_expert_location_metadata: ExpertLocationMetadata, update_layer_ids: List[int], - nnodes: int, tp_rank: int, use_flat_topology: bool = False, expert_backup_client, @@ -314,7 +312,7 @@ def update_expert_location_with_recovery( model.routed_experts_weights_of_layer, new_expert_location_metadata, update_layer_ids=update_layer_ids, - nnodes=nnodes, + nnodes=get_parallel().nnodes, rank=tp_rank, use_flat_topology=use_flat_topology, ) diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index f3c97fc2b3..84dd8d0a95 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -1089,9 +1089,7 @@ class LogitsProcessor(nn.Module): get_parallel().tp_group.all_to_all_single( all_to_all_output.view(-1), logits.view(-1) ) - return _reassemble_tp_lm_head_all_to_all_output( - all_to_all_output, get_parallel().tp_size - ) + return _reassemble_tp_lm_head_all_to_all_output(all_to_all_output) def _scatter_dp_attn_logits( self, @@ -1252,7 +1250,7 @@ class LogitsProcessor(nn.Module): def _reassemble_tp_lm_head_all_to_all_output( - all_to_all_output: torch.Tensor, tp_size: int + all_to_all_output: torch.Tensor, ) -> torch.Tensor: """Convert source-major all-to-all output to row-major full-vocab logits. @@ -1261,6 +1259,7 @@ def _reassemble_tp_lm_head_all_to_all_output( along dim 0, while the sampler expects the vocab shards concatenated along dim 1. """ + tp_size = get_parallel().tp_size assert all_to_all_output.shape[0] % tp_size == 0 local_rows = all_to_all_output.shape[0] // tp_size vocab_shard = all_to_all_output.shape[1] diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index d1f0fc55ef..8c1b940c9c 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -467,7 +467,7 @@ class DataParallelController: ) else: # Other nodes: Receive worker ports from node 0 - return self._receive_ports_as_client(endpoint, get_parallel().node_rank) + return self._receive_ports_as_client(endpoint) def _broadcast_ports_as_server( self, endpoint: str, expected_clients: int, worker_ports: list[int] @@ -522,9 +522,10 @@ class DataParallelController: sock_send(rep_socket, wrap_as_pickle(worker_ports)) logger.debug(f"Sent worker ports to node {client_rank}") - def _receive_ports_as_client(self, endpoint: str, node_rank: int) -> list[int]: + def _receive_ports_as_client(self, endpoint: str) -> list[int]: """Receive worker ports from the server node.""" logger.debug("Connecting to node 0 to receive worker ports") + node_rank = get_parallel().node_rank req_socket = get_zmq_socket(self.context, zmq.REQ, endpoint, False) req_socket.setsockopt(zmq.RCVTIMEO, 600 * 1000) # 10 minute timeout @@ -567,9 +568,7 @@ class DataParallelController: primary_endpoint = NetworkAddress( primary.host, primary.port + DP_ATTENTION_HANDSHAKE_PORT_DELTA ).to_tcp() - all_ports = self._receive_ports_as_client( - primary_endpoint, get_parallel().node_rank - ) + all_ports = self._receive_ports_as_client(primary_endpoint) offset = self._joiner_slot_offset(server_args) local_tp_span = self._joiner_local_tp_span(server_args) broadcasted_ports = all_ports[offset : offset + local_tp_span] @@ -636,8 +635,6 @@ class DataParallelController: tp_size_per_node * (get_parallel().node_rank % nnodes_per_tp_group + 1), ) - attn_cp_rank = 0 - moe_dp_rank = 0 for pp_rank in pp_rank_range: for tp_rank in tp_rank_range: rank_port_args = port_args @@ -679,22 +676,7 @@ class DataParallelController: + ((pp_rank % pp_size_per_node) * tp_size_per_node) + (tp_rank % tp_size_per_node) * get_device().gpu_id_step ) - attn_dp_size = ( - get_parallel().dp_size if get_parallel().enable_dp_attention else 1 - ) - - # Parallelism hierarchy (outermost to innermost): - # - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost) - # - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost) - attn_tp_size = ( - get_parallel().tp_size - // attn_dp_size - // get_parallel().attn_cp_size - ) - attn_cp_rank = (tp_rank // attn_tp_size) % get_parallel().attn_cp_size - moe_dp_rank = tp_rank // ( - get_parallel().tp_size // get_parallel().moe_dp_size - ) + # Derive the child's EP rank for its display label. moe_ep_rank = ( tp_rank % (get_parallel().tp_size // get_parallel().moe_dp_size) @@ -719,9 +701,6 @@ class DataParallelController: rank_port_args, gpu_id, tp_rank, - attn_cp_rank, - moe_dp_rank, - moe_ep_rank, pp_rank, dp_rank, writer, diff --git a/python/sglang/srt/managers/hisparse_coordinator.py b/python/sglang/srt/managers/hisparse_coordinator.py index d05187e54c..37e87d29db 100644 --- a/python/sglang/srt/managers/hisparse_coordinator.py +++ b/python/sglang/srt/managers/hisparse_coordinator.py @@ -47,6 +47,7 @@ from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool, ReqToTokenPool from sglang.srt.mem_cache.memory_pool_host import DeepSeekV4PagedHostPool from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.runtime_context import get_parallel device_module = get_device_module() @@ -72,7 +73,6 @@ class HiSparseTokenStats(NamedTuple): def resolve_shared_index_layers( *, hf_text_config, - pp_size: int, is_speculative: bool, ) -> Optional[List[bool]]: """Per-layer "reuses the previous layer's top-k index" pattern, or None. @@ -91,7 +91,7 @@ def resolve_shared_index_layers( pattern = [dsa_layer_skips_topk(hf_text_config, i) for i in range(num_layers)] if not any(pattern): return None - if pp_size != 1 or is_speculative: + if get_parallel().pp_size != 1 or is_speculative: logger.warning( "HiSparse shared-index prefetch is unsupported under pipeline " "parallelism / speculative decoding; falling back to synchronous " diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index c8408e843c..edcfa66969 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -75,8 +75,6 @@ class _NegotiateOutput(NamedTuple): class PrefillDelayer: def __init__( self, - dp_size: int, - attn_tp_size: int, cpu_group, max_delay_passes: int, token_usage_low_watermark: Optional[float], @@ -106,9 +104,10 @@ class PrefillDelayer: f"max_delay_ms={self._max_delay_ms} " f"queue_trigger_enabled={self._queue_trigger_enabled}" ) - self.dp_size = dp_size - self.enable_dp_attention = get_parallel().enable_dp_attention - dp_size_dim = dp_size if self.enable_dp_attention else 1 + parallel = get_parallel() + self.dp_size = parallel.dp_size + self.enable_dp_attention = parallel.enable_dp_attention + dp_size_dim = self.dp_size if self.enable_dp_attention else 1 # Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the # env flag is on (or overlap scheduling is disabled), ride the NCCL @@ -131,7 +130,7 @@ class PrefillDelayer: # token_watermark_force_allow, running_batch, max_prefill_bs, # waiting_queue_len. self._global_info_buffer = torch.empty( - (dp_size_dim, attn_tp_size, 5), + (dp_size_dim, parallel.attn_tp_size, 5), dtype=torch.int64, device=self._gather_device, ) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index ccdd14f191..8dc1616171 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -449,9 +449,6 @@ class Scheduler( self, server_args: ServerArgs, port_args: PortArgs, - tp_rank: int, - pp_rank: int, - dp_rank: Optional[int], ): # NOTE: KEEP THE FOLLOWING CODE STYLE for this function: # Keep __init__ as an orchestrator: sequence init_* and maybe_init_* calls @@ -526,7 +523,7 @@ class Scheduler( bootstrap.init_layer_runtime(model_config=self.model_config) # Init metrics stats - self.init_metrics_collector(tp_rank, pp_rank, dp_rank) + self.init_metrics_collector() # Init inter-process communication self.init_ipc_channels(port_args) @@ -637,7 +634,7 @@ class Scheduler( # Init diffusion LLM self.init_diffusion_llm() - self.init_metrics_reporter(tp_rank, pp_rank, dp_rank) + self.init_metrics_reporter() self.scheduler_stage_metrics = self.metrics_reporter.scheduler_stage_metrics # Init schedule policy and new token estimation @@ -772,14 +769,9 @@ class Scheduler( else None ) - def init_metrics_collector( - self, tp_rank: int, pp_rank: int, dp_rank: Optional[int] - ) -> None: + def init_metrics_collector(self) -> None: self.metrics_collector_context = SchedulerMetricsCollector.init_new( server_args=self.server_args, - tp_rank=tp_rank, - pp_rank=pp_rank, - dp_rank=dp_rank, enable_priority_scheduling=self.enable_priority_scheduling, enable_lora=self.enable_lora, enable_hierarchical_cache=self.enable_hierarchical_cache, @@ -1340,9 +1332,6 @@ class Scheduler( max_prefill_tokens=self.max_prefill_tokens, page_size=self.page_size, device=self.device, - pp_group=self.pp_group, - world_group=self.world_group, - pp_rank=get_parallel().pp_rank, ) if sizer.profile_and_fit(): self.dynamic_chunk_sizer = sizer @@ -1369,15 +1358,10 @@ class Scheduler( if is_extend: self._prefill_decode_interval_remaining = self.prefill_decode_interval - def init_metrics_reporter( - self, tp_rank: int, pp_rank: int, dp_rank: Optional[int] - ) -> None: + def init_metrics_reporter(self) -> None: # Override point for deployments that need a specialized reporter. self.metrics_reporter = SchedulerMetricsReporter( scheduler=self, - tp_rank=tp_rank, - pp_rank=pp_rank, - dp_rank=dp_rank, metrics_collector_context=self.metrics_collector_context, metrics_collector=self.metrics_collector, ) @@ -1404,8 +1388,6 @@ class Scheduler( ) else: self.prefill_delayer = PrefillDelayer( - dp_size=get_parallel().dp_size, - attn_tp_size=get_parallel().attn_tp_size, cpu_group=self.tp_cpu_group, device_group=self.tp_group.device_group, metrics_collector=( @@ -1536,7 +1518,6 @@ class Scheduler( self.disagg_decode_transfer_queue = DecodeTransferQueue( gloo_group=self.attn_tp_cpu_group, req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator, - tp_rank=get_parallel().tp_rank, metadata_buffers=self.disagg_metadata_buffers, scheduler=self, tree_cache=self.tree_cache, @@ -1553,13 +1534,9 @@ class Scheduler( transfer_queue=self.disagg_decode_transfer_queue, tree_cache=self.tree_cache, gloo_group=self.attn_tp_cpu_group, - tp_rank=get_parallel().tp_rank, - tp_size=get_parallel().tp_size, - dp_size=get_parallel().dp_size, gpu_id=get_device().gpu_id, bootstrap_port=get_disagg().disaggregation_bootstrap_port, max_total_num_tokens=self.max_total_num_tokens, - pp_rank=get_parallel().pp_rank, num_reserved_decode_tokens=get_disagg().num_reserved_decode_tokens, transfer_backend=self.transfer_backend, ) @@ -1585,16 +1562,12 @@ class Scheduler( draft_token_to_kv_pool=draft_token_to_kv_pool, req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator, metadata_buffers=self.disagg_metadata_buffers, - tp_rank=get_parallel().tp_rank, - tp_size=get_parallel().tp_size, gpu_id=get_device().gpu_id, bootstrap_port=get_disagg().disaggregation_bootstrap_port, gloo_group=self.attn_tp_cpu_group, max_total_num_tokens=self.max_total_num_tokens, scheduler=self, scheduler_stage_metrics=self.scheduler_stage_metrics, - pp_rank=get_parallel().pp_rank, - pp_size=get_parallel().pp_size, transfer_backend=self.transfer_backend, ) # The prefill requests that are in the middle of kv sending @@ -2337,7 +2310,6 @@ class Scheduler( ) self.dp_attn_adapter = SchedulerDPAttnAdapter( model_runner=target_worker.model_runner, - tp_group=self.tp_group, req_to_token_pool=self.req_to_token_pool, token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, tree_cache=self.tree_cache, @@ -2398,10 +2370,6 @@ class Scheduler( def init_kv_events_publisher(self) -> None: self.kv_events_publisher = SchedulerKvEventsPublisher( kv_events_config=get_observability().kv_events_config, - attn_tp_rank=get_parallel().attn_tp_rank, - attn_cp_rank=get_parallel().attn_cp_rank, - attn_dp_rank=get_parallel().attn_dp_rank, - dp_rank=get_parallel().dp_rank, tree_cache=self.tree_cache, send_metrics_from_scheduler=self.ipc_channels.send_metrics_from_scheduler, max_running_requests=self.max_running_requests, @@ -5954,42 +5922,37 @@ def resolve_spawn_dp_rank(dp_rank: Optional[int]) -> Optional[int]: def configure_scheduler_process( server_args: ServerArgs, gpu_id: int, - tp_rank: int, - attn_cp_rank: int, - moe_dp_rank: int, - moe_ep_rank: int, - pp_rank: int, - dp_rank: Optional[int], display_tp_rank: Optional[int] = None, display_dp_rank: Optional[int] = None, display_moe_ep_rank: Optional[int] = None, ) -> None: """Configure scheduler worker logging and process title. - display_* ranks are cosmetic; runtime ranks stay local. `dp_rank` arrives - already resolved -- see `resolve_spawn_dp_rank`. + Runs after `publish`, so every rank it labels the process with comes from + the context. display_* ranks are cosmetic; runtime ranks stay local. """ kill_itself_when_parent_died() # Generate the logger prefix - shown_dp = display_dp_rank if display_dp_rank is not None else dp_rank - shown_tp = display_tp_rank if display_tp_rank is not None else tp_rank + parallel = get_parallel() + shown_dp = display_dp_rank if display_dp_rank is not None else parallel.dp_rank + shown_tp = display_tp_rank if display_tp_rank is not None else parallel.tp_rank shown_moe_ep = ( - display_moe_ep_rank if display_moe_ep_rank is not None else moe_ep_rank + display_moe_ep_rank if display_moe_ep_rank is not None else parallel.moe_ep_rank ) prefix = "" if shown_dp is not None: prefix += f" DP{shown_dp}" - if get_parallel().pp_size > 1: - prefix += f" PP{pp_rank}" - if get_parallel().attn_cp_size > 1: - prefix += f" ATTN_CP{attn_cp_rank}" - if get_parallel().moe_dp_size > 1: - prefix += f" MOE_DP{moe_dp_rank}" - if get_parallel().tp_size > 1: + if parallel.pp_size > 1: + prefix += f" PP{parallel.pp_rank}" + if parallel.attn_cp_size > 1: + prefix += f" ATTN_CP{parallel.attn_cp_rank}" + if parallel.moe_dp_size > 1: + prefix += f" MOE_DP{parallel.moe_dp_rank}" + if parallel.tp_size > 1: prefix += f" TP{shown_tp}" - if get_parallel().ep_size > 1: + if parallel.ep_size > 1: prefix += f" EP{shown_moe_ep}" # Config the process @@ -6002,12 +5965,7 @@ def configure_scheduler_process( # Set cpu affinity to this gpu process if envs.SGLANG_SET_CPU_AFFINITY.get(): - set_gpu_proc_affinity( - get_parallel().pp_size, - get_parallel().tp_size, - get_parallel().nnodes, - gpu_id, - ) + set_gpu_proc_affinity(gpu_id) if not envs.SGLANG_NUMA_BIND_V2.get(): numa_node = get_numa_node_if_available(server_args, gpu_id) if numa_node is not None: @@ -6019,9 +5977,6 @@ def run_scheduler_process( port_args: PortArgs, gpu_id: int, tp_rank: int, - attn_cp_rank: int, - moe_dp_rank: int, - moe_ep_rank: int, pp_rank: int, dp_rank: Optional[int], pipe_writer, @@ -6032,6 +5987,8 @@ def run_scheduler_process( # Load plugins so hooks can override Scheduler and its dependencies. load_plugins() dp_rank = resolve_spawn_dp_rank(dp_rank) + # Placement is needed before publish(): TP/PP select a WORLD rank; + # ordinary DP replicas have separate WORLDs and need an explicit DP rank. publish( server_args, role="scheduler", @@ -6044,12 +6001,6 @@ def run_scheduler_process( configure_scheduler_process( server_args, gpu_id, - tp_rank, - attn_cp_rank, - moe_dp_rank, - moe_ep_rank, - pp_rank, - dp_rank, display_tp_rank=display_tp_rank, display_dp_rank=display_dp_rank, display_moe_ep_rank=display_moe_ep_rank, @@ -6073,13 +6024,7 @@ def run_scheduler_process( # Create a scheduler and run the event loop scheduler = None try: - scheduler = Scheduler( - server_args, - port_args, - tp_rank, - pp_rank, - dp_rank, - ) + scheduler = Scheduler(server_args, port_args) # Send initialization info back to the parent process pipe_writer.send(scheduler.get_init_info()) diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index cc6c8aa94d..57b6c6e1c1 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -38,7 +38,6 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils.common import require_mlp_tp_gather if TYPE_CHECKING: - from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.model_executor.model_runner import ModelRunner @@ -363,10 +362,6 @@ def _local_prefill_cuda_graph_vote( def prepare_mlp_sync_batch_raw( local_batch: ScheduleBatch, model_runner: ModelRunner, - dp_size: int, - attn_tp_size: int, - attn_cp_size: int, - tp_group: GroupCoordinator, get_idle_batch: Callable[[], ScheduleBatch], disable_cuda_graph: bool, require_mlp_tp_gather: bool, @@ -374,6 +369,10 @@ def prepare_mlp_sync_batch_raw( offload_tags: set[str], dwdp: bool = False, ): + parallel = get_parallel() + dp_size = parallel.dp_size + attn_tp_size = parallel.attn_tp_size + tp_group = parallel.tp_group # Check if other DP workers have running batches if ( local_batch is None @@ -432,9 +431,7 @@ def prepare_mlp_sync_batch_raw( tbo_preparer = TboDPAttentionPreparer() use_world_group = world_dp_gather_enabled() if use_world_group: - from sglang.srt.runtime_context import get_parallel - - world = get_parallel().world_group + world = parallel.world_group group = torch.distributed.group.WORLD device = world.device elif len(offload_tags) == 0 and ( @@ -460,7 +457,7 @@ def prepare_mlp_sync_batch_raw( mlp_sync_info = MLPSyncBatchInfo( dp_size=dp_size, tp_size=attn_tp_size, - cp_size=attn_cp_size, + cp_size=parallel.attn_cp_size, num_tokens=num_tokens, num_tokens_for_logprob=num_tokens_for_logprob, can_run_decode_cuda_graph=can_run_decode_cuda_graph, @@ -529,7 +526,6 @@ def prepare_mlp_sync_batch_raw( @dataclass(kw_only=True, slots=True, frozen=True) class SchedulerDPAttnAdapter: model_runner: ModelRunner - tp_group: GroupCoordinator req_to_token_pool: ReqToTokenPool token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator tree_cache: BasePrefixCache @@ -543,10 +539,6 @@ class SchedulerDPAttnAdapter: return prepare_mlp_sync_batch_raw( local_batch, model_runner=self.model_runner, - dp_size=get_parallel().dp_size, - attn_tp_size=get_parallel().attn_tp_size, - attn_cp_size=get_parallel().attn_cp_size, - tp_group=self.tp_group, get_idle_batch=self.get_idle_batch, disable_cuda_graph=cuda_graph_fully_disabled(), require_mlp_tp_gather=require_mlp_tp_gather(), diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index e6df01c923..318675488d 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -25,7 +25,6 @@ from sglang.srt.utils.common import get_device_module if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig - from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool @@ -51,9 +50,6 @@ class DynamicChunkSizer: max_prefill_tokens: int, page_size: int, device: str, - pp_group: GroupCoordinator, - world_group: GroupCoordinator, - pp_rank: int, ): self.model_runner = model_runner self.model_config = model_config @@ -65,9 +61,7 @@ class DynamicChunkSizer: self.max_prefill_tokens = max_prefill_tokens self.page_size = page_size self.device = device - self.pp_group = pp_group - self.world_group = world_group - self.pp_rank = pp_rank + self.pp_rank = get_parallel().pp_rank self.predictor = ChunkSizePredictor() def profile_and_fit(self) -> bool: @@ -75,7 +69,8 @@ class DynamicChunkSizer: returns whether the predictor is ready.""" samples: Optional[Tuple[List[int], List[float]]] = None - if self.pp_group.is_first_rank: + parallel = get_parallel() + if parallel.pp_group.is_first_rank: try: samples = self._profile_prefill_latency() except Exception as e: @@ -86,8 +81,9 @@ class DynamicChunkSizer: # The samples are global, so one broadcast from global rank 0 (a PP0 rank) # reaches every stage and attention rank; a failure travels as None. + world_group = parallel.world_group samples = broadcast_pyobj( - [samples], self.world_group.rank, self.world_group.cpu_group, src=0 + [samples], world_group.rank, world_group.cpu_group, src=0 )[0] if samples is None: diff --git a/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py b/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py index 7ed38d252b..0bde503f4b 100644 --- a/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py +++ b/python/sglang/srt/managers/scheduler_components/kv_events_publisher.py @@ -45,10 +45,6 @@ hook_custom_types(KvMetrics) @dataclass(kw_only=True, slots=True) class SchedulerKvEventsPublisher: kv_events_config: Optional[str] - attn_tp_rank: int - attn_cp_rank: int - attn_dp_rank: int - dp_rank: Optional[int] tree_cache: BasePrefixCache send_metrics_from_scheduler: Optional[zmq.Socket] max_running_requests: int diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index 748dc19fc7..e371915adb 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -203,9 +203,6 @@ class _ForwardOccupancyLogWindow(msgspec.Struct): @dataclass(kw_only=True) class SchedulerMetricsReporter: scheduler: Scheduler - tp_rank: int - pp_rank: int - dp_rank: Optional[int] metrics_collector_context: SchedulerMetricsCollectorContext metrics_collector: Optional[SchedulerMetricsCollector] num_retracted_reqs: int = 0 @@ -222,7 +219,7 @@ class SchedulerMetricsReporter: self.enable_kv_cache_events = ( self.metrics_collector_context.enable_kv_cache_events ) - self._init_metrics(self.tp_rank, self.pp_rank, self.dp_rank) + self._init_metrics() self._install_device_timer_on_runners() # Keep log history after the existing async result copy so reporting does # not synchronize the model stream once per generated token. @@ -244,12 +241,7 @@ class SchedulerMetricsReporter: self.last_gen_throughput = 0.0 return self.last_gen_throughput - def _init_metrics( - self, - tp_rank: int, - pp_rank: int, - dp_rank: Optional[int], - ): + def _init_metrics(self): # Basic stats self.forward_ct_decode = 0 self.num_generated_tokens = 0 diff --git a/python/sglang/srt/mem_cache/embedding_cache_controller.py b/python/sglang/srt/mem_cache/embedding_cache_controller.py index 4a95d36351..8437303e07 100644 --- a/python/sglang/srt/mem_cache/embedding_cache_controller.py +++ b/python/sglang/srt/mem_cache/embedding_cache_controller.py @@ -18,6 +18,7 @@ except ImportError: from sglang.srt.managers.schedule_batch import Modality from sglang.srt.mem_cache.embedding_store import EmbeddingStore +from sglang.srt.runtime_context import get_parallel logger = logging.getLogger(__name__) @@ -344,20 +345,18 @@ class EmbeddingInsertOperation: class EmbeddingCacheController: def __init__( self, - tp_rank, - tp_size, embedding_store: EmbeddingStore, max_pool_size_gb=4.0, hidden_dims: dict = None, - tp_group=None, all_rank_get=False, enable_eviction: bool = True, max_eviction_batch: int = 100, dtype: torch.dtype = torch.float32, ): - self.tp_world_size = tp_size - self.tp_group = tp_group - self.tp_rank = tp_rank + parallel = get_parallel() + self.tp_world_size = parallel.tp_size + self.tp_group = parallel.tp_group.cpu_group + self.tp_rank = parallel.tp_rank self.all_rank_get = all_rank_get self.hidden_dims = hidden_dims or {} # Pool dtype must match the model's embedding dtype so that pool views, diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index a273884f23..52cda28703 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -35,6 +35,7 @@ from sglang.srt.model_executor.input_buffers import ( INDEX_SEMANTIC_BUFFERS, share_input_buffer, ) +from sglang.srt.runtime_context import get_parallel if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -1011,7 +1012,6 @@ def build_eager_registry( is_encoder_decoder: bool = False, encoder_len_fill_value: int = 0, encoder_lens_dtype: torch.dtype = torch.int32, - dp_size: int = 1, ) -> CudaGraphBufferRegistry: """One fixed-max input registry for the ``EagerRunner``, serving BOTH eager decode and eager prefill. @@ -1042,7 +1042,7 @@ def build_eager_registry( register_global_num_tokens=False, require_gathered_buffer=False, require_mlp_tp_gather=False, - dp_size=dp_size, + dp_size=get_parallel().dp_size, share_pool=True, source=None, ) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index bd0da74046..5a30720783 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -437,7 +437,7 @@ class ModelRunner: self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None # CPU offload - set_offloader(create_offloader(dp_rank=get_parallel().dp_rank)) + set_offloader(create_offloader()) self._weight_checker = WeightChecker(get_model=lambda: self.model) @@ -582,7 +582,6 @@ class ModelRunner: def init_remote_instance_weight_transporter(self): self.remote_instance_weight_transporter = RemoteInstanceWeightTransporter( get_model=lambda: self.model, - tp_rank=get_parallel().tp_rank, gpu_id=self.gpu_id, ) @@ -653,12 +652,7 @@ class ModelRunner: self.init_token_oracle() self.sampler = create_sampler() self.load_model() - prepare_moe_topk( - model=self.model, - model_config=self.model_config, - moe_ep_size=get_parallel().moe_ep_size, - moe_ep_rank=get_parallel().moe_ep_rank, - ) + prepare_moe_topk(model=self.model, model_config=self.model_config) self.maybe_init_dwdp() @@ -754,8 +748,6 @@ class ModelRunner: self.expert_backup_client = ( ExpertBackupClient( model_config=self.model_config, - moe_ep_size=get_parallel().moe_ep_size, - moe_ep_rank=get_parallel().moe_ep_rank, get_model=lambda: self.model, ) if ( @@ -797,16 +789,12 @@ class ModelRunner: def get_pp_proxy_topk_size(self) -> Optional[int]: return misc_utils.resolve_pp_proxy_topk_size( model_config=self.model_config, - pp_size=get_parallel().pp_size, - pp_rank=get_parallel().pp_rank, start_layer=self.layer_info.start_layer, ) def get_pp_proxy_residual_num_blocks(self) -> Optional[int]: return misc_utils.resolve_pp_proxy_residual_num_blocks( model_config=self.model_config, - pp_size=get_parallel().pp_size, - pp_rank=get_parallel().pp_rank, start_layer=self.layer_info.start_layer, ) @@ -974,7 +962,6 @@ class ModelRunner: swap_in_block_size=hisparse_cfg.swap_in_block_size, shared_index_layers=resolve_shared_index_layers( hf_text_config=self.model_config.hf_text_config, - pp_size=get_parallel().pp_size, is_speculative=self.spec_algorithm.is_speculative(), ), ) diff --git a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py index 026ca04344..f510693e54 100644 --- a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py +++ b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py @@ -12,6 +12,7 @@ from sglang.srt.runtime_context import ( attention_backends, get_context, get_observability, + get_parallel, get_schedule, ) from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS @@ -66,12 +67,13 @@ def create_msprobe_debugger() -> Optional[Any]: def resolve_pp_proxy_topk_size( - *, model_config: ModelConfig, pp_size: int, pp_rank: int, start_layer: int + *, model_config: ModelConfig, start_layer: int ) -> Optional[int]: hf_config = model_config.hf_text_config + parallel = get_parallel() if ( - pp_size <= 1 - or pp_rank == 0 + parallel.pp_size <= 1 + or parallel.pp_rank == 0 or not is_deepseek_dsa(hf_config) or not dsa_layer_skips_topk(hf_config, start_layer) ): @@ -80,10 +82,15 @@ def resolve_pp_proxy_topk_size( def resolve_pp_proxy_residual_num_blocks( - *, model_config: ModelConfig, pp_size: int, pp_rank: int, start_layer: int + *, model_config: ModelConfig, start_layer: int ) -> Optional[int]: """Return the inherited Kimi K3 attention-residual bank width.""" - if pp_size <= 1 or pp_rank == 0 or not is_kimi_k3(model_config.hf_config): + parallel = get_parallel() + if ( + parallel.pp_size <= 1 + or parallel.pp_rank == 0 + or not is_kimi_k3(model_config.hf_config) + ): return None block_size = getattr(model_config.hf_text_config, "attn_res_block_size", None) diff --git a/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py b/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py index 016fd2a383..4f734ac096 100644 --- a/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py @@ -13,7 +13,7 @@ from sglang.srt.eplb.lplb_solver import ( ) from sglang.srt.layers.moe.hash_topk import HashTopK from sglang.srt.layers.moe.topk import TopK -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0 if TYPE_CHECKING: @@ -24,13 +24,8 @@ logger = logging.getLogger(__name__) _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() -def prepare_moe_topk( - *, - model, - model_config: ModelConfig, - moe_ep_size: int, - moe_ep_rank: int, -) -> None: +def prepare_moe_topk(*, model, model_config: ModelConfig) -> None: + parallel = get_parallel() balancer_cls = None num_prepared = 0 num_routed_experts = None @@ -61,8 +56,8 @@ def prepare_moe_topk( routed_scaling_factor = module.routed_scaling_factor module.waterfill_balancer = balancer_cls( num_routed_experts=num_physical_routed_experts, - world_size=moe_ep_size, - rank=moe_ep_rank, + world_size=parallel.moe_ep_size, + rank=parallel.moe_ep_rank, layer_id=module.layer_id, routed_scaling_factor=( routed_scaling_factor if routed_scaling_factor is not None else 1.0 diff --git a/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py b/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py index 14690ab217..8d10417696 100644 --- a/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py +++ b/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Callable, Optional import torch @@ -24,7 +24,8 @@ logger = logging.getLogger(__name__) @dataclass(slots=True, kw_only=True) class RemoteInstanceWeightTransporter: get_model: Callable[[], torch.nn.Module] - tp_rank: int + # Registration may run after the runner's construction scope has exited. + tp_rank: int = field(init=False, default_factory=lambda: get_parallel().tp_rank) gpu_id: int engine: Optional[Any] = None session_id: str = "" diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 39e1970c6c..1c3084f635 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -76,8 +76,6 @@ def _allocate_decode_buffers( hidden_size: int, vocab_size: int, dtype: torch.dtype, - dp_size: int, - pp_size: int, is_encoder_decoder: bool, require_mlp_tp_gather: bool, seq_len_fill_value: int, @@ -92,6 +90,7 @@ def _allocate_decode_buffers( allocate_logits_buffer: bool = True, ) -> SimpleNamespace: """Allocate the FB-shared decode buffers.""" + parallel = get_parallel() with torch.device(device): input_ids = torch.zeros((max_num_token,), dtype=torch.int64) input_embeds = torch.zeros((max_num_token, hidden_size), dtype=dtype) @@ -127,7 +126,7 @@ def _allocate_decode_buffers( torch.zeros((max_bs,), dtype=torch.bool) if enable_mamba_track else None ) - if pp_size > 1: + if parallel.pp_size > 1: # mHC (e.g. DSV4) flattens residual into hidden_states (size = hc_hidden_size). is_mhc = hc_hidden_size is not None hs = hc_hidden_size if is_mhc else hidden_size @@ -163,9 +162,9 @@ def _allocate_decode_buffers( encoder_lens = None if require_mlp_tp_gather: - global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32) + global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32) global_num_tokens_for_logprob_gpu = torch.zeros( - (dp_size,), dtype=torch.int32 + (parallel.dp_size,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) @@ -367,8 +366,6 @@ class BaseRunner(ABC): hidden_size=mr.model_config.hidden_size, vocab_size=mr.model_config.vocab_size, dtype=mr.model_config.dtype, - dp_size=get_parallel().dp_size, - pp_size=get_parallel().pp_size, is_encoder_decoder=mr.model_config.is_encoder_decoder, require_mlp_tp_gather=require_mlp_tp_gather(), seq_len_fill_value=mr.attn_backend.get_cuda_graph_seq_len_fill_value(), diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index d43667cbff..605ec0d911 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -849,7 +849,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): export_cuda_graph_capture_trace( prof_context, runner_name=type(self).__name__, - tp_rank=get_parallel().tp_rank, ) def capture_prepare( diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index 11a3df8016..acb438eb57 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -59,7 +59,6 @@ from sglang.srt.model_executor.runner_utils import ( ) from sglang.srt.runtime_context import ( get_exec, - get_parallel, get_spec, max_prefill_buffer_tokens, max_speculative_num_draft_tokens, @@ -150,7 +149,6 @@ class EagerRunner(BaseRunner): encoder_lens_dtype=( torch.int64 if torch.device(mr.device).type == "cpu" else torch.int32 ), - dp_size=get_parallel().dp_size, ) # Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up). if run_warmup: diff --git a/python/sglang/srt/models/granitemoe.py b/python/sglang/srt/models/granitemoe.py index 7ee66789ab..5c6d4d91c7 100644 --- a/python/sglang/srt/models/granitemoe.py +++ b/python/sglang/srt/models/granitemoe.py @@ -117,7 +117,6 @@ class GraniteMoeMoE(nn.Module): layer_id: int, params_dtype: Optional[torch.dtype] = None, quant_config: Optional[QuantizationConfig] = None, - tp_size: Optional[int] = None, prefix: str = "", ): super().__init__() diff --git a/python/sglang/srt/models/granitemoehybrid.py b/python/sglang/srt/models/granitemoehybrid.py index 01021ebba4..a0a7804f44 100644 --- a/python/sglang/srt/models/granitemoehybrid.py +++ b/python/sglang/srt/models/granitemoehybrid.py @@ -66,7 +66,6 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, layer_id=layer_idx, quant_config=quant_config, - tp_size=get_parallel().tp_size, prefix=f"{prefix}.block_sparse_moe", ) @@ -252,7 +251,6 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, layer_id=layer_idx, quant_config=quant_config, - tp_size=get_parallel().tp_size, prefix=f"{prefix}.block_sparse_moe", ) diff --git a/python/sglang/srt/models/inkling_common/moe.py b/python/sglang/srt/models/inkling_common/moe.py index dd5dd4459f..6f095de4af 100644 --- a/python/sglang/srt/models/inkling_common/moe.py +++ b/python/sglang/srt/models/inkling_common/moe.py @@ -713,12 +713,12 @@ def _build_inkling_shared_experts( intermediate_size: int, layer_id: int, prefix: str, - moe_tp_rank: int, - moe_tp_size: int, quant_config: QuantizationConfig | None = None, ) -> nn.Module | None: if n_shared_experts <= 0: return None + # The shared dense MLP is reconstructed by the full-TP all-reduce in forward. + parallel = get_parallel() if shared_expert_sink: shared_prefix = add_prefix("shared_experts", prefix) shared_sink_serves_fp4 = InklingBatchDenseMLP._resolve_fp4_strategy( @@ -729,8 +729,6 @@ def _build_inkling_shared_experts( shared_sink_serves_fp4=shared_sink_serves_fp4, ) if use_fused_shared: - # moe_tp_rank/moe_tp_size are derived internally; kept as args only - # for the legacy and non-sink InklingDenseMLP paths below. return InklingSharedFusedMoE( n_shared_experts=n_shared_experts, hidden_size=hidden_size, @@ -748,8 +746,8 @@ def _build_inkling_shared_experts( prefix=shared_prefix, quant_config=quant_config, inference_moe_w13_interleaved=inference_moe_w13_interleaved, - tp_rank=moe_tp_rank, - tp_size=moe_tp_size, + tp_rank=parallel.tp_rank, + tp_size=parallel.tp_size, tp_group=get_parallel().tp_group, ) return InklingBatchDenseMLP( @@ -767,8 +765,8 @@ def _build_inkling_shared_experts( layer_id=layer_id, prefix=add_prefix("shared_experts", prefix), quant_config=quant_config, - tp_rank=moe_tp_rank, - tp_size=moe_tp_size, + tp_rank=parallel.tp_rank, + tp_size=parallel.tp_size, tp_group=get_parallel().tp_group, ) @@ -870,10 +868,6 @@ class InklingMoE(nn.Module): intermediate_size=self.intermediate_dim, layer_id=layer_id, prefix=prefix, - # Shared expert is a replicated dense MLP: shard over the full tp group, not - # moe_tp (the single full-tp all_reduce in forward() reconstructs it). - moe_tp_rank=get_parallel().tp_rank, - moe_tp_size=get_parallel().tp_size, quant_config=self.quant_config, ) if isinstance(self.shared_experts, InklingSharedFusedMoE): diff --git a/python/sglang/srt/models/transformers.py b/python/sglang/srt/models/transformers.py index 192f001a21..46b8bd32f2 100644 --- a/python/sglang/srt/models/transformers.py +++ b/python/sglang/srt/models/transformers.py @@ -646,10 +646,9 @@ class TransformersBase(nn.Module): # Pipeline parallel self.pipeline_parallel() # Module replacement (Linear → TP, RMSNorm → fused, MoE overridden by MoEMixin) - tp_size = get_parallel().tp_size self.recursive_replace() # Attention instances - self.attention_instances = self._create_attention_instances(tp_size) + self.attention_instances = self._create_attention_instances() # Vocab embeddings self.replace_vocab_embed_class(self.model) @@ -902,7 +901,8 @@ class TransformersBase(nn.Module): self._register_missing_prefix(maybe_prefix("model", name)) # -- Attention instances ------------------------------------------------ - def _create_attention_instances(self, tp_size: int) -> dict[int, RadixAttention]: + def _create_attention_instances(self) -> dict[int, RadixAttention]: + tp_size = get_parallel().tp_size num_heads = self.text_config.num_attention_heads num_kv_heads = getattr(self.text_config, "num_key_value_heads", num_heads) hidden_size = self.text_config.hidden_size diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 32d6a5642d..0f4a92ca69 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1103,9 +1103,6 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): cls, *, server_args: ServerArgs, - tp_rank: int, - pp_rank: int, - dp_rank: Optional[int], enable_priority_scheduling: bool, enable_lora: bool, enable_hierarchical_cache: bool, @@ -1134,14 +1131,14 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): labels = { "model_name": get_serving().served_model_name, "engine_type": engine_type, - "tp_rank": tp_rank, - "pp_rank": pp_rank, + "tp_rank": parallel.tp_rank, + "pp_rank": parallel.pp_rank, "moe_ep_rank": parallel.moe_ep_rank, } if enable_priority_scheduling: labels["priority"] = "" - if dp_rank is not None: - labels["dp_rank"] = dp_rank + if parallel.dp_rank is not None: + labels["dp_rank"] = parallel.dp_rank if get_observability().extra_metric_labels: labels.update(get_observability().extra_metric_labels) scheduler_collector_cls = resolve_collector_class( diff --git a/python/sglang/srt/ray/data_parallel_controller.py b/python/sglang/srt/ray/data_parallel_controller.py index e0528e191b..7522f84c54 100644 --- a/python/sglang/srt/ray/data_parallel_controller.py +++ b/python/sglang/srt/ray/data_parallel_controller.py @@ -146,10 +146,7 @@ class RayDataParallelController(DataParallelController): for node_idx in range(nnodes): bundle_idx = self.bundle_for_node[node_idx] pp_range, tp_range, pp_per_node, tp_per_node = _calculate_rank_ranges( - nnodes, - get_parallel().pp_size, - get_parallel().tp_size, - node_rank=node_idx, + node_rank=node_idx ) for pp_rank in pp_range: for tp_rank in tp_range: diff --git a/python/sglang/srt/ray/engine.py b/python/sglang/srt/ray/engine.py index 41f667d9e7..0d6b8e3d53 100644 --- a/python/sglang/srt/ray/engine.py +++ b/python/sglang/srt/ray/engine.py @@ -29,7 +29,6 @@ from sglang.srt.entrypoints.engine import ( Engine, SchedulerInitResult, _calculate_rank_ranges, - _compute_parallelism_ranks, ) from sglang.srt.environ import envs from sglang.srt.ray.scheduler_actor import SchedulerActor @@ -204,8 +203,6 @@ def _create_scheduler_actor( rank0_node_ip: IP of rank-0's node, used for NCCL rendezvous. dist_init_addr: Distributed init address (tcp://rank0_node_ip:nccl_port). """ - attn_cp_rank, moe_dp_rank, moe_ep_rank = _compute_parallelism_ranks(tp_rank) - return SchedulerActor.options( num_cpus=0, num_gpus=1, @@ -223,9 +220,6 @@ def _create_scheduler_actor( port_args=port_args, gpu_id=gpu_id, tp_rank=tp_rank, - attn_cp_rank=attn_cp_rank, - moe_dp_rank=moe_dp_rank, - moe_ep_rank=moe_ep_rank, pp_rank=pp_rank, dp_rank=dp_rank, dist_init_addr=dist_init_addr, @@ -340,12 +334,7 @@ class RayEngine(Engine): for node_idx in range(nnodes): bundle_idx = bundle_for_node[node_idx] pp_range, tp_range, pp_per_node, tp_per_node = ( - _calculate_rank_ranges( - nnodes, - get_parallel().pp_size, - get_parallel().tp_size, - node_rank=node_idx, - ) + _calculate_rank_ranges(node_rank=node_idx) ) for pp_rank in pp_range: for tp_rank in tp_range: diff --git a/python/sglang/srt/ray/scheduler_actor.py b/python/sglang/srt/ray/scheduler_actor.py index 5a13dd1e98..8ec28b8d1d 100644 --- a/python/sglang/srt/ray/scheduler_actor.py +++ b/python/sglang/srt/ray/scheduler_actor.py @@ -41,9 +41,6 @@ class SchedulerActor: port_args: PortArgs, gpu_id: int, tp_rank: int, - attn_cp_rank: int, - moe_dp_rank: int, - moe_ep_rank: int, pp_rank: int, dp_rank: Optional[int], dist_init_addr: Optional[str] = None, @@ -98,16 +95,7 @@ class SchedulerActor: ) # Configure worker (logging, process title, etc.) - configure_scheduler_process( - server_args, - actual_gpu_id, - tp_rank, - attn_cp_rank, - moe_dp_rank, - moe_ep_rank, - pp_rank, - dp_rank, - ) + configure_scheduler_process(server_args, actual_gpu_id) # Ray actors can't use the numactl subprocess-wrapping approach # (SGLANG_NUMA_BIND_V2's normal path), so bind in-process via libnuma. @@ -125,9 +113,6 @@ class SchedulerActor: self.scheduler = Scheduler( server_args=server_args, port_args=port_args, - tp_rank=tp_rank, - pp_rank=pp_rank, - dp_rank=dp_rank, ) self._tp_rank = tp_rank diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index f951e40d83..bf29bb7641 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2962,18 +2962,14 @@ def direct_register_custom_op( raise error -def set_gpu_proc_affinity( - pp_size: int, - tp_size: int, - nnodes: int, - gpu_id: int, -): +def set_gpu_proc_affinity(gpu_id: int): # current process pid = os.getpid() p = psutil.Process(pid) - nnodes_per_tp_group = max(nnodes // pp_size, 1) - tp_size_per_node = tp_size // nnodes_per_tp_group + parallel = get_parallel() + nnodes_per_tp_group = max(parallel.nnodes // parallel.pp_size, 1) + tp_size_per_node = parallel.tp_size // nnodes_per_tp_group # total physical cores total_pcores = psutil.cpu_count(logical=False) diff --git a/python/sglang/srt/utils/offloader.py b/python/sglang/srt/utils/offloader.py index acee05d38a..100e116398 100644 --- a/python/sglang/srt/utils/offloader.py +++ b/python/sglang/srt/utils/offloader.py @@ -65,7 +65,7 @@ def set_offloader(instance: BaseOffloader): _instance = instance -def create_offloader(dp_rank: int): +def create_offloader(): if get_exec().offload.cpu_offload_gb > 0: return OffloaderV1( cpu_offload_max_bytes=int(get_exec().offload.cpu_offload_gb * 1024**3) @@ -79,8 +79,6 @@ def create_offloader(dp_rank: int): num_in_group=get_exec().offload.offload_num_in_group, prefetch_step=get_exec().offload.offload_prefetch_step, mode=get_exec().offload.offload_mode, - dp_rank=dp_rank, - dp_size=get_parallel().dp_size, ) return NoopOffloader() @@ -210,9 +208,9 @@ class OffloaderV2(BaseOffloader): num_in_group: int, prefetch_step: int, mode: str, - dp_rank: int, - dp_size: int, ): + parallel = get_parallel() + dp_rank, dp_size = parallel.dp_rank, parallel.dp_size self.group_size = group_size self.num_in_group = num_in_group self.prefetch_step = prefetch_step diff --git a/python/sglang/srt/utils/profile_utils.py b/python/sglang/srt/utils/profile_utils.py index 6b011f12c6..a7d6ab08f3 100644 --- a/python/sglang/srt/utils/profile_utils.py +++ b/python/sglang/srt/utils/profile_utils.py @@ -48,7 +48,7 @@ def graph_capture_profile_dir() -> str: ) -def export_cuda_graph_capture_trace(prof_context, *, runner_name: str, tp_rank: int): +def export_cuda_graph_capture_trace(prof_context, *, runner_name: str): """Persist a CUDA-graph capture profiler trace (chrome trace) to disk. Opt-in via ``SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE`` (no-op otherwise). The @@ -60,6 +60,7 @@ def export_cuda_graph_capture_trace(prof_context, *, runner_name: str, tp_rank: """ if not envs.SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE.get(): return + tp_rank = get_parallel().tp_rank output_dir = graph_capture_profile_dir() os.makedirs(output_dir, exist_ok=True) path = os.path.join( diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index 5696248f65..4b348e51db 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -39,6 +39,18 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.override(False), patch.object(dp_attn, "TboDPAttentionPreparer", return_value=tbo_preparer), patch.object(dp_attn, "world_dp_gather_enabled", return_value=False), + patch.object( + dp_attn, + "get_parallel", + return_value=SimpleNamespace( + dp_size=1, + attn_tp_size=4, + attn_cp_size=1, + tp_group=SimpleNamespace( + device_group=object(), device="cpu", cpu_group=object() + ), + ), + ), patch.object(dp_attn, "check_cuda_graph_backend", return_value=False), patch.object(dp_attn.MLPSyncBatchInfo, "all_gather") as all_gather, ): @@ -49,12 +61,6 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): spec_algorithm=SpeculativeAlgorithm.NONE, model_config=object(), ), - dp_size=1, - attn_tp_size=4, - attn_cp_size=1, - tp_group=SimpleNamespace( - device_group=object(), device="cpu", cpu_group=object() - ), get_idle_batch=Mock( side_effect=AssertionError("DP1 must not emit idle batch") ), diff --git a/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py b/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py index 03990cd682..51999cd602 100644 --- a/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py @@ -248,14 +248,19 @@ class TestOriginalTraceExport(CustomTestCase): def test_writes_named_trace_when_flag_set(self): with tempfile.TemporaryDirectory() as tmp: - with mock.patch.dict( - os.environ, - {"SGLANG_TORCH_PROFILER_DIR": tmp, _CAPTURE_TRACE: "1"}, - clear=False, + with ( + mock.patch.dict( + os.environ, + {"SGLANG_TORCH_PROFILER_DIR": tmp, _CAPTURE_TRACE: "1"}, + clear=False, + ), + mock.patch.object( + putils, "get_parallel", return_value=SimpleNamespace(tp_rank=2) + ), ): prof = mock.Mock() putils.export_cuda_graph_capture_trace( - prof, runner_name="DecodeCudaGraphRunner", tp_rank=2 + prof, runner_name="DecodeCudaGraphRunner" ) expected = os.path.join( tmp, @@ -275,7 +280,7 @@ class TestOriginalTraceExport(CustomTestCase): os.environ.pop(_CAPTURE_TRACE, None) prof = mock.Mock() putils.export_cuda_graph_capture_trace( - prof, runner_name="DecodeCudaGraphRunner", tp_rank=0 + prof, runner_name="DecodeCudaGraphRunner" ) prof.export_chrome_trace.assert_not_called() self.assertFalse( diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index 1f8b1e171a..6c27c4cd46 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -117,9 +117,6 @@ def _make_reporter(test, scheduler) -> SchedulerMetricsReporter: ) return SchedulerMetricsReporter( scheduler=scheduler, - tp_rank=0, - pp_rank=0, - dp_rank=0, metrics_collector_context=context, metrics_collector=None, ) diff --git a/test/registered/unit/test_parallel_consumers.py b/test/registered/unit/test_parallel_consumers.py new file mode 100644 index 0000000000..87bb1fe127 --- /dev/null +++ b/test/registered/unit/test_parallel_consumers.py @@ -0,0 +1,59 @@ +"""Cross-node placement and scoped logits layout regressions.""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +from sglang.srt.entrypoints.engine import _calculate_rank_ranges +from sglang.srt.layers.logits_processor import _reassemble_tp_lm_head_all_to_all_output +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class TestParallelConsumers(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def _publish(self, **config): + publish( + ServerArgs(model_path="dummy", **config), + role="scheduler", + ranks=SpawnRanks(world_rank=0), + ) + + def test_rank_ranges_can_describe_another_node(self): + self._publish(tp_size=4, pp_size=2, nnodes=4, node_rank=0) + expected = [ + ([0], [0, 1]), + ([0], [2, 3]), + ([1], [0, 1]), + ([1], [2, 3]), + ] + for node_rank, (pp, tp) in enumerate(expected): + with self.subTest(node_rank=node_rank): + pp_range, tp_range, pp_per_node, tp_per_node = _calculate_rank_ranges( + node_rank + ) + self.assertEqual((list(pp_range), list(tp_range)), (pp, tp)) + self.assertEqual((pp_per_node, tp_per_node), (1, 2)) + + def test_logits_reassembly_uses_the_active_tp_scope(self): + self._publish(tp_size=4) + source = torch.arange(24).reshape(8, 3) + with get_parallel().override(tp_size=2, attn_tp_size=2, moe_tp_size=2): + result = _reassemble_tp_lm_head_all_to_all_output(source) + torch.testing.assert_close(result, torch.cat(source.chunk(2), dim=1)) + result = _reassemble_tp_lm_head_all_to_all_output(source) + torch.testing.assert_close(result, torch.cat(source.chunk(4), dim=1)) + + +if __name__ == "__main__": + unittest.main()