From 91c329cc866957a4750a929d158978a3ac69f45d Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:55:21 -0700 Subject: [PATCH] Take the model config out of the parallel group build, and finish retiring the parallel getters (#40707) --- .../kernels/all_reduce/benchmark_mscclpp.py | 10 +- .../all_reduce/benchmark_torch_symm_mem.py | 10 +- python/sglang/benchmark/one_batch.py | 2 +- .../srt/disaggregation/encoder/server.py | 8 +- python/sglang/srt/distributed/bootstrap.py | 20 +- .../sglang/srt/distributed/parallel_state.py | 2 + python/sglang/srt/layers/dp_attention.py | 49 ++-- python/sglang/srt/managers/scheduler.py | 2 +- python/sglang/srt/models/inkling.py | 29 ++- .../sglang/srt/models/inkling_common/moe.py | 11 +- python/sglang/srt/runtime_context.py | 24 -- python/sglang/srt/weight_cache/daemon.py | 8 +- test/manual/test_custom_allreduce.py | 6 +- test/manual/test_forward_split_prefill.py | 2 +- test/manual/test_quick_allreduce.py | 6 +- test/manual/test_vlm_accuracy.py | 2 +- test/registered/unit/test_runtime_context.py | 215 ++++++++++++++++-- 17 files changed, 278 insertions(+), 128 deletions(-) diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py index 87dfd17e5f..433d404e50 100644 --- a/benchmark/kernels/all_reduce/benchmark_mscclpp.py +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -25,11 +25,11 @@ from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommu from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator from sglang.srt.distributed.parallel_state import ( cleanup_dist_env_and_memory, - get_tensor_model_parallel_group, graph_capture, initialize_model_parallel, set_mscclpp_all_reduce, ) +from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import publish_build_topology @@ -176,10 +176,10 @@ if __name__ == "__main__": ) publish_build_topology(world_rank=rank, tp_size=world_size) initialize_model_parallel() - group = get_tensor_model_parallel_group().device_group - cpu_group = get_tensor_model_parallel_group().cpu_group - pynccl_comm = get_tensor_model_parallel_group().pynccl_comm - pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm + group = get_parallel().tp_group.device_group + cpu_group = get_parallel().tp_group.cpu_group + pynccl_comm = get_parallel().tp_group.pynccl_comm + pymscclpp_comm = get_parallel().tp_group.pymscclpp_comm dist.barrier() profile = False dtype = torch.bfloat16 diff --git a/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py b/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py index 3d2c658706..b13d859ff3 100644 --- a/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py +++ b/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py @@ -39,11 +39,11 @@ from sglang.srt.distributed.device_communicators.torch_symm_mem import ( TorchSymmMemCommunicator, ) from sglang.srt.distributed.parallel_state import ( - get_tensor_model_parallel_group, graph_capture, initialize_model_parallel, set_torch_symm_mem_all_reduce, ) +from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import publish_build_topology from sglang.utils import is_in_ci @@ -191,10 +191,10 @@ if __name__ == "__main__": ) publish_build_topology(world_rank=rank, tp_size=world_size) initialize_model_parallel() - group = get_tensor_model_parallel_group().device_group - cpu_group = get_tensor_model_parallel_group().cpu_group - pynccl_comm = get_tensor_model_parallel_group().pynccl_comm - torch_symm_mem_comm = get_tensor_model_parallel_group().torch_symm_mem_comm + group = get_parallel().tp_group.device_group + cpu_group = get_parallel().tp_group.cpu_group + pynccl_comm = get_parallel().tp_group.pynccl_comm + torch_symm_mem_comm = get_parallel().tp_group.torch_symm_mem_comm dist.barrier() profile = False dtype = torch.bfloat16 diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index 309add1b93..9240a7df21 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -329,10 +329,10 @@ def load_model(server_args, port_args, gpu_id, tp_rank): bootstrap.init_parallel_runtime( server_args=server_args, - model_config=model_config, device=get_device().device, dist_port=port_args.nccl_port, ) + bootstrap.init_layer_runtime(model_config=model_config) _use_mlx = use_mlx() if _use_mlx: diff --git a/python/sglang/srt/disaggregation/encoder/server.py b/python/sglang/srt/disaggregation/encoder/server.py index 40a4e67102..e13e254da5 100644 --- a/python/sglang/srt/disaggregation/encoder/server.py +++ b/python/sglang/srt/disaggregation/encoder/server.py @@ -39,7 +39,10 @@ from sglang.srt.distributed.parallel_state import ( initialize_model_parallel, ) from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import initialize_dp_attention +from sglang.srt.layers.dp_attention import ( + init_dp_gathered_buffer, + initialize_dp_attention_flags, +) from sglang.srt.managers.io_struct import ( ProfileReq, ProfileReqType, @@ -610,7 +613,8 @@ class MMEncoder: moe_tp_size=parallel.tp_size, ) initialize_model_parallel() - initialize_dp_attention(server_args, self.model_config) + initialize_dp_attention_flags(server_args) + init_dp_gathered_buffer(self.model_config) self.model = load_model( model_config=self.model_config, diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index db988f3a49..6f82a92f3e 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -24,7 +24,10 @@ from sglang.srt.distributed.parallel_state import ( set_torch_symm_mem_all_reduce, ) from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import initialize_dp_attention +from sglang.srt.layers.dp_attention import ( + init_dp_gathered_buffer, + initialize_dp_attention, +) from sglang.srt.layers.layernorm_sp import initialize_layernorm_sp from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( @@ -90,7 +93,6 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]": def init_parallel_runtime( *, server_args: ServerArgs, - model_config: ModelConfig, device: str, dist_port: int, ) -> None: @@ -150,7 +152,6 @@ def init_parallel_runtime( backend=backend, dist_init_method=dist_init_method, server_args=server_args, - model_config=model_config, gpu_id=get_device().gpu_id, ) @@ -292,7 +293,6 @@ def _init_parallel_groups( backend: str, dist_init_method: str, server_args: ServerArgs, - model_config: ModelConfig, gpu_id: int, ) -> None: parallel = get_parallel() @@ -327,15 +327,17 @@ def _init_parallel_groups( max_world_size=None if is_scale_joiner else get_parallel().max_ep_size, ) _tag_groups_for_flashinfer_allreduce_only() - initialize_dp_attention( - server_args=server_args, - model_config=model_config, - ) - initialize_layernorm_sp(model_config=model_config) + initialize_dp_attention(server_args=server_args) if is_npu(): register_sgl_tp_rank(gpu_id) +def init_layer_runtime(*, model_config: ModelConfig) -> None: + """Size the DP gathered buffer and resolve layernorm SP from the model.""" + init_dp_gathered_buffer(model_config) + initialize_layernorm_sp(model_config=model_config) + + def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None: warmup_start = time.perf_counter() tp_group_handle = get_tp_group().device_group diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 539a4f1844..e7bcc1d88e 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -3412,7 +3412,9 @@ _EXEMPT_CALLERS = ("sglang.srt.distributed.",) _CONTEXT_NAME_OF = { "get_world_group": "world_group", "get_tp_group": "tp_group", + "get_tensor_model_parallel_group": "tp_group", "get_pp_group": "pp_group", + "get_pipeline_model_parallel_group": "pp_group", "get_moe_ep_group": "moe_ep_group", "get_moe_dp_group": "moe_dp_group", "get_moe_tp_group": "moe_tp_group", diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 2693d4f8b9..8c4f1eead3 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -394,40 +394,35 @@ def compute_dp_attention_world_info( return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size -def initialize_dp_attention( - server_args: ServerArgs, - model_config: ModelConfig, -): +def initialize_dp_attention_flags(server_args: ServerArgs): + """Initialize DP runtime flags without changing the worker's placement.""" dp = get_flags().dp - dp.max_len_with_idle = ( - getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None - ) - enable_dp_attention = get_parallel().enable_dp_attention - dp_size = get_parallel().dp_size - attn_cp_size = get_parallel().attn_cp_size - - dp.enabled = enable_dp_attention - - tp_rank = get_parallel().tp_rank - tp_size = get_parallel().tp_size - - _, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info( - enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size - ) + dp.enabled = get_parallel().enable_dp_attention if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size: - # Reads the resolution, not a bag: this runs under - # `initialize_dp_attention`, which the weight-cache daemon calls from - # `_init_distributed` -- and other callers reach it from processes - # whose publish is not guaranteed to have happened yet. (The daemon - # itself publishes first, at `daemon.py:284`, before `:320`.) if ep_scale_joiner_of(resolving_view(server_args)): dp.joiner_skip_all_gather = True - get_parallel().override_permanently( - attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank - ) +def initialize_dp_attention(server_args: ServerArgs): + """Initialize DP flags and state placement from the published topology.""" + initialize_dp_attention_flags(server_args) + parallel = get_parallel() + _, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info( + parallel.enable_dp_attention, + parallel.tp_rank, + parallel.tp_size, + parallel.dp_size, + parallel.attn_cp_size, + ) + parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank) + + +def init_dp_gathered_buffer(model_config: ModelConfig): + """Size the gathered buffer from the model this worker is about to run.""" + get_flags().dp.max_len_with_idle = ( + getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None + ) _DpGatheredBufferWrapper.set_metadata( hidden_size=model_config.hidden_size, dtype=model_config.dtype, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index b694ba6355..c134c098eb 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -519,10 +519,10 @@ class Scheduler( bootstrap.init_parallel_runtime( server_args=server_args, - model_config=self.model_config, device=get_device().device, dist_port=self.nccl_port, ) + bootstrap.init_layer_runtime(model_config=self.model_config) # Init metrics stats self.init_metrics_collector(tp_rank, pp_rank, dp_rank) diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index ebad2504ff..aad174ee75 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -14,9 +14,6 @@ from sglang.srt.configs.inkling import ( InklingModelConfig, InklingVisionConfig, ) -from sglang.srt.distributed import ( - get_tensor_model_parallel_group, -) from sglang.srt.environ import envs from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor @@ -308,7 +305,7 @@ class InklingDecoderLayer(nn.Module): hs, prev_mlp_sconv, forward_batch, - get_tensor_model_parallel_group(), + get_parallel().tp_group, norm=self.attn_norm, norm_residual=res, ) @@ -317,7 +314,7 @@ class InklingDecoderLayer(nn.Module): # MoE's unreduced partials; the kernel returns the gathered # post-conv [T, H], and the norm runs unfused below. hs = ar_scattered_sconv_fused( - hs, prev_mlp_sconv, forward_batch, get_tensor_model_parallel_group() + hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group ) hs, res = self.attn_norm(hs, res) elif prev_mlp_partial: @@ -331,13 +328,13 @@ class InklingDecoderLayer(nn.Module): prev_mlp_sconv, self.attn_norm, forward_batch, - get_tensor_model_parallel_group(), + get_parallel().tp_group, ) else: # Fused extend {AR + full-width sconv + cache update} # (non-scattered); norm runs unfused on the gathered [T, H]. hs = ar_fullwidth_sconv_fused( - hs, prev_mlp_sconv, forward_batch, get_tensor_model_parallel_group() + hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group ) hs, res = self.attn_norm(hs, res) else: @@ -618,7 +615,7 @@ class InklingCausalLLM(nn.Module): # the prefill graph disabled and --skip-server-warmup there is no eager # forward to build them lazily; decode capture would bake the fallback). if envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get(): - ensure_inkling_ar_resources(get_tensor_model_parallel_group()) + ensure_inkling_ar_resources(get_parallel().tp_group) ensure_inkling_ar_resources(get_parallel().attn_tp_group) # Warm the fused decode {AR -> mlp_sconv -> norm} JIT module (both @@ -763,7 +760,7 @@ class InklingCausalLLM(nn.Module): fuse_ar_sconv = ( not forward_batch.forward_mode.is_idle() and ar_sconv_norm_fusable( - get_tensor_model_parallel_group(), + get_parallel().tp_group, forward_batch, hidden_states.shape[0], hidden_states.shape[-1], @@ -788,7 +785,7 @@ class InklingCausalLLM(nn.Module): # kernel. Mutually exclusive with ar_sconv_norm_fusable by mode # (extend vs decode/verify) and by the scattered gate inside it. if not forward_batch.forward_mode.is_idle() and scattered_ar_sconv_fusable( - get_tensor_model_parallel_group(), + get_parallel().tp_group, forward_batch, hidden_states.shape[0], hidden_states.shape[-1], @@ -803,7 +800,7 @@ class InklingCausalLLM(nn.Module): # (mode for ar_sconv_norm_fusable, the scattered flag for # scattered_ar_sconv_fusable). if not forward_batch.forward_mode.is_idle() and fullwidth_ar_sconv_fusable( - get_tensor_model_parallel_group(), + get_parallel().tp_group, forward_batch, hidden_states.shape[0], hidden_states.shape[-1], @@ -856,7 +853,7 @@ class InklingCausalLLM(nn.Module): hidden_states, prev_mlp_sconv, forward_batch, - get_tensor_model_parallel_group(), + get_parallel().tp_group, norm=self.norm, norm_residual=residual, ) @@ -871,7 +868,7 @@ class InklingCausalLLM(nn.Module): hidden_states, prev_mlp_sconv, forward_batch, - get_tensor_model_parallel_group(), + get_parallel().tp_group, ) hidden_states, _ = self.norm(hidden_states, residual) return ( @@ -890,7 +887,7 @@ class InklingCausalLLM(nn.Module): prev_mlp_sconv, self.norm, forward_batch, - get_tensor_model_parallel_group(), + get_parallel().tp_group, ) return ( (hidden_states, aux_hidden_states) @@ -903,7 +900,7 @@ class InklingCausalLLM(nn.Module): hidden_states, prev_mlp_sconv, forward_batch, - get_tensor_model_parallel_group(), + get_parallel().tp_group, ) hidden_states, _ = self.norm(hidden_states, residual) return ( @@ -1787,7 +1784,7 @@ class InklingMTPLayer(nn.Module): h_inproj.dtype, ) if not fm_idle and scattered_ar_sconv_fusable( - get_tensor_model_parallel_group(), + get_parallel().tp_group, forward_batch, h_inproj.shape[0], h_inproj.shape[-1], diff --git a/python/sglang/srt/models/inkling_common/moe.py b/python/sglang/srt/models/inkling_common/moe.py index 6713f5bd36..dd5dd4459f 100644 --- a/python/sglang/srt/models/inkling_common/moe.py +++ b/python/sglang/srt/models/inkling_common/moe.py @@ -31,9 +31,6 @@ from sglang.kernels.ops.moe.sigmoid_gate_topk_renorm import ( sigmoid_gate_topk_renorm, ) from sglang.srt.configs.inkling import InklingModelConfig -from sglang.srt.distributed import ( - get_tensor_model_parallel_group, -) from sglang.srt.environ import GateGemvMode, envs from sglang.srt.layers.moe import get_moe_runner_backend from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -753,7 +750,7 @@ def _build_inkling_shared_experts( inference_moe_w13_interleaved=inference_moe_w13_interleaved, tp_rank=moe_tp_rank, tp_size=moe_tp_size, - tp_group=get_tensor_model_parallel_group(), + tp_group=get_parallel().tp_group, ) return InklingBatchDenseMLP( **dense_kwargs, @@ -772,7 +769,7 @@ def _build_inkling_shared_experts( quant_config=quant_config, tp_rank=moe_tp_rank, tp_size=moe_tp_size, - tp_group=get_tensor_model_parallel_group(), + tp_group=get_parallel().tp_group, ) @@ -1071,7 +1068,7 @@ class InklingMoE(nn.Module): # {routed + shared} add on the fold paths. stash_ar_shared(shared_out) return out - tp = get_tensor_model_parallel_group() + tp = get_parallel().tp_group buf = get_ar_buffer(tp, out.shape[0], out.shape[1], out.dtype) if buf is not None: torch.add(out, shared_out, out=buf) @@ -1079,7 +1076,7 @@ class InklingMoE(nn.Module): return out + shared_out return out - tp = get_tensor_model_parallel_group() + tp = get_parallel().tp_group if shared_out is not None: if self._fused_ar_shared and not self.scattered_sconv: # The AR dispatch folds in-kernel on the fold paths and pre-adds diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index f44b0928f8..8395248230 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -63,27 +63,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# Imported lazily so this module has no import-time dependencies: any module can -# import get_parallel at module level without risking an import cycle. -_PARALLEL_STATE = None - - -def _ps(): - """Lazily import and cache the parallel-state module.""" - global _PARALLEL_STATE - if _PARALLEL_STATE is None: - from sglang.srt.distributed import parallel_state - - _PARALLEL_STATE = parallel_state - return _PARALLEL_STATE - - -def _dp(): - from sglang.srt.layers import dp_attention - - return dp_attention - - @functools.lru_cache(maxsize=1) def _parallel_config_leaves() -> frozenset: """Return configured parallel field names, including before publication.""" @@ -97,9 +76,6 @@ def _parallel_config_leaves() -> frozenset: ) -_MISSING_READ = object() - - @functools.lru_cache(maxsize=1) def _parallel_fields() -> frozenset: """Return configured and derived parallel names accepted by overrides.""" diff --git a/python/sglang/srt/weight_cache/daemon.py b/python/sglang/srt/weight_cache/daemon.py index c1bd2bf0db..2fe9a16480 100644 --- a/python/sglang/srt/weight_cache/daemon.py +++ b/python/sglang/srt/weight_cache/daemon.py @@ -235,9 +235,13 @@ class WeightCacheDaemon: initialize_model_parallel() # Initialize DP attention state (required by some models like Qwen3 MoE) - from sglang.srt.layers.dp_attention import initialize_dp_attention + from sglang.srt.layers.dp_attention import ( + init_dp_gathered_buffer, + initialize_dp_attention, + ) - initialize_dp_attention(server_args, model_config) + initialize_dp_attention(server_args) + init_dp_gathered_buffer(model_config) logger.info( f"[WeightCacheDaemon gpu={self.gpu_id} tp_rank={self.tp_rank}] " diff --git a/test/manual/test_custom_allreduce.py b/test/manual/test_custom_allreduce.py index e1d91d690d..e1aedc140f 100644 --- a/test/manual/test_custom_allreduce.py +++ b/test/manual/test_custom_allreduce.py @@ -13,10 +13,10 @@ from sglang.srt.distributed.communication_op import ( # noqa tensor_model_parallel_all_reduce, ) from sglang.srt.distributed.parallel_state import ( - get_tensor_model_parallel_group, graph_capture, initialize_model_parallel, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler from sglang.test.test_utils import CustomTestCase, publish_build_topology @@ -100,7 +100,7 @@ class TestCustomAllReduce(CustomTestCase): ) publish_build_topology(tp_size=world_size, world_rank=rank) initialize_model_parallel() - group = get_tensor_model_parallel_group().device_group + group = get_parallel().tp_group.device_group # Set global server args to avoid "Global server args is not set yet!" error set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) @@ -164,7 +164,7 @@ class TestCustomAllReduce(CustomTestCase): ) publish_build_topology(tp_size=world_size, world_rank=rank) initialize_model_parallel() - group = get_tensor_model_parallel_group().device_group + group = get_parallel().tp_group.device_group # Set global server args to avoid "Global server args is not set yet!" error set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) diff --git a/test/manual/test_forward_split_prefill.py b/test/manual/test_forward_split_prefill.py index 68e99672cc..ef95db318f 100644 --- a/test/manual/test_forward_split_prefill.py +++ b/test/manual/test_forward_split_prefill.py @@ -67,10 +67,10 @@ class TestForwardSplitPrefill(CustomTestCase): cls.model_config = ModelConfig.from_server_args(cls.server_args) bootstrap.init_parallel_runtime( server_args=cls.server_args, - model_config=cls.model_config, device=cls.device, dist_port=cls.port_args.nccl_port, ) + bootstrap.init_layer_runtime(model_config=cls.model_config) cls.model_runner = ModelRunner( model_config=cls.model_config, mem_fraction_static=cls.server_args.mem_fraction_static, diff --git a/test/manual/test_quick_allreduce.py b/test/manual/test_quick_allreduce.py index 3f4e3cd0b9..83640eb123 100644 --- a/test/manual/test_quick_allreduce.py +++ b/test/manual/test_quick_allreduce.py @@ -19,10 +19,10 @@ from sglang.srt.distributed.device_communicators.quick_all_reduce import ( qr_rocm_arch_available, ) from sglang.srt.distributed.parallel_state import ( - get_tensor_model_parallel_group, graph_capture, initialize_model_parallel, ) +from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import CustomTestCase, publish_build_topology torch.manual_seed(42) @@ -119,7 +119,7 @@ class TestQuickAllReduce(CustomTestCase): ) publish_build_topology(tp_size=world_size, world_rank=rank) initialize_model_parallel() - group = get_tensor_model_parallel_group().device_group + group = get_parallel().tp_group.device_group # A small all_reduce for warmup. # this is needed because device communicators might be created lazily @@ -189,7 +189,7 @@ class TestQuickAllReduce(CustomTestCase): ) publish_build_topology(tp_size=world_size, world_rank=rank) initialize_model_parallel() - group = get_tensor_model_parallel_group().device_group + group = get_parallel().tp_group.device_group for sz in self.TEST_SIZES: for dtype in [torch.float16, torch.bfloat16]: diff --git a/test/manual/test_vlm_accuracy.py b/test/manual/test_vlm_accuracy.py index f35ee1d5d7..62353352ed 100644 --- a/test/manual/test_vlm_accuracy.py +++ b/test/manual/test_vlm_accuracy.py @@ -154,10 +154,10 @@ class VisionLLMLogitsBase(unittest.IsolatedAsyncioTestCase): model_config = ModelConfig(self.model_path, model_override_args="{}") bootstrap.init_parallel_runtime( server_args=server_args, - model_config=model_config, device=get_device().device, dist_port=12435, ) + bootstrap.init_layer_runtime(model_config=model_config) self.model_runner = ModelRunner( model_config=model_config, mem_fraction_static=0.8, diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index b7bd5faa9b..4f4f5b3678 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -245,7 +245,17 @@ class TestTheBuildStatesEveryGroup(_IsolatedOverrides): and isinstance(node.func, ast.Attribute) and node.func.attr == "override_permanently" ) - self.assertIn("is not None", ast.unparse(stamp)) + self.assertEqual([keyword.arg for keyword in stamp.keywords], [None]) + mapping = stamp.keywords[0].value + self.assertIsInstance(mapping, ast.DictComp) + self.assertEqual(len(mapping.generators), 1) + guards = mapping.generators[0].ifs + self.assertEqual(len(guards), 1) + guard = guards[0] + self.assertIsInstance(guard, ast.Compare) + self.assertEqual([type(op) for op in guard.ops], [ast.IsNot]) + self.assertEqual([c.value for c in guard.comparators], [None]) + self.assertEqual(ast.unparse(guard.left), ast.unparse(mapping.value)) class TestParallelDelegation(_IsolatedOverrides): @@ -2585,27 +2595,39 @@ class TestTheTopologyIdentities(CustomTestCase): def test_a_refused_write_leaves_nothing_behind(self): self._publish_square() + written = dict(tp_size=4, attn_tp_size=3, attn_dp_size=1, attn_cp_size=1) + before = {name: getattr(get_parallel(), name) for name in written} with self.assertRaises(ValueError): - with get_parallel().override( - tp_size=4, attn_tp_size=3, attn_dp_size=1, attn_cp_size=1 - ): + with get_parallel().override(**written): pass - self.assertEqual(get_parallel().attn_tp_size, 2) + self.assertEqual( + {name: getattr(get_parallel(), name) for name in written}, before + ) + self.assertEqual(before["attn_tp_size"], 2) def test_a_group_built_at_another_width_is_refused(self): from sglang.srt.distributed.parallel_state import GroupCoordinator + from sglang.srt.runtime_context import _WIDTH_AND_GROUP - self._publish_square() - wrong = GroupCoordinator.__new__(GroupCoordinator) - wrong.world_size = 8 - wrong.rank_in_group = 0 - with self.assertRaises(ValueError) as caught: - get_parallel().override_permanently(tp_group=wrong) - message = str(caught.exception) - self.assertIn("tp_group.world_size == tp_size", message) - self.assertIn("built 8, configured 4", message) - with self.assertRaises(RuntimeError): - get_parallel().tp_group + exercised = set() + for size_name, group_name in _WIDTH_AND_GROUP: + with self.subTest(group=group_name): + self._publish_square() + configured = getattr(get_parallel(), size_name) + wrong = GroupCoordinator.__new__(GroupCoordinator) + wrong.world_size = configured + 4 + wrong.rank_in_group = 0 + with self.assertRaises(ValueError) as caught: + get_parallel().override_permanently(**{group_name: wrong}) + message = str(caught.exception) + self.assertIn(f"{group_name}.world_size == {size_name}", message) + self.assertIn( + f"built {configured + 4}, configured {configured}", message + ) + with self.assertRaises(RuntimeError): + getattr(get_parallel(), group_name) + exercised.add((size_name, group_name)) + self.assertEqual(exercised, set(_WIDTH_AND_GROUP)) def test_a_group_built_at_the_configured_width_is_quiet(self): from sglang.srt.distributed.parallel_state import GroupCoordinator @@ -2632,6 +2654,101 @@ class TestTheTopologyIdentities(CustomTestCase): class TestTheParallelPhase(CustomTestCase): """Entry points initialize parallel groups before constructing runners.""" + def test_the_layer_phase_leaves_a_stated_placement_alone(self): + """A server that runs its own WORLD states its attention placement, and + the layer phase reads the model's shape only.""" + import torch + + from sglang.srt.distributed import bootstrap + from sglang.srt.layers.dp_attention import initialize_dp_attention_flags + + reset_context() + self.addCleanup(reset_context) + publish( + ServerArgs( + model_path="dummy", + tp_size=4, + dp_size=2, + enable_dp_attention=True, + device="cpu", + ), + role="test", + ranks=SpawnRanks(world_rank=3, dp_rank=1), + ) + get_parallel().override_permanently( + attn_dp_size=1, + attn_dp_rank=0, + attn_tp_size=4, + attn_tp_rank=3, + attn_cp_size=1, + attn_cp_rank=0, + ) + + initialize_dp_attention_flags(get_server_args()) + bootstrap.init_layer_runtime( + model_config=SimpleNamespace( + hf_config=SimpleNamespace(architectures=["Qwen2ForCausalLM"]), + hidden_size=8, + dtype=torch.float16, + ) + ) + + self.assertTrue(get_flags().dp.enabled) + self.assertEqual(get_parallel().attn_dp_size, 1) + self.assertEqual(get_parallel().attn_dp_rank, 0) + + def test_encoder_cp_mlp_reduces_over_attention_tp(self): + from unittest.mock import Mock + + import torch + + from sglang.srt.layers.dp_attention import ( + init_dp_gathered_buffer, + initialize_dp_attention_flags, + ) + from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP + + reset_context() + self.addCleanup(reset_context) + server_args = ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + attn_cp_size=2, + enable_dp_attention=True, + ) + publish(server_args, role="test", ranks=SpawnRanks(world_rank=0)) + # Identical shards contribute equally; reducing across CP replicas + # would double the output even though the layer uses attention TP. + tp_group = SimpleNamespace( + world_size=4, + rank_in_group=0, + all_reduce=Mock(side_effect=lambda x: x * 4), + ) + attn_tp_group = SimpleNamespace( + world_size=2, + rank_in_group=0, + all_reduce=Mock(side_effect=lambda x: x * 2), + ) + get_parallel().override_permanently( + tp_group=tp_group, attn_tp_group=attn_tp_group + ) + initialize_dp_attention_flags(server_args) + init_dp_gathered_buffer( + SimpleNamespace( + hf_config=SimpleNamespace(), hidden_size=4, dtype=torch.float32 + ) + ) + mlp = Qwen3_VisionMLP(4, 4, bias=False, hidden_act="relu") + with torch.no_grad(): + for parameter in mlp.parameters(): + parameter.fill_(1) + output = mlp(torch.ones(1, 4)) + + torch.testing.assert_close(output, torch.full((1, 4), 16.0)) + attn_tp_group.all_reduce.assert_called_once() + tp_group.all_reduce.assert_not_called() + def test_building_twice_is_refused(self): from sglang.srt.distributed import bootstrap @@ -2656,7 +2773,6 @@ class TestTheParallelPhase(CustomTestCase): ) kwargs = dict( server_args=ServerArgs(model_path="dummy"), - model_config=None, device="cpu", dist_port=12345, ) @@ -2682,13 +2798,16 @@ class TestTheParallelPhase(CustomTestCase): ) if not builds: continue - if "init_parallel_runtime(" not in text: - offenders.append(str(path)) + for phase in ("init_parallel_runtime(", "init_layer_runtime("): + if phase not in text: + offenders.append(f"{path} ({phase[:-1]})") self.assertEqual( offenders, [], - "these publish and then build a ModelRunner without bringing the " - "parallel runtime up first:\n " + "\n ".join(offenders), + "these publish and then build a ModelRunner without running both " + "phases first -- the group build derives the topology, and the " + "layer phase sizes what the model's shape decides:\n " + + "\n ".join(offenders), ) @@ -2888,11 +3007,65 @@ class TestTheRecordIsNeverWrittenTo(CustomTestCase): class TestTheRetiredNamesAreGoneEverywhere(CustomTestCase): """Reject retired getter imports and group-initialization width arguments.""" + #: Getters that answer something other than a place in the topology. + NOT_A_PLACEMENT = { + "get_default_distributed_backend", + "get_mooncake_transfer_engine", + "get_torch_distributed_pg_options", + } + #: Widths whose group is not in ``_WIDTH_AND_GROUP``. + WIDTH_WITHOUT_A_CHECKED_GROUP = { + "get_dcp_world_size", + "get_moe_data_parallel_world_size", + "get_moe_tensor_parallel_world_size", + } + #: Variants that answer a group the map already covers. + VARIANT_OF_A_MAPPED_GROUP = { + "get_dcp_group_no_assert", + "get_self_pp_group", + } + def _retired(self): from sglang.srt.distributed.parallel_state import _CONTEXT_NAME_OF return set(_CONTEXT_NAME_OF) + def test_every_getter_the_module_defines_is_classified(self): + """Every getter the module defines is deprecated or classified here.""" + from sglang.srt.distributed import parallel_state + + defined = { + name + for name in dir(parallel_state) + if name.startswith("get_") + and callable(getattr(parallel_state, name)) + and getattr(getattr(parallel_state, name), "__module__", None) + == parallel_state.__name__ + } + self.assertTrue(defined, "no getters found; this proves nothing") + unclassified = ( + defined + - self._retired() + - self.NOT_A_PLACEMENT + - self.WIDTH_WITHOUT_A_CHECKED_GROUP + - self.VARIANT_OF_A_MAPPED_GROUP + ) + self.assertEqual( + unclassified, + set(), + "these getters are neither deprecated nor classified; say which " + "kind each one is, or route it through get_parallel():\n " + + "\n ".join(sorted(unclassified)), + ) + stale = ( + self.NOT_A_PLACEMENT + | self.WIDTH_WITHOUT_A_CHECKED_GROUP + | self.VARIANT_OF_A_MAPPED_GROUP + ) - defined + self.assertEqual( + stale, set(), f"these are named here but no longer defined: {stale}" + ) + def test_nothing_imports_a_retired_name_from_the_package(self): import ast as _ast