diff --git a/python/sglang/kernels/ops/communication/configs/sp_collective/world=16,H=7168,device_name=NVIDIA_GB200.json b/python/sglang/kernels/ops/communication/configs/sp_collective/world=16,H=7168,device_name=NVIDIA_GB200.json new file mode 100644 index 0000000000..90c74e29b2 --- /dev/null +++ b/python/sglang/kernels/ops/communication/configs/sp_collective/world=16,H=7168,device_name=NVIDIA_GB200.json @@ -0,0 +1,203 @@ +{ + "source": { + "date": "2026-09-01", + "device": "NVIDIA GB200", + "nodes": 4, + "gpus_per_node": 4, + "push_slot_bytes": 33554432, + "raw_results": [ + "world16-raw.json" + ] + }, + "selection": { + "rule": "nearest global-token bucket at or below the workload", + "fallback": "nccl", + "note": "Fresh world-16 GB200 sweep. Each candidate was correctness-checked against NCCL; custom is selected only when its measured latency is strictly lower. Push candidates are limited to the default 256 KiB per-rank workspace. Attention-residual fusion was not tuned." + }, + "configs": { + "reduce_scatter": { + "16": { + "strategy": "push", + "num_blocks": 32, + "block_size": 512 + }, + "32": { + "strategy": "push", + "num_blocks": 96, + "block_size": 512 + }, + "64": { + "strategy": "push", + "num_blocks": 96, + "block_size": 128 + }, + "128": { + "strategy": "push", + "num_blocks": 128, + "block_size": 128 + }, + "256": { + "strategy": "push", + "num_blocks": 32, + "block_size": 128 + }, + "512": { + "strategy": "pull", + "num_blocks": 32, + "block_size": 1024 + }, + "1024": { + "strategy": "pull", + "num_blocks": 96, + "block_size": 1024 + }, + "2048": { + "strategy": "pull", + "num_blocks": 96, + "block_size": 1024 + }, + "4096": { + "strategy": "pull", + "num_blocks": 32, + "block_size": 1024 + }, + "8192": { + "strategy": "pull", + "num_blocks": 64, + "block_size": 1024 + }, + "16384": { + "strategy": "pull", + "num_blocks": 96, + "block_size": 512 + } + }, + "all_gather": { + "16": { + "strategy": "push", + "num_blocks": 32, + "block_size": 256 + }, + "32": { + "strategy": "push", + "num_blocks": 4, + "block_size": 512 + }, + "64": { + "strategy": "push", + "num_blocks": 16, + "block_size": 128 + }, + "128": { + "strategy": "push", + "num_blocks": 128, + "block_size": 512 + }, + "256": { + "strategy": "push", + "num_blocks": 128, + "block_size": 128 + }, + "512": { + "strategy": "direct", + "num_blocks": 2, + "block_size": 512 + }, + "1024": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "2048": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "4096": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "8192": { + "strategy": "direct", + "num_blocks": 4, + "block_size": 512 + }, + "16384": { + "strategy": "direct", + "num_blocks": 4, + "block_size": 512 + } + }, + "reduce_scatter_attn_res": { + "16": { + "strategy": "separate" + }, + "32": { + "strategy": "separate" + }, + "64": { + "strategy": "separate" + }, + "128": { + "strategy": "separate" + }, + "256": { + "strategy": "separate" + }, + "512": { + "strategy": "separate" + }, + "1024": { + "strategy": "separate" + }, + "2048": { + "strategy": "separate" + }, + "4096": { + "strategy": "separate" + }, + "8192": { + "strategy": "separate" + }, + "16384": { + "strategy": "separate" + } + }, + "attn_res_all_gather": { + "16": { + "strategy": "separate" + }, + "32": { + "strategy": "separate" + }, + "64": { + "strategy": "separate" + }, + "128": { + "strategy": "separate" + }, + "256": { + "strategy": "separate" + }, + "512": { + "strategy": "separate" + }, + "1024": { + "strategy": "separate" + }, + "2048": { + "strategy": "separate" + }, + "4096": { + "strategy": "separate" + }, + "8192": { + "strategy": "separate" + }, + "16384": { + "strategy": "separate" + } + } + } +} diff --git a/python/sglang/kernels/ops/communication/configs/sp_collective/world=16,H=7168,device_name=NVIDIA_GB300.json b/python/sglang/kernels/ops/communication/configs/sp_collective/world=16,H=7168,device_name=NVIDIA_GB300.json new file mode 100644 index 0000000000..84c31e373e --- /dev/null +++ b/python/sglang/kernels/ops/communication/configs/sp_collective/world=16,H=7168,device_name=NVIDIA_GB300.json @@ -0,0 +1,203 @@ +{ + "source": { + "date": "2026-09-01", + "device": "NVIDIA GB300", + "nodes": 4, + "gpus_per_node": 4, + "push_slot_bytes": 33554432, + "raw_results": [ + "world16-raw.json" + ] + }, + "selection": { + "rule": "nearest global-token bucket at or below the workload", + "fallback": "nccl", + "note": "Fresh world-16 GB300 sweep. Each candidate was correctness-checked against NCCL; custom is selected only when its measured latency is strictly lower. Push candidates are limited to the default 256 KiB per-rank workspace. Attention-residual fusion was not tuned." + }, + "configs": { + "reduce_scatter": { + "16": { + "strategy": "push", + "num_blocks": 128, + "block_size": 128 + }, + "32": { + "strategy": "push", + "num_blocks": 4, + "block_size": 512 + }, + "64": { + "strategy": "push", + "num_blocks": 128, + "block_size": 512 + }, + "128": { + "strategy": "push", + "num_blocks": 16, + "block_size": 256 + }, + "256": { + "strategy": "push", + "num_blocks": 128, + "block_size": 512 + }, + "512": { + "strategy": "pull", + "num_blocks": 32, + "block_size": 1024 + }, + "1024": { + "strategy": "pull", + "num_blocks": 96, + "block_size": 1024 + }, + "2048": { + "strategy": "pull", + "num_blocks": 96, + "block_size": 1024 + }, + "4096": { + "strategy": "pull", + "num_blocks": 32, + "block_size": 1024 + }, + "8192": { + "strategy": "pull", + "num_blocks": 64, + "block_size": 1024 + }, + "16384": { + "strategy": "pull", + "num_blocks": 32, + "block_size": 1024 + } + }, + "all_gather": { + "16": { + "strategy": "direct", + "num_blocks": 64, + "block_size": 1024 + }, + "32": { + "strategy": "push", + "num_blocks": 4, + "block_size": 256 + }, + "64": { + "strategy": "push", + "num_blocks": 32, + "block_size": 128 + }, + "128": { + "strategy": "push", + "num_blocks": 96, + "block_size": 256 + }, + "256": { + "strategy": "push", + "num_blocks": 128, + "block_size": 256 + }, + "512": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "1024": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "2048": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "4096": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "8192": { + "strategy": "direct", + "num_blocks": 1, + "block_size": 1024 + }, + "16384": { + "strategy": "direct", + "num_blocks": 8, + "block_size": 512 + } + }, + "reduce_scatter_attn_res": { + "16": { + "strategy": "separate" + }, + "32": { + "strategy": "separate" + }, + "64": { + "strategy": "separate" + }, + "128": { + "strategy": "separate" + }, + "256": { + "strategy": "separate" + }, + "512": { + "strategy": "separate" + }, + "1024": { + "strategy": "separate" + }, + "2048": { + "strategy": "separate" + }, + "4096": { + "strategy": "separate" + }, + "8192": { + "strategy": "separate" + }, + "16384": { + "strategy": "separate" + } + }, + "attn_res_all_gather": { + "16": { + "strategy": "separate" + }, + "32": { + "strategy": "separate" + }, + "64": { + "strategy": "separate" + }, + "128": { + "strategy": "separate" + }, + "256": { + "strategy": "separate" + }, + "512": { + "strategy": "separate" + }, + "1024": { + "strategy": "separate" + }, + "2048": { + "strategy": "separate" + }, + "4096": { + "strategy": "separate" + }, + "8192": { + "strategy": "separate" + }, + "16384": { + "strategy": "separate" + } + } + } +} diff --git a/python/sglang/kernels/ops/communication/sp_collective.py b/python/sglang/kernels/ops/communication/sp_collective.py index 647f109ef1..2e269605b3 100644 --- a/python/sglang/kernels/ops/communication/sp_collective.py +++ b/python/sglang/kernels/ops/communication/sp_collective.py @@ -68,8 +68,11 @@ def get_dispatch( hidden_size: int, num_tokens: int, device: torch.device, + *, + element_size: int = 2, + max_push_size: Optional[int] = None, ) -> Optional[Dispatch]: - """Return the tuned strategy, or None when the table selects NCCL.""" + """Return a tuned strategy that fits, or None when NCCL should run.""" table = _table(world_size, hidden_size, device) if table is None: return None @@ -77,13 +80,35 @@ def get_dispatch( if raw_configs is None: return None configs = {int(k): v for k, v in raw_configs.items()} - bucket = min(configs) - for candidate in sorted(configs): + buckets = sorted(configs) + bucket_index = 0 + for index, candidate in enumerate(buckets): if candidate <= num_tokens: - bucket = candidate + bucket_index = index else: break - config = configs[bucket] + config = configs[buckets[bucket_index]] + if ( + max_push_size is not None + and config["strategy"] == "push" + and num_tokens * hidden_size * element_size // world_size > max_push_size + ): + # The selected push bucket does not fit. Advance to the next measured + # operation-specific fallback rather than immediately using NCCL. + fallback_strategy = { + "reduce_scatter": "pull", + "all_gather": "direct", + }.get(kind) + if fallback_strategy is None: + return None + for candidate in buckets[bucket_index + 1 :]: + config = configs[candidate] + if config["strategy"] == "nccl": + return None + if config["strategy"] == fallback_strategy: + break + else: + return None if config["strategy"] == "nccl": return None return Dispatch( diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 37c6732744..d3277f4386 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -2799,7 +2799,10 @@ def initialize_model_parallel( get_world_group().local_rank, backend, use_pynccl=SYNC_TOKEN_IDS_ACROSS_TP or enable_symm_mem, - use_custom_allreduce=False, + # Attention TP can be a derived subgroup of the full TP group. + # Inherit the global policy and let per-group capability detection + # select a custom communicator or fall back. + use_custom_allreduce=None, use_torch_symm_mem_allreduce=False, use_message_queue_broadcaster=envs.SGLANG_USE_MESSAGE_QUEUE_BROADCASTER.get(), group_name="attention_tp", diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index c2efb1299e..4425032a8a 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1772,8 +1772,8 @@ class Envs: # either direction. See srt/layers/communication/k3_ar_fusion.py. SGLANG_K3_AR_FUSION = EnvBool(False) # K3 SP-MoE fused residual + reduce-scatter and matching all-gather over - # CustomAllReduceV2's MNNVL push workspace. Auto-probed for the validated - # TP8 GB300 configuration; set 0/1 to override. See + # CustomAllReduceV2's MNNVL push workspace. Auto-probed for checked-in + # TP4/TP8 GB300 and TP16 GB200/GB300 tuning; set 0/1 to override. See # srt/layers/communication/k3_sp_collective.py. SGLANG_K3_SP_COLLECTIVE = EnvBool(False) # Keep K3's post-MoE residual stream token-sharded between consecutive diff --git a/python/sglang/srt/layers/communication/k3_sp_collective.py b/python/sglang/srt/layers/communication/k3_sp_collective.py index a21deaf2ef..6185ff7978 100644 --- a/python/sglang/srt/layers/communication/k3_sp_collective.py +++ b/python/sglang/srt/layers/communication/k3_sp_collective.py @@ -3,7 +3,7 @@ The reduce-scatter folds the pending attention residual into its reduction epilogue. The matching all-gather reassembles the token shards after MoE. Both reuse CustomAllReduceV2's push workspace and fall back as a pair outside -the checked-in GB300 tuning envelope. +the checked-in GB200/GB300 tuning envelope. """ from __future__ import annotations @@ -28,7 +28,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) _HIDDEN_SIZE = 7168 -_SUPPORTED_WORLD_SIZES = (4, 8) +_SUPPORTED_WORLD_SIZES = (4, 8, 16) # Named persistent symmetric buffers, one per NVLS-aliased tensor. Every rank # must resolve the same (buffer, offset) for these, which a per-forward @@ -70,17 +70,35 @@ def _init_state() -> Optional[_State]: a2a = get_exec().moe.moe_a2a_backend group = get_parallel().attn_tp_group comm = group.ca_comm + device_sm = get_device_sm() + is_custom_allreduce = isinstance(comm, CustomAllReduceV2) + if is_custom_allreduce: + comm_disabled = comm.disabled + has_multicast = comm.has_multicast + else: + comm_disabled = None + has_multicast = None if ( - get_device_sm() != 103 + device_sm not in (100, 103) or group.world_size not in _SUPPORTED_WORLD_SIZES or a2a not in ("megamoe", "deepep") - or not isinstance(comm, CustomAllReduceV2) - or comm.disabled - or not comm.has_multicast + or not is_custom_allreduce + or comm_disabled + or not has_multicast ): message = ( - "K3 SP collective requires SM103, TP4/TP8, MegaMoE/DeepEP, and " - "CustomAllReduceV2 with multicast; using NCCL." + "K3 SP collective requires SM100/SM103, TP4/TP8/TP16, " + "MegaMoE/DeepEP, and CustomAllReduceV2 with multicast; using NCCL " + "(sm=%s, world_size=%s, a2a=%s, comm=%s, disabled=%s, " + "has_multicast=%s)." + % ( + device_sm, + group.world_size, + a2a, + type(comm).__name__ if comm is not None else None, + comm_disabled, + has_multicast, + ) ) (logger.warning if explicit else logger.info)(message) return None @@ -133,7 +151,9 @@ def _symm_buffer( ) -def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool: +def requires_symmetric_rs( + num_tokens: int, device: torch.device, element_size: int +) -> bool: """Whether standalone or fused RS reads o_proj through its NVLS alias.""" state = _init_state() if state is None: @@ -146,6 +166,8 @@ def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool: _HIDDEN_SIZE, num_tokens, device, + element_size=element_size, + max_push_size=state.comm.max_push_size, ) if dispatch is not None and dispatch.strategy == "pull": return True @@ -247,8 +269,9 @@ def _eligible( or not residual.is_contiguous() ): return False - local_bytes = tensor.numel() * tensor.element_size() // state.group.world_size - return local_bytes <= state.comm.max_push_size + # Note(ajit283): Push capacity participates in dispatch selection so an + # oversized push can advance to a tuned pull/direct configuration. + return True def reduce_scatter_res( @@ -266,6 +289,8 @@ def reduce_scatter_res( tensor.shape[1], tensor.shape[0], tensor.device, + element_size=tensor.element_size(), + max_push_size=state.comm.max_push_size, ) if dispatch is None: return None @@ -374,7 +399,6 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]: or tensor.ndim != 2 or tensor.shape[1] != _HIDDEN_SIZE or tensor.shape[0] <= 0 - or tensor.numel() * tensor.element_size() > state.comm.max_push_size ): return None from sglang.kernels.ops.communication import sp_collective @@ -385,6 +409,8 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]: tensor.shape[1], global_tokens, tensor.device, + element_size=tensor.element_size(), + max_push_size=state.comm.max_push_size, ) if dispatch is None: return None diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index b04ac8b40c..9f3f44b003 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -2515,7 +2515,9 @@ class KimiK3DecoderLayer(nn.Module): def _sp_o_proj_forward(x, *args, **kwargs): output_rows = k3_sp_collective.get_o_proj_output_rows(x.shape[0]) - if k3_sp_collective.requires_symmetric_rs(output_rows, x.device): + if k3_sp_collective.requires_symmetric_rs( + output_rows, x.device, x.element_size() + ): output = k3_sp_collective.get_o_proj_output_buffer( output_rows, x.dtype, o_proj.output_size ) diff --git a/test/registered/unit/distributed/test_parallel_state.py b/test/registered/unit/distributed/test_parallel_state.py index 652f909f8f..6a995d4f8a 100644 --- a/test/registered/unit/distributed/test_parallel_state.py +++ b/test/registered/unit/distributed/test_parallel_state.py @@ -26,9 +26,9 @@ then figures out which specific group(s) it belongs to. Our tests: 1. Mock the distributed backend (no real GPUs needed) -2. Mock init_model_parallel_group to capture the group_ranks parameter +2. Mock GroupCoordinator construction to capture the effective group configuration 3. Call the real initialize_model_parallel() -4. Verify group_ranks contains the expected complete group structure +4. Verify the resulting group structure and policies We only need to simulate rank 0 because we're testing the group creation logic, not the per-rank group membership logic. @@ -172,7 +172,8 @@ def test_custom_allreduce_precedes_symmetric_memory_pynccl(): coordinator.pynccl_comm.all_reduce.assert_not_called() -def test_parallel_group_construction_tp8_attn_cp2(): +@pytest.mark.parametrize("custom_allreduce_enabled", [False, True]) +def test_parallel_group_construction_tp8_attn_cp2(custom_allreduce_enabled): """ Test parallel group construction for 8 GPU configuration with: - tensor_model_parallel_size = 8 @@ -187,8 +188,8 @@ def test_parallel_group_construction_tp8_attn_cp2(): This test calls the ACTUAL initialize_model_parallel() and verifies the groups. Note: We simulate only rank 0 here, but initialize_model_parallel() creates - ALL groups for ALL ranks in a single call. We capture these groups via mocking - and verify the complete group structure. + ALL groups for ALL ranks in a single call. We capture GroupCoordinator + construction and verify the complete group structure and effective policy. """ world_size = 8 @@ -201,28 +202,35 @@ def test_parallel_group_construction_tp8_attn_cp2(): patch.object(parallel_state, "_ATTN_CP", None), patch.object(parallel_state, "_ATTN_TP", None), patch.object(parallel_state, "_PP", None), + patch.object( + parallel_state, + "_ENABLE_CUSTOM_ALL_REDUCE", + custom_allreduce_enabled, + ), patch("torch.distributed.is_initialized", return_value=True), patch("torch.distributed.get_world_size", return_value=world_size), patch("torch.distributed.get_rank", return_value=0), patch("torch.distributed.get_backend", return_value="nccl"), ): - # Mock init_model_parallel_group to capture the groups being created + # Capture the effective configuration after init_model_parallel_group + # has resolved inherited policies. created_groups = {} - def mock_init_model_parallel_group(group_ranks, local_rank, backend, **kwargs): - group_name = kwargs.get("group_name", "unknown") + def mock_group_coordinator( + *, group_ranks, group_name, use_custom_allreduce, **kwargs + ): created_groups[group_name] = group_ranks - # Create a mock group object mock_group = Mock() mock_group.device_group = Mock() + mock_group.use_custom_allreduce = use_custom_allreduce return mock_group with ( patch.object( parallel_state, - "init_model_parallel_group", - side_effect=mock_init_model_parallel_group, + "GroupCoordinator", + side_effect=mock_group_coordinator, ), patch.object(parallel_state, "get_world_group") as mock_world_group, ): @@ -265,6 +273,13 @@ def test_parallel_group_construction_tp8_attn_cp2(): f"Wrong ATTN_CP groups: {attn_cp_groups}" ) + # Derived attention-TP groups inherit the effective global policy; + # the communicator performs the final per-group capability check. + assert ( + parallel_state.get_attn_tp_group().use_custom_allreduce + is custom_allreduce_enabled + ) + print("TP=8, Attn CP=2 group construction verified") # Cleanup @@ -452,7 +467,8 @@ if __name__ == "__main__": import sys try: - test_parallel_group_construction_tp8_attn_cp2() + test_parallel_group_construction_tp8_attn_cp2(False) + test_parallel_group_construction_tp8_attn_cp2(True) test_parallel_group_construction_tp8_moe_ep4_cp2() test_group_desc_propagated_via_real_new_group("tp") test_group_desc_propagated_via_real_new_group("pp")