[Kimi] Enable GB300 TP4 and GB200/GB300 TP16 SP collectives (#35330)

Co-authored-by: Ajit Mistry <ajmistry@amistry.nvidia.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
Ajit Mistry
2026-09-29 10:18:11 -07:00
committed by GitHub
co-authored by Ajit Mistry Po-Han Huang
parent fa090f7755
commit ced5e9fe49
8 changed files with 511 additions and 33 deletions
@@ -0,0 +1,203 @@
{
"source": {
"date": "2026-09-01",
"device": "NVIDIA GB200",
"nodes": 4,
"gpus_per_node": 4,
"push_slot_bytes": 33554432,
"raw_results": [
"world16-raw.json"
]
},
"selection": {
"rule": "nearest global-token bucket at or below the workload",
"fallback": "nccl",
"note": "Fresh world-16 GB200 sweep. Each candidate was correctness-checked against NCCL; custom is selected only when its measured latency is strictly lower. Push candidates are limited to the default 256 KiB per-rank workspace. Attention-residual fusion was not tuned."
},
"configs": {
"reduce_scatter": {
"16": {
"strategy": "push",
"num_blocks": 32,
"block_size": 512
},
"32": {
"strategy": "push",
"num_blocks": 96,
"block_size": 512
},
"64": {
"strategy": "push",
"num_blocks": 96,
"block_size": 128
},
"128": {
"strategy": "push",
"num_blocks": 128,
"block_size": 128
},
"256": {
"strategy": "push",
"num_blocks": 32,
"block_size": 128
},
"512": {
"strategy": "pull",
"num_blocks": 32,
"block_size": 1024
},
"1024": {
"strategy": "pull",
"num_blocks": 96,
"block_size": 1024
},
"2048": {
"strategy": "pull",
"num_blocks": 96,
"block_size": 1024
},
"4096": {
"strategy": "pull",
"num_blocks": 32,
"block_size": 1024
},
"8192": {
"strategy": "pull",
"num_blocks": 64,
"block_size": 1024
},
"16384": {
"strategy": "pull",
"num_blocks": 96,
"block_size": 512
}
},
"all_gather": {
"16": {
"strategy": "push",
"num_blocks": 32,
"block_size": 256
},
"32": {
"strategy": "push",
"num_blocks": 4,
"block_size": 512
},
"64": {
"strategy": "push",
"num_blocks": 16,
"block_size": 128
},
"128": {
"strategy": "push",
"num_blocks": 128,
"block_size": 512
},
"256": {
"strategy": "push",
"num_blocks": 128,
"block_size": 128
},
"512": {
"strategy": "direct",
"num_blocks": 2,
"block_size": 512
},
"1024": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"2048": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"4096": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"8192": {
"strategy": "direct",
"num_blocks": 4,
"block_size": 512
},
"16384": {
"strategy": "direct",
"num_blocks": 4,
"block_size": 512
}
},
"reduce_scatter_attn_res": {
"16": {
"strategy": "separate"
},
"32": {
"strategy": "separate"
},
"64": {
"strategy": "separate"
},
"128": {
"strategy": "separate"
},
"256": {
"strategy": "separate"
},
"512": {
"strategy": "separate"
},
"1024": {
"strategy": "separate"
},
"2048": {
"strategy": "separate"
},
"4096": {
"strategy": "separate"
},
"8192": {
"strategy": "separate"
},
"16384": {
"strategy": "separate"
}
},
"attn_res_all_gather": {
"16": {
"strategy": "separate"
},
"32": {
"strategy": "separate"
},
"64": {
"strategy": "separate"
},
"128": {
"strategy": "separate"
},
"256": {
"strategy": "separate"
},
"512": {
"strategy": "separate"
},
"1024": {
"strategy": "separate"
},
"2048": {
"strategy": "separate"
},
"4096": {
"strategy": "separate"
},
"8192": {
"strategy": "separate"
},
"16384": {
"strategy": "separate"
}
}
}
}
@@ -0,0 +1,203 @@
{
"source": {
"date": "2026-09-01",
"device": "NVIDIA GB300",
"nodes": 4,
"gpus_per_node": 4,
"push_slot_bytes": 33554432,
"raw_results": [
"world16-raw.json"
]
},
"selection": {
"rule": "nearest global-token bucket at or below the workload",
"fallback": "nccl",
"note": "Fresh world-16 GB300 sweep. Each candidate was correctness-checked against NCCL; custom is selected only when its measured latency is strictly lower. Push candidates are limited to the default 256 KiB per-rank workspace. Attention-residual fusion was not tuned."
},
"configs": {
"reduce_scatter": {
"16": {
"strategy": "push",
"num_blocks": 128,
"block_size": 128
},
"32": {
"strategy": "push",
"num_blocks": 4,
"block_size": 512
},
"64": {
"strategy": "push",
"num_blocks": 128,
"block_size": 512
},
"128": {
"strategy": "push",
"num_blocks": 16,
"block_size": 256
},
"256": {
"strategy": "push",
"num_blocks": 128,
"block_size": 512
},
"512": {
"strategy": "pull",
"num_blocks": 32,
"block_size": 1024
},
"1024": {
"strategy": "pull",
"num_blocks": 96,
"block_size": 1024
},
"2048": {
"strategy": "pull",
"num_blocks": 96,
"block_size": 1024
},
"4096": {
"strategy": "pull",
"num_blocks": 32,
"block_size": 1024
},
"8192": {
"strategy": "pull",
"num_blocks": 64,
"block_size": 1024
},
"16384": {
"strategy": "pull",
"num_blocks": 32,
"block_size": 1024
}
},
"all_gather": {
"16": {
"strategy": "direct",
"num_blocks": 64,
"block_size": 1024
},
"32": {
"strategy": "push",
"num_blocks": 4,
"block_size": 256
},
"64": {
"strategy": "push",
"num_blocks": 32,
"block_size": 128
},
"128": {
"strategy": "push",
"num_blocks": 96,
"block_size": 256
},
"256": {
"strategy": "push",
"num_blocks": 128,
"block_size": 256
},
"512": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"1024": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"2048": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"4096": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"8192": {
"strategy": "direct",
"num_blocks": 1,
"block_size": 1024
},
"16384": {
"strategy": "direct",
"num_blocks": 8,
"block_size": 512
}
},
"reduce_scatter_attn_res": {
"16": {
"strategy": "separate"
},
"32": {
"strategy": "separate"
},
"64": {
"strategy": "separate"
},
"128": {
"strategy": "separate"
},
"256": {
"strategy": "separate"
},
"512": {
"strategy": "separate"
},
"1024": {
"strategy": "separate"
},
"2048": {
"strategy": "separate"
},
"4096": {
"strategy": "separate"
},
"8192": {
"strategy": "separate"
},
"16384": {
"strategy": "separate"
}
},
"attn_res_all_gather": {
"16": {
"strategy": "separate"
},
"32": {
"strategy": "separate"
},
"64": {
"strategy": "separate"
},
"128": {
"strategy": "separate"
},
"256": {
"strategy": "separate"
},
"512": {
"strategy": "separate"
},
"1024": {
"strategy": "separate"
},
"2048": {
"strategy": "separate"
},
"4096": {
"strategy": "separate"
},
"8192": {
"strategy": "separate"
},
"16384": {
"strategy": "separate"
}
}
}
}
@@ -68,8 +68,11 @@ def get_dispatch(
hidden_size: int,
num_tokens: int,
device: torch.device,
*,
element_size: int = 2,
max_push_size: Optional[int] = None,
) -> Optional[Dispatch]:
"""Return the tuned strategy, or None when the table selects NCCL."""
"""Return a tuned strategy that fits, or None when NCCL should run."""
table = _table(world_size, hidden_size, device)
if table is None:
return None
@@ -77,13 +80,35 @@ def get_dispatch(
if raw_configs is None:
return None
configs = {int(k): v for k, v in raw_configs.items()}
bucket = min(configs)
for candidate in sorted(configs):
buckets = sorted(configs)
bucket_index = 0
for index, candidate in enumerate(buckets):
if candidate <= num_tokens:
bucket = candidate
bucket_index = index
else:
break
config = configs[bucket]
config = configs[buckets[bucket_index]]
if (
max_push_size is not None
and config["strategy"] == "push"
and num_tokens * hidden_size * element_size // world_size > max_push_size
):
# The selected push bucket does not fit. Advance to the next measured
# operation-specific fallback rather than immediately using NCCL.
fallback_strategy = {
"reduce_scatter": "pull",
"all_gather": "direct",
}.get(kind)
if fallback_strategy is None:
return None
for candidate in buckets[bucket_index + 1 :]:
config = configs[candidate]
if config["strategy"] == "nccl":
return None
if config["strategy"] == fallback_strategy:
break
else:
return None
if config["strategy"] == "nccl":
return None
return Dispatch(
@@ -2799,7 +2799,10 @@ def initialize_model_parallel(
get_world_group().local_rank,
backend,
use_pynccl=SYNC_TOKEN_IDS_ACROSS_TP or enable_symm_mem,
use_custom_allreduce=False,
# Attention TP can be a derived subgroup of the full TP group.
# Inherit the global policy and let per-group capability detection
# select a custom communicator or fall back.
use_custom_allreduce=None,
use_torch_symm_mem_allreduce=False,
use_message_queue_broadcaster=envs.SGLANG_USE_MESSAGE_QUEUE_BROADCASTER.get(),
group_name="attention_tp",
+2 -2
View File
@@ -1772,8 +1772,8 @@ class Envs:
# either direction. See srt/layers/communication/k3_ar_fusion.py.
SGLANG_K3_AR_FUSION = EnvBool(False)
# K3 SP-MoE fused residual + reduce-scatter and matching all-gather over
# CustomAllReduceV2's MNNVL push workspace. Auto-probed for the validated
# TP8 GB300 configuration; set 0/1 to override. See
# CustomAllReduceV2's MNNVL push workspace. Auto-probed for checked-in
# TP4/TP8 GB300 and TP16 GB200/GB300 tuning; set 0/1 to override. See
# srt/layers/communication/k3_sp_collective.py.
SGLANG_K3_SP_COLLECTIVE = EnvBool(False)
# Keep K3's post-MoE residual stream token-sharded between consecutive
@@ -3,7 +3,7 @@
The reduce-scatter folds the pending attention residual into its reduction
epilogue. The matching all-gather reassembles the token shards after MoE.
Both reuse CustomAllReduceV2's push workspace and fall back as a pair outside
the checked-in GB300 tuning envelope.
the checked-in GB200/GB300 tuning envelope.
"""
from __future__ import annotations
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_HIDDEN_SIZE = 7168
_SUPPORTED_WORLD_SIZES = (4, 8)
_SUPPORTED_WORLD_SIZES = (4, 8, 16)
# Named persistent symmetric buffers, one per NVLS-aliased tensor. Every rank
# must resolve the same (buffer, offset) for these, which a per-forward
@@ -70,17 +70,35 @@ def _init_state() -> Optional[_State]:
a2a = get_exec().moe.moe_a2a_backend
group = get_parallel().attn_tp_group
comm = group.ca_comm
device_sm = get_device_sm()
is_custom_allreduce = isinstance(comm, CustomAllReduceV2)
if is_custom_allreduce:
comm_disabled = comm.disabled
has_multicast = comm.has_multicast
else:
comm_disabled = None
has_multicast = None
if (
get_device_sm() != 103
device_sm not in (100, 103)
or group.world_size not in _SUPPORTED_WORLD_SIZES
or a2a not in ("megamoe", "deepep")
or not isinstance(comm, CustomAllReduceV2)
or comm.disabled
or not comm.has_multicast
or not is_custom_allreduce
or comm_disabled
or not has_multicast
):
message = (
"K3 SP collective requires SM103, TP4/TP8, MegaMoE/DeepEP, and "
"CustomAllReduceV2 with multicast; using NCCL."
"K3 SP collective requires SM100/SM103, TP4/TP8/TP16, "
"MegaMoE/DeepEP, and CustomAllReduceV2 with multicast; using NCCL "
"(sm=%s, world_size=%s, a2a=%s, comm=%s, disabled=%s, "
"has_multicast=%s)."
% (
device_sm,
group.world_size,
a2a,
type(comm).__name__ if comm is not None else None,
comm_disabled,
has_multicast,
)
)
(logger.warning if explicit else logger.info)(message)
return None
@@ -133,7 +151,9 @@ def _symm_buffer(
)
def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool:
def requires_symmetric_rs(
num_tokens: int, device: torch.device, element_size: int
) -> bool:
"""Whether standalone or fused RS reads o_proj through its NVLS alias."""
state = _init_state()
if state is None:
@@ -146,6 +166,8 @@ def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool:
_HIDDEN_SIZE,
num_tokens,
device,
element_size=element_size,
max_push_size=state.comm.max_push_size,
)
if dispatch is not None and dispatch.strategy == "pull":
return True
@@ -247,8 +269,9 @@ def _eligible(
or not residual.is_contiguous()
):
return False
local_bytes = tensor.numel() * tensor.element_size() // state.group.world_size
return local_bytes <= state.comm.max_push_size
# Note(ajit283): Push capacity participates in dispatch selection so an
# oversized push can advance to a tuned pull/direct configuration.
return True
def reduce_scatter_res(
@@ -266,6 +289,8 @@ def reduce_scatter_res(
tensor.shape[1],
tensor.shape[0],
tensor.device,
element_size=tensor.element_size(),
max_push_size=state.comm.max_push_size,
)
if dispatch is None:
return None
@@ -374,7 +399,6 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]:
or tensor.ndim != 2
or tensor.shape[1] != _HIDDEN_SIZE
or tensor.shape[0] <= 0
or tensor.numel() * tensor.element_size() > state.comm.max_push_size
):
return None
from sglang.kernels.ops.communication import sp_collective
@@ -385,6 +409,8 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]:
tensor.shape[1],
global_tokens,
tensor.device,
element_size=tensor.element_size(),
max_push_size=state.comm.max_push_size,
)
if dispatch is None:
return None
+3 -1
View File
@@ -2515,7 +2515,9 @@ class KimiK3DecoderLayer(nn.Module):
def _sp_o_proj_forward(x, *args, **kwargs):
output_rows = k3_sp_collective.get_o_proj_output_rows(x.shape[0])
if k3_sp_collective.requires_symmetric_rs(output_rows, x.device):
if k3_sp_collective.requires_symmetric_rs(
output_rows, x.device, x.element_size()
):
output = k3_sp_collective.get_o_proj_output_buffer(
output_rows, x.dtype, o_proj.output_size
)