[Kimi] Enable GB300 TP4 and GB200/GB300 TP16 SP collectives (#35330)
Co-authored-by: Ajit Mistry <ajmistry@amistry.nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Ajit Mistry
Po-Han Huang
parent
fa090f7755
commit
ced5e9fe49
+203
@@ -0,0 +1,203 @@
|
||||
{
|
||||
"source": {
|
||||
"date": "2026-09-01",
|
||||
"device": "NVIDIA GB200",
|
||||
"nodes": 4,
|
||||
"gpus_per_node": 4,
|
||||
"push_slot_bytes": 33554432,
|
||||
"raw_results": [
|
||||
"world16-raw.json"
|
||||
]
|
||||
},
|
||||
"selection": {
|
||||
"rule": "nearest global-token bucket at or below the workload",
|
||||
"fallback": "nccl",
|
||||
"note": "Fresh world-16 GB200 sweep. Each candidate was correctness-checked against NCCL; custom is selected only when its measured latency is strictly lower. Push candidates are limited to the default 256 KiB per-rank workspace. Attention-residual fusion was not tuned."
|
||||
},
|
||||
"configs": {
|
||||
"reduce_scatter": {
|
||||
"16": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 32,
|
||||
"block_size": 512
|
||||
},
|
||||
"32": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 96,
|
||||
"block_size": 512
|
||||
},
|
||||
"64": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 96,
|
||||
"block_size": 128
|
||||
},
|
||||
"128": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 128
|
||||
},
|
||||
"256": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 32,
|
||||
"block_size": 128
|
||||
},
|
||||
"512": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 32,
|
||||
"block_size": 1024
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 96,
|
||||
"block_size": 1024
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 96,
|
||||
"block_size": 1024
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 32,
|
||||
"block_size": 1024
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 64,
|
||||
"block_size": 1024
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 96,
|
||||
"block_size": 512
|
||||
}
|
||||
},
|
||||
"all_gather": {
|
||||
"16": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 32,
|
||||
"block_size": 256
|
||||
},
|
||||
"32": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 4,
|
||||
"block_size": 512
|
||||
},
|
||||
"64": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 16,
|
||||
"block_size": 128
|
||||
},
|
||||
"128": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 512
|
||||
},
|
||||
"256": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 128
|
||||
},
|
||||
"512": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 2,
|
||||
"block_size": 512
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 4,
|
||||
"block_size": 512
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 4,
|
||||
"block_size": 512
|
||||
}
|
||||
},
|
||||
"reduce_scatter_attn_res": {
|
||||
"16": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"32": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"64": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"128": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"256": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"512": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "separate"
|
||||
}
|
||||
},
|
||||
"attn_res_all_gather": {
|
||||
"16": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"32": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"64": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"128": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"256": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"512": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "separate"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
{
|
||||
"source": {
|
||||
"date": "2026-09-01",
|
||||
"device": "NVIDIA GB300",
|
||||
"nodes": 4,
|
||||
"gpus_per_node": 4,
|
||||
"push_slot_bytes": 33554432,
|
||||
"raw_results": [
|
||||
"world16-raw.json"
|
||||
]
|
||||
},
|
||||
"selection": {
|
||||
"rule": "nearest global-token bucket at or below the workload",
|
||||
"fallback": "nccl",
|
||||
"note": "Fresh world-16 GB300 sweep. Each candidate was correctness-checked against NCCL; custom is selected only when its measured latency is strictly lower. Push candidates are limited to the default 256 KiB per-rank workspace. Attention-residual fusion was not tuned."
|
||||
},
|
||||
"configs": {
|
||||
"reduce_scatter": {
|
||||
"16": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 128
|
||||
},
|
||||
"32": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 4,
|
||||
"block_size": 512
|
||||
},
|
||||
"64": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 512
|
||||
},
|
||||
"128": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 16,
|
||||
"block_size": 256
|
||||
},
|
||||
"256": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 512
|
||||
},
|
||||
"512": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 32,
|
||||
"block_size": 1024
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 96,
|
||||
"block_size": 1024
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 96,
|
||||
"block_size": 1024
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 32,
|
||||
"block_size": 1024
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 64,
|
||||
"block_size": 1024
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "pull",
|
||||
"num_blocks": 32,
|
||||
"block_size": 1024
|
||||
}
|
||||
},
|
||||
"all_gather": {
|
||||
"16": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 64,
|
||||
"block_size": 1024
|
||||
},
|
||||
"32": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 4,
|
||||
"block_size": 256
|
||||
},
|
||||
"64": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 32,
|
||||
"block_size": 128
|
||||
},
|
||||
"128": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 96,
|
||||
"block_size": 256
|
||||
},
|
||||
"256": {
|
||||
"strategy": "push",
|
||||
"num_blocks": 128,
|
||||
"block_size": 256
|
||||
},
|
||||
"512": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 1,
|
||||
"block_size": 1024
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "direct",
|
||||
"num_blocks": 8,
|
||||
"block_size": 512
|
||||
}
|
||||
},
|
||||
"reduce_scatter_attn_res": {
|
||||
"16": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"32": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"64": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"128": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"256": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"512": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "separate"
|
||||
}
|
||||
},
|
||||
"attn_res_all_gather": {
|
||||
"16": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"32": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"64": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"128": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"256": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"512": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"1024": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"2048": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"4096": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"8192": {
|
||||
"strategy": "separate"
|
||||
},
|
||||
"16384": {
|
||||
"strategy": "separate"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,8 +68,11 @@ def get_dispatch(
|
||||
hidden_size: int,
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
*,
|
||||
element_size: int = 2,
|
||||
max_push_size: Optional[int] = None,
|
||||
) -> Optional[Dispatch]:
|
||||
"""Return the tuned strategy, or None when the table selects NCCL."""
|
||||
"""Return a tuned strategy that fits, or None when NCCL should run."""
|
||||
table = _table(world_size, hidden_size, device)
|
||||
if table is None:
|
||||
return None
|
||||
@@ -77,13 +80,35 @@ def get_dispatch(
|
||||
if raw_configs is None:
|
||||
return None
|
||||
configs = {int(k): v for k, v in raw_configs.items()}
|
||||
bucket = min(configs)
|
||||
for candidate in sorted(configs):
|
||||
buckets = sorted(configs)
|
||||
bucket_index = 0
|
||||
for index, candidate in enumerate(buckets):
|
||||
if candidate <= num_tokens:
|
||||
bucket = candidate
|
||||
bucket_index = index
|
||||
else:
|
||||
break
|
||||
config = configs[bucket]
|
||||
config = configs[buckets[bucket_index]]
|
||||
if (
|
||||
max_push_size is not None
|
||||
and config["strategy"] == "push"
|
||||
and num_tokens * hidden_size * element_size // world_size > max_push_size
|
||||
):
|
||||
# The selected push bucket does not fit. Advance to the next measured
|
||||
# operation-specific fallback rather than immediately using NCCL.
|
||||
fallback_strategy = {
|
||||
"reduce_scatter": "pull",
|
||||
"all_gather": "direct",
|
||||
}.get(kind)
|
||||
if fallback_strategy is None:
|
||||
return None
|
||||
for candidate in buckets[bucket_index + 1 :]:
|
||||
config = configs[candidate]
|
||||
if config["strategy"] == "nccl":
|
||||
return None
|
||||
if config["strategy"] == fallback_strategy:
|
||||
break
|
||||
else:
|
||||
return None
|
||||
if config["strategy"] == "nccl":
|
||||
return None
|
||||
return Dispatch(
|
||||
|
||||
@@ -2799,7 +2799,10 @@ def initialize_model_parallel(
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
use_pynccl=SYNC_TOKEN_IDS_ACROSS_TP or enable_symm_mem,
|
||||
use_custom_allreduce=False,
|
||||
# Attention TP can be a derived subgroup of the full TP group.
|
||||
# Inherit the global policy and let per-group capability detection
|
||||
# select a custom communicator or fall back.
|
||||
use_custom_allreduce=None,
|
||||
use_torch_symm_mem_allreduce=False,
|
||||
use_message_queue_broadcaster=envs.SGLANG_USE_MESSAGE_QUEUE_BROADCASTER.get(),
|
||||
group_name="attention_tp",
|
||||
|
||||
@@ -1772,8 +1772,8 @@ class Envs:
|
||||
# either direction. See srt/layers/communication/k3_ar_fusion.py.
|
||||
SGLANG_K3_AR_FUSION = EnvBool(False)
|
||||
# K3 SP-MoE fused residual + reduce-scatter and matching all-gather over
|
||||
# CustomAllReduceV2's MNNVL push workspace. Auto-probed for the validated
|
||||
# TP8 GB300 configuration; set 0/1 to override. See
|
||||
# CustomAllReduceV2's MNNVL push workspace. Auto-probed for checked-in
|
||||
# TP4/TP8 GB300 and TP16 GB200/GB300 tuning; set 0/1 to override. See
|
||||
# srt/layers/communication/k3_sp_collective.py.
|
||||
SGLANG_K3_SP_COLLECTIVE = EnvBool(False)
|
||||
# Keep K3's post-MoE residual stream token-sharded between consecutive
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The reduce-scatter folds the pending attention residual into its reduction
|
||||
epilogue. The matching all-gather reassembles the token shards after MoE.
|
||||
Both reuse CustomAllReduceV2's push workspace and fall back as a pair outside
|
||||
the checked-in GB300 tuning envelope.
|
||||
the checked-in GB200/GB300 tuning envelope.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HIDDEN_SIZE = 7168
|
||||
_SUPPORTED_WORLD_SIZES = (4, 8)
|
||||
_SUPPORTED_WORLD_SIZES = (4, 8, 16)
|
||||
|
||||
# Named persistent symmetric buffers, one per NVLS-aliased tensor. Every rank
|
||||
# must resolve the same (buffer, offset) for these, which a per-forward
|
||||
@@ -70,17 +70,35 @@ def _init_state() -> Optional[_State]:
|
||||
a2a = get_exec().moe.moe_a2a_backend
|
||||
group = get_parallel().attn_tp_group
|
||||
comm = group.ca_comm
|
||||
device_sm = get_device_sm()
|
||||
is_custom_allreduce = isinstance(comm, CustomAllReduceV2)
|
||||
if is_custom_allreduce:
|
||||
comm_disabled = comm.disabled
|
||||
has_multicast = comm.has_multicast
|
||||
else:
|
||||
comm_disabled = None
|
||||
has_multicast = None
|
||||
if (
|
||||
get_device_sm() != 103
|
||||
device_sm not in (100, 103)
|
||||
or group.world_size not in _SUPPORTED_WORLD_SIZES
|
||||
or a2a not in ("megamoe", "deepep")
|
||||
or not isinstance(comm, CustomAllReduceV2)
|
||||
or comm.disabled
|
||||
or not comm.has_multicast
|
||||
or not is_custom_allreduce
|
||||
or comm_disabled
|
||||
or not has_multicast
|
||||
):
|
||||
message = (
|
||||
"K3 SP collective requires SM103, TP4/TP8, MegaMoE/DeepEP, and "
|
||||
"CustomAllReduceV2 with multicast; using NCCL."
|
||||
"K3 SP collective requires SM100/SM103, TP4/TP8/TP16, "
|
||||
"MegaMoE/DeepEP, and CustomAllReduceV2 with multicast; using NCCL "
|
||||
"(sm=%s, world_size=%s, a2a=%s, comm=%s, disabled=%s, "
|
||||
"has_multicast=%s)."
|
||||
% (
|
||||
device_sm,
|
||||
group.world_size,
|
||||
a2a,
|
||||
type(comm).__name__ if comm is not None else None,
|
||||
comm_disabled,
|
||||
has_multicast,
|
||||
)
|
||||
)
|
||||
(logger.warning if explicit else logger.info)(message)
|
||||
return None
|
||||
@@ -133,7 +151,9 @@ def _symm_buffer(
|
||||
)
|
||||
|
||||
|
||||
def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool:
|
||||
def requires_symmetric_rs(
|
||||
num_tokens: int, device: torch.device, element_size: int
|
||||
) -> bool:
|
||||
"""Whether standalone or fused RS reads o_proj through its NVLS alias."""
|
||||
state = _init_state()
|
||||
if state is None:
|
||||
@@ -146,6 +166,8 @@ def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool:
|
||||
_HIDDEN_SIZE,
|
||||
num_tokens,
|
||||
device,
|
||||
element_size=element_size,
|
||||
max_push_size=state.comm.max_push_size,
|
||||
)
|
||||
if dispatch is not None and dispatch.strategy == "pull":
|
||||
return True
|
||||
@@ -247,8 +269,9 @@ def _eligible(
|
||||
or not residual.is_contiguous()
|
||||
):
|
||||
return False
|
||||
local_bytes = tensor.numel() * tensor.element_size() // state.group.world_size
|
||||
return local_bytes <= state.comm.max_push_size
|
||||
# Note(ajit283): Push capacity participates in dispatch selection so an
|
||||
# oversized push can advance to a tuned pull/direct configuration.
|
||||
return True
|
||||
|
||||
|
||||
def reduce_scatter_res(
|
||||
@@ -266,6 +289,8 @@ def reduce_scatter_res(
|
||||
tensor.shape[1],
|
||||
tensor.shape[0],
|
||||
tensor.device,
|
||||
element_size=tensor.element_size(),
|
||||
max_push_size=state.comm.max_push_size,
|
||||
)
|
||||
if dispatch is None:
|
||||
return None
|
||||
@@ -374,7 +399,6 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
or tensor.ndim != 2
|
||||
or tensor.shape[1] != _HIDDEN_SIZE
|
||||
or tensor.shape[0] <= 0
|
||||
or tensor.numel() * tensor.element_size() > state.comm.max_push_size
|
||||
):
|
||||
return None
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
@@ -385,6 +409,8 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
tensor.shape[1],
|
||||
global_tokens,
|
||||
tensor.device,
|
||||
element_size=tensor.element_size(),
|
||||
max_push_size=state.comm.max_push_size,
|
||||
)
|
||||
if dispatch is None:
|
||||
return None
|
||||
|
||||
@@ -2515,7 +2515,9 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
|
||||
def _sp_o_proj_forward(x, *args, **kwargs):
|
||||
output_rows = k3_sp_collective.get_o_proj_output_rows(x.shape[0])
|
||||
if k3_sp_collective.requires_symmetric_rs(output_rows, x.device):
|
||||
if k3_sp_collective.requires_symmetric_rs(
|
||||
output_rows, x.device, x.element_size()
|
||||
):
|
||||
output = k3_sp_collective.get_o_proj_output_buffer(
|
||||
output_rows, x.dtype, o_proj.output_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user