[Refactor] Retire the model-specific Kimi K3 kernel namespace (#40922)

This commit is contained in:
Xiaoyu Zhang
2026-09-24 21:25:00 +08:00
committed by GitHub
parent 28ec6700da
commit ea5baf4022
52 changed files with 856 additions and 593 deletions
+4
View File
@@ -29,6 +29,10 @@ Operator groups (all populated): `activation`, `attention`, `communication`,
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `moe`, `quantization`,
`sampling`, `speculative`.
Place model-specific implementations and tuning data inside the corresponding
logical operator group, rather than creating a model-specific top-level group
under `ops/`.
As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has
been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit`
and each JIT-backed operator into its group as
-1
View File
@@ -23,7 +23,6 @@ _GROUPS = (
"embeddings",
"gemm",
"grammar",
"kimi_k3",
"kvcache",
"layernorm",
"mamba",
@@ -305,7 +305,19 @@ def gelu_quick(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch
return _GELU_QUICK(input, out)
def situ_and_mul(
input: torch.Tensor,
out: Optional[torch.Tensor],
beta: float,
linear_beta: Optional[float],
) -> torch.Tensor:
from ._jit_situ_and_mul import situ_and_mul as impl
return impl(input, out, beta, linear_beta)
__all__ = [
"situ_and_mul",
"SiluAndMulOp",
"GeluAndMulOp",
"GeluTanhAndMulOp",
@@ -330,3 +342,14 @@ for _fn in ("softcap_out", "softcap_inplace_logits"):
)
)
del _fn
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
register_kernel(
KernelSpec(
op="activation.situ_and_mul",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
@@ -235,3 +235,55 @@ for _mod, _fn in [
)
)
del _mod, _fn
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
for _mod, _fn, _backend, _device in [
("attn_res", "attn_res_fused_tma", KernelBackend.JIT, CapabilityRequirement.CUDA),
(
"attn_res",
"attn_res_fused_pull_rs",
KernelBackend.JIT,
CapabilityRequirement.CUDA,
),
(
"attn_res",
"attn_res_fused_direct_ag",
KernelBackend.JIT,
CapabilityRequirement.CUDA,
),
("attn_res_hip", "attn_res_hip", KernelBackend.TRITON, CapabilityRequirement.HIP),
(
"mla_output_gate",
"kimi_k3_mla_output_gate",
KernelBackend.JIT,
CapabilityRequirement.CUDA,
),
(
"kda_decode_mtp",
"fused_kda_decode_mtp_dspark",
KernelBackend.CUTE_DSL,
CapabilityRequirement.CUDA,
),
(
"kda_flydsl.kimi_k3_kda_decode",
"flydsl_kimi_k3_kda_decode",
KernelBackend.FLYDSL,
CapabilityRequirement.HIP,
),
(
"kda_flydsl.kimi_k3_kda_decode",
"flydsl_kimi_k3_kda_decode_with_f_b",
KernelBackend.FLYDSL,
CapabilityRequirement.HIP,
),
]:
register_kernel(
KernelSpec(
op=f"attention.{_fn}",
backend=_backend,
target=f"sglang.kernels.ops.attention.{_mod}:{_fn}",
capabilities=frozenset({_device}),
)
)
del _mod, _fn, _backend, _device
@@ -103,7 +103,7 @@ _COMM_MAP: dict[int, Communicator] = {}
def register_comm(comm: Communicator) -> None:
# One communicator per world_size per process -- see the note in
# kimi_k3/all_reduce.py::register_comm. The ops key only on world_size, so an
# communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an
# overwrite here would hand the old group's callers the new group's peer
# pointers.
prev = _COMM_MAP.get(comm.world_size)
@@ -20,10 +20,10 @@ def enabled() -> bool:
def _ops():
try:
from sglang.kernels.ops.kimi_k3.flydsl.source import load_module
from sglang.kernels.ops.attention.kda_flydsl.source import load_module
module = load_module(
"sglang.kernels.ops.kimi_k3.flydsl.kimi_k3_kda_decode",
"sglang.kernels.ops.attention.kda_flydsl.kimi_k3_kda_decode",
"aiter.ops.flydsl.kimi_k3_kda_decode",
)
except (ImportError, ModuleNotFoundError):
@@ -1,12 +1,36 @@
"""Collective-communication kernels (custom all-reduce, ...).
"""Collective-communication kernels and fused compute/communication kernels.
Reserved group in the ``sglang.kernels`` namespace (RFC #29630). No thin
wrappers are exposed here: the collective ops (custom all-reduce and friends)
are stateful — they manage workspaces / IPC handles and are driven through a
``CustomAllreduce``-style object and ``torch.ops.sgl_kernel.*`` bindings rather
than standalone callable kernels, so a thin ``sglang.kernels.ops`` forwarder
would be misleading. Import them from ``sgl_kernel`` / ``sglang.kernels.jit``
directly until a proper stateful-op interface is designed.
Stateful operators are imported from their modules so communicator registration,
workspace ownership, and tuning helpers stay together. The registry below only
records metadata; importing this group does not load the implementations.
"""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
__all__ = []
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
for _mod, _fn in [
("all_reduce_residual", "all_reduce_push_res"),
("all_reduce_residual", "all_reduce_push_norm"),
("all_reduce_residual", "finalize_all_reduce_push_norm"),
("all_reduce_residual", "all_reduce_pull_res"),
("all_reduce_residual", "all_reduce_pull_norm"),
("gemm_ag", "gemm_ag_up_proj"),
("gemm_ar", "o_proj_gemm_ar"),
("sp_collective", "reduce_scatter_res"),
("sp_collective", "reduce_scatter_pull"),
("sp_collective", "all_gather"),
("sp_collective", "all_gather_direct"),
]:
register_kernel(
KernelSpec(
op=f"communication.{_fn}",
backend=KernelBackend.JIT,
target=f"sglang.kernels.ops.communication.{_mod}:{_fn}",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
del _mod, _fn
@@ -7,7 +7,7 @@ full weight itself), multicast-stores it into the CustomAllReduceV2 push
workspace (one more user of its double-buffer phase protocol), and a
Lamport-spin consumer assembles ``out = up_proj(x) + b (+ c)`` — reading
1/8 of the weight bytes per rank instead of all of them. Needs
:func:`sglang.kernels.ops.kimi_k3.all_reduce.register_comm` once beforehand
:func:`sglang.kernels.ops.communication.all_reduce_residual.register_comm` once beforehand
(the same registration the push all-reduce uses).
"""
@@ -23,7 +23,7 @@ from sglang.kernels.jit.utils import (
load_jit,
make_cpp_args,
)
from sglang.kernels.ops.kimi_k3.all_reduce import _COMM_MAP
from sglang.kernels.ops.communication.all_reduce_residual import _COMM_MAP
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
@@ -156,7 +156,7 @@ _COMM_MAP: dict[int, Communicator] = {}
def register_comm(comm: Communicator) -> None:
# One communicator per world_size per process -- see the note in
# kimi_k3/all_reduce.py::register_comm. The ops key only on world_size, so an
# communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an
# overwrite here would hand the old group's callers the new group's peer
# pointers.
prev = _COMM_MAP.get(comm.world_size)
@@ -376,7 +376,15 @@ def try_sm120_fp8_linear(
)
def kimi_k3_tiny_gemm(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
"""Shape-tuned tiny GEMM for Kimi-K3, with the existing torch fallback."""
from .kimi_k3 import kimi_k3_tiny_gemm as impl
return impl(x, w)
__all__ = [
"kimi_k3_tiny_gemm",
"Fp8ScaledMMOp",
"bmm_fp8",
"dsv3_fused_a_gemm",
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from sglang.srt.utils import is_npu
if TYPE_CHECKING:
import torch
_is_npu = is_npu()
# (n, k) -> the largest num_tokens where the tiny GEMM still beats cuBLAS.
# Doubles as the compile-time max_m: one kernel is built per m up to it.
_K3_TINY_GEMM_MAX_TOKENS = {
(144, 7168): 16,
(896, 7168): 8,
(1536, 128): 12,
}
def kimi_k3_tiny_gemm(
x: torch.Tensor,
w: torch.Tensor,
) -> torch.Tensor:
import torch
from .tiny_gemm import tiny_gemm_bf16
m, k = x.shape
n, _ = w.shape
max_num_tokens = _K3_TINY_GEMM_MAX_TOKENS.get((n, k))
if not _is_npu and max_num_tokens is not None and 0 < m <= max_num_tokens:
return tiny_gemm_bf16(x, w, max_m=max_num_tokens)
return torch.nn.functional.linear(x, w)
@@ -1,82 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
from sglang.srt.utils import is_npu
if TYPE_CHECKING:
import torch
_is_npu = is_npu()
# (n, k) -> the largest num_tokens where the tiny GEMM still beats cuBLAS.
# Doubles as the compile-time max_m: one kernel is built per m up to it.
_K3_TINY_GEMM_MAX_TOKENS = {
(144, 7168): 16,
(896, 7168): 8,
(1536, 128): 12,
}
def situ_and_mul(
input: torch.Tensor,
out: Optional[torch.Tensor],
beta: float,
linear_beta: Optional[float],
) -> torch.Tensor:
from .activation import situ_and_mul as impl
return impl(input, out, beta, linear_beta)
def situ_and_mul_masked_post_quant(
input: torch.Tensor,
output: torch.Tensor,
output_scale: torch.Tensor,
quant_group_size: int,
masked_m: torch.Tensor,
beta: float,
linear_beta: float,
scale_ue8m0: bool = False,
topk: int = 8,
transposed: bool = False,
swizzle: bool = False,
) -> None:
from .moe import situ_and_mul_masked_post_quant as impl
return impl(
input,
output,
output_scale,
quant_group_size,
masked_m,
beta,
linear_beta,
scale_ue8m0,
topk,
transposed,
swizzle,
)
def kimi_k3_tiny_gemm(
x: torch.Tensor,
w: torch.Tensor,
) -> torch.Tensor:
import torch
from ..gemm.tiny_gemm import tiny_gemm_bf16
m, k = x.shape
n, _ = w.shape
max_num_tokens = _K3_TINY_GEMM_MAX_TOKENS.get((n, k))
if not _is_npu and max_num_tokens is not None and 0 < m <= max_num_tokens:
return tiny_gemm_bf16(x, w, max_m=max_num_tokens)
return torch.nn.functional.linear(x, w)
__all__ = [
"situ_and_mul",
"situ_and_mul_masked_post_quant",
"kimi_k3_tiny_gemm",
]
+42 -1
View File
@@ -128,7 +128,37 @@ def topk_softmax(
)
__all__ = ["moe_align_block_size", "topk_softmax"]
def situ_and_mul_masked_post_quant(
input: torch.Tensor,
output: torch.Tensor,
output_scale: torch.Tensor,
quant_group_size: int,
masked_m: torch.Tensor,
beta: float,
linear_beta: float,
scale_ue8m0: bool = False,
topk: int = 8,
transposed: bool = False,
swizzle: bool = False,
) -> None:
from ._jit_situ_and_mul_quant import situ_and_mul_masked_post_quant as impl
return impl(
input,
output,
output_scale,
quant_group_size,
masked_m,
beta,
linear_beta,
scale_ue8m0,
topk,
transposed,
swizzle,
)
__all__ = ["situ_and_mul_masked_post_quant", "moe_align_block_size", "topk_softmax"]
# Fused MoE-LoRA Triton kernels migrated into this group (from lora/triton_ops);
@@ -204,3 +234,14 @@ register_kernel(
description="Row gather of quantized values plus their scales, one launch.",
)
)
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
register_kernel(
KernelSpec(
op="moe.situ_and_mul_masked_post_quant",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.moe._jit_situ_and_mul_quant:situ_and_mul_masked_post_quant",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
+1 -1
View File
@@ -207,7 +207,7 @@ class SituAndMul(BaseFusedOp):
return (gate * up).to(x.dtype)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
from sglang.kernels.ops.kimi_k3.activation import situ_and_mul
from sglang.kernels.ops.activation import situ_and_mul
return situ_and_mul(x, None, self.beta, self.linear_beta)
@@ -645,7 +645,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
# The model deferred f_b only after publishing static fallback
# weights. Materialize the original gate before entering the
# unchanged conv + packed-KDA fallback chain.
from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm
from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm
if fused_static is None:
raise RuntimeError("K3 deferred f_b is missing fallback weights")
@@ -1544,7 +1544,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
replayssm_g: Optional[torch.Tensor] = None,
replayssm_beta: Optional[torch.Tensor] = None,
) -> torch.Tensor:
from sglang.kernels.ops.kimi_k3.kda_decode_mtp import (
from sglang.kernels.ops.attention.kda_decode_mtp import (
fused_kda_decode_mtp_dspark,
)
+3 -3
View File
@@ -57,7 +57,7 @@ def _use_hip_fused(hidden_size: int, nvb: int) -> bool:
return False
global _HIP_SHAPE_GATE
if _HIP_SHAPE_GATE is None:
from sglang.kernels.ops.kimi_k3.attn_res_hip import supports_attn_res_hip
from sglang.kernels.ops.attention.attn_res_hip import supports_attn_res_hip
_HIP_SHAPE_GATE = supports_attn_res_hip
return _HIP_SHAPE_GATE(hidden_size, nvb)
@@ -97,7 +97,7 @@ def _aggregate_fast(
config (GB300 benchmark winner across nvb). With write_bank_row the kernel
also snapshots the prefix row into bank[:, nvb, :] (bit-exact, zero extra
reads — the row streams through the score pass anyway)."""
from sglang.kernels.ops.kimi_k3.attn_res import attn_res_fused_tma
from sglang.kernels.ops.attention.attn_res import attn_res_fused_tma
# The kernel applies one eps to both the score norm and the output norm.
assert score_norm.variance_epsilon == out_norm.variance_epsilon
@@ -294,7 +294,7 @@ def _aggregate_hip(
mixing share one read, and the pending residual add, the bank snapshot and
the output RMSNorm all fold into the same launch. out_norm None returns the
pre-norm mixture instead. Returns (result, prefix)."""
from sglang.kernels.ops.kimi_k3.attn_res_hip import attn_res_hip
from sglang.kernels.ops.attention.attn_res_hip import attn_res_hip
cw = get_cw(score_proj, score_norm)
prefix = prefix_sum if addend is None else torch.empty_like(prefix_sum)
+10 -9
View File
@@ -4,8 +4,9 @@ Auto-enabled on SM100/SM103 when CustomAllReduceV2 with multicast is
available; ``SGLANG_K3_AR_FUSION`` overrides in either direction (0 = off,
1 = attempt anywhere and warn when unavailable).
Glue between the model and the ``kernels.ops.kimi_k3.all_reduce`` kernels. Small
messages take the 1shot multicast-push through the group's CustomAllReduceV2
Glue between the model and the
``kernels.ops.communication.all_reduce_residual`` kernels. Small messages take
the 1shot multicast-push through the group's CustomAllReduceV2
workspace; large ones take the in-place NVLS 2shot, which needs its input to be a
:func:`symm_buffer` slice and otherwise falls back to the regular all-reduce.
@@ -105,7 +106,7 @@ def _get_state() -> Optional[_State]:
"multicast is unavailable; using the regular all-reduce path."
)
return None
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
from sglang.kernels.ops.communication import all_reduce_residual as mod
mod.register_comm(comm.obj)
logger.info("K3 all-reduce fusion enabled (world_size=%d)", comm.world_size)
@@ -229,7 +230,7 @@ def all_reduce(
residual: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""In-place ``x = allreduce(x) [+ residual]``; returns ``x``."""
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
from sglang.kernels.ops.communication import all_reduce_residual as mod
state = _get_state()
assert state is not None
@@ -255,7 +256,7 @@ def all_reduce_low_sm(
For side-stream use: push would fan out over many blocks and steal SMs from
the GEMMs it overlaps. ``num_blocks`` / ``unroll`` default to the tuned tables.
"""
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
from sglang.kernels.ops.communication import all_reduce_residual as mod
state = _get_state()
assert state is not None
@@ -280,7 +281,7 @@ def all_reduce_norm(
) -> torch.Tensor:
"""In-place ``x = allreduce(x)`` with a fused RMSNorm over the first
``num_tokens`` rows of the 2D ``[rows, NORM_DIM]`` input."""
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
from sglang.kernels.ops.communication import all_reduce_residual as mod
state = _get_state()
assert state is not None
@@ -317,7 +318,7 @@ def finalize_push_fits(num_tokens: int) -> bool:
def gemm_ag_up_fits(num_tokens: int) -> bool:
"""Whether :func:`gemm_ag_up_proj` covers this decode batch: TP8, within the
kernel's win range, and the staging slice fits a push slot."""
from sglang.kernels.ops.kimi_k3 import gemm_ag as mod
from sglang.kernels.ops.communication import gemm_ag as mod
state = _get_state()
assert state is not None
@@ -342,7 +343,7 @@ def gemm_ag_up_proj(
"""``up_proj(x) + b (+ c)`` via column-parallel GEMV + multicast all-gather +
fused add3. ``weight`` is the FULL replicated up_proj (the kernel slices this
rank's rows). Caller checked :func:`gemm_ag_up_fits`."""
from sglang.kernels.ops.kimi_k3 import gemm_ag as mod
from sglang.kernels.ops.communication import gemm_ag as mod
state = _get_state()
assert state is not None
@@ -366,7 +367,7 @@ def finalize_all_reduce_push_norm(
) -> torch.Tensor:
"""Deferred MoE finalize fused into the 1shot push AR + RMSNorm; ``out`` is
output-only. Caller checked :func:`finalize_push_fits`."""
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
from sglang.kernels.ops.communication import all_reduce_residual as mod
state = _get_state()
assert state is not None
+3 -4
View File
@@ -1,11 +1,10 @@
"""K3 fused o_proj GEMM+AR dispatch (``SGLANG_K3_GEMM_AR``).
Glue between the model and ``kernels.ops.kimi_k3.gemm_ar``: lazily allocates
Glue between the model and ``kernels.ops.communication.gemm_ar``: lazily allocates
the P2P comm region on the TP group and swaps the o_proj RowParallelLinear
forward for the single fused GEMM+all-reduce kernel at decode shapes
(falling through to the regular GEMM + AR path whenever the input doesn't
fit — prefill, capture, non-2D input). Eager-mode only; see
kernels/ops/kimi_k3/GEMM_AR_README.md.
fit — prefill, capture, non-2D input). Eager-mode only.
"""
from __future__ import annotations
@@ -55,7 +54,7 @@ def maybe_wrap_o_proj(o_proj: RowParallelLinear) -> None:
ready-to-launch path."""
if not _init():
return
from sglang.kernels.ops.kimi_k3 import gemm_ar as mod
from sglang.kernels.ops.communication import gemm_ar as mod
from sglang.srt.runtime_context import get_parallel
parallel = get_parallel()
+9 -6
View File
@@ -85,7 +85,8 @@ def _init_state() -> Optional[_State]:
(logger.warning if explicit else logger.info)(message)
return None
from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective
from sglang.kernels.ops.attention import attn_res
from sglang.kernels.ops.communication import sp_collective
# Refuse to enable without a checked-in table for this exact device.
if (
@@ -137,7 +138,7 @@ def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool:
state = _init_state()
if state is None:
return False
from sglang.kernels.ops.kimi_k3 import sp_collective
from sglang.kernels.ops.communication import sp_collective
dispatch = sp_collective.get_dispatch(
"reduce_scatter",
@@ -257,7 +258,7 @@ def reduce_scatter_res(
state = _init_state()
if state is None or not _eligible(state, tensor, residual):
return None
from sglang.kernels.ops.kimi_k3 import sp_collective
from sglang.kernels.ops.communication import sp_collective
dispatch = sp_collective.get_dispatch(
"reduce_scatter",
@@ -324,7 +325,8 @@ def reduce_scatter_attn_res(
or ow.shape != (_HIDDEN_SIZE,)
):
return None
from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective
from sglang.kernels.ops.attention import attn_res
from sglang.kernels.ops.communication import sp_collective
dispatch = sp_collective.get_fusion_dispatch(
"reduce_scatter_attn_res",
@@ -375,7 +377,7 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]:
or tensor.numel() * tensor.element_size() > state.comm.max_push_size
):
return None
from sglang.kernels.ops.kimi_k3 import sp_collective
from sglang.kernels.ops.communication import sp_collective
dispatch = sp_collective.get_dispatch(
"all_gather",
@@ -437,7 +439,8 @@ def attn_res_all_gather(
or ow.shape != (_HIDDEN_SIZE,)
):
return None
from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective
from sglang.kernels.ops.attention import attn_res
from sglang.kernels.ops.communication import sp_collective
global_tokens = prefix.shape[0] * state.group.world_size
dispatch = sp_collective.get_fusion_dispatch(
@@ -1493,7 +1493,7 @@ def _varlen_deep_gemm_situ_mul_quant(
linear_beta: float,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Fused SiTU activation + per-group fp8 quant via CUDA JIT kernel."""
from sglang.kernels.ops.kimi_k3 import situ_and_mul_masked_post_quant
from sglang.kernels.ops.moe import situ_and_mul_masked_post_quant
E, N, D_2 = gateup_output.shape
D = D_2 // 2
+2 -2
View File
@@ -2014,7 +2014,7 @@ class KimiK3DeltaAttention(nn.Module):
if self._bfa_w is not None:
w = self._bfa_w
n_fa, n_b = self._bfa_fa_size, self._bfa_b_size
from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm as gemm
from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm as gemm
if (
_is_hip
@@ -2297,7 +2297,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
self._gate_hidden_states = None
if gate_input is not None and not isinstance(x, tuple):
gate = self._compute_output_gate(gate_input)
from sglang.kernels.ops.kimi_k3 import mla_output_gate
from sglang.kernels.ops.attention import mla_output_gate
if mla_output_gate.covered(x, gate):
# One kernel for x * sigmoid(gate); double rounding