diff --git a/python/sglang/kernels/README.md b/python/sglang/kernels/README.md index 7bbcc17978..5a9b160f90 100644 --- a/python/sglang/kernels/README.md +++ b/python/sglang/kernels/README.md @@ -29,6 +29,10 @@ Operator groups (all populated): `activation`, `attention`, `communication`, `kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `moe`, `quantization`, `sampling`, `speculative`. +Place model-specific implementations and tuning data inside the corresponding +logical operator group, rather than creating a model-specific top-level group +under `ops/`. + As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit` and each JIT-backed operator into its group as diff --git a/python/sglang/kernels/ops/__init__.py b/python/sglang/kernels/ops/__init__.py index 052c615ff1..42cc5ccf9b 100644 --- a/python/sglang/kernels/ops/__init__.py +++ b/python/sglang/kernels/ops/__init__.py @@ -23,7 +23,6 @@ _GROUPS = ( "embeddings", "gemm", "grammar", - "kimi_k3", "kvcache", "layernorm", "mamba", diff --git a/python/sglang/kernels/ops/activation/__init__.py b/python/sglang/kernels/ops/activation/__init__.py index 39fb60ba9a..5dbd6865ac 100644 --- a/python/sglang/kernels/ops/activation/__init__.py +++ b/python/sglang/kernels/ops/activation/__init__.py @@ -305,7 +305,19 @@ def gelu_quick(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch return _GELU_QUICK(input, out) +def situ_and_mul( + input: torch.Tensor, + out: Optional[torch.Tensor], + beta: float, + linear_beta: Optional[float], +) -> torch.Tensor: + from ._jit_situ_and_mul import situ_and_mul as impl + + return impl(input, out, beta, linear_beta) + + __all__ = [ + "situ_and_mul", "SiluAndMulOp", "GeluAndMulOp", "GeluTanhAndMulOp", @@ -330,3 +342,14 @@ for _fn in ("softcap_out", "softcap_inplace_logits"): ) ) del _fn + + +# Kernels introduced with Kimi-K3, inventoried by logical operator group. +register_kernel( + KernelSpec( + op="activation.situ_and_mul", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/kimi_k3/activation.py b/python/sglang/kernels/ops/activation/_jit_situ_and_mul.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/activation.py rename to python/sglang/kernels/ops/activation/_jit_situ_and_mul.py diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index 3c9e6d30b7..037a927156 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -235,3 +235,55 @@ for _mod, _fn in [ ) ) del _mod, _fn + + +# Kernels introduced with Kimi-K3, inventoried by logical operator group. +for _mod, _fn, _backend, _device in [ + ("attn_res", "attn_res_fused_tma", KernelBackend.JIT, CapabilityRequirement.CUDA), + ( + "attn_res", + "attn_res_fused_pull_rs", + KernelBackend.JIT, + CapabilityRequirement.CUDA, + ), + ( + "attn_res", + "attn_res_fused_direct_ag", + KernelBackend.JIT, + CapabilityRequirement.CUDA, + ), + ("attn_res_hip", "attn_res_hip", KernelBackend.TRITON, CapabilityRequirement.HIP), + ( + "mla_output_gate", + "kimi_k3_mla_output_gate", + KernelBackend.JIT, + CapabilityRequirement.CUDA, + ), + ( + "kda_decode_mtp", + "fused_kda_decode_mtp_dspark", + KernelBackend.CUTE_DSL, + CapabilityRequirement.CUDA, + ), + ( + "kda_flydsl.kimi_k3_kda_decode", + "flydsl_kimi_k3_kda_decode", + KernelBackend.FLYDSL, + CapabilityRequirement.HIP, + ), + ( + "kda_flydsl.kimi_k3_kda_decode", + "flydsl_kimi_k3_kda_decode_with_f_b", + KernelBackend.FLYDSL, + CapabilityRequirement.HIP, + ), +]: + register_kernel( + KernelSpec( + op=f"attention.{_fn}", + backend=_backend, + target=f"sglang.kernels.ops.attention.{_mod}:{_fn}", + capabilities=frozenset({_device}), + ) + ) +del _mod, _fn, _backend, _device diff --git a/python/sglang/kernels/ops/kimi_k3/attn_res.py b/python/sglang/kernels/ops/attention/attn_res.py similarity index 98% rename from python/sglang/kernels/ops/kimi_k3/attn_res.py rename to python/sglang/kernels/ops/attention/attn_res.py index d8c96a18db..2074ca4135 100644 --- a/python/sglang/kernels/ops/kimi_k3/attn_res.py +++ b/python/sglang/kernels/ops/attention/attn_res.py @@ -103,7 +103,7 @@ _COMM_MAP: dict[int, Communicator] = {} def register_comm(comm: Communicator) -> None: # One communicator per world_size per process -- see the note in - # kimi_k3/all_reduce.py::register_comm. The ops key only on world_size, so an + # communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an # overwrite here would hand the old group's callers the new group's peer # pointers. prev = _COMM_MAP.get(comm.world_size) diff --git a/python/sglang/kernels/ops/kimi_k3/attn_res_hip.py b/python/sglang/kernels/ops/attention/attn_res_hip.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/attn_res_hip.py rename to python/sglang/kernels/ops/attention/attn_res_hip.py diff --git a/python/sglang/kernels/ops/kimi_k3/kda_decode_mtp.py b/python/sglang/kernels/ops/attention/kda_decode_mtp.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/kda_decode_mtp.py rename to python/sglang/kernels/ops/attention/kda_decode_mtp.py diff --git a/python/sglang/kernels/ops/kimi_k3/flydsl/__init__.py b/python/sglang/kernels/ops/attention/kda_flydsl/__init__.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/flydsl/__init__.py rename to python/sglang/kernels/ops/attention/kda_flydsl/__init__.py diff --git a/python/sglang/kernels/ops/kimi_k3/flydsl/kernels/__init__.py b/python/sglang/kernels/ops/attention/kda_flydsl/kernels/__init__.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/flydsl/kernels/__init__.py rename to python/sglang/kernels/ops/attention/kda_flydsl/kernels/__init__.py diff --git a/python/sglang/kernels/ops/kimi_k3/flydsl/kernels/kimi_k3_kda_decode.py b/python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/flydsl/kernels/kimi_k3_kda_decode.py rename to python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode.py diff --git a/python/sglang/kernels/ops/kimi_k3/flydsl/kernels/kimi_k3_kda_decode_fb.py b/python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode_fb.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/flydsl/kernels/kimi_k3_kda_decode_fb.py rename to python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode_fb.py diff --git a/python/sglang/kernels/ops/kimi_k3/flydsl/kimi_k3_kda_decode.py b/python/sglang/kernels/ops/attention/kda_flydsl/kimi_k3_kda_decode.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/flydsl/kimi_k3_kda_decode.py rename to python/sglang/kernels/ops/attention/kda_flydsl/kimi_k3_kda_decode.py diff --git a/python/sglang/kernels/ops/kimi_k3/flydsl/source.py b/python/sglang/kernels/ops/attention/kda_flydsl/source.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/flydsl/source.py rename to python/sglang/kernels/ops/attention/kda_flydsl/source.py diff --git a/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py b/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py index bbc9d1617f..b192a2a73f 100644 --- a/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py +++ b/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py @@ -20,10 +20,10 @@ def enabled() -> bool: def _ops(): try: - from sglang.kernels.ops.kimi_k3.flydsl.source import load_module + from sglang.kernels.ops.attention.kda_flydsl.source import load_module module = load_module( - "sglang.kernels.ops.kimi_k3.flydsl.kimi_k3_kda_decode", + "sglang.kernels.ops.attention.kda_flydsl.kimi_k3_kda_decode", "aiter.ops.flydsl.kimi_k3_kda_decode", ) except (ImportError, ModuleNotFoundError): diff --git a/python/sglang/kernels/ops/kimi_k3/mla_output_gate.py b/python/sglang/kernels/ops/attention/mla_output_gate.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/mla_output_gate.py rename to python/sglang/kernels/ops/attention/mla_output_gate.py diff --git a/python/sglang/kernels/ops/communication/__init__.py b/python/sglang/kernels/ops/communication/__init__.py index b932b763d2..2f8ee65a28 100644 --- a/python/sglang/kernels/ops/communication/__init__.py +++ b/python/sglang/kernels/ops/communication/__init__.py @@ -1,12 +1,36 @@ -"""Collective-communication kernels (custom all-reduce, ...). +"""Collective-communication kernels and fused compute/communication kernels. -Reserved group in the ``sglang.kernels`` namespace (RFC #29630). No thin -wrappers are exposed here: the collective ops (custom all-reduce and friends) -are stateful — they manage workspaces / IPC handles and are driven through a -``CustomAllreduce``-style object and ``torch.ops.sgl_kernel.*`` bindings rather -than standalone callable kernels, so a thin ``sglang.kernels.ops`` forwarder -would be misleading. Import them from ``sgl_kernel`` / ``sglang.kernels.jit`` -directly until a proper stateful-op interface is designed. +Stateful operators are imported from their modules so communicator registration, +workspace ownership, and tuning helpers stay together. The registry below only +records metadata; importing this group does not load the implementations. """ +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec + __all__ = [] + + +# Kernels introduced with Kimi-K3, inventoried by logical operator group. +for _mod, _fn in [ + ("all_reduce_residual", "all_reduce_push_res"), + ("all_reduce_residual", "all_reduce_push_norm"), + ("all_reduce_residual", "finalize_all_reduce_push_norm"), + ("all_reduce_residual", "all_reduce_pull_res"), + ("all_reduce_residual", "all_reduce_pull_norm"), + ("gemm_ag", "gemm_ag_up_proj"), + ("gemm_ar", "o_proj_gemm_ar"), + ("sp_collective", "reduce_scatter_res"), + ("sp_collective", "reduce_scatter_pull"), + ("sp_collective", "all_gather"), + ("sp_collective", "all_gather_direct"), +]: + register_kernel( + KernelSpec( + op=f"communication.{_fn}", + backend=KernelBackend.JIT, + target=f"sglang.kernels.ops.communication.{_mod}:{_fn}", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) + ) +del _mod, _fn diff --git a/python/sglang/kernels/ops/kimi_k3/all_reduce.py b/python/sglang/kernels/ops/communication/all_reduce_residual.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/all_reduce.py rename to python/sglang/kernels/ops/communication/all_reduce_residual.py diff --git a/python/sglang/kernels/ops/kimi_k3/configs/sp_collective/world=4,H=7168,device_name=NVIDIA_GB300.json b/python/sglang/kernels/ops/communication/configs/sp_collective/world=4,H=7168,device_name=NVIDIA_GB300.json similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/configs/sp_collective/world=4,H=7168,device_name=NVIDIA_GB300.json rename to python/sglang/kernels/ops/communication/configs/sp_collective/world=4,H=7168,device_name=NVIDIA_GB300.json diff --git a/python/sglang/kernels/ops/kimi_k3/configs/sp_collective/world=8,H=7168,device_name=NVIDIA_GB300.json b/python/sglang/kernels/ops/communication/configs/sp_collective/world=8,H=7168,device_name=NVIDIA_GB300.json similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/configs/sp_collective/world=8,H=7168,device_name=NVIDIA_GB300.json rename to python/sglang/kernels/ops/communication/configs/sp_collective/world=8,H=7168,device_name=NVIDIA_GB300.json diff --git a/python/sglang/kernels/ops/kimi_k3/gemm_ag.py b/python/sglang/kernels/ops/communication/gemm_ag.py similarity index 93% rename from python/sglang/kernels/ops/kimi_k3/gemm_ag.py rename to python/sglang/kernels/ops/communication/gemm_ag.py index e108082c79..da56eaf11d 100644 --- a/python/sglang/kernels/ops/kimi_k3/gemm_ag.py +++ b/python/sglang/kernels/ops/communication/gemm_ag.py @@ -7,7 +7,7 @@ full weight itself), multicast-stores it into the CustomAllReduceV2 push workspace (one more user of its double-buffer phase protocol), and a Lamport-spin consumer assembles ``out = up_proj(x) + b (+ c)`` — reading 1/8 of the weight bytes per rank instead of all of them. Needs -:func:`sglang.kernels.ops.kimi_k3.all_reduce.register_comm` once beforehand +:func:`sglang.kernels.ops.communication.all_reduce_residual.register_comm` once beforehand (the same registration the push all-reduce uses). """ @@ -23,7 +23,7 @@ from sglang.kernels.jit.utils import ( load_jit, make_cpp_args, ) -from sglang.kernels.ops.kimi_k3.all_reduce import _COMM_MAP +from sglang.kernels.ops.communication.all_reduce_residual import _COMM_MAP from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: diff --git a/python/sglang/kernels/ops/kimi_k3/gemm_ar.py b/python/sglang/kernels/ops/communication/gemm_ar.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/gemm_ar.py rename to python/sglang/kernels/ops/communication/gemm_ar.py diff --git a/python/sglang/kernels/ops/kimi_k3/sp_collective.py b/python/sglang/kernels/ops/communication/sp_collective.py similarity index 98% rename from python/sglang/kernels/ops/kimi_k3/sp_collective.py rename to python/sglang/kernels/ops/communication/sp_collective.py index 2d87f84e56..647f109ef1 100644 --- a/python/sglang/kernels/ops/kimi_k3/sp_collective.py +++ b/python/sglang/kernels/ops/communication/sp_collective.py @@ -156,7 +156,7 @@ _COMM_MAP: dict[int, Communicator] = {} def register_comm(comm: Communicator) -> None: # One communicator per world_size per process -- see the note in - # kimi_k3/all_reduce.py::register_comm. The ops key only on world_size, so an + # communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an # overwrite here would hand the old group's callers the new group's peer # pointers. prev = _COMM_MAP.get(comm.world_size) diff --git a/python/sglang/kernels/ops/gemm/__init__.py b/python/sglang/kernels/ops/gemm/__init__.py index f8b6187ee0..9f9cb3b1b5 100644 --- a/python/sglang/kernels/ops/gemm/__init__.py +++ b/python/sglang/kernels/ops/gemm/__init__.py @@ -376,7 +376,15 @@ def try_sm120_fp8_linear( ) +def kimi_k3_tiny_gemm(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + """Shape-tuned tiny GEMM for Kimi-K3, with the existing torch fallback.""" + from .kimi_k3 import kimi_k3_tiny_gemm as impl + + return impl(x, w) + + __all__ = [ + "kimi_k3_tiny_gemm", "Fp8ScaledMMOp", "bmm_fp8", "dsv3_fused_a_gemm", diff --git a/python/sglang/kernels/ops/gemm/kimi_k3.py b/python/sglang/kernels/ops/gemm/kimi_k3.py new file mode 100644 index 0000000000..e9f476fc09 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/kimi_k3.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sglang.srt.utils import is_npu + +if TYPE_CHECKING: + import torch + +_is_npu = is_npu() + +# (n, k) -> the largest num_tokens where the tiny GEMM still beats cuBLAS. +# Doubles as the compile-time max_m: one kernel is built per m up to it. +_K3_TINY_GEMM_MAX_TOKENS = { + (144, 7168): 16, + (896, 7168): 8, + (1536, 128): 12, +} + + +def kimi_k3_tiny_gemm( + x: torch.Tensor, + w: torch.Tensor, +) -> torch.Tensor: + import torch + + from .tiny_gemm import tiny_gemm_bf16 + + m, k = x.shape + n, _ = w.shape + max_num_tokens = _K3_TINY_GEMM_MAX_TOKENS.get((n, k)) + if not _is_npu and max_num_tokens is not None and 0 < m <= max_num_tokens: + return tiny_gemm_bf16(x, w, max_m=max_num_tokens) + return torch.nn.functional.linear(x, w) diff --git a/python/sglang/kernels/ops/kimi_k3/__init__.py b/python/sglang/kernels/ops/kimi_k3/__init__.py deleted file mode 100644 index f6aba29b2b..0000000000 --- a/python/sglang/kernels/ops/kimi_k3/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -from sglang.srt.utils import is_npu - -if TYPE_CHECKING: - import torch - -_is_npu = is_npu() - -# (n, k) -> the largest num_tokens where the tiny GEMM still beats cuBLAS. -# Doubles as the compile-time max_m: one kernel is built per m up to it. -_K3_TINY_GEMM_MAX_TOKENS = { - (144, 7168): 16, - (896, 7168): 8, - (1536, 128): 12, -} - - -def situ_and_mul( - input: torch.Tensor, - out: Optional[torch.Tensor], - beta: float, - linear_beta: Optional[float], -) -> torch.Tensor: - from .activation import situ_and_mul as impl - - return impl(input, out, beta, linear_beta) - - -def situ_and_mul_masked_post_quant( - input: torch.Tensor, - output: torch.Tensor, - output_scale: torch.Tensor, - quant_group_size: int, - masked_m: torch.Tensor, - beta: float, - linear_beta: float, - scale_ue8m0: bool = False, - topk: int = 8, - transposed: bool = False, - swizzle: bool = False, -) -> None: - from .moe import situ_and_mul_masked_post_quant as impl - - return impl( - input, - output, - output_scale, - quant_group_size, - masked_m, - beta, - linear_beta, - scale_ue8m0, - topk, - transposed, - swizzle, - ) - - -def kimi_k3_tiny_gemm( - x: torch.Tensor, - w: torch.Tensor, -) -> torch.Tensor: - import torch - - from ..gemm.tiny_gemm import tiny_gemm_bf16 - - m, k = x.shape - n, _ = w.shape - max_num_tokens = _K3_TINY_GEMM_MAX_TOKENS.get((n, k)) - if not _is_npu and max_num_tokens is not None and 0 < m <= max_num_tokens: - return tiny_gemm_bf16(x, w, max_m=max_num_tokens) - return torch.nn.functional.linear(x, w) - - -__all__ = [ - "situ_and_mul", - "situ_and_mul_masked_post_quant", - "kimi_k3_tiny_gemm", -] diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index 7f06a870b7..aa6000a748 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -128,7 +128,37 @@ def topk_softmax( ) -__all__ = ["moe_align_block_size", "topk_softmax"] +def situ_and_mul_masked_post_quant( + input: torch.Tensor, + output: torch.Tensor, + output_scale: torch.Tensor, + quant_group_size: int, + masked_m: torch.Tensor, + beta: float, + linear_beta: float, + scale_ue8m0: bool = False, + topk: int = 8, + transposed: bool = False, + swizzle: bool = False, +) -> None: + from ._jit_situ_and_mul_quant import situ_and_mul_masked_post_quant as impl + + return impl( + input, + output, + output_scale, + quant_group_size, + masked_m, + beta, + linear_beta, + scale_ue8m0, + topk, + transposed, + swizzle, + ) + + +__all__ = ["situ_and_mul_masked_post_quant", "moe_align_block_size", "topk_softmax"] # Fused MoE-LoRA Triton kernels migrated into this group (from lora/triton_ops); @@ -204,3 +234,14 @@ register_kernel( description="Row gather of quantized values plus their scales, one launch.", ) ) + + +# Kernels introduced with Kimi-K3, inventoried by logical operator group. +register_kernel( + KernelSpec( + op="moe.situ_and_mul_masked_post_quant", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe._jit_situ_and_mul_quant:situ_and_mul_masked_post_quant", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/kimi_k3/moe.py b/python/sglang/kernels/ops/moe/_jit_situ_and_mul_quant.py similarity index 100% rename from python/sglang/kernels/ops/kimi_k3/moe.py rename to python/sglang/kernels/ops/moe/_jit_situ_and_mul_quant.py diff --git a/python/sglang/srt/layers/activation.py b/python/sglang/srt/layers/activation.py index b620fd701e..7c1bf82fed 100644 --- a/python/sglang/srt/layers/activation.py +++ b/python/sglang/srt/layers/activation.py @@ -207,7 +207,7 @@ class SituAndMul(BaseFusedOp): return (gate * up).to(x.dtype) def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: - from sglang.kernels.ops.kimi_k3.activation import situ_and_mul + from sglang.kernels.ops.activation import situ_and_mul return situ_and_mul(x, None, self.beta, self.linear_beta) diff --git a/python/sglang/srt/layers/attention/linear/kda_backend.py b/python/sglang/srt/layers/attention/linear/kda_backend.py index c526d34325..49e373e01e 100644 --- a/python/sglang/srt/layers/attention/linear/kda_backend.py +++ b/python/sglang/srt/layers/attention/linear/kda_backend.py @@ -645,7 +645,7 @@ class KDAAttnBackend(MambaAttnBackendBase): # The model deferred f_b only after publishing static fallback # weights. Materialize the original gate before entering the # unchanged conv + packed-KDA fallback chain. - from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm + from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm if fused_static is None: raise RuntimeError("K3 deferred f_b is missing fallback weights") @@ -1544,7 +1544,7 @@ class KDAAttnBackend(MambaAttnBackendBase): replayssm_g: Optional[torch.Tensor] = None, replayssm_beta: Optional[torch.Tensor] = None, ) -> torch.Tensor: - from sglang.kernels.ops.kimi_k3.kda_decode_mtp import ( + from sglang.kernels.ops.attention.kda_decode_mtp import ( fused_kda_decode_mtp_dspark, ) diff --git a/python/sglang/srt/layers/attn_residual.py b/python/sglang/srt/layers/attn_residual.py index d6f00b589d..cd49676085 100644 --- a/python/sglang/srt/layers/attn_residual.py +++ b/python/sglang/srt/layers/attn_residual.py @@ -57,7 +57,7 @@ def _use_hip_fused(hidden_size: int, nvb: int) -> bool: return False global _HIP_SHAPE_GATE if _HIP_SHAPE_GATE is None: - from sglang.kernels.ops.kimi_k3.attn_res_hip import supports_attn_res_hip + from sglang.kernels.ops.attention.attn_res_hip import supports_attn_res_hip _HIP_SHAPE_GATE = supports_attn_res_hip return _HIP_SHAPE_GATE(hidden_size, nvb) @@ -97,7 +97,7 @@ def _aggregate_fast( config (GB300 benchmark winner across nvb). With write_bank_row the kernel also snapshots the prefix row into bank[:, nvb, :] (bit-exact, zero extra reads — the row streams through the score pass anyway).""" - from sglang.kernels.ops.kimi_k3.attn_res import attn_res_fused_tma + from sglang.kernels.ops.attention.attn_res import attn_res_fused_tma # The kernel applies one eps to both the score norm and the output norm. assert score_norm.variance_epsilon == out_norm.variance_epsilon @@ -294,7 +294,7 @@ def _aggregate_hip( mixing share one read, and the pending residual add, the bank snapshot and the output RMSNorm all fold into the same launch. out_norm None returns the pre-norm mixture instead. Returns (result, prefix).""" - from sglang.kernels.ops.kimi_k3.attn_res_hip import attn_res_hip + from sglang.kernels.ops.attention.attn_res_hip import attn_res_hip cw = get_cw(score_proj, score_norm) prefix = prefix_sum if addend is None else torch.empty_like(prefix_sum) diff --git a/python/sglang/srt/layers/k3_ar_fusion.py b/python/sglang/srt/layers/k3_ar_fusion.py index 111ebb98c3..381c8b77e8 100644 --- a/python/sglang/srt/layers/k3_ar_fusion.py +++ b/python/sglang/srt/layers/k3_ar_fusion.py @@ -4,8 +4,9 @@ Auto-enabled on SM100/SM103 when CustomAllReduceV2 with multicast is available; ``SGLANG_K3_AR_FUSION`` overrides in either direction (0 = off, 1 = attempt anywhere and warn when unavailable). -Glue between the model and the ``kernels.ops.kimi_k3.all_reduce`` kernels. Small -messages take the 1shot multicast-push through the group's CustomAllReduceV2 +Glue between the model and the +``kernels.ops.communication.all_reduce_residual`` kernels. Small messages take +the 1shot multicast-push through the group's CustomAllReduceV2 workspace; large ones take the in-place NVLS 2shot, which needs its input to be a :func:`symm_buffer` slice and otherwise falls back to the regular all-reduce. @@ -105,7 +106,7 @@ def _get_state() -> Optional[_State]: "multicast is unavailable; using the regular all-reduce path." ) return None - from sglang.kernels.ops.kimi_k3 import all_reduce as mod + from sglang.kernels.ops.communication import all_reduce_residual as mod mod.register_comm(comm.obj) logger.info("K3 all-reduce fusion enabled (world_size=%d)", comm.world_size) @@ -229,7 +230,7 @@ def all_reduce( residual: Optional[torch.Tensor] = None, ) -> torch.Tensor: """In-place ``x = allreduce(x) [+ residual]``; returns ``x``.""" - from sglang.kernels.ops.kimi_k3 import all_reduce as mod + from sglang.kernels.ops.communication import all_reduce_residual as mod state = _get_state() assert state is not None @@ -255,7 +256,7 @@ def all_reduce_low_sm( For side-stream use: push would fan out over many blocks and steal SMs from the GEMMs it overlaps. ``num_blocks`` / ``unroll`` default to the tuned tables. """ - from sglang.kernels.ops.kimi_k3 import all_reduce as mod + from sglang.kernels.ops.communication import all_reduce_residual as mod state = _get_state() assert state is not None @@ -280,7 +281,7 @@ def all_reduce_norm( ) -> torch.Tensor: """In-place ``x = allreduce(x)`` with a fused RMSNorm over the first ``num_tokens`` rows of the 2D ``[rows, NORM_DIM]`` input.""" - from sglang.kernels.ops.kimi_k3 import all_reduce as mod + from sglang.kernels.ops.communication import all_reduce_residual as mod state = _get_state() assert state is not None @@ -317,7 +318,7 @@ def finalize_push_fits(num_tokens: int) -> bool: def gemm_ag_up_fits(num_tokens: int) -> bool: """Whether :func:`gemm_ag_up_proj` covers this decode batch: TP8, within the kernel's win range, and the staging slice fits a push slot.""" - from sglang.kernels.ops.kimi_k3 import gemm_ag as mod + from sglang.kernels.ops.communication import gemm_ag as mod state = _get_state() assert state is not None @@ -342,7 +343,7 @@ def gemm_ag_up_proj( """``up_proj(x) + b (+ c)`` via column-parallel GEMV + multicast all-gather + fused add3. ``weight`` is the FULL replicated up_proj (the kernel slices this rank's rows). Caller checked :func:`gemm_ag_up_fits`.""" - from sglang.kernels.ops.kimi_k3 import gemm_ag as mod + from sglang.kernels.ops.communication import gemm_ag as mod state = _get_state() assert state is not None @@ -366,7 +367,7 @@ def finalize_all_reduce_push_norm( ) -> torch.Tensor: """Deferred MoE finalize fused into the 1shot push AR + RMSNorm; ``out`` is output-only. Caller checked :func:`finalize_push_fits`.""" - from sglang.kernels.ops.kimi_k3 import all_reduce as mod + from sglang.kernels.ops.communication import all_reduce_residual as mod state = _get_state() assert state is not None diff --git a/python/sglang/srt/layers/k3_gemm_ar.py b/python/sglang/srt/layers/k3_gemm_ar.py index 2bbc068137..0fd074432b 100644 --- a/python/sglang/srt/layers/k3_gemm_ar.py +++ b/python/sglang/srt/layers/k3_gemm_ar.py @@ -1,11 +1,10 @@ """K3 fused o_proj GEMM+AR dispatch (``SGLANG_K3_GEMM_AR``). -Glue between the model and ``kernels.ops.kimi_k3.gemm_ar``: lazily allocates +Glue between the model and ``kernels.ops.communication.gemm_ar``: lazily allocates the P2P comm region on the TP group and swaps the o_proj RowParallelLinear forward for the single fused GEMM+all-reduce kernel at decode shapes (falling through to the regular GEMM + AR path whenever the input doesn't -fit — prefill, capture, non-2D input). Eager-mode only; see -kernels/ops/kimi_k3/GEMM_AR_README.md. +fit — prefill, capture, non-2D input). Eager-mode only. """ from __future__ import annotations @@ -55,7 +54,7 @@ def maybe_wrap_o_proj(o_proj: RowParallelLinear) -> None: ready-to-launch path.""" if not _init(): return - from sglang.kernels.ops.kimi_k3 import gemm_ar as mod + from sglang.kernels.ops.communication import gemm_ar as mod from sglang.srt.runtime_context import get_parallel parallel = get_parallel() diff --git a/python/sglang/srt/layers/k3_sp_collective.py b/python/sglang/srt/layers/k3_sp_collective.py index 9ab74858ac..e542a56611 100644 --- a/python/sglang/srt/layers/k3_sp_collective.py +++ b/python/sglang/srt/layers/k3_sp_collective.py @@ -85,7 +85,8 @@ def _init_state() -> Optional[_State]: (logger.warning if explicit else logger.info)(message) return None - from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective + from sglang.kernels.ops.attention import attn_res + from sglang.kernels.ops.communication import sp_collective # Refuse to enable without a checked-in table for this exact device. if ( @@ -137,7 +138,7 @@ def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool: state = _init_state() if state is None: return False - from sglang.kernels.ops.kimi_k3 import sp_collective + from sglang.kernels.ops.communication import sp_collective dispatch = sp_collective.get_dispatch( "reduce_scatter", @@ -257,7 +258,7 @@ def reduce_scatter_res( state = _init_state() if state is None or not _eligible(state, tensor, residual): return None - from sglang.kernels.ops.kimi_k3 import sp_collective + from sglang.kernels.ops.communication import sp_collective dispatch = sp_collective.get_dispatch( "reduce_scatter", @@ -324,7 +325,8 @@ def reduce_scatter_attn_res( or ow.shape != (_HIDDEN_SIZE,) ): return None - from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective + from sglang.kernels.ops.attention import attn_res + from sglang.kernels.ops.communication import sp_collective dispatch = sp_collective.get_fusion_dispatch( "reduce_scatter_attn_res", @@ -375,7 +377,7 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]: or tensor.numel() * tensor.element_size() > state.comm.max_push_size ): return None - from sglang.kernels.ops.kimi_k3 import sp_collective + from sglang.kernels.ops.communication import sp_collective dispatch = sp_collective.get_dispatch( "all_gather", @@ -437,7 +439,8 @@ def attn_res_all_gather( or ow.shape != (_HIDDEN_SIZE,) ): return None - from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective + from sglang.kernels.ops.attention import attn_res + from sglang.kernels.ops.communication import sp_collective global_tokens = prefix.shape[0] * state.group.world_size dispatch = sp_collective.get_fusion_dispatch( diff --git a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py index 7fe5311f44..0a3a71fdb9 100644 --- a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py +++ b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py @@ -1493,7 +1493,7 @@ def _varlen_deep_gemm_situ_mul_quant( linear_beta: float, ) -> Tuple[torch.Tensor, torch.Tensor]: """Fused SiTU activation + per-group fp8 quant via CUDA JIT kernel.""" - from sglang.kernels.ops.kimi_k3 import situ_and_mul_masked_post_quant + from sglang.kernels.ops.moe import situ_and_mul_masked_post_quant E, N, D_2 = gateup_output.shape D = D_2 // 2 diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index b4a8291aa9..a8410480e4 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -2014,7 +2014,7 @@ class KimiK3DeltaAttention(nn.Module): if self._bfa_w is not None: w = self._bfa_w n_fa, n_b = self._bfa_fa_size, self._bfa_b_size - from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm as gemm + from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm as gemm if ( _is_hip @@ -2297,7 +2297,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): self._gate_hidden_states = None if gate_input is not None and not isinstance(x, tuple): gate = self._compute_output_gate(gate_input) - from sglang.kernels.ops.kimi_k3 import mla_output_gate + from sglang.kernels.ops.attention import mla_output_gate if mla_output_gate.covered(x, gate): # One kernel for x * sigmoid(gate); double rounding diff --git a/test/registered/amd/test_kimi_k3_kda_inproj_fusion.py b/test/registered/amd/test_kimi_k3_kda_inproj_fusion.py index 3479523e6a..c456503816 100644 --- a/test/registered/amd/test_kimi_k3_kda_inproj_fusion.py +++ b/test/registered/amd/test_kimi_k3_kda_inproj_fusion.py @@ -16,7 +16,7 @@ from types import SimpleNamespace import torch -from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm +from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm from sglang.srt.models.kimi_k3 import ( KimiK3DeltaAttention, _merge_weights_as_views, diff --git a/test/registered/kernels/ops/kimi_k3/flydsl_ops/bench_kimi_k3_kda_decode.py b/test/registered/kernels/benchmark/attention/bench_kimi_k3_kda_decode.py similarity index 93% rename from test/registered/kernels/ops/kimi_k3/flydsl_ops/bench_kimi_k3_kda_decode.py rename to test/registered/kernels/benchmark/attention/bench_kimi_k3_kda_decode.py index e0b8842788..a84494daec 100644 --- a/test/registered/kernels/ops/kimi_k3/flydsl_ops/bench_kimi_k3_kda_decode.py +++ b/test/registered/kernels/benchmark/attention/bench_kimi_k3_kda_decode.py @@ -22,7 +22,12 @@ def main() -> None: parser.add_argument("--mode", choices=("eager", "graph"), default="graph") args = parser.parse_args() - test = runpy.run_path(str(Path(__file__).with_name("test_kimi_k3_kda_decode.py"))) + test = runpy.run_path( + str( + Path(__file__).resolve().parents[2] + / "ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py" + ) + ) f_a, f_b_weight, inputs = test["_make_fb_inputs"](args.batch) out = torch.empty((1, args.batch, 12, 128), dtype=torch.bfloat16, device="cuda") kwargs = dict( diff --git a/test/registered/kernels/ops/activation/test_situ_and_mul.py b/test/registered/kernels/ops/activation/test_situ_and_mul.py new file mode 100644 index 0000000000..0e1b4f39fb --- /dev/null +++ b/test/registered/kernels/ops/activation/test_situ_and_mul.py @@ -0,0 +1,65 @@ +import unittest + +import torch + +from sglang.kernels.ops.activation import situ_and_mul +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +_BETA = 4.0 +_LINEAR_BETA = 25.0 + + +def _situ_reference(gate_up): + gate, up = gate_up.chunk(2, dim=-1) + gate = gate.float() + up = up.float() + return ( + _BETA + * torch.tanh(gate / _BETA) + * torch.sigmoid(gate) + * _LINEAR_BETA + * torch.tanh(up / _LINEAR_BETA) + ) + + +class TestSituAndMul(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + if get_device_sm() < 100: + raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") + + def test_situ_and_mul(self): + generator = torch.Generator(device="cuda").manual_seed(2) + hidden_size = 1024 + storage = torch.randn( + (7, 2 * hidden_size + 16), + generator=generator, + device="cuda", + dtype=torch.bfloat16, + ) + gate_up = storage[:, : 2 * hidden_size] + output = torch.empty( + (gate_up.shape[0], hidden_size), + device="cuda", + dtype=torch.bfloat16, + ) + + returned = situ_and_mul(gate_up, output, beta=_BETA, linear_beta=_LINEAR_BETA) + + self.assertIs(returned, output) + torch.testing.assert_close( + returned.float(), + _situ_reference(gate_up).to(torch.bfloat16).float(), + rtol=2e-2, + atol=4e-2, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/kimi_k3/flydsl_ops/__init__.py b/test/registered/kernels/ops/attention/kda_flydsl/__init__.py similarity index 100% rename from test/registered/kernels/ops/kimi_k3/flydsl_ops/__init__.py rename to test/registered/kernels/ops/attention/kda_flydsl/__init__.py diff --git a/test/registered/kernels/ops/kimi_k3/flydsl_ops/test_kimi_k3_kda_decode.py b/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py similarity index 98% rename from test/registered/kernels/ops/kimi_k3/flydsl_ops/test_kimi_k3_kda_decode.py rename to test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py index 8b13fadc67..dd33997eaa 100644 --- a/test/registered/kernels/ops/kimi_k3/flydsl_ops/test_kimi_k3_kda_decode.py +++ b/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py @@ -15,7 +15,7 @@ pytest.importorskip("flydsl") from aiter.jit.utils.chip_info import get_gfx from aiter.ops.flydsl.utils import is_flydsl_available -from sglang.kernels.ops.kimi_k3.flydsl.kimi_k3_kda_decode import ( +from sglang.kernels.ops.attention.kda_flydsl.kimi_k3_kda_decode import ( _fb_build_options, flydsl_kimi_k3_kda_decode, flydsl_kimi_k3_kda_decode_with_f_b, @@ -349,7 +349,7 @@ def _run_with_f_b( def test_public_api_and_support_predicate() -> None: - import sglang.kernels.ops.kimi_k3.flydsl as flydsl_ops + import sglang.kernels.ops.attention.kda_flydsl as flydsl_ops assert flydsl_ops.flydsl_kimi_k3_kda_decode is flydsl_kimi_k3_kda_decode assert ( @@ -361,7 +361,7 @@ def test_public_api_and_support_predicate() -> None: def test_f_b_public_api() -> None: - import sglang.kernels.ops.kimi_k3.flydsl as flydsl_ops + import sglang.kernels.ops.attention.kda_flydsl as flydsl_ops assert ( flydsl_ops.flydsl_kimi_k3_kda_decode_with_f_b diff --git a/test/registered/kernels/ops/attention/test_attn_res.py b/test/registered/kernels/ops/attention/test_attn_res.py new file mode 100644 index 0000000000..028b0aa1c8 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_attn_res.py @@ -0,0 +1,68 @@ +import unittest + +import torch + +from sglang.kernels.ops.attention.attn_res import attn_res_fused_tma +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +_HIDDEN_SIZE = 7168 + + +class TestAttentionResidual(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + if get_device_sm() < 100: + raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") + + def test_attn_residual_and_prefix_write(self): + generator = torch.Generator(device="cuda").manual_seed(0) + + def randn(*shape): + return torch.randn(*shape, generator=generator, device="cuda") + + num_tokens, num_bank_rows, num_valid_bank_rows = 5, 8, 5 + prefix = randn(num_tokens, _HIDDEN_SIZE).to(torch.bfloat16) + bank = randn(num_tokens, num_bank_rows, _HIDDEN_SIZE).to(torch.bfloat16) + combine_weight = (randn(_HIDDEN_SIZE) * _HIDDEN_SIZE**-0.5).to(torch.bfloat16) + output_weight = (1 + 0.1 * randn(_HIDDEN_SIZE)).to(torch.bfloat16) + output = torch.empty_like(prefix) + + rows = torch.cat( + [ + bank[:, :num_valid_bank_rows].float(), + prefix.unsqueeze(1).float(), + ], + dim=1, + ) + rms = torch.rsqrt(rows.square().mean(-1) + 1e-6) + scores = (rows * combine_weight.float()).sum(-1) * rms + mixed = (torch.softmax(scores, dim=-1).unsqueeze(-1) * rows).sum(1) + expected = ( + mixed + * torch.rsqrt(mixed.square().mean(-1, keepdim=True) + 1e-6) + * output_weight.float() + ) + + attn_res_fused_tma( + prefix, + bank, + combine_weight, + output_weight, + output, + num_valid_bank_rows, + 1e-6, + write_prefix=True, + ) + + torch.testing.assert_close(output.float(), expected, rtol=2e-2, atol=4e-2) + self.assertTrue(torch.equal(bank[:, num_valid_bank_rows], prefix)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_kda_decode_mtp.py b/test/registered/kernels/ops/attention/test_kda_decode_mtp.py new file mode 100644 index 0000000000..365921a77e --- /dev/null +++ b/test/registered/kernels/ops/attention/test_kda_decode_mtp.py @@ -0,0 +1,194 @@ +import unittest + +import torch + +from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( + commit_kda_replayssm_spec, +) +from sglang.kernels.ops.attention.kda_decode_mtp import fused_kda_decode_mtp_dspark +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=50, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +class TestKdaDecodeMtp(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + if get_device_sm() < 100: + raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") + + def test_mtp_replayssm_ring(self): + num_requests, num_heads, num_spec, key_dim = 2, 2, 2, 128 + num_tokens = num_requests * (1 + num_spec) + num_slots, ring_size, conv_width = num_requests + 2, 16, 4 + + def run(cache_ring): + torch.manual_seed(5) + x_q = torch.randn( + 1, + num_tokens, + num_heads, + key_dim, + device="cuda", + dtype=torch.bfloat16, + ) + x_k = torch.randn_like(x_q) + x_v = torch.randn_like(x_q) + gate = torch.randn_like(x_q) + beta = torch.randn( + 1, + num_tokens, + num_heads, + device="cuda", + dtype=torch.bfloat16, + ) + conv_weight = [ + torch.randn( + num_heads * key_dim, + conv_width, + device="cuda", + ) + * 0.1 + for _ in range(3) + ] + conv_state = [ + torch.randn( + num_slots, + num_heads * key_dim, + conv_width - 1, + device="cuda", + dtype=torch.bfloat16, + ) + for _ in range(3) + ] + slots = torch.arange(1, num_requests + 1, device="cuda", dtype=torch.int32) + scratch = torch.arange(num_requests, device="cuda", dtype=torch.int32) + state = torch.randn( + num_slots, + num_heads, + key_dim, + key_dim, + device="cuda", + ) + intermediate_conv = torch.zeros( + num_requests, + 1 + num_spec, + num_heads * key_dim, + conv_width - 1, + device="cuda", + dtype=torch.bfloat16, + ) + kwargs = dict( + x_q=x_q, + x_k=x_k, + x_v=x_v, + w_q=conv_weight[0], + w_k=conv_weight[1], + w_v=conv_weight[2], + cs_q=conv_state[0], + cs_k=conv_state[1], + cs_v=conv_state[2], + g=gate, + beta=beta, + A_log=torch.randn(num_heads, device="cuda"), + dt_bias=torch.randn(num_heads * key_dim, device="cuda"), + recurrent_state=state, + intermediate_state_indices=scratch, + intermediate_conv_q=intermediate_conv.clone(), + intermediate_conv_k=intermediate_conv.clone(), + intermediate_conv_v=intermediate_conv.clone(), + ssm_state_indices=slots, + cu_seqlens=torch.arange( + 0, + num_tokens + 1, + 1 + num_spec, + device="cuda", + dtype=torch.int32, + ), + lower_bound=-5.0, + ) + if not cache_ring: + intermediate = torch.zeros( + num_requests, + 1 + num_spec, + num_heads, + key_dim, + key_dim, + device="cuda", + ) + output = fused_kda_decode_mtp_dspark( + intermediate_ssm=intermediate, + **kwargs, + ) + return output, intermediate, slots, scratch + + raw_v = torch.zeros( + num_slots, + num_heads, + ring_size, + key_dim, + device="cuda", + dtype=torch.bfloat16, + ) + raw_k = torch.zeros_like(raw_v) + ring_gate = torch.zeros( + num_slots, + num_heads, + ring_size, + key_dim, + device="cuda", + ) + ring_beta = torch.zeros( + num_slots, + num_heads, + ring_size, + device="cuda", + ) + output = fused_kda_decode_mtp_dspark( + intermediate_ssm=None, + replayssm_rawv=raw_v, + replayssm_rawk=raw_k, + replayssm_g=ring_gate, + replayssm_beta=ring_beta, + **kwargs, + ) + return ( + output, + state, + slots, + (raw_v, raw_k, ring_gate, ring_beta), + ) + + baseline, intermediate, slots, scratch = run(cache_ring=False) + ring_output, checkpoint, ring_slots, rings = run(cache_ring=True) + self.assertTrue(torch.equal(ring_output, baseline)) + commit_kda_replayssm_spec( + checkpoint, + *rings, + ring_slots, + torch.full( + (num_requests,), + 1 + num_spec, + device="cuda", + dtype=torch.int32, + ), + max_cache_len=ring_size, + num_k_heads=num_heads, + use_qk_l2norm_in_kernel=True, + null_block_id=-1, + ) + for request in range(num_requests): + expected = intermediate[scratch[request], num_spec] + actual = checkpoint[slots[request]] + relative_error = ( + actual - expected + ).abs().max() / expected.abs().max().clamp_min(1e-6) + self.assertLess(relative_error.item(), 2e-2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py b/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py index f83aa6a9aa..86bb5afb38 100644 --- a/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py +++ b/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py @@ -35,7 +35,7 @@ if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( # noqa: E402 commit_kda_replayssm_spec, ) -from sglang.kernels.ops.kimi_k3.kda_decode_mtp import ( # noqa: E402 +from sglang.kernels.ops.attention.kda_decode_mtp import ( # noqa: E402 fused_kda_decode_mtp_dspark, ) diff --git a/test/registered/kernels/ops/attention/test_mla_output_gate.py b/test/registered/kernels/ops/attention/test_mla_output_gate.py new file mode 100644 index 0000000000..a463fc5367 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_mla_output_gate.py @@ -0,0 +1,38 @@ +import unittest + +import torch + +from sglang.kernels.ops.attention.mla_output_gate import ( + covered, + kimi_k3_mla_output_gate, +) +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +class TestMlaOutputGate(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + if get_device_sm() < 100: + raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") + + def test_mla_output_gate(self): + generator = torch.Generator(device="cuda").manual_seed(1) + shape = (5, 12, 128) + x = torch.randn(shape, generator=generator, device="cuda", dtype=torch.bfloat16) + gate = torch.randn( + shape, generator=generator, device="cuda", dtype=torch.bfloat16 + ) + + self.assertTrue(covered(x, gate)) + expected = x * torch.sigmoid(gate).to(torch.bfloat16) + self.assertTrue(torch.equal(kimi_k3_mla_output_gate(x, gate), expected)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/kimi_k3/test_ar_fusion.py b/test/registered/kernels/ops/communication/test_ar_fusion.py similarity index 98% rename from test/registered/kernels/ops/kimi_k3/test_ar_fusion.py rename to test/registered/kernels/ops/communication/test_ar_fusion.py index eac0d2bd9e..cd357a4274 100644 --- a/test/registered/kernels/ops/kimi_k3/test_ar_fusion.py +++ b/test/registered/kernels/ops/communication/test_ar_fusion.py @@ -10,7 +10,7 @@ semaphore window cycling. Usage:: - python test/registered/kernels/ops/kimi_k3/test_ar_fusion.py # relaunches under torchrun (8 GPUs) + python test/registered/kernels/ops/communication/test_ar_fusion.py # relaunches under torchrun (8 GPUs) """ from __future__ import annotations @@ -25,8 +25,8 @@ import torch.distributed as dist import sglang.srt.distributed.parallel_state as ps from sglang.kernels.jit.utils import cache_once, get_ci_test_range +from sglang.kernels.ops.communication import all_reduce_residual as all_reduce from sglang.kernels.ops.communication.mp import register_comm_cleanup -from sglang.kernels.ops.kimi_k3 import all_reduce from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/test/registered/kernels/ops/kimi_k3/test_collectives.py b/test/registered/kernels/ops/communication/test_collectives.py similarity index 98% rename from test/registered/kernels/ops/kimi_k3/test_collectives.py rename to test/registered/kernels/ops/communication/test_collectives.py index 66d228e662..4e58b28ea6 100644 --- a/test/registered/kernels/ops/kimi_k3/test_collectives.py +++ b/test/registered/kernels/ops/communication/test_collectives.py @@ -9,14 +9,14 @@ import torch.distributed as dist import sglang.srt.distributed.parallel_state as ps from sglang.kernels.jit.utils import cache_once -from sglang.kernels.ops.communication.mp import register_comm_cleanup -from sglang.kernels.ops.kimi_k3 import ( - all_reduce, - attn_res, +from sglang.kernels.ops.attention import attn_res +from sglang.kernels.ops.communication import all_reduce_residual as all_reduce +from sglang.kernels.ops.communication import ( gemm_ag, gemm_ar, sp_collective, ) +from sglang.kernels.ops.communication.mp import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/test/registered/kernels/ops/kimi_k3/test_compute.py b/test/registered/kernels/ops/kimi_k3/test_compute.py deleted file mode 100644 index b8ccc88692..0000000000 --- a/test/registered/kernels/ops/kimi_k3/test_compute.py +++ /dev/null @@ -1,453 +0,0 @@ -import unittest - -import torch - -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( - commit_kda_replayssm_spec, -) -from sglang.kernels.ops.kimi_k3 import ( - situ_and_mul, - situ_and_mul_masked_post_quant, -) -from sglang.kernels.ops.kimi_k3.attn_res import attn_res_fused_tma -from sglang.kernels.ops.kimi_k3.kda_decode_mtp import ( - fused_kda_decode_mtp_dspark, -) -from sglang.kernels.ops.kimi_k3.mla_output_gate import ( - covered, - kimi_k3_mla_output_gate, -) -from sglang.kernels.ops.moe.moe_front import ( - NUM_EXPERTS, - TOPK, - fused_front, -) -from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate -from sglang.srt.utils import get_device_sm -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu-b200") - -_HIDDEN_SIZE = 7168 -_GROUP_SIZE = 128 -_BETA = 4.0 -_LINEAR_BETA = 25.0 - - -def _situ_reference(gate_up): - gate, up = gate_up.chunk(2, dim=-1) - gate = gate.float() - up = up.float() - return ( - _BETA - * torch.tanh(gate / _BETA) - * torch.sigmoid(gate) - * _LINEAR_BETA - * torch.tanh(up / _LINEAR_BETA) - ) - - -def _unpack_ue8m0_scales(packed, num_groups): - num_experts, groups_per_word, num_tokens = packed.shape - exponents = packed.contiguous().view(torch.uint8) - exponents = exponents.view(num_experts, groups_per_word, num_tokens, 4) - exponents = exponents.permute(0, 2, 1, 3).reshape( - num_experts, num_tokens, num_groups - ) - return torch.exp2(exponents.float() - 127.0) - - -class TestKimiK3ComputeKernels(CustomTestCase): - @classmethod - def setUpClass(cls): - if not torch.cuda.is_available(): - raise unittest.SkipTest("CUDA is not available") - if get_device_sm() < 100: - raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") - - def test_attn_residual_and_prefix_write(self): - generator = torch.Generator(device="cuda").manual_seed(0) - - def randn(*shape): - return torch.randn(*shape, generator=generator, device="cuda") - - num_tokens, num_bank_rows, num_valid_bank_rows = 5, 8, 5 - prefix = randn(num_tokens, _HIDDEN_SIZE).to(torch.bfloat16) - bank = randn(num_tokens, num_bank_rows, _HIDDEN_SIZE).to(torch.bfloat16) - combine_weight = (randn(_HIDDEN_SIZE) * _HIDDEN_SIZE**-0.5).to(torch.bfloat16) - output_weight = (1 + 0.1 * randn(_HIDDEN_SIZE)).to(torch.bfloat16) - output = torch.empty_like(prefix) - - rows = torch.cat( - [ - bank[:, :num_valid_bank_rows].float(), - prefix.unsqueeze(1).float(), - ], - dim=1, - ) - rms = torch.rsqrt(rows.square().mean(-1) + 1e-6) - scores = (rows * combine_weight.float()).sum(-1) * rms - mixed = (torch.softmax(scores, dim=-1).unsqueeze(-1) * rows).sum(1) - expected = ( - mixed - * torch.rsqrt(mixed.square().mean(-1, keepdim=True) + 1e-6) - * output_weight.float() - ) - - attn_res_fused_tma( - prefix, - bank, - combine_weight, - output_weight, - output, - num_valid_bank_rows, - 1e-6, - write_prefix=True, - ) - - torch.testing.assert_close(output.float(), expected, rtol=2e-2, atol=4e-2) - self.assertTrue(torch.equal(bank[:, num_valid_bank_rows], prefix)) - - def test_mla_output_gate(self): - generator = torch.Generator(device="cuda").manual_seed(1) - shape = (5, 12, 128) - x = torch.randn(shape, generator=generator, device="cuda", dtype=torch.bfloat16) - gate = torch.randn( - shape, generator=generator, device="cuda", dtype=torch.bfloat16 - ) - - self.assertTrue(covered(x, gate)) - expected = x * torch.sigmoid(gate).to(torch.bfloat16) - self.assertTrue(torch.equal(kimi_k3_mla_output_gate(x, gate), expected)) - - def test_situ_and_mul(self): - generator = torch.Generator(device="cuda").manual_seed(2) - hidden_size = 1024 - storage = torch.randn( - (7, 2 * hidden_size + 16), - generator=generator, - device="cuda", - dtype=torch.bfloat16, - ) - gate_up = storage[:, : 2 * hidden_size] - output = torch.empty( - (gate_up.shape[0], hidden_size), - device="cuda", - dtype=torch.bfloat16, - ) - - returned = situ_and_mul(gate_up, output, beta=_BETA, linear_beta=_LINEAR_BETA) - - self.assertIs(returned, output) - torch.testing.assert_close( - returned.float(), - _situ_reference(gate_up).to(torch.bfloat16).float(), - rtol=2e-2, - atol=4e-2, - ) - - def test_situ_mul_quant(self): - torch.cuda.manual_seed_all(3) - num_experts, num_tokens, hidden_size, topk = 8, 32, 1024, 16 - gate_up = ( - torch.randn( - num_experts, - num_tokens, - 2 * hidden_size, - device="cuda", - dtype=torch.float32, - ) - * 2.0 - ).to(torch.bfloat16) - masked_m = torch.randint( - 0, - num_tokens + 1, - (num_experts,), - device="cuda", - dtype=torch.int32, - ) - masked_m[0] = 0 - masked_m[-1] = num_tokens - - output = torch.full( - (num_experts, num_tokens, hidden_size), - 0x7F, - device="cuda", - dtype=torch.uint8, - ).view(torch.float8_e4m3fn) - num_groups = hidden_size // _GROUP_SIZE - output_scale = torch.zeros( - (num_experts, num_groups // 4, num_tokens), - device="cuda", - dtype=torch.int32, - ) - - situ_and_mul_masked_post_quant( - input=gate_up, - output=output, - output_scale=output_scale, - quant_group_size=_GROUP_SIZE, - masked_m=masked_m, - beta=_BETA, - linear_beta=_LINEAR_BETA, - scale_ue8m0=True, - topk=topk, - transposed=True, - ) - - scales = _unpack_ue8m0_scales(output_scale, num_groups) - expanded_scales = scales.repeat_interleave(_GROUP_SIZE, dim=-1) - dequantized = output.float() * expanded_scales - expected = _situ_reference(gate_up) - error_bound = expanded_scales * 17.0 - raw_output = output.view(torch.uint8) - for expert in range(num_experts): - valid_tokens = int(masked_m[expert].item()) - self.assertTrue( - bool( - ( - ( - dequantized[expert, :valid_tokens] - - expected[expert, :valid_tokens] - ).abs() - <= error_bound[expert, :valid_tokens] - ).all() - ) - ) - self.assertTrue(bool((raw_output[expert, valid_tokens:] == 0x7F).all())) - - def test_moe_front(self): - torch.manual_seed(4) - num_tokens, latent_dim = 1, 128 - hidden = ( - torch.randn( - num_tokens, - _HIDDEN_SIZE, - device="cuda", - dtype=torch.bfloat16, - ) - / 32 - ) - weight = ( - torch.randn( - NUM_EXPERTS + latent_dim, - _HIDDEN_SIZE, - device="cuda", - dtype=torch.bfloat16, - ) - / 32 - ) - bias = torch.randn(NUM_EXPERTS, device="cuda") - - weights, ids, routed = fused_front( - hidden, - weight, - bias, - latent_dim, - renormalize=True, - routed_scaling_factor=2.5, - apply_routed_scaling_factor_on_output=True, - ) - merged = torch.mm(hidden, weight.t(), out_dtype=torch.float32) - ref_weights, ref_ids = moe_fused_gate( - merged[:, :NUM_EXPERTS], - bias, - topk=TOPK, - scoring_func="sigmoid", - renormalize=True, - routed_scaling_factor=2.5, - apply_routed_scaling_factor_on_output=True, - ) - order = ids.argsort(dim=-1) - ref_order = ref_ids.argsort(dim=-1) - self.assertTrue( - torch.equal( - ids.gather(1, order), - ref_ids.to(torch.int32).gather(1, ref_order), - ) - ) - torch.testing.assert_close( - weights.gather(1, order), - ref_weights.gather(1, ref_order), - rtol=1e-6, - atol=0, - ) - self.assertTrue( - torch.equal( - routed, - merged[:, NUM_EXPERTS:].to(torch.bfloat16), - ) - ) - - def test_mtp_replayssm_ring(self): - num_requests, num_heads, num_spec, key_dim = 2, 2, 2, 128 - num_tokens = num_requests * (1 + num_spec) - num_slots, ring_size, conv_width = num_requests + 2, 16, 4 - - def run(cache_ring): - torch.manual_seed(5) - x_q = torch.randn( - 1, - num_tokens, - num_heads, - key_dim, - device="cuda", - dtype=torch.bfloat16, - ) - x_k = torch.randn_like(x_q) - x_v = torch.randn_like(x_q) - gate = torch.randn_like(x_q) - beta = torch.randn( - 1, - num_tokens, - num_heads, - device="cuda", - dtype=torch.bfloat16, - ) - conv_weight = [ - torch.randn( - num_heads * key_dim, - conv_width, - device="cuda", - ) - * 0.1 - for _ in range(3) - ] - conv_state = [ - torch.randn( - num_slots, - num_heads * key_dim, - conv_width - 1, - device="cuda", - dtype=torch.bfloat16, - ) - for _ in range(3) - ] - slots = torch.arange(1, num_requests + 1, device="cuda", dtype=torch.int32) - scratch = torch.arange(num_requests, device="cuda", dtype=torch.int32) - state = torch.randn( - num_slots, - num_heads, - key_dim, - key_dim, - device="cuda", - ) - intermediate_conv = torch.zeros( - num_requests, - 1 + num_spec, - num_heads * key_dim, - conv_width - 1, - device="cuda", - dtype=torch.bfloat16, - ) - kwargs = dict( - x_q=x_q, - x_k=x_k, - x_v=x_v, - w_q=conv_weight[0], - w_k=conv_weight[1], - w_v=conv_weight[2], - cs_q=conv_state[0], - cs_k=conv_state[1], - cs_v=conv_state[2], - g=gate, - beta=beta, - A_log=torch.randn(num_heads, device="cuda"), - dt_bias=torch.randn(num_heads * key_dim, device="cuda"), - recurrent_state=state, - intermediate_state_indices=scratch, - intermediate_conv_q=intermediate_conv.clone(), - intermediate_conv_k=intermediate_conv.clone(), - intermediate_conv_v=intermediate_conv.clone(), - ssm_state_indices=slots, - cu_seqlens=torch.arange( - 0, - num_tokens + 1, - 1 + num_spec, - device="cuda", - dtype=torch.int32, - ), - lower_bound=-5.0, - ) - if not cache_ring: - intermediate = torch.zeros( - num_requests, - 1 + num_spec, - num_heads, - key_dim, - key_dim, - device="cuda", - ) - output = fused_kda_decode_mtp_dspark( - intermediate_ssm=intermediate, - **kwargs, - ) - return output, intermediate, slots, scratch - - raw_v = torch.zeros( - num_slots, - num_heads, - ring_size, - key_dim, - device="cuda", - dtype=torch.bfloat16, - ) - raw_k = torch.zeros_like(raw_v) - ring_gate = torch.zeros( - num_slots, - num_heads, - ring_size, - key_dim, - device="cuda", - ) - ring_beta = torch.zeros( - num_slots, - num_heads, - ring_size, - device="cuda", - ) - output = fused_kda_decode_mtp_dspark( - intermediate_ssm=None, - replayssm_rawv=raw_v, - replayssm_rawk=raw_k, - replayssm_g=ring_gate, - replayssm_beta=ring_beta, - **kwargs, - ) - return ( - output, - state, - slots, - (raw_v, raw_k, ring_gate, ring_beta), - ) - - baseline, intermediate, slots, scratch = run(cache_ring=False) - ring_output, checkpoint, ring_slots, rings = run(cache_ring=True) - self.assertTrue(torch.equal(ring_output, baseline)) - commit_kda_replayssm_spec( - checkpoint, - *rings, - ring_slots, - torch.full( - (num_requests,), - 1 + num_spec, - device="cuda", - dtype=torch.int32, - ), - max_cache_len=ring_size, - num_k_heads=num_heads, - use_qk_l2norm_in_kernel=True, - null_block_id=-1, - ) - for request in range(num_requests): - expected = intermediate[scratch[request], num_spec] - actual = checkpoint[slots[request]] - relative_error = ( - actual - expected - ).abs().max() / expected.abs().max().clamp_min(1e-6) - self.assertLess(relative_error.item(), 2e-2) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_kernels_namespace.py b/test/registered/kernels/ops/layernorm/test_kernels_namespace.py index ae05236314..948307eab3 100644 --- a/test/registered/kernels/ops/layernorm/test_kernels_namespace.py +++ b/test/registered/kernels/ops/layernorm/test_kernels_namespace.py @@ -168,7 +168,9 @@ def test_import_stays_metadata_only(): # the JIT compilation infra (sglang.kernels.jit), which import torch / nvcc. code = ( "import sys, sglang.kernels.ops; " - "print('DIRTY' if 'sgl_kernel' in sys.modules or any(" + "print('DIRTY' if any(m in sys.modules for m in " + "('sgl_kernel', 'cutlass', 'flydsl', 'aiter', " + "'sglang.kernels.ops.gemm.kimi_k3')) or any(" "m.startswith('sglang.kernels.jit') for m in sys.modules) else 'CLEAN')" ) r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) @@ -176,5 +178,34 @@ def test_import_stays_metadata_only(): assert "CLEAN" in r.stdout +@pytest.mark.parametrize( + "op, backend, device", + [ + ("activation.situ_and_mul", KernelBackend.JIT, "cuda"), + ("moe.situ_and_mul_masked_post_quant", KernelBackend.JIT, "cuda"), + ("attention.attn_res_fused_tma", KernelBackend.JIT, "cuda"), + ("attention.attn_res_hip", KernelBackend.TRITON, "hip"), + ("attention.kimi_k3_mla_output_gate", KernelBackend.JIT, "cuda"), + ("attention.fused_kda_decode_mtp_dspark", KernelBackend.CUTE_DSL, "cuda"), + ("attention.flydsl_kimi_k3_kda_decode", KernelBackend.FLYDSL, "hip"), + ("attention.flydsl_kimi_k3_kda_decode_with_f_b", KernelBackend.FLYDSL, "hip"), + ("communication.all_reduce_push_res", KernelBackend.JIT, "cuda"), + ("communication.gemm_ag_up_proj", KernelBackend.JIT, "cuda"), + ("communication.o_proj_gemm_ar", KernelBackend.JIT, "cuda"), + ("communication.reduce_scatter_res", KernelBackend.JIT, "cuda"), + ], +) +def test_kimi_k3_kernels_are_inventoried_by_operator(op, backend, device): + spec = K.select_kernel(op, backend=backend) + group = op.split(".")[0] + assert spec.target.startswith(f"sglang.kernels.ops.{group}.") + assert K.capabilities_satisfied(spec.capabilities, PlatformInfo(device_type=device)) + assert not K.capabilities_satisfied(spec.capabilities, _CPU) + + +def test_kimi_k3_model_namespace_is_retired(): + assert importlib.util.find_spec("sglang.kernels.ops.kimi_k3") is None + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/moe/test_moe_front.py b/test/registered/kernels/ops/moe/test_moe_front.py new file mode 100644 index 0000000000..e4dd38d403 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_moe_front.py @@ -0,0 +1,89 @@ +import unittest + +import torch + +from sglang.kernels.ops.moe.moe_front import NUM_EXPERTS, TOPK, fused_front +from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +_HIDDEN_SIZE = 7168 + + +class TestMoeFront(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + if get_device_sm() < 100: + raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") + + def test_moe_front(self): + torch.manual_seed(4) + num_tokens, latent_dim = 1, 128 + hidden = ( + torch.randn( + num_tokens, + _HIDDEN_SIZE, + device="cuda", + dtype=torch.bfloat16, + ) + / 32 + ) + weight = ( + torch.randn( + NUM_EXPERTS + latent_dim, + _HIDDEN_SIZE, + device="cuda", + dtype=torch.bfloat16, + ) + / 32 + ) + bias = torch.randn(NUM_EXPERTS, device="cuda") + + weights, ids, routed = fused_front( + hidden, + weight, + bias, + latent_dim, + renormalize=True, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=True, + ) + merged = torch.mm(hidden, weight.t(), out_dtype=torch.float32) + ref_weights, ref_ids = moe_fused_gate( + merged[:, :NUM_EXPERTS], + bias, + topk=TOPK, + scoring_func="sigmoid", + renormalize=True, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=True, + ) + order = ids.argsort(dim=-1) + ref_order = ref_ids.argsort(dim=-1) + self.assertTrue( + torch.equal( + ids.gather(1, order), + ref_ids.to(torch.int32).gather(1, ref_order), + ) + ) + torch.testing.assert_close( + weights.gather(1, order), + ref_weights.gather(1, ref_order), + rtol=1e-6, + atol=0, + ) + self.assertTrue( + torch.equal( + routed, + merged[:, NUM_EXPERTS:].to(torch.bfloat16), + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_situ_and_mul_quant.py b/test/registered/kernels/ops/moe/test_situ_and_mul_quant.py new file mode 100644 index 0000000000..d07c6d64f8 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_situ_and_mul_quant.py @@ -0,0 +1,120 @@ +import unittest + +import torch + +from sglang.kernels.ops.moe import situ_and_mul_masked_post_quant +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +_GROUP_SIZE = 128 +_BETA = 4.0 +_LINEAR_BETA = 25.0 + + +def _situ_reference(gate_up): + gate, up = gate_up.chunk(2, dim=-1) + gate = gate.float() + up = up.float() + return ( + _BETA + * torch.tanh(gate / _BETA) + * torch.sigmoid(gate) + * _LINEAR_BETA + * torch.tanh(up / _LINEAR_BETA) + ) + + +def _unpack_ue8m0_scales(packed, num_groups): + num_experts, groups_per_word, num_tokens = packed.shape + exponents = packed.contiguous().view(torch.uint8) + exponents = exponents.view(num_experts, groups_per_word, num_tokens, 4) + exponents = exponents.permute(0, 2, 1, 3).reshape( + num_experts, num_tokens, num_groups + ) + return torch.exp2(exponents.float() - 127.0) + + +class TestSituAndMulQuant(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + if get_device_sm() < 100: + raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") + + def test_situ_mul_quant(self): + torch.cuda.manual_seed_all(3) + num_experts, num_tokens, hidden_size, topk = 8, 32, 1024, 16 + gate_up = ( + torch.randn( + num_experts, + num_tokens, + 2 * hidden_size, + device="cuda", + dtype=torch.float32, + ) + * 2.0 + ).to(torch.bfloat16) + masked_m = torch.randint( + 0, + num_tokens + 1, + (num_experts,), + device="cuda", + dtype=torch.int32, + ) + masked_m[0] = 0 + masked_m[-1] = num_tokens + + output = torch.full( + (num_experts, num_tokens, hidden_size), + 0x7F, + device="cuda", + dtype=torch.uint8, + ).view(torch.float8_e4m3fn) + num_groups = hidden_size // _GROUP_SIZE + output_scale = torch.zeros( + (num_experts, num_groups // 4, num_tokens), + device="cuda", + dtype=torch.int32, + ) + + situ_and_mul_masked_post_quant( + input=gate_up, + output=output, + output_scale=output_scale, + quant_group_size=_GROUP_SIZE, + masked_m=masked_m, + beta=_BETA, + linear_beta=_LINEAR_BETA, + scale_ue8m0=True, + topk=topk, + transposed=True, + ) + + scales = _unpack_ue8m0_scales(output_scale, num_groups) + expanded_scales = scales.repeat_interleave(_GROUP_SIZE, dim=-1) + dequantized = output.float() * expanded_scales + expected = _situ_reference(gate_up) + error_bound = expanded_scales * 17.0 + raw_output = output.view(torch.uint8) + for expert in range(num_experts): + valid_tokens = int(masked_m[expert].item()) + self.assertTrue( + bool( + ( + ( + dequantized[expert, :valid_tokens] + - expected[expert, :valid_tokens] + ).abs() + <= error_bound[expert, :valid_tokens] + ).all() + ) + ) + self.assertTrue(bool((raw_output[expert, valid_tokens:] == 0x7F).all())) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_kda_decode_mtp_slot_stride.py b/test/registered/unit/layers/test_kda_decode_mtp_slot_stride.py index 3e2899e9c9..a25ba0f5da 100644 --- a/test/registered/unit/layers/test_kda_decode_mtp_slot_stride.py +++ b/test/registered/unit/layers/test_kda_decode_mtp_slot_stride.py @@ -48,7 +48,7 @@ class TestKdaDecodeMtpSlotStride(unittest.TestCase): small slot id and asserts bitwise parity against a contiguous pool.""" def test_wrap_regime_matches_contiguous(self): - from sglang.kernels.ops.kimi_k3.kda_decode_mtp import ( + from sglang.kernels.ops.attention.kda_decode_mtp import ( fused_kda_decode_mtp_dspark, )