[Refactor] Retire the model-specific Kimi K3 kernel namespace (#40922)
This commit is contained in:
@@ -29,6 +29,10 @@ Operator groups (all populated): `activation`, `attention`, `communication`,
|
||||
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `moe`, `quantization`,
|
||||
`sampling`, `speculative`.
|
||||
|
||||
Place model-specific implementations and tuning data inside the corresponding
|
||||
logical operator group, rather than creating a model-specific top-level group
|
||||
under `ops/`.
|
||||
|
||||
As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has
|
||||
been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit`
|
||||
and each JIT-backed operator into its group as
|
||||
|
||||
@@ -23,7 +23,6 @@ _GROUPS = (
|
||||
"embeddings",
|
||||
"gemm",
|
||||
"grammar",
|
||||
"kimi_k3",
|
||||
"kvcache",
|
||||
"layernorm",
|
||||
"mamba",
|
||||
|
||||
@@ -305,7 +305,19 @@ def gelu_quick(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch
|
||||
return _GELU_QUICK(input, out)
|
||||
|
||||
|
||||
def situ_and_mul(
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor],
|
||||
beta: float,
|
||||
linear_beta: Optional[float],
|
||||
) -> torch.Tensor:
|
||||
from ._jit_situ_and_mul import situ_and_mul as impl
|
||||
|
||||
return impl(input, out, beta, linear_beta)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"situ_and_mul",
|
||||
"SiluAndMulOp",
|
||||
"GeluAndMulOp",
|
||||
"GeluTanhAndMulOp",
|
||||
@@ -330,3 +342,14 @@ for _fn in ("softcap_out", "softcap_inplace_logits"):
|
||||
)
|
||||
)
|
||||
del _fn
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="activation.situ_and_mul",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -235,3 +235,55 @@ for _mod, _fn in [
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
for _mod, _fn, _backend, _device in [
|
||||
("attn_res", "attn_res_fused_tma", KernelBackend.JIT, CapabilityRequirement.CUDA),
|
||||
(
|
||||
"attn_res",
|
||||
"attn_res_fused_pull_rs",
|
||||
KernelBackend.JIT,
|
||||
CapabilityRequirement.CUDA,
|
||||
),
|
||||
(
|
||||
"attn_res",
|
||||
"attn_res_fused_direct_ag",
|
||||
KernelBackend.JIT,
|
||||
CapabilityRequirement.CUDA,
|
||||
),
|
||||
("attn_res_hip", "attn_res_hip", KernelBackend.TRITON, CapabilityRequirement.HIP),
|
||||
(
|
||||
"mla_output_gate",
|
||||
"kimi_k3_mla_output_gate",
|
||||
KernelBackend.JIT,
|
||||
CapabilityRequirement.CUDA,
|
||||
),
|
||||
(
|
||||
"kda_decode_mtp",
|
||||
"fused_kda_decode_mtp_dspark",
|
||||
KernelBackend.CUTE_DSL,
|
||||
CapabilityRequirement.CUDA,
|
||||
),
|
||||
(
|
||||
"kda_flydsl.kimi_k3_kda_decode",
|
||||
"flydsl_kimi_k3_kda_decode",
|
||||
KernelBackend.FLYDSL,
|
||||
CapabilityRequirement.HIP,
|
||||
),
|
||||
(
|
||||
"kda_flydsl.kimi_k3_kda_decode",
|
||||
"flydsl_kimi_k3_kda_decode_with_f_b",
|
||||
KernelBackend.FLYDSL,
|
||||
CapabilityRequirement.HIP,
|
||||
),
|
||||
]:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"attention.{_fn}",
|
||||
backend=_backend,
|
||||
target=f"sglang.kernels.ops.attention.{_mod}:{_fn}",
|
||||
capabilities=frozenset({_device}),
|
||||
)
|
||||
)
|
||||
del _mod, _fn, _backend, _device
|
||||
|
||||
+1
-1
@@ -103,7 +103,7 @@ _COMM_MAP: dict[int, Communicator] = {}
|
||||
|
||||
def register_comm(comm: Communicator) -> None:
|
||||
# One communicator per world_size per process -- see the note in
|
||||
# kimi_k3/all_reduce.py::register_comm. The ops key only on world_size, so an
|
||||
# communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an
|
||||
# overwrite here would hand the old group's callers the new group's peer
|
||||
# pointers.
|
||||
prev = _COMM_MAP.get(comm.world_size)
|
||||
@@ -20,10 +20,10 @@ def enabled() -> bool:
|
||||
|
||||
def _ops():
|
||||
try:
|
||||
from sglang.kernels.ops.kimi_k3.flydsl.source import load_module
|
||||
from sglang.kernels.ops.attention.kda_flydsl.source import load_module
|
||||
|
||||
module = load_module(
|
||||
"sglang.kernels.ops.kimi_k3.flydsl.kimi_k3_kda_decode",
|
||||
"sglang.kernels.ops.attention.kda_flydsl.kimi_k3_kda_decode",
|
||||
"aiter.ops.flydsl.kimi_k3_kda_decode",
|
||||
)
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
|
||||
@@ -1,12 +1,36 @@
|
||||
"""Collective-communication kernels (custom all-reduce, ...).
|
||||
"""Collective-communication kernels and fused compute/communication kernels.
|
||||
|
||||
Reserved group in the ``sglang.kernels`` namespace (RFC #29630). No thin
|
||||
wrappers are exposed here: the collective ops (custom all-reduce and friends)
|
||||
are stateful — they manage workspaces / IPC handles and are driven through a
|
||||
``CustomAllreduce``-style object and ``torch.ops.sgl_kernel.*`` bindings rather
|
||||
than standalone callable kernels, so a thin ``sglang.kernels.ops`` forwarder
|
||||
would be misleading. Import them from ``sgl_kernel`` / ``sglang.kernels.jit``
|
||||
directly until a proper stateful-op interface is designed.
|
||||
Stateful operators are imported from their modules so communicator registration,
|
||||
workspace ownership, and tuning helpers stay together. The registry below only
|
||||
records metadata; importing this group does not load the implementations.
|
||||
"""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
for _mod, _fn in [
|
||||
("all_reduce_residual", "all_reduce_push_res"),
|
||||
("all_reduce_residual", "all_reduce_push_norm"),
|
||||
("all_reduce_residual", "finalize_all_reduce_push_norm"),
|
||||
("all_reduce_residual", "all_reduce_pull_res"),
|
||||
("all_reduce_residual", "all_reduce_pull_norm"),
|
||||
("gemm_ag", "gemm_ag_up_proj"),
|
||||
("gemm_ar", "o_proj_gemm_ar"),
|
||||
("sp_collective", "reduce_scatter_res"),
|
||||
("sp_collective", "reduce_scatter_pull"),
|
||||
("sp_collective", "all_gather"),
|
||||
("sp_collective", "all_gather_direct"),
|
||||
]:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"communication.{_fn}",
|
||||
backend=KernelBackend.JIT,
|
||||
target=f"sglang.kernels.ops.communication.{_mod}:{_fn}",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ full weight itself), multicast-stores it into the CustomAllReduceV2 push
|
||||
workspace (one more user of its double-buffer phase protocol), and a
|
||||
Lamport-spin consumer assembles ``out = up_proj(x) + b (+ c)`` — reading
|
||||
1/8 of the weight bytes per rank instead of all of them. Needs
|
||||
:func:`sglang.kernels.ops.kimi_k3.all_reduce.register_comm` once beforehand
|
||||
:func:`sglang.kernels.ops.communication.all_reduce_residual.register_comm` once beforehand
|
||||
(the same registration the push all-reduce uses).
|
||||
"""
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.kernels.jit.utils import (
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.kernels.ops.kimi_k3.all_reduce import _COMM_MAP
|
||||
from sglang.kernels.ops.communication.all_reduce_residual import _COMM_MAP
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
+1
-1
@@ -156,7 +156,7 @@ _COMM_MAP: dict[int, Communicator] = {}
|
||||
|
||||
def register_comm(comm: Communicator) -> None:
|
||||
# One communicator per world_size per process -- see the note in
|
||||
# kimi_k3/all_reduce.py::register_comm. The ops key only on world_size, so an
|
||||
# communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an
|
||||
# overwrite here would hand the old group's callers the new group's peer
|
||||
# pointers.
|
||||
prev = _COMM_MAP.get(comm.world_size)
|
||||
@@ -376,7 +376,15 @@ def try_sm120_fp8_linear(
|
||||
)
|
||||
|
||||
|
||||
def kimi_k3_tiny_gemm(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
|
||||
"""Shape-tuned tiny GEMM for Kimi-K3, with the existing torch fallback."""
|
||||
from .kimi_k3 import kimi_k3_tiny_gemm as impl
|
||||
|
||||
return impl(x, w)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"kimi_k3_tiny_gemm",
|
||||
"Fp8ScaledMMOp",
|
||||
"bmm_fp8",
|
||||
"dsv3_fused_a_gemm",
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# (n, k) -> the largest num_tokens where the tiny GEMM still beats cuBLAS.
|
||||
# Doubles as the compile-time max_m: one kernel is built per m up to it.
|
||||
_K3_TINY_GEMM_MAX_TOKENS = {
|
||||
(144, 7168): 16,
|
||||
(896, 7168): 8,
|
||||
(1536, 128): 12,
|
||||
}
|
||||
|
||||
|
||||
def kimi_k3_tiny_gemm(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
import torch
|
||||
|
||||
from .tiny_gemm import tiny_gemm_bf16
|
||||
|
||||
m, k = x.shape
|
||||
n, _ = w.shape
|
||||
max_num_tokens = _K3_TINY_GEMM_MAX_TOKENS.get((n, k))
|
||||
if not _is_npu and max_num_tokens is not None and 0 < m <= max_num_tokens:
|
||||
return tiny_gemm_bf16(x, w, max_m=max_num_tokens)
|
||||
return torch.nn.functional.linear(x, w)
|
||||
@@ -1,82 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# (n, k) -> the largest num_tokens where the tiny GEMM still beats cuBLAS.
|
||||
# Doubles as the compile-time max_m: one kernel is built per m up to it.
|
||||
_K3_TINY_GEMM_MAX_TOKENS = {
|
||||
(144, 7168): 16,
|
||||
(896, 7168): 8,
|
||||
(1536, 128): 12,
|
||||
}
|
||||
|
||||
|
||||
def situ_and_mul(
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor],
|
||||
beta: float,
|
||||
linear_beta: Optional[float],
|
||||
) -> torch.Tensor:
|
||||
from .activation import situ_and_mul as impl
|
||||
|
||||
return impl(input, out, beta, linear_beta)
|
||||
|
||||
|
||||
def situ_and_mul_masked_post_quant(
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
quant_group_size: int,
|
||||
masked_m: torch.Tensor,
|
||||
beta: float,
|
||||
linear_beta: float,
|
||||
scale_ue8m0: bool = False,
|
||||
topk: int = 8,
|
||||
transposed: bool = False,
|
||||
swizzle: bool = False,
|
||||
) -> None:
|
||||
from .moe import situ_and_mul_masked_post_quant as impl
|
||||
|
||||
return impl(
|
||||
input,
|
||||
output,
|
||||
output_scale,
|
||||
quant_group_size,
|
||||
masked_m,
|
||||
beta,
|
||||
linear_beta,
|
||||
scale_ue8m0,
|
||||
topk,
|
||||
transposed,
|
||||
swizzle,
|
||||
)
|
||||
|
||||
|
||||
def kimi_k3_tiny_gemm(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
import torch
|
||||
|
||||
from ..gemm.tiny_gemm import tiny_gemm_bf16
|
||||
|
||||
m, k = x.shape
|
||||
n, _ = w.shape
|
||||
max_num_tokens = _K3_TINY_GEMM_MAX_TOKENS.get((n, k))
|
||||
if not _is_npu and max_num_tokens is not None and 0 < m <= max_num_tokens:
|
||||
return tiny_gemm_bf16(x, w, max_m=max_num_tokens)
|
||||
return torch.nn.functional.linear(x, w)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"situ_and_mul",
|
||||
"situ_and_mul_masked_post_quant",
|
||||
"kimi_k3_tiny_gemm",
|
||||
]
|
||||
@@ -128,7 +128,37 @@ def topk_softmax(
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["moe_align_block_size", "topk_softmax"]
|
||||
def situ_and_mul_masked_post_quant(
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
quant_group_size: int,
|
||||
masked_m: torch.Tensor,
|
||||
beta: float,
|
||||
linear_beta: float,
|
||||
scale_ue8m0: bool = False,
|
||||
topk: int = 8,
|
||||
transposed: bool = False,
|
||||
swizzle: bool = False,
|
||||
) -> None:
|
||||
from ._jit_situ_and_mul_quant import situ_and_mul_masked_post_quant as impl
|
||||
|
||||
return impl(
|
||||
input,
|
||||
output,
|
||||
output_scale,
|
||||
quant_group_size,
|
||||
masked_m,
|
||||
beta,
|
||||
linear_beta,
|
||||
scale_ue8m0,
|
||||
topk,
|
||||
transposed,
|
||||
swizzle,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["situ_and_mul_masked_post_quant", "moe_align_block_size", "topk_softmax"]
|
||||
|
||||
|
||||
# Fused MoE-LoRA Triton kernels migrated into this group (from lora/triton_ops);
|
||||
@@ -204,3 +234,14 @@ register_kernel(
|
||||
description="Row gather of quantized values plus their scales, one launch.",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.situ_and_mul_masked_post_quant",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.moe._jit_situ_and_mul_quant:situ_and_mul_masked_post_quant",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -207,7 +207,7 @@ class SituAndMul(BaseFusedOp):
|
||||
return (gate * up).to(x.dtype)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
from sglang.kernels.ops.kimi_k3.activation import situ_and_mul
|
||||
from sglang.kernels.ops.activation import situ_and_mul
|
||||
|
||||
return situ_and_mul(x, None, self.beta, self.linear_beta)
|
||||
|
||||
|
||||
@@ -645,7 +645,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
# The model deferred f_b only after publishing static fallback
|
||||
# weights. Materialize the original gate before entering the
|
||||
# unchanged conv + packed-KDA fallback chain.
|
||||
from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm
|
||||
from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm
|
||||
|
||||
if fused_static is None:
|
||||
raise RuntimeError("K3 deferred f_b is missing fallback weights")
|
||||
@@ -1544,7 +1544,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
replayssm_g: Optional[torch.Tensor] = None,
|
||||
replayssm_beta: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.kimi_k3.kda_decode_mtp import (
|
||||
from sglang.kernels.ops.attention.kda_decode_mtp import (
|
||||
fused_kda_decode_mtp_dspark,
|
||||
)
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def _use_hip_fused(hidden_size: int, nvb: int) -> bool:
|
||||
return False
|
||||
global _HIP_SHAPE_GATE
|
||||
if _HIP_SHAPE_GATE is None:
|
||||
from sglang.kernels.ops.kimi_k3.attn_res_hip import supports_attn_res_hip
|
||||
from sglang.kernels.ops.attention.attn_res_hip import supports_attn_res_hip
|
||||
|
||||
_HIP_SHAPE_GATE = supports_attn_res_hip
|
||||
return _HIP_SHAPE_GATE(hidden_size, nvb)
|
||||
@@ -97,7 +97,7 @@ def _aggregate_fast(
|
||||
config (GB300 benchmark winner across nvb). With write_bank_row the kernel
|
||||
also snapshots the prefix row into bank[:, nvb, :] (bit-exact, zero extra
|
||||
reads — the row streams through the score pass anyway)."""
|
||||
from sglang.kernels.ops.kimi_k3.attn_res import attn_res_fused_tma
|
||||
from sglang.kernels.ops.attention.attn_res import attn_res_fused_tma
|
||||
|
||||
# The kernel applies one eps to both the score norm and the output norm.
|
||||
assert score_norm.variance_epsilon == out_norm.variance_epsilon
|
||||
@@ -294,7 +294,7 @@ def _aggregate_hip(
|
||||
mixing share one read, and the pending residual add, the bank snapshot and
|
||||
the output RMSNorm all fold into the same launch. out_norm None returns the
|
||||
pre-norm mixture instead. Returns (result, prefix)."""
|
||||
from sglang.kernels.ops.kimi_k3.attn_res_hip import attn_res_hip
|
||||
from sglang.kernels.ops.attention.attn_res_hip import attn_res_hip
|
||||
|
||||
cw = get_cw(score_proj, score_norm)
|
||||
prefix = prefix_sum if addend is None else torch.empty_like(prefix_sum)
|
||||
|
||||
@@ -4,8 +4,9 @@ Auto-enabled on SM100/SM103 when CustomAllReduceV2 with multicast is
|
||||
available; ``SGLANG_K3_AR_FUSION`` overrides in either direction (0 = off,
|
||||
1 = attempt anywhere and warn when unavailable).
|
||||
|
||||
Glue between the model and the ``kernels.ops.kimi_k3.all_reduce`` kernels. Small
|
||||
messages take the 1shot multicast-push through the group's CustomAllReduceV2
|
||||
Glue between the model and the
|
||||
``kernels.ops.communication.all_reduce_residual`` kernels. Small messages take
|
||||
the 1shot multicast-push through the group's CustomAllReduceV2
|
||||
workspace; large ones take the in-place NVLS 2shot, which needs its input to be a
|
||||
:func:`symm_buffer` slice and otherwise falls back to the regular all-reduce.
|
||||
|
||||
@@ -105,7 +106,7 @@ def _get_state() -> Optional[_State]:
|
||||
"multicast is unavailable; using the regular all-reduce path."
|
||||
)
|
||||
return None
|
||||
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
|
||||
from sglang.kernels.ops.communication import all_reduce_residual as mod
|
||||
|
||||
mod.register_comm(comm.obj)
|
||||
logger.info("K3 all-reduce fusion enabled (world_size=%d)", comm.world_size)
|
||||
@@ -229,7 +230,7 @@ def all_reduce(
|
||||
residual: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""In-place ``x = allreduce(x) [+ residual]``; returns ``x``."""
|
||||
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
|
||||
from sglang.kernels.ops.communication import all_reduce_residual as mod
|
||||
|
||||
state = _get_state()
|
||||
assert state is not None
|
||||
@@ -255,7 +256,7 @@ def all_reduce_low_sm(
|
||||
For side-stream use: push would fan out over many blocks and steal SMs from
|
||||
the GEMMs it overlaps. ``num_blocks`` / ``unroll`` default to the tuned tables.
|
||||
"""
|
||||
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
|
||||
from sglang.kernels.ops.communication import all_reduce_residual as mod
|
||||
|
||||
state = _get_state()
|
||||
assert state is not None
|
||||
@@ -280,7 +281,7 @@ def all_reduce_norm(
|
||||
) -> torch.Tensor:
|
||||
"""In-place ``x = allreduce(x)`` with a fused RMSNorm over the first
|
||||
``num_tokens`` rows of the 2D ``[rows, NORM_DIM]`` input."""
|
||||
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
|
||||
from sglang.kernels.ops.communication import all_reduce_residual as mod
|
||||
|
||||
state = _get_state()
|
||||
assert state is not None
|
||||
@@ -317,7 +318,7 @@ def finalize_push_fits(num_tokens: int) -> bool:
|
||||
def gemm_ag_up_fits(num_tokens: int) -> bool:
|
||||
"""Whether :func:`gemm_ag_up_proj` covers this decode batch: TP8, within the
|
||||
kernel's win range, and the staging slice fits a push slot."""
|
||||
from sglang.kernels.ops.kimi_k3 import gemm_ag as mod
|
||||
from sglang.kernels.ops.communication import gemm_ag as mod
|
||||
|
||||
state = _get_state()
|
||||
assert state is not None
|
||||
@@ -342,7 +343,7 @@ def gemm_ag_up_proj(
|
||||
"""``up_proj(x) + b (+ c)`` via column-parallel GEMV + multicast all-gather +
|
||||
fused add3. ``weight`` is the FULL replicated up_proj (the kernel slices this
|
||||
rank's rows). Caller checked :func:`gemm_ag_up_fits`."""
|
||||
from sglang.kernels.ops.kimi_k3 import gemm_ag as mod
|
||||
from sglang.kernels.ops.communication import gemm_ag as mod
|
||||
|
||||
state = _get_state()
|
||||
assert state is not None
|
||||
@@ -366,7 +367,7 @@ def finalize_all_reduce_push_norm(
|
||||
) -> torch.Tensor:
|
||||
"""Deferred MoE finalize fused into the 1shot push AR + RMSNorm; ``out`` is
|
||||
output-only. Caller checked :func:`finalize_push_fits`."""
|
||||
from sglang.kernels.ops.kimi_k3 import all_reduce as mod
|
||||
from sglang.kernels.ops.communication import all_reduce_residual as mod
|
||||
|
||||
state = _get_state()
|
||||
assert state is not None
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
"""K3 fused o_proj GEMM+AR dispatch (``SGLANG_K3_GEMM_AR``).
|
||||
|
||||
Glue between the model and ``kernels.ops.kimi_k3.gemm_ar``: lazily allocates
|
||||
Glue between the model and ``kernels.ops.communication.gemm_ar``: lazily allocates
|
||||
the P2P comm region on the TP group and swaps the o_proj RowParallelLinear
|
||||
forward for the single fused GEMM+all-reduce kernel at decode shapes
|
||||
(falling through to the regular GEMM + AR path whenever the input doesn't
|
||||
fit — prefill, capture, non-2D input). Eager-mode only; see
|
||||
kernels/ops/kimi_k3/GEMM_AR_README.md.
|
||||
fit — prefill, capture, non-2D input). Eager-mode only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -55,7 +54,7 @@ def maybe_wrap_o_proj(o_proj: RowParallelLinear) -> None:
|
||||
ready-to-launch path."""
|
||||
if not _init():
|
||||
return
|
||||
from sglang.kernels.ops.kimi_k3 import gemm_ar as mod
|
||||
from sglang.kernels.ops.communication import gemm_ar as mod
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
parallel = get_parallel()
|
||||
|
||||
@@ -85,7 +85,8 @@ def _init_state() -> Optional[_State]:
|
||||
(logger.warning if explicit else logger.info)(message)
|
||||
return None
|
||||
|
||||
from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective
|
||||
from sglang.kernels.ops.attention import attn_res
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
|
||||
# Refuse to enable without a checked-in table for this exact device.
|
||||
if (
|
||||
@@ -137,7 +138,7 @@ def requires_symmetric_rs(num_tokens: int, device: torch.device) -> bool:
|
||||
state = _init_state()
|
||||
if state is None:
|
||||
return False
|
||||
from sglang.kernels.ops.kimi_k3 import sp_collective
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
|
||||
dispatch = sp_collective.get_dispatch(
|
||||
"reduce_scatter",
|
||||
@@ -257,7 +258,7 @@ def reduce_scatter_res(
|
||||
state = _init_state()
|
||||
if state is None or not _eligible(state, tensor, residual):
|
||||
return None
|
||||
from sglang.kernels.ops.kimi_k3 import sp_collective
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
|
||||
dispatch = sp_collective.get_dispatch(
|
||||
"reduce_scatter",
|
||||
@@ -324,7 +325,8 @@ def reduce_scatter_attn_res(
|
||||
or ow.shape != (_HIDDEN_SIZE,)
|
||||
):
|
||||
return None
|
||||
from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective
|
||||
from sglang.kernels.ops.attention import attn_res
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
|
||||
dispatch = sp_collective.get_fusion_dispatch(
|
||||
"reduce_scatter_attn_res",
|
||||
@@ -375,7 +377,7 @@ def all_gather(tensor: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
or tensor.numel() * tensor.element_size() > state.comm.max_push_size
|
||||
):
|
||||
return None
|
||||
from sglang.kernels.ops.kimi_k3 import sp_collective
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
|
||||
dispatch = sp_collective.get_dispatch(
|
||||
"all_gather",
|
||||
@@ -437,7 +439,8 @@ def attn_res_all_gather(
|
||||
or ow.shape != (_HIDDEN_SIZE,)
|
||||
):
|
||||
return None
|
||||
from sglang.kernels.ops.kimi_k3 import attn_res, sp_collective
|
||||
from sglang.kernels.ops.attention import attn_res
|
||||
from sglang.kernels.ops.communication import sp_collective
|
||||
|
||||
global_tokens = prefix.shape[0] * state.group.world_size
|
||||
dispatch = sp_collective.get_fusion_dispatch(
|
||||
|
||||
@@ -1493,7 +1493,7 @@ def _varlen_deep_gemm_situ_mul_quant(
|
||||
linear_beta: float,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused SiTU activation + per-group fp8 quant via CUDA JIT kernel."""
|
||||
from sglang.kernels.ops.kimi_k3 import situ_and_mul_masked_post_quant
|
||||
from sglang.kernels.ops.moe import situ_and_mul_masked_post_quant
|
||||
|
||||
E, N, D_2 = gateup_output.shape
|
||||
D = D_2 // 2
|
||||
|
||||
@@ -2014,7 +2014,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
if self._bfa_w is not None:
|
||||
w = self._bfa_w
|
||||
n_fa, n_b = self._bfa_fa_size, self._bfa_b_size
|
||||
from sglang.kernels.ops.kimi_k3 import kimi_k3_tiny_gemm as gemm
|
||||
from sglang.kernels.ops.gemm import kimi_k3_tiny_gemm as gemm
|
||||
|
||||
if (
|
||||
_is_hip
|
||||
@@ -2297,7 +2297,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
self._gate_hidden_states = None
|
||||
if gate_input is not None and not isinstance(x, tuple):
|
||||
gate = self._compute_output_gate(gate_input)
|
||||
from sglang.kernels.ops.kimi_k3 import mla_output_gate
|
||||
from sglang.kernels.ops.attention import mla_output_gate
|
||||
|
||||
if mla_output_gate.covered(x, gate):
|
||||
# One kernel for x * sigmoid(gate); double rounding
|
||||
|
||||
Reference in New Issue
Block a user