[Refactor] Restore logical kernel groups and test organization (#41243)

This commit is contained in:
Xiaoyu Zhang
2026-09-26 12:30:44 +08:00
committed by GitHub
parent e538489e0e
commit 38ec649048
357 changed files with 4018 additions and 4973 deletions
+26 -3
View File
@@ -26,7 +26,7 @@ sglang/kernels/
Operator groups (all populated): `activation`, `attention`, `communication`,
`diffusion`, `elementwise`, `embeddings`, `gemm`, `grammar`, `kv_canary`,
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `moe`, `quantization`,
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `mm`, `moe`, `quantization`,
`sampling`, `speculative`.
Place model-specific implementations and tuning data inside the corresponding
@@ -36,9 +36,32 @@ under `ops/`.
As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has
been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit`
and each JIT-backed operator into its group as
`sglang.kernels.ops.<group>._jit_<op>`. Tests and benchmarks live under
`sglang.kernels.ops.<group>.<op>` (the old `_jit_` filename prefix was removed
in #32148). Tests and benchmarks live under
`test/registered/kernels/` (`ops/<group>/` for tests, `benchmark/<group>/` for
benchmarks); shared test helpers are in `sglang.test.kernels`.
benchmarks); shared test helpers are in `sglang.test.kernels`. Standalone scripts
that are not CI-registered belong in `test/manual/kernels/`, outside the shipped
operator package. Keep vendored upstream trees under their existing maintenance
policy.
Classify the public computation, not the model that first used it: expert
routing belongs in `moe`, matrix multiplication (including quantized GEMM and
its tuning data) in `gemm`, and attention index selection in `attention`.
Keep a fused operation intact; an attention kernel that fuses RoPE with a
projection may stay in `attention`. A model name inside a logical group is fine.
Runtime integration that owns process groups, symmetric buffers, or model
dispatch stays in `srt`; the K3 communication adapters live in
`srt/layers/communication/`, while their callable kernels live in
`kernels/ops/communication/`. Tests of this runtime state belong with the
runtime subsystem, even when they need a GPU. Kernel inventory/dispatch tests
belong in `test/registered/unit/kernels/`.
When moving an operator, update its callers, lazy `KernelSpec` targets and op
ids, tests, benchmark imports, and tuning-data paths together. Preserve CI
stage/runner registrations and test cases when splitting files. See the
[kernel organization skill](../../../.claude/skills/kernel-organization/SKILL.md)
for the review checklist.
## How it works
File diff suppressed because it is too large Load Diff
-1
View File
@@ -27,7 +27,6 @@ _GROUPS = (
"layernorm",
"mamba",
"memory",
"minicpm_sala",
"mm",
"moe",
"quantization",
@@ -184,7 +184,6 @@ for _mod, _fn in [
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
("dsv4.rms_normalize_hip", "rms_normalize_triton"),
]:
register_kernel(
KernelSpec(
@@ -287,3 +286,64 @@ for _mod, _fn, _backend, _device in [
)
)
del _mod, _fn, _backend, _device
# Public entry points inventoried by logical operator group (RFC #29630).
register_kernel(
KernelSpec(
op="attention.get_block_table",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.minicpm_sala.get_block_table:get_block_table",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.fast_kpool_topk_transform_fused",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsa.kpool_topk_transform:fast_kpool_topk_transform_fused",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.fast_topk",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.fast_topk:fast_topk",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.fused_k_indexer_norm_rope",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.fused_k_indexer_norm_rope_store",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope_store",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="attention.fused_rope_wo_a_bf16",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsv4.wo_a:fused_rope_wo_a_bf16",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
),
)
)
register_kernel(
KernelSpec(
op="attention.fp4_index_logits_decode",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.attention.dsv4.fp4_indexer:fp4_index_logits_decode",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
@@ -557,10 +557,6 @@ def _gather_dequant_requant_fp8_paged_kernel(
tl.store(dst_ptr, data, mask=mask)
if __name__ == "__main__":
raise Exception("UT is in quant_k_cache.py")
@triton.jit
def _concat_cast_kv_fp8_pad_kernel(
out_ptr,
@@ -397,128 +397,3 @@ def _quantize_k_cache_fast_kernel(
data = tl.load(src_ptr, mask=mask)
tl.store(dst_ptr, data, mask=mask)
if __name__ == "__main__":
import dequant_k_cache
for num_blocks, block_size in [
(1, 1),
(10, 64),
]:
dim_nope_and_rope = 512 + 64
input_k_cache = torch.randn(
(num_blocks, block_size, 1, dim_nope_and_rope),
dtype=torch.bfloat16,
device="cuda",
)
ref_quant = _quantize_k_cache_ref(input_k_cache)
actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache)
ref_ref_dequant = dequant_k_cache._dequantize_k_cache_slow(ref_quant)
ref_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped(ref_quant)
actual_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped(
actual_quant
)
print(f"{ref_ref_dequant=}")
print(f"{actual_actual_dequant=}")
print(f"{actual_actual_dequant - ref_ref_dequant=}")
print(f"{torch.mean(ref_ref_dequant - actual_actual_dequant)=}")
# TODO too different?
torch.testing.assert_close(
ref_ref_dequant, ref_actual_dequant, atol=0.2, rtol=0.2
)
torch.testing.assert_close(
ref_ref_dequant, actual_actual_dequant, atol=0.2, rtol=0.2
)
# test dequant_k_cache_paged
page_table_1 = torch.arange(
num_blocks * block_size, dtype=torch.int32, device="cuda"
)
actual_dequant_paged = dequant_k_cache.dequantize_k_cache_paged(
actual_quant, page_table_1
).reshape(actual_actual_dequant.shape)
print(f"{torch.mean(actual_actual_dequant - actual_dequant_paged)=}")
torch.testing.assert_close(
ref_ref_dequant, actual_dequant_paged, atol=0.2, rtol=0.2
)
print("Passed")
# Test quantize_k_cache_separate: verify output matches concat path
print("\nTesting quantize_k_cache_separate...")
for num_tokens in [64, 100]:
dim_nope = 512
dim_rope = 64
k_nope = torch.randn(
num_tokens, 1, dim_nope, dtype=torch.bfloat16, device="cuda"
)
k_rope = torch.randn(
num_tokens, 1, dim_rope, dtype=torch.bfloat16, device="cuda"
)
# Old path: concat then quantize
k_concat = torch.cat([k_nope, k_rope], dim=-1).squeeze(1) # (num_tokens, 576)
old_output = quantize_k_cache(k_concat.unsqueeze(1).unsqueeze(1)) # 4D input
old_output = old_output.squeeze(1).squeeze(1) # Back to (num_tokens, 656)
# New path: quantize separately
nope_part, rope_part = quantize_k_cache_separate(k_nope, k_rope)
new_bytes = torch.cat([nope_part.squeeze(1), rope_part.squeeze(1)], dim=-1)
# Compare byte-level equality
old_bytes = old_output.view(torch.uint8)
if old_bytes.shape != new_bytes.shape:
raise RuntimeError(
f"Shape mismatch: {old_bytes.shape} vs {new_bytes.shape}"
)
diff_bytes = (old_bytes != new_bytes).sum().item()
if diff_bytes > 0:
max_diff = (old_bytes.float() - new_bytes.float()).abs().max().item()
raise RuntimeError(
f"quantize_k_cache_separate output doesn't match concat path: "
f"{diff_bytes} differing bytes, max_diff={max_diff}"
)
print(f" num_tokens={num_tokens}: PASSED (outputs match byte-wise)")
print("quantize_k_cache_separate tests passed!")
print("\nDo benchmark...")
for num_blocks, block_size in [
(1, 64),
(64, 64),
(128, 64),
(256, 64),
(512, 64),
(1024, 64),
(2048, 64),
]:
dim_nope_and_rope = 512 + 64
input_k_cache = torch.randn(
(num_blocks, block_size, 1, dim_nope_and_rope),
dtype=torch.bfloat16,
device="cuda",
)
actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache)
page_table_1 = torch.arange(
num_blocks * block_size, dtype=torch.int32, device="cuda"
)
def run_ans():
return dequant_k_cache.dequantize_k_cache_paged(actual_quant, page_table_1)
ans_time: float = triton.testing.do_bench(run_ans, warmup=10, rep=20) / 1000 # type: ignore
print(f"seq_kv: {num_blocks * block_size}, time: {ans_time * 1e6: 4.0f} us")
@@ -375,14 +375,3 @@ def transform_index_page_table_prefill_ref(
)
offset += l
return result
if __name__ == "__main__":
bs, topk, max_seqlen = 10, 2048, 3000
page_table = torch.randint(0, 100, (bs, max_seqlen), device="cuda")
topk_indices = torch.full((bs, topk), -1, device="cuda")
topk_indices[:, :1600] = torch.arange(1600).unsqueeze(0).repeat(bs, 1)
ref_result = transform_index_page_table_decode_ref(page_table, topk_indices)
result = transform_index_page_table_decode_fast(page_table, topk_indices)
assert torch.all(result == ref_result)
print("Passed")
@@ -23,15 +23,6 @@ from .elementwise import (
fused_rope_inplace,
)
from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a
from .gemm import linear_bf16_fp32
from .moe import (
hash_topk,
mask_topk_ids,
mega_moe_pre_dispatch,
silu_and_mul_clamp,
silu_and_mul_contig_post_quant,
silu_and_mul_masked_post_quant,
)
from .topk import (
plan_topk_v2,
topk_transform_paged,
@@ -56,17 +47,10 @@ __all__ = [
"fused_k_norm_rope_flashmla",
"sglang_per_token_group_quant_fp8_dsv4_wo_a",
"make_name",
"linear_bf16_fp32",
"get_paged_mqa_logits_metadata",
"triton_create_paged_compress_data",
"topk_transform_paged",
"topk_transform_paged_v2",
"topk_transform_ragged_v2",
"plan_topk_v2",
"hash_topk",
"mega_moe_pre_dispatch",
"mask_topk_ids",
"silu_and_mul_clamp",
"silu_and_mul_masked_post_quant",
"silu_and_mul_contig_post_quant",
]
@@ -572,33 +572,3 @@ def gather_dequant_requant_fp8_paged_ref(
)
out[: active.shape[0]] = active
return out
if __name__ == "__main__":
assert torch.cuda.is_available(), "this self-test needs a CUDA device"
torch.manual_seed(0)
device = "cuda"
page_size = 64
num_pages = 8
num_tokens = 333
raw_bytes = page_size * (NOPE_ROPE_BYTES + PADDED_SCALE_PER_TOKEN)
bytes_per_page = (
(raw_bytes + NOPE_ROPE_BYTES - 1) // NOPE_ROPE_BYTES
) * NOPE_ROPE_BYTES
quant_k_cache = torch.randint(
0, 256, (num_pages, bytes_per_page), dtype=torch.uint8, device=device
)
page_table = torch.randint(
0, num_pages * page_size, (num_tokens,), dtype=torch.int32, device=device
)
out_kernel = dequantize_k_cache_paged(quant_k_cache, page_table, page_size)
out_ref = dequantize_k_cache_paged_ref(quant_k_cache, page_table, page_size)
torch.testing.assert_close(out_kernel, out_ref, atol=0, rtol=0, equal_nan=True)
print(
f"OK: kernel matches torch ref for {num_tokens} tokens "
f"(page_size={page_size}, bytes_per_page={bytes_per_page})"
)
@@ -3,11 +3,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Sequence
import torch
import triton
import triton.language as tl
from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit
from sglang.kernels.ops.layernorm.mxfp8_epilogue import ue8m0_scale
if TYPE_CHECKING:
from tvm_ffi.module import Module
@@ -52,133 +49,3 @@ def fused_rope_wo_a_bf16(
y = x.new_empty((num_tokens, N_OUT))
module.run(x, weight, freqs_cis, positions, None, None, y)
return (y,)
@triton.jit
def _wo_a_bf16_gemv_kernel(X, W, Y, R: tl.constexpr, D: tl.constexpr, BN: tl.constexpr):
group = tl.program_id(1)
rows = tl.program_id(0) * BN + tl.arange(0, BN)
columns = tl.arange(0, D)
x = tl.load(X + group * D + columns).to(tl.float32)
w = tl.load(
W + (group * R + rows[:, None]) * D + columns[None, :],
rows[:, None] < R,
0,
).to(tl.float32)
result = tl.sum(w * x[None, :], axis=1)
tl.store(Y + group * R + rows, result, rows < R)
def wo_a_bf16_gemv(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
"""Compute ``einsum('tgd,grd->tgr', x, weight)`` for one token."""
assert x.shape[0] == 1 and x.ndim == weight.ndim == 3
assert x.dtype == weight.dtype == torch.bfloat16
assert x.is_cuda and x.device == weight.device
assert x.is_contiguous() and weight.is_contiguous()
groups, rows, dim = weight.shape
assert x.shape[1:] == (groups, dim) and dim == triton.next_power_of_2(dim)
result = torch.empty((1, groups, rows), dtype=x.dtype, device=x.device)
# One output row per CTA keeps register use low and exposes enough
# independent weight loads for single-token decode.
_wo_a_bf16_gemv_kernel[(rows, groups)](
x,
weight,
result,
rows,
dim,
1,
num_warps=4,
enable_fp_fusion=False,
)
return result
@triton.jit
def _wo_a_partial(X, W, P, M: tl.constexpr, SX: tl.constexpr):
tile, group, split = tl.program_id(0), tl.program_id(1), tl.program_id(2)
m = tl.arange(0, 16)
n = tile * 64 + tl.arange(0, 64)
k = split * 512 + tl.arange(0, 128)
acc = tl.zeros((16, 64), tl.float32)
for i in range(4):
offsets = k + i * 128
x = tl.load(
X + m[:, None] * SX + group * 4096 + offsets[None, :], m[:, None] < M, 0
)
w = tl.load(W + (group * 1024 + n[None, :]) * 4096 + offsets[:, None])
acc += tl.dot(x, w)
tl.store(
P + ((split * M + m[:, None]) * 2 + group) * 1024 + n[None, :],
acc,
m[:, None] < M,
)
@triton.jit
def _wo_a_reduce(P, Y, E: tl.constexpr):
i = tl.program_id(0) * 256 + tl.arange(0, 256)
split = tl.arange(0, 8)
values = tl.load(P + split[:, None] * E + i[None, :], i[None, :] < E, 0)
tl.store(Y + i, tl.sum(values, 0), i < E)
def wo_a_bf16_small_batch(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
"""Compute ``einsum('tgd,grd->tgr', x, weight)`` for the TP4 WO-A shape.
Partial sums stay in FP32 until the final BF16, token-major store."""
m = x.shape[0]
assert 2 <= m <= 8 and x.shape[1:] == (2, 4096)
assert weight.shape == (2, 1024, 4096) and weight.is_contiguous()
assert x.dtype == weight.dtype == torch.bfloat16
assert x.is_cuda and x.device == weight.device
assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192
result = torch.empty((m, 2, 1024), dtype=x.dtype, device=x.device)
partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device)
_wo_a_partial[(16, 2, 8)](
x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3
)
_wo_a_reduce[(triton.cdiv(m * 2048, 256),)](partial, result, m * 2048, num_warps=4)
return result
@triton.jit
def _wo_a_reduce_quant(P, Q, S, M: tl.constexpr):
row, tile = tl.program_id(0), tl.program_id(1)
i = tile * 256 + tl.arange(0, 256)
split = tl.arange(0, 8)
v = tl.load(P + split[:, None] * (M * 2048) + row * 2048 + i[None, :])
y = tl.sum(v, 0).to(tl.bfloat16).to(tl.float32).reshape((8, 32))
amax = tl.max(tl.abs(y), 1)
sf, inv = ue8m0_scale(amax)
quant = tl.minimum(tl.maximum(y * inv[:, None], -448.0), 448.0).to(tl.float8e4nv)
tl.store(Q + row * 2048 + i, quant.reshape((256,)))
col = tile * 8 + tl.arange(0, 8)
off = (col // 4) * 512 + row * 16 + col % 4
tl.store(S + off, sf.to(tl.uint8))
# Zero only padding rows; valid scale bytes have disjoint writers above.
for z in tl.static_range(triton.cdiv(8192, M * 8 * 256)):
s = (row * 8 + tile) * 256 + tl.arange(0, 256) + z * (M * 8 * 256)
sr = (s % 512) // 16 + ((s % 16) // 4) * 32
tl.store(S + s, 0, (s < 8192) & (sr >= M))
def _quantize_partial(p):
m = p.shape[1]
q = torch.empty((m, 2048), device=p.device, dtype=torch.float8_e4m3fn)
s = torch.empty(8192, device=p.device, dtype=torch.uint8)
_wo_a_reduce_quant[(m, 8)](p, q, s, m, num_warps=4)
return q, s
def wo_a_bf16_small_batch_mxfp8(x: torch.Tensor, weight: torch.Tensor):
"""WO-A with BF16 rounding followed by FlashInfer-compatible MXFP8 quantization."""
m = x.shape[0]
assert 2 <= m <= 8 and x.shape[1:] == (2, 4096)
assert x.dtype == weight.dtype == torch.bfloat16
assert weight.shape == (2, 1024, 4096) and weight.is_contiguous()
assert x.is_cuda and x.device == weight.device
assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192
partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device)
_wo_a_partial[(16, 2, 8)](
x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3
)
return _quantize_partial(partial)
@@ -1,127 +0,0 @@
"""Microbenchmark for ``commit_gdn_replayssm_fold_all_layers`` (defaults match
Qwen3.5-397B at TP4). The kernel is bound by the mandatory checkpoint
read+write (``bs * layers * HV * K * V * 4 B * 2``), so the reported GB/s
approximates achieved HBM bandwidth.
Run: ``python -m sglang.kernels.ops.attention.fla.bench_gdn_replayssm_fold``
"""
from __future__ import annotations
import argparse
import torch
import triton
from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import (
commit_gdn_replayssm_fold_all_layers,
)
def _make_pool(num_layers, num_slots, HV, H, K, V, RL, device):
state = torch.randn(
num_layers, num_slots, HV, K, V, device=device, dtype=torch.float32
)
rawv = torch.randn(
num_layers, num_slots, HV, RL, V, device=device, dtype=torch.bfloat16
)
rawk = torch.randn(
num_layers, num_slots, H, RL, K, device=device, dtype=torch.bfloat16
)
g_ring = (
-torch.rand(num_layers, num_slots, HV, RL, device=device, dtype=torch.float32)
* 0.5
)
beta = torch.rand(num_layers, num_slots, HV, RL, device=device, dtype=torch.float32)
return state, rawv, rawk, g_ring, beta
def _bench_bs(*, pool, bs, num_slots, num_layers, HV, H, K, V, RL, device, with_track):
state, rawv, rawk, g_ring, beta = pool
gen = torch.Generator(device=device).manual_seed(bs)
slots = torch.randperm(num_slots, device=device, generator=gen)[:bs].to(torch.int32)
accept_lens = torch.randint(
1, RL + 1, (bs,), device=device, dtype=torch.int32, generator=gen
)
if with_track:
track_indices = torch.randperm(num_slots, device=device, generator=gen)[:bs].to(
torch.int64
)
track_steps = torch.where(
torch.rand(bs, device=device, generator=gen) < 0.25,
accept_lens.to(torch.int64) - 1,
torch.full((bs,), -1, dtype=torch.int64, device=device),
)
else:
track_indices = None
track_steps = None
def run():
commit_gdn_replayssm_fold_all_layers(
checkpoint_state=state,
rawv_cache=rawv,
rawk_cache=rawk,
g_cache=g_ring,
beta_cache=beta,
ssm_state_indices=slots,
accept_lens=accept_lens,
max_cache_len=RL,
num_k_heads=H,
mamba_track_indices=track_indices,
mamba_steps_to_track=track_steps,
null_block_id=-1,
)
ms = triton.testing.do_bench(run, warmup=25, rep=100)
traffic_gb = bs * num_layers * HV * K * V * 4 * 2 / 1e9
return ms * 1000, traffic_gb / (ms / 1000) if ms > 0 else 0.0
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--num-layers", type=int, default=45)
parser.add_argument("--num-slots", type=int, default=475)
parser.add_argument("--hv", type=int, default=16)
parser.add_argument("--h", type=int, default=4)
parser.add_argument("--k", type=int, default=128)
parser.add_argument("--v", type=int, default=128)
parser.add_argument("--ring-len", type=int, default=4)
parser.add_argument("--batch-sizes", type=int, nargs="+", default=[1, 8, 32, 128])
args = parser.parse_args()
device = "cuda"
pool = _make_pool(
args.num_layers,
args.num_slots,
args.hv,
args.h,
args.k,
args.v,
args.ring_len,
device,
)
print(
f"config: layers={args.num_layers} slots={args.num_slots} HV={args.hv} "
f"H={args.h} K={args.k} V={args.v} ring_len={args.ring_len}"
)
print(f"{'bs':>4} {'track':>6} {'us/launch':>10} {'achieved GB/s':>14}")
for bs in args.batch_sizes:
for with_track in (False, True):
us, gbs = _bench_bs(
pool=pool,
bs=bs,
num_slots=args.num_slots,
num_layers=args.num_layers,
HV=args.hv,
H=args.h,
K=args.k,
V=args.v,
RL=args.ring_len,
device=device,
with_track=with_track,
)
print(f"{bs:>4} {str(with_track):>6} {us:>10.1f} {gbs:>14.0f}")
if __name__ == "__main__":
main()
@@ -50,7 +50,7 @@ def apply_log_scaling_tau(x: torch.Tensor, tau: torch.Tensor) -> torch.Tensor:
# Vectorized JIT kernel (16B loads, one row divide per vector) --
# bit-identical output (same fp32-mul + bf16-round), ~2-3x the
# scalar triton kernel below at every size.
from sglang.kernels.ops.attention.inkling_row_scale import row_scale_bf16
from sglang.kernels.ops.elementwise.row_scale import row_scale_bf16
x2d = torch.as_strided(x, (rows, inner), (x.stride(0), 1))
return row_scale_bf16(x2d, tau.reshape(rows).float()).view(x.shape)
@@ -5,12 +5,16 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table
from sglang.kernels.ops.attention.minicpm_sala.get_block_table import (
get_block_table,
)
def __getattr__(name: str) -> Any:
if name == "get_block_table":
from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table
from sglang.kernels.ops.attention.minicpm_sala.get_block_table import (
get_block_table,
)
globals()[name] = get_block_table
return get_block_table
@@ -19,7 +19,6 @@ for _mod, _fn in [
("all_reduce_residual", "all_reduce_pull_res"),
("all_reduce_residual", "all_reduce_pull_norm"),
("gemm_ag", "gemm_ag_up_proj"),
("gemm_ar", "o_proj_gemm_ar"),
("sp_collective", "reduce_scatter_res"),
("sp_collective", "reduce_scatter_pull"),
("sp_collective", "all_gather"),
@@ -1,216 +0,0 @@
"""K3 fused o_proj GEMM + all-reduce for decode (bf16, TP row-parallel).
One entry point over ``csrc/kimi_k3/comm/gemm_ar.cuh``: a single kernel per
rank computes the local ``x_r [M, K] @ W_r [7168, K]^T`` partial AND the
cross-rank sum — the epilogue pushes finished tiles straight into a
peer-mapped P2P comm region, one flag boundary, then a tile-local reduce
writes the fully reduced ``out [M, 7168]`` on every rank. Replaces the
o_proj GEMM + NCCL all-reduce pair with one launch (see GEMM_AR_README.md).
Contracts:
* bf16 only; ``out = sum_r bf16(x_r @ W_r^T)`` (partials round to bf16
pre-sum — same numerics as the unfused bf16 GEMM + ring AR).
* M in [1, 512]; internally rounded up to a tuned cell {8, 16, 32, 64,
128, 256, 512}. ``out`` is allocated with ``cell`` rows and sliced.
* SM100+ with full NVLink P2P (fabric/MNNVL across nodes); perf-tuned on
GB300 (sm_103a).
* CUDA-graph compatible: the per-cell launch epoch lives in device memory
(read at kernel entry, bumped by a trailing kernel), so replays advance
it naturally. Each dispatch cell owns its own flag-ring family — no
host-side ring reset, ever.
* All TP ranks must call :func:`o_proj_gemm_ar` with the same M in
lockstep (same stream order of cells on every rank).
"""
from __future__ import annotations
from typing import TYPE_CHECKING, NamedTuple, Optional
import torch
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
N = 7168 # K3 hidden size (OPROJ_N compile-time default in gemm_ar.cuh)
MAX_TOKENS = 512 # kMMax
@cache_once
def _jit_module(k: int, world_size: int) -> Module:
args = make_cpp_args(
k,
world_size,
is_arch_support_pdl(),
)
cls = f"GemmArKernel<{args}>"
return load_jit(
"kimi_k3_gemm_ar",
*args,
cuda_files=["kimi_k3/comm/gemm_ar.cuh"],
cuda_wrappers=[
("run", f"{cls}::run"),
("set_bases", f"{cls}::set_bases"),
("region_nbytes", f"{cls}::region_nbytes"),
("gather_words", f"{cls}::gather_words"),
("num_fams", f"{cls}::num_fams"),
],
extra_cuda_cflags=["-O3"],
extra_dependencies=["cutlass"],
)
class _State(NamedTuple):
world_size: int
rank: int
region: tuple # (slab tensor, peer buffer views) — keeps mappings alive
uc_bases: torch.Tensor # [R] int64 CPU: per-rank UC VAs of the region
gather: torch.Tensor # [kFams * 2 * kRing] int32 CUDA, device-local
epochs: torch.Tensor # [kFams] int32 CUDA: device-resident CTA ticket counters
_STATE: Optional[_State] = None
def init(
*,
world_size: int,
rank: int,
group: torch.distributed.ProcessGroup,
k: int,
) -> None:
"""Allocate + rendezvous the P2P comm region (collective; call once from
every TP rank, BEFORE any CUDA-graph capture). ``group`` is the TP CPU
(gloo) group used for the symm-mem rendezvous."""
global _STATE
if _STATE is not None:
return
# the empty_strided_p2p + get_buffer path is the one that exchanges
# fabric handles and maps every peer (incl. cross-node MNNVL) into this
# process — the mem-pool rendezvous(tensor, group_name) API leaves
# remote-node (and sometimes even local) peers unmapped.
from torch._C._distributed_c10d import _SymmetricMemory
mod = _jit_module(k, world_size)
nbytes = int(mod.region_nbytes())
device = torch.device("cuda", torch.cuda.current_device())
if torch.__version__ < "2.11.0":
import torch.distributed._symmetric_memory as torch_symm_mem
torch_symm_mem.enable_symm_mem_for_group(group.group_name)
region = _SymmetricMemory.empty_strided_p2p(
(nbytes,), [1], torch.uint8, device, group.group_name
)
symm = _SymmetricMemory.rendezvous(region)
region.zero_()
torch.cuda.synchronize()
torch.distributed.barrier(group=group)
# keep the peer buffer tensors alive alongside the region
peer_bufs = [symm.get_buffer(r, [nbytes], torch.uint8) for r in range(world_size)]
ptrs = [t.data_ptr() for t in peer_bufs]
import logging
logging.getLogger(__name__).info(
"gemm_ar comm region: rank=%d nbytes=%d uc_bases=%s",
rank,
nbytes,
[hex(p) for p in ptrs],
)
assert all(p != 0 for p in ptrs), f"gemm_ar: null peer pointers {ptrs}"
# explicit cpu: model build may run under a cuda default-device context,
# and a silently-cuda tensor here means the host-side deref in set_bases
# reads a device pointer (segfault)
uc_bases = torch.tensor(ptrs, dtype=torch.int64, device="cpu")
gather = torch.zeros(int(mod.gather_words()), dtype=torch.int32, device=device)
epochs = torch.zeros(int(mod.num_fams()), dtype=torch.int32, device=device)
torch.cuda.synchronize()
_STATE = _State(
world_size=world_size,
rank=rank,
region=(region, peer_bufs),
uc_bases=uc_bases,
gather=gather,
epochs=epochs,
)
def initialized() -> bool:
return _STATE is not None
@cache_once
def _module_with_bases(k: int, world_size: int) -> Module:
"""The per-K JIT module with the comm-region base addresses stashed
host-side (per-call CPU-tensor derefs from inside the custom op segfault
under the sglang scheduler, so the module holds them in a static)."""
state = _STATE
assert state is not None
mod = _jit_module(k, world_size)
mod.set_bases(state.uc_bases)
return mod
def fits(x: torch.Tensor) -> bool:
"""Whether this o_proj input can take the fused GEMM+AR path."""
return (
_STATE is not None
and x.dim() == 2
and x.dtype == torch.bfloat16
and 0 < x.shape[0] <= MAX_TOKENS
and x.stride(1) == 1
and x.stride(0) == x.shape[1]
)
@register_custom_op(mutates_args=["out", "epochs"])
def _gemm_ar_op(
k: int,
world_size: int,
out: torch.Tensor,
x: torch.Tensor,
weight: torch.Tensor,
gather: torch.Tensor,
epochs: torch.Tensor,
my_rank: int,
) -> None:
_module_with_bases(k, world_size).run(out, x, weight, gather, epochs, my_rank)
def _cell_of(m: int) -> int:
for c in (8, 16, 32, 64, 128, 256, 512):
if m <= c:
return c
raise ValueError(f"gemm_ar: M={m} outside [1, {MAX_TOKENS}]")
def o_proj_gemm_ar(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
"""Fully reduced ``sum_r x_r @ weight_r^T`` on every rank, one kernel.
``x`` is the TP-local [M, K] o_proj input, ``weight`` the TP-local
[7168, K] o_proj weight shard. Caller checked :func:`fits`; all ranks
call in lockstep with the same M.
"""
state = _STATE
assert state is not None
m = x.shape[0]
cell = _cell_of(m)
out = torch.empty((cell, N), dtype=torch.bfloat16, device=x.device)
_gemm_ar_op(
weight.shape[1],
state.world_size,
out,
x,
weight,
state.gather,
state.epochs,
state.rank,
)
return out[:m]
@@ -8,4 +8,34 @@ Individual functions register (or are imported) under the functional op id
they logically belong to.
"""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
__all__ = []
# Public entry points inventoried by logical operator group (RFC #29630).
register_kernel(
KernelSpec(
op="elementwise.fused_qwen4_gate_value",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_value",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="elementwise.fused_qwen4_gate_reduce",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_reduce",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="elementwise.row_scale_bf16",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.elementwise.row_scale:row_scale_bf16",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
@@ -0,0 +1,155 @@
from __future__ import annotations
import torch
import triton
import triton.language as tl
_QWEN4_HC_COUNT = 4
_QWEN4_HIDDEN_SIZE = 2560
@triton.jit
def _round_bf16_to_fp32(value):
"""RNE-round fp32 to BF16 precision while retaining an fp32 register."""
bits = value.to(tl.int32, bitcast=True)
rounding_bias = 0x7FFF + ((bits >> 16) & 1)
rounded_bits = (bits + rounding_bias) & -65536
return rounded_bits.to(tl.float32, bitcast=True)
@triton.jit
def _qwen4_gate_value_kernel(
gate_ptr,
value_ptr,
output_ptr,
num_tokens,
HC_COUNT: tl.constexpr,
HIDDEN_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
token_group = tl.program_id(0)
token = token_group // HC_COUNT
hidden = tl.arange(0, BLOCK_SIZE)
mask = (token < num_tokens) & (hidden < HIDDEN_SIZE)
# `gate` arrives already rounded to bf16 by the eager multiply/reduce/divide;
# every remaining eager bf16 rounding boundary is reproduced below.
gate = tl.load(gate_ptr + token_group).to(tl.float32)
magnitude = tl.maximum(tl.abs(gate), 1.0e-6)
root = _round_bf16_to_fp32(tl.sqrt(magnitude))
sign = tl.where(gate > 0.0, 1.0, tl.where(gate < 0.0, -1.0, 0.0))
transformed = _round_bf16_to_fp32(root * sign)
activated = _round_bf16_to_fp32(tl.sigmoid(transformed))
value = tl.load(value_ptr + token * HIDDEN_SIZE + hidden, mask=mask, other=0.0).to(
tl.float32
)
output_offset = token_group * HIDDEN_SIZE + hidden
tl.store(output_ptr + output_offset, activated * value, mask=mask)
def can_fuse_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> bool:
"""Return whether inputs match Qwen4's fixed BF16 gate/value contract."""
return (
gate.is_cuda
and gate.dtype == torch.bfloat16
and gate.dim() == 3
and gate.shape[1:] == (_QWEN4_HC_COUNT, 1)
and gate.is_contiguous()
and value.is_cuda
and value.dtype == gate.dtype
and value.shape == (gate.shape[0], _QWEN4_HIDDEN_SIZE)
and value.is_contiguous()
)
def fused_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
"""Apply Qwen4's post-reduction gate and value broadcast in one kernel."""
if not can_fuse_qwen4_gate_value(gate, value):
raise ValueError("unsupported input for fused Qwen4 PLE gate/value")
output = torch.empty(
(gate.shape[0], _QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE),
dtype=value.dtype,
device=value.device,
)
if gate.shape[0]:
_qwen4_gate_value_kernel[(gate.shape[0] * _QWEN4_HC_COUNT,)](
gate,
value,
output,
gate.shape[0],
HC_COUNT=_QWEN4_HC_COUNT,
HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE,
BLOCK_SIZE=4096,
num_warps=8,
)
return output
@triton.jit
def _qwen4_gate_reduce_kernel(
key_ptr,
query_ptr,
value_ptr,
output_ptr,
HIDDEN_SIZE: tl.constexpr,
HC_COUNT: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
group = tl.program_id(0)
hidden = tl.arange(0, BLOCK_SIZE)
mask = hidden < HIDDEN_SIZE
key = tl.load(key_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(tl.float32)
query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(
tl.float32
)
product = _round_bf16_to_fp32(key * query)
gate = _round_bf16_to_fp32(tl.sum(product, 0))
gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5))
magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6))
root = _round_bf16_to_fp32(tl.sqrt(magnitude))
sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0))
activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign)))
value = tl.load(
value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0
).to(tl.float32)
tl.store(output_ptr + group * HIDDEN_SIZE + hidden, activated * value, mask)
def can_fuse_qwen4_gate_reduce(key, query, value):
return (
key.is_cuda
and key.dtype == torch.bfloat16
and key.ndim == 3
and key.shape[1:] == (_QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE)
and key.is_contiguous()
and query.shape == key.shape
and query.dtype == key.dtype
and query.device == key.device
and query.is_contiguous()
and value.shape == (key.shape[0], _QWEN4_HIDDEN_SIZE)
and value.dtype == key.dtype
and value.device == key.device
and value.is_contiguous()
)
def fused_qwen4_gate_reduce(key, query, value):
if not can_fuse_qwen4_gate_reduce(key, query, value):
raise ValueError("unsupported input for Qwen4 PLE gate reduction")
output = torch.empty_like(key)
if key.shape[0]:
_qwen4_gate_reduce_kernel[(key.shape[0] * _QWEN4_HC_COUNT,)](
key,
query,
value,
output,
HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE,
HC_COUNT=_QWEN4_HC_COUNT,
BLOCK_SIZE=triton.next_power_of_2(_QWEN4_HIDDEN_SIZE),
enable_fp_fusion=False,
)
return output
@@ -45,15 +45,3 @@ def row_scale_bf16(
module = _jit_row_scale_module(is_arch_support_pdl())
module.run(x, tau, out)
return out
def row_compact_bf16(x: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
"""Contiguous copy of row-strided ``x`` ([rows, inner] bf16, inner
contiguous, inner % 8 == 0, 16B-aligned rows) -- the tau-less flavor of
``row_scale_bf16``. Beats the TensorIterator strided copy that einsum's
reshape would otherwise run on such inputs."""
if out is None:
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
module = _jit_row_scale_module(is_arch_support_pdl())
module.run_compact(x, out)
return out
@@ -1,7 +1,7 @@
"""Embedding kernels."""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import KernelBackend, KernelSpec
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
register_kernel(
KernelSpec(
@@ -55,3 +55,14 @@ register_kernel(
)
__all__ = []
# Public entry points inventoried by logical operator group (RFC #29630).
register_kernel(
KernelSpec(
op="embeddings.fused_qwen4_ngram_hash",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.embeddings.qwen4_ngram:fused_qwen4_ngram_hash",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
@@ -0,0 +1,114 @@
from __future__ import annotations
import torch
import triton
import triton.language as tl
_QWEN4_NGRAM_SIZE = 3
_QWEN4_HEADS_PER_NGRAM = 8
_QWEN4_NGRAM_HEADS = 16
@triton.jit
def _qwen4_ngram_hash_kernel(
contexts_ptr,
multipliers_ptr,
vocab_sizes_ptr,
offsets_ptr,
output_ptr,
num_outputs,
eos_token_id,
NGRAM_SIZE: tl.constexpr,
HEADS_PER_NGRAM: tl.constexpr,
NGRAM_HEADS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
output_idx = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = output_idx < num_outputs
token_idx = output_idx // NGRAM_HEADS
head_idx = output_idx % NGRAM_HEADS
context_base = token_idx * NGRAM_SIZE
token_0 = tl.load(contexts_ptr + context_base, mask=mask, other=0)
token_1 = tl.load(contexts_ptr + context_base + 1, mask=mask, other=0)
token_2 = tl.load(contexts_ptr + context_base + 2, mask=mask, other=0)
multiplier_0 = tl.load(multipliers_ptr)
multiplier_1 = tl.load(multipliers_ptr + 1)
multiplier_2 = tl.load(multipliers_ptr + 2)
# Only the final position of each three-token context is materialized.
previous_1 = tl.where(token_1 == eos_token_id, eos_token_id, token_1)
previous_2 = tl.where(
(token_0 == eos_token_id) | (token_1 == eos_token_id),
eos_token_id,
token_0,
)
mixed = (token_2 * multiplier_0) ^ (previous_1 * multiplier_1)
mixed_3 = mixed ^ (previous_2 * multiplier_2)
mixed = tl.where(head_idx < HEADS_PER_NGRAM, mixed, mixed_3)
vocab_size = tl.load(vocab_sizes_ptr + head_idx, mask=mask, other=1)
offset = tl.load(offsets_ptr + head_idx, mask=mask, other=0)
tl.store(output_ptr + output_idx, mixed % vocab_size + offset, mask=mask)
def can_fuse_qwen4_ngram_hash(
contexts: torch.Tensor,
multipliers: torch.Tensor,
vocab_sizes: torch.Tensor,
offsets: torch.Tensor,
) -> bool:
"""Return whether inputs match the fixed Qwen4 PLE hash contract."""
return (
contexts.is_cuda
and contexts.dtype == torch.long
and contexts.dim() == 2
and contexts.shape[1] == _QWEN4_NGRAM_SIZE
and contexts.is_contiguous()
and multipliers.is_cuda
and multipliers.dtype == torch.long
and multipliers.numel() == _QWEN4_NGRAM_SIZE
and vocab_sizes.is_cuda
and vocab_sizes.dtype == torch.long
and vocab_sizes.numel() == _QWEN4_NGRAM_HEADS
and offsets.is_cuda
and offsets.dtype == torch.long
and offsets.numel() == _QWEN4_NGRAM_HEADS
)
def fused_qwen4_ngram_hash(
contexts: torch.Tensor,
multipliers: torch.Tensor,
vocab_sizes: torch.Tensor,
offsets: torch.Tensor,
eos_token_id: int,
) -> torch.Tensor:
"""Return the 16 Qwen4 PLE N-gram IDs in one kernel launch."""
if not can_fuse_qwen4_ngram_hash(contexts, multipliers, vocab_sizes, offsets):
raise ValueError("unsupported input for fused Qwen4 PLE N-gram hash")
output = torch.empty(
(contexts.shape[0], _QWEN4_NGRAM_HEADS),
dtype=torch.long,
device=contexts.device,
)
num_outputs = output.numel()
if num_outputs:
block_size = 256
_qwen4_ngram_hash_kernel[(triton.cdiv(num_outputs, block_size),)](
contexts,
multipliers,
vocab_sizes,
offsets,
output,
num_outputs,
eos_token_id,
NGRAM_SIZE=_QWEN4_NGRAM_SIZE,
HEADS_PER_NGRAM=_QWEN4_HEADS_PER_NGRAM,
NGRAM_HEADS=_QWEN4_NGRAM_HEADS,
BLOCK_SIZE=block_size,
num_warps=4,
)
return output
+137
View File
@@ -422,3 +422,140 @@ for _mod, _fn in _TRITON_KERNELS:
)
)
del _mod, _fn
# Public entry points inventoried by logical operator group (RFC #29630).
register_kernel(
KernelSpec(
op="gemm.linear_bf16_fp32",
backend=KernelBackend.TORCH,
target="sglang.kernels.ops.gemm.bf16_fp32:_linear_bf16_fp32_cublas",
description=(
"Torch/cuBLAS implementation. The direct linear_bf16_fp32 entry also "
"retains its environment-controlled HPC-Ops, AITER and DeepGEMM dispatch."
),
)
)
register_kernel(
KernelSpec(
op="gemm.wo_a_bf16_gemv",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_gemv",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="gemm.wo_a_bf16_small_batch",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="gemm.wo_a_bf16_small_batch_mxfp8",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch_mxfp8",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="gemm.hopper_bf16_gemv",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.hopper_bf16_gemv:hopper_bf16_gemv",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 0))}
),
)
)
register_kernel(
KernelSpec(
op="gemm.fp8_blockwise_scaled_mm",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.fp8_blockwise_gemm:fp8_blockwise_scaled_mm",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))}
),
)
)
register_kernel(
KernelSpec(
op="gemm.sm120_fp8_gemv",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.sm120_fp8_gemv:sm120_fp8_gemv",
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))}
),
)
)
register_kernel(
KernelSpec(
op="gemm.gptq_marlin_gemm",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.gptq_marlin:gptq_marlin_gemm",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="gemm.rel_proj_small_t",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.inkling_rel_proj:rel_proj_small_t",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="gemm.fused_hc_mix",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.hc_mix:fused_hc_mix",
capabilities=frozenset({CapabilityRequirement.CUDA}),
)
)
register_kernel(
KernelSpec(
op="gemm.w8a8_block_fp8_matmul",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul",
)
)
register_kernel(
KernelSpec(
op="gemm.w8a8_block_fp8_matmul_triton",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul_triton",
capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}),
)
)
register_kernel(
KernelSpec(
op="gemm.w8a8_block_fp8_matmul_deepgemm",
backend=KernelBackend.DEEPGEMM,
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul_deepgemm",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
)
)
register_kernel(
KernelSpec(
op="gemm.w8a8_mxfp8_matmul_deepgemm",
backend=KernelBackend.DEEPGEMM,
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_mxfp8_matmul_deepgemm",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}),
)
)
register_kernel(
KernelSpec(
op="gemm.triton_scaled_mm",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.fp8_kernel:triton_scaled_mm",
)
)
register_kernel(
KernelSpec(
op="gemm.w8a8_block_int8_matmul",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.gemm.int8_kernel:w8a8_block_int8_matmul",
)
)

Some files were not shown because too many files have changed in this diff Show More