[Refactor] Restore logical kernel groups and test organization (#41243)
This commit is contained in:
@@ -26,7 +26,7 @@ sglang/kernels/
|
||||
|
||||
Operator groups (all populated): `activation`, `attention`, `communication`,
|
||||
`diffusion`, `elementwise`, `embeddings`, `gemm`, `grammar`, `kv_canary`,
|
||||
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `moe`, `quantization`,
|
||||
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `mm`, `moe`, `quantization`,
|
||||
`sampling`, `speculative`.
|
||||
|
||||
Place model-specific implementations and tuning data inside the corresponding
|
||||
@@ -36,9 +36,32 @@ under `ops/`.
|
||||
As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has
|
||||
been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit`
|
||||
and each JIT-backed operator into its group as
|
||||
`sglang.kernels.ops.<group>._jit_<op>`. Tests and benchmarks live under
|
||||
`sglang.kernels.ops.<group>.<op>` (the old `_jit_` filename prefix was removed
|
||||
in #32148). Tests and benchmarks live under
|
||||
`test/registered/kernels/` (`ops/<group>/` for tests, `benchmark/<group>/` for
|
||||
benchmarks); shared test helpers are in `sglang.test.kernels`.
|
||||
benchmarks); shared test helpers are in `sglang.test.kernels`. Standalone scripts
|
||||
that are not CI-registered belong in `test/manual/kernels/`, outside the shipped
|
||||
operator package. Keep vendored upstream trees under their existing maintenance
|
||||
policy.
|
||||
|
||||
Classify the public computation, not the model that first used it: expert
|
||||
routing belongs in `moe`, matrix multiplication (including quantized GEMM and
|
||||
its tuning data) in `gemm`, and attention index selection in `attention`.
|
||||
Keep a fused operation intact; an attention kernel that fuses RoPE with a
|
||||
projection may stay in `attention`. A model name inside a logical group is fine.
|
||||
|
||||
Runtime integration that owns process groups, symmetric buffers, or model
|
||||
dispatch stays in `srt`; the K3 communication adapters live in
|
||||
`srt/layers/communication/`, while their callable kernels live in
|
||||
`kernels/ops/communication/`. Tests of this runtime state belong with the
|
||||
runtime subsystem, even when they need a GPU. Kernel inventory/dispatch tests
|
||||
belong in `test/registered/unit/kernels/`.
|
||||
|
||||
When moving an operator, update its callers, lazy `KernelSpec` targets and op
|
||||
ids, tests, benchmark imports, and tuning-data paths together. Preserve CI
|
||||
stage/runner registrations and test cases when splitting files. See the
|
||||
[kernel organization skill](../../../.claude/skills/kernel-organization/SKILL.md)
|
||||
for the review checklist.
|
||||
|
||||
## How it works
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -27,7 +27,6 @@ _GROUPS = (
|
||||
"layernorm",
|
||||
"mamba",
|
||||
"memory",
|
||||
"minicpm_sala",
|
||||
"mm",
|
||||
"moe",
|
||||
"quantization",
|
||||
|
||||
@@ -184,7 +184,6 @@ for _mod, _fn in [
|
||||
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
|
||||
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
|
||||
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
|
||||
("dsv4.rms_normalize_hip", "rms_normalize_triton"),
|
||||
]:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
@@ -287,3 +286,64 @@ for _mod, _fn, _backend, _device in [
|
||||
)
|
||||
)
|
||||
del _mod, _fn, _backend, _device
|
||||
|
||||
|
||||
# Public entry points inventoried by logical operator group (RFC #29630).
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.get_block_table",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.minicpm_sala.get_block_table:get_block_table",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fast_kpool_topk_transform_fused",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsa.kpool_topk_transform:fast_kpool_topk_transform_fused",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fast_topk",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.fast_topk:fast_topk",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fused_k_indexer_norm_rope",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fused_k_indexer_norm_rope_store",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope_store",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fused_rope_wo_a_bf16",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsv4.wo_a:fused_rope_wo_a_bf16",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="attention.fp4_index_logits_decode",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.attention.dsv4.fp4_indexer:fp4_index_logits_decode",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -557,10 +557,6 @@ def _gather_dequant_requant_fp8_paged_kernel(
|
||||
tl.store(dst_ptr, data, mask=mask)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise Exception("UT is in quant_k_cache.py")
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _concat_cast_kv_fp8_pad_kernel(
|
||||
out_ptr,
|
||||
|
||||
@@ -397,128 +397,3 @@ def _quantize_k_cache_fast_kernel(
|
||||
|
||||
data = tl.load(src_ptr, mask=mask)
|
||||
tl.store(dst_ptr, data, mask=mask)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import dequant_k_cache
|
||||
|
||||
for num_blocks, block_size in [
|
||||
(1, 1),
|
||||
(10, 64),
|
||||
]:
|
||||
dim_nope_and_rope = 512 + 64
|
||||
|
||||
input_k_cache = torch.randn(
|
||||
(num_blocks, block_size, 1, dim_nope_and_rope),
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
ref_quant = _quantize_k_cache_ref(input_k_cache)
|
||||
actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache)
|
||||
|
||||
ref_ref_dequant = dequant_k_cache._dequantize_k_cache_slow(ref_quant)
|
||||
ref_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped(ref_quant)
|
||||
actual_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped(
|
||||
actual_quant
|
||||
)
|
||||
|
||||
print(f"{ref_ref_dequant=}")
|
||||
print(f"{actual_actual_dequant=}")
|
||||
print(f"{actual_actual_dequant - ref_ref_dequant=}")
|
||||
print(f"{torch.mean(ref_ref_dequant - actual_actual_dequant)=}")
|
||||
|
||||
# TODO too different?
|
||||
torch.testing.assert_close(
|
||||
ref_ref_dequant, ref_actual_dequant, atol=0.2, rtol=0.2
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_ref_dequant, actual_actual_dequant, atol=0.2, rtol=0.2
|
||||
)
|
||||
|
||||
# test dequant_k_cache_paged
|
||||
page_table_1 = torch.arange(
|
||||
num_blocks * block_size, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
actual_dequant_paged = dequant_k_cache.dequantize_k_cache_paged(
|
||||
actual_quant, page_table_1
|
||||
).reshape(actual_actual_dequant.shape)
|
||||
print(f"{torch.mean(actual_actual_dequant - actual_dequant_paged)=}")
|
||||
torch.testing.assert_close(
|
||||
ref_ref_dequant, actual_dequant_paged, atol=0.2, rtol=0.2
|
||||
)
|
||||
|
||||
print("Passed")
|
||||
|
||||
# Test quantize_k_cache_separate: verify output matches concat path
|
||||
print("\nTesting quantize_k_cache_separate...")
|
||||
for num_tokens in [64, 100]:
|
||||
dim_nope = 512
|
||||
dim_rope = 64
|
||||
|
||||
k_nope = torch.randn(
|
||||
num_tokens, 1, dim_nope, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
k_rope = torch.randn(
|
||||
num_tokens, 1, dim_rope, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
# Old path: concat then quantize
|
||||
k_concat = torch.cat([k_nope, k_rope], dim=-1).squeeze(1) # (num_tokens, 576)
|
||||
old_output = quantize_k_cache(k_concat.unsqueeze(1).unsqueeze(1)) # 4D input
|
||||
old_output = old_output.squeeze(1).squeeze(1) # Back to (num_tokens, 656)
|
||||
|
||||
# New path: quantize separately
|
||||
nope_part, rope_part = quantize_k_cache_separate(k_nope, k_rope)
|
||||
new_bytes = torch.cat([nope_part.squeeze(1), rope_part.squeeze(1)], dim=-1)
|
||||
|
||||
# Compare byte-level equality
|
||||
old_bytes = old_output.view(torch.uint8)
|
||||
|
||||
if old_bytes.shape != new_bytes.shape:
|
||||
raise RuntimeError(
|
||||
f"Shape mismatch: {old_bytes.shape} vs {new_bytes.shape}"
|
||||
)
|
||||
|
||||
diff_bytes = (old_bytes != new_bytes).sum().item()
|
||||
if diff_bytes > 0:
|
||||
max_diff = (old_bytes.float() - new_bytes.float()).abs().max().item()
|
||||
raise RuntimeError(
|
||||
f"quantize_k_cache_separate output doesn't match concat path: "
|
||||
f"{diff_bytes} differing bytes, max_diff={max_diff}"
|
||||
)
|
||||
|
||||
print(f" num_tokens={num_tokens}: PASSED (outputs match byte-wise)")
|
||||
|
||||
print("quantize_k_cache_separate tests passed!")
|
||||
|
||||
print("\nDo benchmark...")
|
||||
|
||||
for num_blocks, block_size in [
|
||||
(1, 64),
|
||||
(64, 64),
|
||||
(128, 64),
|
||||
(256, 64),
|
||||
(512, 64),
|
||||
(1024, 64),
|
||||
(2048, 64),
|
||||
]:
|
||||
dim_nope_and_rope = 512 + 64
|
||||
|
||||
input_k_cache = torch.randn(
|
||||
(num_blocks, block_size, 1, dim_nope_and_rope),
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache)
|
||||
|
||||
page_table_1 = torch.arange(
|
||||
num_blocks * block_size, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
def run_ans():
|
||||
return dequant_k_cache.dequantize_k_cache_paged(actual_quant, page_table_1)
|
||||
|
||||
ans_time: float = triton.testing.do_bench(run_ans, warmup=10, rep=20) / 1000 # type: ignore
|
||||
print(f"seq_kv: {num_blocks * block_size}, time: {ans_time * 1e6: 4.0f} us")
|
||||
|
||||
@@ -375,14 +375,3 @@ def transform_index_page_table_prefill_ref(
|
||||
)
|
||||
offset += l
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
bs, topk, max_seqlen = 10, 2048, 3000
|
||||
page_table = torch.randint(0, 100, (bs, max_seqlen), device="cuda")
|
||||
topk_indices = torch.full((bs, topk), -1, device="cuda")
|
||||
topk_indices[:, :1600] = torch.arange(1600).unsqueeze(0).repeat(bs, 1)
|
||||
ref_result = transform_index_page_table_decode_ref(page_table, topk_indices)
|
||||
result = transform_index_page_table_decode_fast(page_table, topk_indices)
|
||||
assert torch.all(result == ref_result)
|
||||
print("Passed")
|
||||
|
||||
@@ -23,15 +23,6 @@ from .elementwise import (
|
||||
fused_rope_inplace,
|
||||
)
|
||||
from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a
|
||||
from .gemm import linear_bf16_fp32
|
||||
from .moe import (
|
||||
hash_topk,
|
||||
mask_topk_ids,
|
||||
mega_moe_pre_dispatch,
|
||||
silu_and_mul_clamp,
|
||||
silu_and_mul_contig_post_quant,
|
||||
silu_and_mul_masked_post_quant,
|
||||
)
|
||||
from .topk import (
|
||||
plan_topk_v2,
|
||||
topk_transform_paged,
|
||||
@@ -56,17 +47,10 @@ __all__ = [
|
||||
"fused_k_norm_rope_flashmla",
|
||||
"sglang_per_token_group_quant_fp8_dsv4_wo_a",
|
||||
"make_name",
|
||||
"linear_bf16_fp32",
|
||||
"get_paged_mqa_logits_metadata",
|
||||
"triton_create_paged_compress_data",
|
||||
"topk_transform_paged",
|
||||
"topk_transform_paged_v2",
|
||||
"topk_transform_ragged_v2",
|
||||
"plan_topk_v2",
|
||||
"hash_topk",
|
||||
"mega_moe_pre_dispatch",
|
||||
"mask_topk_ids",
|
||||
"silu_and_mul_clamp",
|
||||
"silu_and_mul_masked_post_quant",
|
||||
"silu_and_mul_contig_post_quant",
|
||||
]
|
||||
|
||||
@@ -572,33 +572,3 @@ def gather_dequant_requant_fp8_paged_ref(
|
||||
)
|
||||
out[: active.shape[0]] = active
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
assert torch.cuda.is_available(), "this self-test needs a CUDA device"
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
|
||||
page_size = 64
|
||||
num_pages = 8
|
||||
num_tokens = 333
|
||||
raw_bytes = page_size * (NOPE_ROPE_BYTES + PADDED_SCALE_PER_TOKEN)
|
||||
bytes_per_page = (
|
||||
(raw_bytes + NOPE_ROPE_BYTES - 1) // NOPE_ROPE_BYTES
|
||||
) * NOPE_ROPE_BYTES
|
||||
|
||||
quant_k_cache = torch.randint(
|
||||
0, 256, (num_pages, bytes_per_page), dtype=torch.uint8, device=device
|
||||
)
|
||||
page_table = torch.randint(
|
||||
0, num_pages * page_size, (num_tokens,), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
out_kernel = dequantize_k_cache_paged(quant_k_cache, page_table, page_size)
|
||||
out_ref = dequantize_k_cache_paged_ref(quant_k_cache, page_table, page_size)
|
||||
|
||||
torch.testing.assert_close(out_kernel, out_ref, atol=0, rtol=0, equal_nan=True)
|
||||
print(
|
||||
f"OK: kernel matches torch ref for {num_tokens} tokens "
|
||||
f"(page_size={page_size}, bytes_per_page={bytes_per_page})"
|
||||
)
|
||||
|
||||
@@ -3,11 +3,8 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING, Sequence
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit
|
||||
from sglang.kernels.ops.layernorm.mxfp8_epilogue import ue8m0_scale
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
@@ -52,133 +49,3 @@ def fused_rope_wo_a_bf16(
|
||||
y = x.new_empty((num_tokens, N_OUT))
|
||||
module.run(x, weight, freqs_cis, positions, None, None, y)
|
||||
return (y,)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _wo_a_bf16_gemv_kernel(X, W, Y, R: tl.constexpr, D: tl.constexpr, BN: tl.constexpr):
|
||||
group = tl.program_id(1)
|
||||
rows = tl.program_id(0) * BN + tl.arange(0, BN)
|
||||
columns = tl.arange(0, D)
|
||||
x = tl.load(X + group * D + columns).to(tl.float32)
|
||||
w = tl.load(
|
||||
W + (group * R + rows[:, None]) * D + columns[None, :],
|
||||
rows[:, None] < R,
|
||||
0,
|
||||
).to(tl.float32)
|
||||
result = tl.sum(w * x[None, :], axis=1)
|
||||
tl.store(Y + group * R + rows, result, rows < R)
|
||||
|
||||
|
||||
def wo_a_bf16_gemv(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute ``einsum('tgd,grd->tgr', x, weight)`` for one token."""
|
||||
assert x.shape[0] == 1 and x.ndim == weight.ndim == 3
|
||||
assert x.dtype == weight.dtype == torch.bfloat16
|
||||
assert x.is_cuda and x.device == weight.device
|
||||
assert x.is_contiguous() and weight.is_contiguous()
|
||||
groups, rows, dim = weight.shape
|
||||
assert x.shape[1:] == (groups, dim) and dim == triton.next_power_of_2(dim)
|
||||
result = torch.empty((1, groups, rows), dtype=x.dtype, device=x.device)
|
||||
# One output row per CTA keeps register use low and exposes enough
|
||||
# independent weight loads for single-token decode.
|
||||
_wo_a_bf16_gemv_kernel[(rows, groups)](
|
||||
x,
|
||||
weight,
|
||||
result,
|
||||
rows,
|
||||
dim,
|
||||
1,
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _wo_a_partial(X, W, P, M: tl.constexpr, SX: tl.constexpr):
|
||||
tile, group, split = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
m = tl.arange(0, 16)
|
||||
n = tile * 64 + tl.arange(0, 64)
|
||||
k = split * 512 + tl.arange(0, 128)
|
||||
acc = tl.zeros((16, 64), tl.float32)
|
||||
for i in range(4):
|
||||
offsets = k + i * 128
|
||||
x = tl.load(
|
||||
X + m[:, None] * SX + group * 4096 + offsets[None, :], m[:, None] < M, 0
|
||||
)
|
||||
w = tl.load(W + (group * 1024 + n[None, :]) * 4096 + offsets[:, None])
|
||||
acc += tl.dot(x, w)
|
||||
tl.store(
|
||||
P + ((split * M + m[:, None]) * 2 + group) * 1024 + n[None, :],
|
||||
acc,
|
||||
m[:, None] < M,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _wo_a_reduce(P, Y, E: tl.constexpr):
|
||||
i = tl.program_id(0) * 256 + tl.arange(0, 256)
|
||||
split = tl.arange(0, 8)
|
||||
values = tl.load(P + split[:, None] * E + i[None, :], i[None, :] < E, 0)
|
||||
tl.store(Y + i, tl.sum(values, 0), i < E)
|
||||
|
||||
|
||||
def wo_a_bf16_small_batch(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute ``einsum('tgd,grd->tgr', x, weight)`` for the TP4 WO-A shape.
|
||||
Partial sums stay in FP32 until the final BF16, token-major store."""
|
||||
m = x.shape[0]
|
||||
assert 2 <= m <= 8 and x.shape[1:] == (2, 4096)
|
||||
assert weight.shape == (2, 1024, 4096) and weight.is_contiguous()
|
||||
assert x.dtype == weight.dtype == torch.bfloat16
|
||||
assert x.is_cuda and x.device == weight.device
|
||||
assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192
|
||||
result = torch.empty((m, 2, 1024), dtype=x.dtype, device=x.device)
|
||||
partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device)
|
||||
_wo_a_partial[(16, 2, 8)](
|
||||
x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3
|
||||
)
|
||||
_wo_a_reduce[(triton.cdiv(m * 2048, 256),)](partial, result, m * 2048, num_warps=4)
|
||||
return result
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _wo_a_reduce_quant(P, Q, S, M: tl.constexpr):
|
||||
row, tile = tl.program_id(0), tl.program_id(1)
|
||||
i = tile * 256 + tl.arange(0, 256)
|
||||
split = tl.arange(0, 8)
|
||||
v = tl.load(P + split[:, None] * (M * 2048) + row * 2048 + i[None, :])
|
||||
y = tl.sum(v, 0).to(tl.bfloat16).to(tl.float32).reshape((8, 32))
|
||||
amax = tl.max(tl.abs(y), 1)
|
||||
sf, inv = ue8m0_scale(amax)
|
||||
quant = tl.minimum(tl.maximum(y * inv[:, None], -448.0), 448.0).to(tl.float8e4nv)
|
||||
tl.store(Q + row * 2048 + i, quant.reshape((256,)))
|
||||
col = tile * 8 + tl.arange(0, 8)
|
||||
off = (col // 4) * 512 + row * 16 + col % 4
|
||||
tl.store(S + off, sf.to(tl.uint8))
|
||||
# Zero only padding rows; valid scale bytes have disjoint writers above.
|
||||
for z in tl.static_range(triton.cdiv(8192, M * 8 * 256)):
|
||||
s = (row * 8 + tile) * 256 + tl.arange(0, 256) + z * (M * 8 * 256)
|
||||
sr = (s % 512) // 16 + ((s % 16) // 4) * 32
|
||||
tl.store(S + s, 0, (s < 8192) & (sr >= M))
|
||||
|
||||
|
||||
def _quantize_partial(p):
|
||||
m = p.shape[1]
|
||||
q = torch.empty((m, 2048), device=p.device, dtype=torch.float8_e4m3fn)
|
||||
s = torch.empty(8192, device=p.device, dtype=torch.uint8)
|
||||
_wo_a_reduce_quant[(m, 8)](p, q, s, m, num_warps=4)
|
||||
return q, s
|
||||
|
||||
|
||||
def wo_a_bf16_small_batch_mxfp8(x: torch.Tensor, weight: torch.Tensor):
|
||||
"""WO-A with BF16 rounding followed by FlashInfer-compatible MXFP8 quantization."""
|
||||
m = x.shape[0]
|
||||
assert 2 <= m <= 8 and x.shape[1:] == (2, 4096)
|
||||
assert x.dtype == weight.dtype == torch.bfloat16
|
||||
assert weight.shape == (2, 1024, 4096) and weight.is_contiguous()
|
||||
assert x.is_cuda and x.device == weight.device
|
||||
assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192
|
||||
partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device)
|
||||
_wo_a_partial[(16, 2, 8)](
|
||||
x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3
|
||||
)
|
||||
return _quantize_partial(partial)
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
"""Microbenchmark for ``commit_gdn_replayssm_fold_all_layers`` (defaults match
|
||||
Qwen3.5-397B at TP4). The kernel is bound by the mandatory checkpoint
|
||||
read+write (``bs * layers * HV * K * V * 4 B * 2``), so the reported GB/s
|
||||
approximates achieved HBM bandwidth.
|
||||
|
||||
Run: ``python -m sglang.kernels.ops.attention.fla.bench_gdn_replayssm_fold``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import (
|
||||
commit_gdn_replayssm_fold_all_layers,
|
||||
)
|
||||
|
||||
|
||||
def _make_pool(num_layers, num_slots, HV, H, K, V, RL, device):
|
||||
state = torch.randn(
|
||||
num_layers, num_slots, HV, K, V, device=device, dtype=torch.float32
|
||||
)
|
||||
rawv = torch.randn(
|
||||
num_layers, num_slots, HV, RL, V, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
rawk = torch.randn(
|
||||
num_layers, num_slots, H, RL, K, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
g_ring = (
|
||||
-torch.rand(num_layers, num_slots, HV, RL, device=device, dtype=torch.float32)
|
||||
* 0.5
|
||||
)
|
||||
beta = torch.rand(num_layers, num_slots, HV, RL, device=device, dtype=torch.float32)
|
||||
return state, rawv, rawk, g_ring, beta
|
||||
|
||||
|
||||
def _bench_bs(*, pool, bs, num_slots, num_layers, HV, H, K, V, RL, device, with_track):
|
||||
state, rawv, rawk, g_ring, beta = pool
|
||||
gen = torch.Generator(device=device).manual_seed(bs)
|
||||
slots = torch.randperm(num_slots, device=device, generator=gen)[:bs].to(torch.int32)
|
||||
accept_lens = torch.randint(
|
||||
1, RL + 1, (bs,), device=device, dtype=torch.int32, generator=gen
|
||||
)
|
||||
if with_track:
|
||||
track_indices = torch.randperm(num_slots, device=device, generator=gen)[:bs].to(
|
||||
torch.int64
|
||||
)
|
||||
track_steps = torch.where(
|
||||
torch.rand(bs, device=device, generator=gen) < 0.25,
|
||||
accept_lens.to(torch.int64) - 1,
|
||||
torch.full((bs,), -1, dtype=torch.int64, device=device),
|
||||
)
|
||||
else:
|
||||
track_indices = None
|
||||
track_steps = None
|
||||
|
||||
def run():
|
||||
commit_gdn_replayssm_fold_all_layers(
|
||||
checkpoint_state=state,
|
||||
rawv_cache=rawv,
|
||||
rawk_cache=rawk,
|
||||
g_cache=g_ring,
|
||||
beta_cache=beta,
|
||||
ssm_state_indices=slots,
|
||||
accept_lens=accept_lens,
|
||||
max_cache_len=RL,
|
||||
num_k_heads=H,
|
||||
mamba_track_indices=track_indices,
|
||||
mamba_steps_to_track=track_steps,
|
||||
null_block_id=-1,
|
||||
)
|
||||
|
||||
ms = triton.testing.do_bench(run, warmup=25, rep=100)
|
||||
traffic_gb = bs * num_layers * HV * K * V * 4 * 2 / 1e9
|
||||
return ms * 1000, traffic_gb / (ms / 1000) if ms > 0 else 0.0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num-layers", type=int, default=45)
|
||||
parser.add_argument("--num-slots", type=int, default=475)
|
||||
parser.add_argument("--hv", type=int, default=16)
|
||||
parser.add_argument("--h", type=int, default=4)
|
||||
parser.add_argument("--k", type=int, default=128)
|
||||
parser.add_argument("--v", type=int, default=128)
|
||||
parser.add_argument("--ring-len", type=int, default=4)
|
||||
parser.add_argument("--batch-sizes", type=int, nargs="+", default=[1, 8, 32, 128])
|
||||
args = parser.parse_args()
|
||||
|
||||
device = "cuda"
|
||||
pool = _make_pool(
|
||||
args.num_layers,
|
||||
args.num_slots,
|
||||
args.hv,
|
||||
args.h,
|
||||
args.k,
|
||||
args.v,
|
||||
args.ring_len,
|
||||
device,
|
||||
)
|
||||
print(
|
||||
f"config: layers={args.num_layers} slots={args.num_slots} HV={args.hv} "
|
||||
f"H={args.h} K={args.k} V={args.v} ring_len={args.ring_len}"
|
||||
)
|
||||
print(f"{'bs':>4} {'track':>6} {'us/launch':>10} {'achieved GB/s':>14}")
|
||||
for bs in args.batch_sizes:
|
||||
for with_track in (False, True):
|
||||
us, gbs = _bench_bs(
|
||||
pool=pool,
|
||||
bs=bs,
|
||||
num_slots=args.num_slots,
|
||||
num_layers=args.num_layers,
|
||||
HV=args.hv,
|
||||
H=args.h,
|
||||
K=args.k,
|
||||
V=args.v,
|
||||
RL=args.ring_len,
|
||||
device=device,
|
||||
with_track=with_track,
|
||||
)
|
||||
print(f"{bs:>4} {str(with_track):>6} {us:>10.1f} {gbs:>14.0f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -50,7 +50,7 @@ def apply_log_scaling_tau(x: torch.Tensor, tau: torch.Tensor) -> torch.Tensor:
|
||||
# Vectorized JIT kernel (16B loads, one row divide per vector) --
|
||||
# bit-identical output (same fp32-mul + bf16-round), ~2-3x the
|
||||
# scalar triton kernel below at every size.
|
||||
from sglang.kernels.ops.attention.inkling_row_scale import row_scale_bf16
|
||||
from sglang.kernels.ops.elementwise.row_scale import row_scale_bf16
|
||||
|
||||
x2d = torch.as_strided(x, (rows, inner), (x.stride(0), 1))
|
||||
return row_scale_bf16(x2d, tau.reshape(rows).float()).view(x.shape)
|
||||
|
||||
+6
-2
@@ -5,12 +5,16 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table
|
||||
from sglang.kernels.ops.attention.minicpm_sala.get_block_table import (
|
||||
get_block_table,
|
||||
)
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "get_block_table":
|
||||
from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table
|
||||
from sglang.kernels.ops.attention.minicpm_sala.get_block_table import (
|
||||
get_block_table,
|
||||
)
|
||||
|
||||
globals()[name] = get_block_table
|
||||
return get_block_table
|
||||
@@ -19,7 +19,6 @@ for _mod, _fn in [
|
||||
("all_reduce_residual", "all_reduce_pull_res"),
|
||||
("all_reduce_residual", "all_reduce_pull_norm"),
|
||||
("gemm_ag", "gemm_ag_up_proj"),
|
||||
("gemm_ar", "o_proj_gemm_ar"),
|
||||
("sp_collective", "reduce_scatter_res"),
|
||||
("sp_collective", "reduce_scatter_pull"),
|
||||
("sp_collective", "all_gather"),
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
"""K3 fused o_proj GEMM + all-reduce for decode (bf16, TP row-parallel).
|
||||
|
||||
One entry point over ``csrc/kimi_k3/comm/gemm_ar.cuh``: a single kernel per
|
||||
rank computes the local ``x_r [M, K] @ W_r [7168, K]^T`` partial AND the
|
||||
cross-rank sum — the epilogue pushes finished tiles straight into a
|
||||
peer-mapped P2P comm region, one flag boundary, then a tile-local reduce
|
||||
writes the fully reduced ``out [M, 7168]`` on every rank. Replaces the
|
||||
o_proj GEMM + NCCL all-reduce pair with one launch (see GEMM_AR_README.md).
|
||||
|
||||
Contracts:
|
||||
|
||||
* bf16 only; ``out = sum_r bf16(x_r @ W_r^T)`` (partials round to bf16
|
||||
pre-sum — same numerics as the unfused bf16 GEMM + ring AR).
|
||||
* M in [1, 512]; internally rounded up to a tuned cell {8, 16, 32, 64,
|
||||
128, 256, 512}. ``out`` is allocated with ``cell`` rows and sliced.
|
||||
* SM100+ with full NVLink P2P (fabric/MNNVL across nodes); perf-tuned on
|
||||
GB300 (sm_103a).
|
||||
* CUDA-graph compatible: the per-cell launch epoch lives in device memory
|
||||
(read at kernel entry, bumped by a trailing kernel), so replays advance
|
||||
it naturally. Each dispatch cell owns its own flag-ring family — no
|
||||
host-side ring reset, ever.
|
||||
* All TP ranks must call :func:`o_proj_gemm_ar` with the same M in
|
||||
lockstep (same stream order of cells on every rank).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
N = 7168 # K3 hidden size (OPROJ_N compile-time default in gemm_ar.cuh)
|
||||
MAX_TOKENS = 512 # kMMax
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_module(k: int, world_size: int) -> Module:
|
||||
args = make_cpp_args(
|
||||
k,
|
||||
world_size,
|
||||
is_arch_support_pdl(),
|
||||
)
|
||||
cls = f"GemmArKernel<{args}>"
|
||||
return load_jit(
|
||||
"kimi_k3_gemm_ar",
|
||||
*args,
|
||||
cuda_files=["kimi_k3/comm/gemm_ar.cuh"],
|
||||
cuda_wrappers=[
|
||||
("run", f"{cls}::run"),
|
||||
("set_bases", f"{cls}::set_bases"),
|
||||
("region_nbytes", f"{cls}::region_nbytes"),
|
||||
("gather_words", f"{cls}::gather_words"),
|
||||
("num_fams", f"{cls}::num_fams"),
|
||||
],
|
||||
extra_cuda_cflags=["-O3"],
|
||||
extra_dependencies=["cutlass"],
|
||||
)
|
||||
|
||||
|
||||
class _State(NamedTuple):
|
||||
world_size: int
|
||||
rank: int
|
||||
region: tuple # (slab tensor, peer buffer views) — keeps mappings alive
|
||||
uc_bases: torch.Tensor # [R] int64 CPU: per-rank UC VAs of the region
|
||||
gather: torch.Tensor # [kFams * 2 * kRing] int32 CUDA, device-local
|
||||
epochs: torch.Tensor # [kFams] int32 CUDA: device-resident CTA ticket counters
|
||||
|
||||
|
||||
_STATE: Optional[_State] = None
|
||||
|
||||
|
||||
def init(
|
||||
*,
|
||||
world_size: int,
|
||||
rank: int,
|
||||
group: torch.distributed.ProcessGroup,
|
||||
k: int,
|
||||
) -> None:
|
||||
"""Allocate + rendezvous the P2P comm region (collective; call once from
|
||||
every TP rank, BEFORE any CUDA-graph capture). ``group`` is the TP CPU
|
||||
(gloo) group used for the symm-mem rendezvous."""
|
||||
global _STATE
|
||||
if _STATE is not None:
|
||||
return
|
||||
# the empty_strided_p2p + get_buffer path is the one that exchanges
|
||||
# fabric handles and maps every peer (incl. cross-node MNNVL) into this
|
||||
# process — the mem-pool rendezvous(tensor, group_name) API leaves
|
||||
# remote-node (and sometimes even local) peers unmapped.
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
mod = _jit_module(k, world_size)
|
||||
nbytes = int(mod.region_nbytes())
|
||||
device = torch.device("cuda", torch.cuda.current_device())
|
||||
if torch.__version__ < "2.11.0":
|
||||
import torch.distributed._symmetric_memory as torch_symm_mem
|
||||
|
||||
torch_symm_mem.enable_symm_mem_for_group(group.group_name)
|
||||
region = _SymmetricMemory.empty_strided_p2p(
|
||||
(nbytes,), [1], torch.uint8, device, group.group_name
|
||||
)
|
||||
symm = _SymmetricMemory.rendezvous(region)
|
||||
region.zero_()
|
||||
torch.cuda.synchronize()
|
||||
torch.distributed.barrier(group=group)
|
||||
# keep the peer buffer tensors alive alongside the region
|
||||
peer_bufs = [symm.get_buffer(r, [nbytes], torch.uint8) for r in range(world_size)]
|
||||
ptrs = [t.data_ptr() for t in peer_bufs]
|
||||
import logging
|
||||
|
||||
logging.getLogger(__name__).info(
|
||||
"gemm_ar comm region: rank=%d nbytes=%d uc_bases=%s",
|
||||
rank,
|
||||
nbytes,
|
||||
[hex(p) for p in ptrs],
|
||||
)
|
||||
assert all(p != 0 for p in ptrs), f"gemm_ar: null peer pointers {ptrs}"
|
||||
# explicit cpu: model build may run under a cuda default-device context,
|
||||
# and a silently-cuda tensor here means the host-side deref in set_bases
|
||||
# reads a device pointer (segfault)
|
||||
uc_bases = torch.tensor(ptrs, dtype=torch.int64, device="cpu")
|
||||
gather = torch.zeros(int(mod.gather_words()), dtype=torch.int32, device=device)
|
||||
epochs = torch.zeros(int(mod.num_fams()), dtype=torch.int32, device=device)
|
||||
torch.cuda.synchronize()
|
||||
_STATE = _State(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
region=(region, peer_bufs),
|
||||
uc_bases=uc_bases,
|
||||
gather=gather,
|
||||
epochs=epochs,
|
||||
)
|
||||
|
||||
|
||||
def initialized() -> bool:
|
||||
return _STATE is not None
|
||||
|
||||
|
||||
@cache_once
|
||||
def _module_with_bases(k: int, world_size: int) -> Module:
|
||||
"""The per-K JIT module with the comm-region base addresses stashed
|
||||
host-side (per-call CPU-tensor derefs from inside the custom op segfault
|
||||
under the sglang scheduler, so the module holds them in a static)."""
|
||||
state = _STATE
|
||||
assert state is not None
|
||||
mod = _jit_module(k, world_size)
|
||||
mod.set_bases(state.uc_bases)
|
||||
return mod
|
||||
|
||||
|
||||
def fits(x: torch.Tensor) -> bool:
|
||||
"""Whether this o_proj input can take the fused GEMM+AR path."""
|
||||
return (
|
||||
_STATE is not None
|
||||
and x.dim() == 2
|
||||
and x.dtype == torch.bfloat16
|
||||
and 0 < x.shape[0] <= MAX_TOKENS
|
||||
and x.stride(1) == 1
|
||||
and x.stride(0) == x.shape[1]
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out", "epochs"])
|
||||
def _gemm_ar_op(
|
||||
k: int,
|
||||
world_size: int,
|
||||
out: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
gather: torch.Tensor,
|
||||
epochs: torch.Tensor,
|
||||
my_rank: int,
|
||||
) -> None:
|
||||
_module_with_bases(k, world_size).run(out, x, weight, gather, epochs, my_rank)
|
||||
|
||||
|
||||
def _cell_of(m: int) -> int:
|
||||
for c in (8, 16, 32, 64, 128, 256, 512):
|
||||
if m <= c:
|
||||
return c
|
||||
raise ValueError(f"gemm_ar: M={m} outside [1, {MAX_TOKENS}]")
|
||||
|
||||
|
||||
def o_proj_gemm_ar(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Fully reduced ``sum_r x_r @ weight_r^T`` on every rank, one kernel.
|
||||
|
||||
``x`` is the TP-local [M, K] o_proj input, ``weight`` the TP-local
|
||||
[7168, K] o_proj weight shard. Caller checked :func:`fits`; all ranks
|
||||
call in lockstep with the same M.
|
||||
"""
|
||||
state = _STATE
|
||||
assert state is not None
|
||||
m = x.shape[0]
|
||||
cell = _cell_of(m)
|
||||
out = torch.empty((cell, N), dtype=torch.bfloat16, device=x.device)
|
||||
_gemm_ar_op(
|
||||
weight.shape[1],
|
||||
state.world_size,
|
||||
out,
|
||||
x,
|
||||
weight,
|
||||
state.gather,
|
||||
state.epochs,
|
||||
state.rank,
|
||||
)
|
||||
return out[:m]
|
||||
@@ -8,4 +8,34 @@ Individual functions register (or are imported) under the functional op id
|
||||
they logically belong to.
|
||||
"""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
# Public entry points inventoried by logical operator group (RFC #29630).
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="elementwise.fused_qwen4_gate_value",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_value",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="elementwise.fused_qwen4_gate_reduce",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_reduce",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="elementwise.row_scale_bf16",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.elementwise.row_scale:row_scale_bf16",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
_QWEN4_HC_COUNT = 4
|
||||
_QWEN4_HIDDEN_SIZE = 2560
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _round_bf16_to_fp32(value):
|
||||
"""RNE-round fp32 to BF16 precision while retaining an fp32 register."""
|
||||
|
||||
bits = value.to(tl.int32, bitcast=True)
|
||||
rounding_bias = 0x7FFF + ((bits >> 16) & 1)
|
||||
rounded_bits = (bits + rounding_bias) & -65536
|
||||
return rounded_bits.to(tl.float32, bitcast=True)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qwen4_gate_value_kernel(
|
||||
gate_ptr,
|
||||
value_ptr,
|
||||
output_ptr,
|
||||
num_tokens,
|
||||
HC_COUNT: tl.constexpr,
|
||||
HIDDEN_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_group = tl.program_id(0)
|
||||
token = token_group // HC_COUNT
|
||||
hidden = tl.arange(0, BLOCK_SIZE)
|
||||
mask = (token < num_tokens) & (hidden < HIDDEN_SIZE)
|
||||
|
||||
# `gate` arrives already rounded to bf16 by the eager multiply/reduce/divide;
|
||||
# every remaining eager bf16 rounding boundary is reproduced below.
|
||||
gate = tl.load(gate_ptr + token_group).to(tl.float32)
|
||||
magnitude = tl.maximum(tl.abs(gate), 1.0e-6)
|
||||
root = _round_bf16_to_fp32(tl.sqrt(magnitude))
|
||||
sign = tl.where(gate > 0.0, 1.0, tl.where(gate < 0.0, -1.0, 0.0))
|
||||
transformed = _round_bf16_to_fp32(root * sign)
|
||||
activated = _round_bf16_to_fp32(tl.sigmoid(transformed))
|
||||
|
||||
value = tl.load(value_ptr + token * HIDDEN_SIZE + hidden, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
output_offset = token_group * HIDDEN_SIZE + hidden
|
||||
tl.store(output_ptr + output_offset, activated * value, mask=mask)
|
||||
|
||||
|
||||
def can_fuse_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> bool:
|
||||
"""Return whether inputs match Qwen4's fixed BF16 gate/value contract."""
|
||||
|
||||
return (
|
||||
gate.is_cuda
|
||||
and gate.dtype == torch.bfloat16
|
||||
and gate.dim() == 3
|
||||
and gate.shape[1:] == (_QWEN4_HC_COUNT, 1)
|
||||
and gate.is_contiguous()
|
||||
and value.is_cuda
|
||||
and value.dtype == gate.dtype
|
||||
and value.shape == (gate.shape[0], _QWEN4_HIDDEN_SIZE)
|
||||
and value.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def fused_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply Qwen4's post-reduction gate and value broadcast in one kernel."""
|
||||
|
||||
if not can_fuse_qwen4_gate_value(gate, value):
|
||||
raise ValueError("unsupported input for fused Qwen4 PLE gate/value")
|
||||
output = torch.empty(
|
||||
(gate.shape[0], _QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE),
|
||||
dtype=value.dtype,
|
||||
device=value.device,
|
||||
)
|
||||
if gate.shape[0]:
|
||||
_qwen4_gate_value_kernel[(gate.shape[0] * _QWEN4_HC_COUNT,)](
|
||||
gate,
|
||||
value,
|
||||
output,
|
||||
gate.shape[0],
|
||||
HC_COUNT=_QWEN4_HC_COUNT,
|
||||
HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE,
|
||||
BLOCK_SIZE=4096,
|
||||
num_warps=8,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qwen4_gate_reduce_kernel(
|
||||
key_ptr,
|
||||
query_ptr,
|
||||
value_ptr,
|
||||
output_ptr,
|
||||
HIDDEN_SIZE: tl.constexpr,
|
||||
HC_COUNT: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
group = tl.program_id(0)
|
||||
hidden = tl.arange(0, BLOCK_SIZE)
|
||||
mask = hidden < HIDDEN_SIZE
|
||||
key = tl.load(key_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(tl.float32)
|
||||
query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
product = _round_bf16_to_fp32(key * query)
|
||||
gate = _round_bf16_to_fp32(tl.sum(product, 0))
|
||||
gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5))
|
||||
magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6))
|
||||
root = _round_bf16_to_fp32(tl.sqrt(magnitude))
|
||||
sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0))
|
||||
activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign)))
|
||||
value = tl.load(
|
||||
value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0
|
||||
).to(tl.float32)
|
||||
tl.store(output_ptr + group * HIDDEN_SIZE + hidden, activated * value, mask)
|
||||
|
||||
|
||||
def can_fuse_qwen4_gate_reduce(key, query, value):
|
||||
return (
|
||||
key.is_cuda
|
||||
and key.dtype == torch.bfloat16
|
||||
and key.ndim == 3
|
||||
and key.shape[1:] == (_QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE)
|
||||
and key.is_contiguous()
|
||||
and query.shape == key.shape
|
||||
and query.dtype == key.dtype
|
||||
and query.device == key.device
|
||||
and query.is_contiguous()
|
||||
and value.shape == (key.shape[0], _QWEN4_HIDDEN_SIZE)
|
||||
and value.dtype == key.dtype
|
||||
and value.device == key.device
|
||||
and value.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def fused_qwen4_gate_reduce(key, query, value):
|
||||
if not can_fuse_qwen4_gate_reduce(key, query, value):
|
||||
raise ValueError("unsupported input for Qwen4 PLE gate reduction")
|
||||
output = torch.empty_like(key)
|
||||
if key.shape[0]:
|
||||
_qwen4_gate_reduce_kernel[(key.shape[0] * _QWEN4_HC_COUNT,)](
|
||||
key,
|
||||
query,
|
||||
value,
|
||||
output,
|
||||
HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE,
|
||||
HC_COUNT=_QWEN4_HC_COUNT,
|
||||
BLOCK_SIZE=triton.next_power_of_2(_QWEN4_HIDDEN_SIZE),
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return output
|
||||
-12
@@ -45,15 +45,3 @@ def row_scale_bf16(
|
||||
module = _jit_row_scale_module(is_arch_support_pdl())
|
||||
module.run(x, tau, out)
|
||||
return out
|
||||
|
||||
|
||||
def row_compact_bf16(x: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""Contiguous copy of row-strided ``x`` ([rows, inner] bf16, inner
|
||||
contiguous, inner % 8 == 0, 16B-aligned rows) -- the tau-less flavor of
|
||||
``row_scale_bf16``. Beats the TensorIterator strided copy that einsum's
|
||||
reshape would otherwise run on such inputs."""
|
||||
if out is None:
|
||||
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
module = _jit_row_scale_module(is_arch_support_pdl())
|
||||
module.run_compact(x, out)
|
||||
return out
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Embedding kernels."""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import KernelBackend, KernelSpec
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
@@ -55,3 +55,14 @@ register_kernel(
|
||||
)
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
# Public entry points inventoried by logical operator group (RFC #29630).
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="embeddings.fused_qwen4_ngram_hash",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.embeddings.qwen4_ngram:fused_qwen4_ngram_hash",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
_QWEN4_NGRAM_SIZE = 3
|
||||
_QWEN4_HEADS_PER_NGRAM = 8
|
||||
_QWEN4_NGRAM_HEADS = 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qwen4_ngram_hash_kernel(
|
||||
contexts_ptr,
|
||||
multipliers_ptr,
|
||||
vocab_sizes_ptr,
|
||||
offsets_ptr,
|
||||
output_ptr,
|
||||
num_outputs,
|
||||
eos_token_id,
|
||||
NGRAM_SIZE: tl.constexpr,
|
||||
HEADS_PER_NGRAM: tl.constexpr,
|
||||
NGRAM_HEADS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
output_idx = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = output_idx < num_outputs
|
||||
token_idx = output_idx // NGRAM_HEADS
|
||||
head_idx = output_idx % NGRAM_HEADS
|
||||
context_base = token_idx * NGRAM_SIZE
|
||||
|
||||
token_0 = tl.load(contexts_ptr + context_base, mask=mask, other=0)
|
||||
token_1 = tl.load(contexts_ptr + context_base + 1, mask=mask, other=0)
|
||||
token_2 = tl.load(contexts_ptr + context_base + 2, mask=mask, other=0)
|
||||
multiplier_0 = tl.load(multipliers_ptr)
|
||||
multiplier_1 = tl.load(multipliers_ptr + 1)
|
||||
multiplier_2 = tl.load(multipliers_ptr + 2)
|
||||
|
||||
# Only the final position of each three-token context is materialized.
|
||||
previous_1 = tl.where(token_1 == eos_token_id, eos_token_id, token_1)
|
||||
previous_2 = tl.where(
|
||||
(token_0 == eos_token_id) | (token_1 == eos_token_id),
|
||||
eos_token_id,
|
||||
token_0,
|
||||
)
|
||||
mixed = (token_2 * multiplier_0) ^ (previous_1 * multiplier_1)
|
||||
mixed_3 = mixed ^ (previous_2 * multiplier_2)
|
||||
mixed = tl.where(head_idx < HEADS_PER_NGRAM, mixed, mixed_3)
|
||||
|
||||
vocab_size = tl.load(vocab_sizes_ptr + head_idx, mask=mask, other=1)
|
||||
offset = tl.load(offsets_ptr + head_idx, mask=mask, other=0)
|
||||
tl.store(output_ptr + output_idx, mixed % vocab_size + offset, mask=mask)
|
||||
|
||||
|
||||
def can_fuse_qwen4_ngram_hash(
|
||||
contexts: torch.Tensor,
|
||||
multipliers: torch.Tensor,
|
||||
vocab_sizes: torch.Tensor,
|
||||
offsets: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Return whether inputs match the fixed Qwen4 PLE hash contract."""
|
||||
|
||||
return (
|
||||
contexts.is_cuda
|
||||
and contexts.dtype == torch.long
|
||||
and contexts.dim() == 2
|
||||
and contexts.shape[1] == _QWEN4_NGRAM_SIZE
|
||||
and contexts.is_contiguous()
|
||||
and multipliers.is_cuda
|
||||
and multipliers.dtype == torch.long
|
||||
and multipliers.numel() == _QWEN4_NGRAM_SIZE
|
||||
and vocab_sizes.is_cuda
|
||||
and vocab_sizes.dtype == torch.long
|
||||
and vocab_sizes.numel() == _QWEN4_NGRAM_HEADS
|
||||
and offsets.is_cuda
|
||||
and offsets.dtype == torch.long
|
||||
and offsets.numel() == _QWEN4_NGRAM_HEADS
|
||||
)
|
||||
|
||||
|
||||
def fused_qwen4_ngram_hash(
|
||||
contexts: torch.Tensor,
|
||||
multipliers: torch.Tensor,
|
||||
vocab_sizes: torch.Tensor,
|
||||
offsets: torch.Tensor,
|
||||
eos_token_id: int,
|
||||
) -> torch.Tensor:
|
||||
"""Return the 16 Qwen4 PLE N-gram IDs in one kernel launch."""
|
||||
|
||||
if not can_fuse_qwen4_ngram_hash(contexts, multipliers, vocab_sizes, offsets):
|
||||
raise ValueError("unsupported input for fused Qwen4 PLE N-gram hash")
|
||||
output = torch.empty(
|
||||
(contexts.shape[0], _QWEN4_NGRAM_HEADS),
|
||||
dtype=torch.long,
|
||||
device=contexts.device,
|
||||
)
|
||||
num_outputs = output.numel()
|
||||
if num_outputs:
|
||||
block_size = 256
|
||||
_qwen4_ngram_hash_kernel[(triton.cdiv(num_outputs, block_size),)](
|
||||
contexts,
|
||||
multipliers,
|
||||
vocab_sizes,
|
||||
offsets,
|
||||
output,
|
||||
num_outputs,
|
||||
eos_token_id,
|
||||
NGRAM_SIZE=_QWEN4_NGRAM_SIZE,
|
||||
HEADS_PER_NGRAM=_QWEN4_HEADS_PER_NGRAM,
|
||||
NGRAM_HEADS=_QWEN4_NGRAM_HEADS,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=4,
|
||||
)
|
||||
return output
|
||||
@@ -422,3 +422,140 @@ for _mod, _fn in _TRITON_KERNELS:
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
|
||||
# Public entry points inventoried by logical operator group (RFC #29630).
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.linear_bf16_fp32",
|
||||
backend=KernelBackend.TORCH,
|
||||
target="sglang.kernels.ops.gemm.bf16_fp32:_linear_bf16_fp32_cublas",
|
||||
description=(
|
||||
"Torch/cuBLAS implementation. The direct linear_bf16_fp32 entry also "
|
||||
"retains its environment-controlled HPC-Ops, AITER and DeepGEMM dispatch."
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.wo_a_bf16_gemv",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_gemv",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.wo_a_bf16_small_batch",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.wo_a_bf16_small_batch_mxfp8",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch_mxfp8",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.hopper_bf16_gemv",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.hopper_bf16_gemv:hopper_bf16_gemv",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 0))}
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.fp8_blockwise_scaled_mm",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.fp8_blockwise_gemm:fp8_blockwise_scaled_mm",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))}
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.sm120_fp8_gemv",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.sm120_fp8_gemv:sm120_fp8_gemv",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))}
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.gptq_marlin_gemm",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.gptq_marlin:gptq_marlin_gemm",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.rel_proj_small_t",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.inkling_rel_proj:rel_proj_small_t",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.fused_hc_mix",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.hc_mix:fused_hc_mix",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.w8a8_block_fp8_matmul",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.w8a8_block_fp8_matmul_triton",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul_triton",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.w8a8_block_fp8_matmul_deepgemm",
|
||||
backend=KernelBackend.DEEPGEMM,
|
||||
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul_deepgemm",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.w8a8_mxfp8_matmul_deepgemm",
|
||||
backend=KernelBackend.DEEPGEMM,
|
||||
target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_mxfp8_matmul_deepgemm",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.triton_scaled_mm",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.fp8_kernel:triton_scaled_mm",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.w8a8_block_int8_matmul",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.gemm.int8_kernel:w8a8_block_int8_matmul",
|
||||
)
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user