[Refactor] kernels/diffusion/tests: Prune dead code and fix tests that never ran (#44690)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Xiaoyu Zhang
2026-10-19 08:55:47 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 59fec011dd
commit 329072d99a
414 changed files with 3522 additions and 26178 deletions
+10 -7
View File
@@ -4,8 +4,8 @@ This package is the public in-tree import surface for callable kernels, per
[RFC #29630](https://github.com/sgl-project/sglang/issues/29630).
```python
from sglang.kernels.ops.layernorm import rmsnorm
from sglang.kernels.ops.activation import silu_and_mul
from sglang.kernels.ops.gemm import fp8_scaled_mm
from sglang.kernels.ops.kvcache import reshape_and_cache_flash
```
@@ -13,8 +13,8 @@ from sglang.kernels.ops.kvcache import reshape_and_cache_flash
```
sglang/kernels/
spec.py # KernelSpec, KernelBackend, FormatSignature,
# CapabilityRequirement, PlatformInfo
spec.py # KernelSpec, KernelBackend, CapabilityRequirement,
# PlatformInfo
registry.py # process-wide KernelRegistry + register_kernel()
selector.py # heuristic select_kernel() and cached get_kernel()
fused_op.py # BaseFusedOp: per-operator multi-backend contract
@@ -94,7 +94,7 @@ explicitly, e.g.:
```python
from sglang.kernels import select_kernel, KernelBackend
jit_rmsnorm = select_kernel("layernorm.rmsnorm", backend=KernelBackend.JIT).load()
jit_moe_align = select_kernel("moe.moe_align_block_size", backend=KernelBackend.JIT).load()
```
## `BaseFusedOp` — the unified operator contract
@@ -128,8 +128,11 @@ Dispatch priority, highest first: explicit `forward(..., backend=...)` →
global forced backend (`SGLANG_FORCE_FUSED_OP_BACKEND`) → OOT platform
override → declared optimized kernel backends by `priority` (filtered by
`backend_eligible()`, a `CapabilityRequirement`-vs-`PlatformInfo` check
extensible with per-call shape/dtype gates) → platform-specific forward →
`forward_native`. The static part of the decision is resolved once and cached
extensible with per-call applicability gates) → platform-specific forward →
`forward_native`. A gate only answers whether the kernel covers these operands,
so that falling through is a correct way to compute the same result; operands
that violate the op's contract (mismatched shapes or devices) are a caller bug
that the backend's launcher must raise, not a reason to fall back. The static part of the decision is resolved once and cached
on the instance, so the hot path stays a single indirect call.
`BaseFusedOp` also owns the torch.compile mode protocol
@@ -148,7 +151,7 @@ What this buys (see the
- **Unified correctness testing** — a generic harness enumerates
`available_backends()` and asserts each one matches `forward_native`
(`test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py`); new
(`test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py`); new
backends are picked up automatically.
- **One-switch debugging** — `SGLANG_FORCE_FUSED_OP_BACKEND=torch` (or
`set_fused_op_backend(KernelBackend.TORCH)`) flips *every* fused op to its
+1 -3
View File
@@ -3,8 +3,8 @@
SGLang runtime code and tests should import callable kernels from
``sglang.kernels.ops.<group>``, e.g.::
from sglang.kernels.ops.layernorm import rmsnorm
from sglang.kernels.ops.activation import silu_and_mul
from sglang.kernels.ops.gemm import fp8_scaled_mm
from sglang.kernels.ops.kvcache import reshape_and_cache_flash
Implementations still live in ``sglang.kernels.jit`` (JIT CUDA), the
@@ -39,7 +39,6 @@ from sglang.kernels.selector import get_kernel, select_kernel
from sglang.kernels.spec import (
CapabilityRequirement,
DeviceType,
FormatSignature,
KernelBackend,
KernelSpec,
PlatformInfo,
@@ -57,7 +56,6 @@ __all__ = [
"CapabilityRequirement",
"DeviceType",
"capabilities_satisfied",
"FormatSignature",
"FusedOpTraceRecord",
"KernelBackend",
"KernelRegistry",
@@ -1,75 +0,0 @@
"""Benchmark for DeepSeek-V4 fused norm + RoPE kernels."""
import itertools
import sgl_kernel
import torch
import triton
import triton.testing
try:
from sglang.utils import is_in_ci
IS_CI = is_in_ci()
except ImportError:
IS_CI = False
batch_sizes = [1] if IS_CI else [1, 4, 16, 64, 256]
num_heads_list = [8] if IS_CI else [8, 16, 64]
head_dims = [192] if IS_CI else [128, 192]
configs = list(itertools.product(batch_sizes, num_heads_list, head_dims))
def torch_rmsnorm_rope(
q: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, eps: float
) -> torch.Tensor:
"""Naive PyTorch reference: RMSNorm + RoPE."""
rms = torch.sqrt(q.float().pow(2).mean(dim=-1, keepdim=True) + eps)
q_normed = (q.float() / rms).to(q.dtype)
return q_normed
@triton.testing.perf_report(
triton.testing.Benchmark(
x_names=["batch_size", "num_heads", "head_dim"],
x_vals=configs,
line_arg="provider",
line_vals=["sglang", "torch"],
line_names=["SGL Kernel", "PyTorch"],
styles=[("green", "-"), ("red", "--")],
ylabel="µs (median)",
plot_name="dsv4-q-norm-rope-performance",
args={},
)
)
def benchmark_q_norm_rope(batch_size, num_heads, head_dim, provider):
torch.manual_seed(42)
eps = 1e-6
max_pos = 8192
rope_dim = 64
q_input = torch.randn(
batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda"
)
q_output = torch.empty_like(q_input)
freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda")
positions = torch.randint(
0, max_pos, (batch_size,), dtype=torch.int32, device="cuda"
)
if provider == "sglang":
fn = lambda: sgl_kernel.dsv4_fused_q_norm_rope(
q_input, freqs_cis, positions, eps, q_output
)
else:
fn = lambda: torch_rmsnorm_rope(q_input, freqs_cis, positions, eps)
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
fn, quantiles=[0.5, 0.2, 0.8]
)
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
if __name__ == "__main__":
benchmark_q_norm_rope.run(print_data=True)
@@ -1,85 +0,0 @@
import torch
import triton
from sglang.kernels.ops.moe.ep_moe_kernels import post_reorder_triton_kernel
from sglang.utils import is_in_ci
IS_CI = is_in_ci()
# CI environment uses simplified parameters
if IS_CI:
batch_sizes = [64, 128] # Only test 2 values in CI
else:
batch_sizes = [64, 128, 256, 512, 640, 768, 1024, 2048, 4096]
configs = [(bs,) for bs in batch_sizes]
@triton.testing.perf_report(
triton.testing.Benchmark(
x_names=["batch_size"],
x_vals=[list(_) for _ in configs],
line_arg="provider",
line_vals=["triton"],
line_names=["Triton Kernel"],
styles=[("orange", "-")],
ylabel="us",
plot_name="ep-moe-post-reorder-performance",
args={},
)
)
def benchmark(batch_size, provider):
dtype = torch.bfloat16
device = torch.device("cuda")
hidden_size, topk, start_expert_id, end_expert_id, block_size = 4096, 8, 0, 255, 512
def alloc_tensors():
down_output = torch.randn(
batch_size * topk, hidden_size, dtype=dtype, device=device
)
output = torch.zeros(batch_size, hidden_size, dtype=dtype, device=device)
src2dst = torch.randint(
0, batch_size * topk, (batch_size, topk), dtype=torch.int32, device=device
)
topk_ids = torch.randint(
start_expert_id,
end_expert_id + 1,
(batch_size, topk),
dtype=torch.int32,
device=device,
)
topk_weights = torch.rand(batch_size, topk, dtype=dtype, device=device)
return down_output, output, src2dst, topk_ids, topk_weights
quantiles = [0.5, 0.2, 0.8]
if provider == "triton":
d_out, out, s2d, tk_ids, tk_weights = alloc_tensors()
def run_triton():
post_reorder_triton_kernel[(batch_size,)](
d_out.view(-1),
out.view(-1),
s2d.view(-1),
tk_ids.view(-1),
tk_weights.view(-1),
start_expert_id,
end_expert_id,
topk,
hidden_size,
0,
block_size,
)
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
run_triton, quantiles=quantiles
)
else:
raise ValueError(f"Unknown provider: {provider}")
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
if __name__ == "__main__":
benchmark.run(print_data=True)
@@ -5,14 +5,12 @@ import torch
import triton
from sgl_kernel.test_utils import create_per_token_group_quant_test_data
from sglang.kernels.ops.quantization.fp8_kernel import (
per_token_group_quant_8bit as triton_per_token_group_quant_8bit,
)
from sglang.kernels.ops.quantization.fp8_kernel import (
sglang_per_token_group_quant_8bit,
)
from sglang.srt.utils import is_hip
from sglang.srt.utils.bench_utils import bench_kineto
from sglang.test.kernels.per_token_group_quant import (
sglang_per_token_group_quant_8bit,
triton_per_token_group_quant_8bit,
)
from sglang.utils import is_in_ci
IS_CI = is_in_ci()
@@ -178,16 +178,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.def("apply_shuffle_mul_sum(Tensor input, Tensor output, Tensor permutation, Tensor? factors) -> ()");
m.impl("apply_shuffle_mul_sum", torch::kCUDA, &apply_shuffle_mul_sum);
// DeepSeek-V4 fused norm + rope
m.def(
"dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()");
m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope);
m.def(
"dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, "
"Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()");
m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla);
// DeepSeek-V4 fused indexer rope + hadamard + quant
m.def(
"dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, "
"Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()");
@@ -52,15 +52,6 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) {
"page_indices, int page_size, Tensor!? raw_indices, bool sort_output=False) -> ()");
m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512);
m.def(
"dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()");
m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope);
m.def(
"dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, "
"Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()");
m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla);
m.def(
"dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, "
"Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()");
@@ -13,9 +13,8 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// DeepSeek-V4 fused norm + RoPE kernels, ported from JIT kernel
// python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh
// to sgl-kernel AOT compilation with CUDA + HIP (ROCm) support.
// DeepSeek-V4 indexer Q RoPE + Hadamard + FP8 quant kernel, ported from the JIT kernel in
// python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh for the ROCm build.
#ifndef USE_ROCM
#include <cuda_bf16.h>
@@ -59,14 +58,6 @@ using fp8x2_e4m3_t = uint16_t;
static constexpr uint32_t kWarpSize = 32;
template <uint32_t kNumThreads = kWarpSize>
__device__ __forceinline__ float warp_reduce_sum(float val) {
#pragma unroll
for (uint32_t mask = kNumThreads / 2; mask > 0; mask >>= 1)
val += SGLANG_SHFL_XOR_SYNC(FULL_MASK, val, mask);
return val;
}
__device__ __forceinline__ float warp_reduce_max(float val) {
#pragma unroll
for (uint32_t mask = kWarpSize / 2; mask > 0; mask >>= 1)
@@ -96,31 +87,10 @@ __device__ __forceinline__ float bf16_to_float(bf16_t v) {
return __bfloat162float(v);
}
__device__ __forceinline__ bf16_t float_to_bf16(float v) {
#ifndef USE_ROCM
return __float2bfloat16_rn(v);
#else
return __float2bfloat16(v);
#endif
}
// ============================================================================
// FP8 E4M3 helpers (portable CUDA + HIP)
// ============================================================================
// UE8M0 scale: round a positive float to the nearest power-of-two
// representable in UE8M0 (unsigned 8-bit exponent, no mantissa).
__device__ __forceinline__ int32_t cast_to_ue8m0(float x) {
uint32_t u = __float_as_uint(x);
int32_t exp = static_cast<int32_t>((u >> 23) & 0xFFu);
uint32_t mant = u & 0x7FFFFFu;
return exp + (mant != 0);
}
__device__ __forceinline__ float inv_scale_ue8m0(int32_t exp) {
return __uint_as_float(static_cast<uint32_t>((127 + 127 - exp) << 23));
}
static constexpr float kFP8Max = 448.0f;
#ifndef USE_ROCM
@@ -188,236 +158,13 @@ __device__ __forceinline__ fp8x2_e4m3_t pack_fp8(float x, float y) {
}
#endif
// ============================================================================
// Kernel 1: Fused Q Norm + RoPE
// warp-per-(token, head), rmsnorm-self (no weight) + RoPE + write to q_out.
// ============================================================================
namespace {
constexpr uint32_t kFusedQBlockSize = 128;
constexpr uint32_t kFusedQNumWarps = kFusedQBlockSize / kWarpSize;
constexpr uint32_t kFusedKBlockSize = 256;
constexpr uint32_t kFusedKNumWarps = kFusedKBlockSize / kWarpSize;
struct FusedQNormRopeParams {
const void* __restrict__ q_input;
void* __restrict__ q_output;
const float* __restrict__ freqs_cis;
const int32_t* __restrict__ positions;
int64_t q_input_stride_batch;
int64_t q_output_stride_batch;
uint32_t batch_size;
uint32_t num_q_heads;
float eps;
};
// Compute the largest power-of-2 vec size that divides both kHeadDim and
// fits in 16 bytes, while also dividing kRopeDim.
template <int64_t kHeadDim, int64_t kRopeDim>
struct QKernelTraits {
static constexpr int64_t kMaxVecSize = 16 / sizeof(bf16_t); // 8
// Use kRopeDim/kWarpSize (=2 for kRopeDim=64) as the vec size.
// This guarantees kRopeDim % kVecSize == 0 and works for all head dims
// that are multiples of kWarpSize*kVecSize.
static constexpr int64_t kVecSize = kRopeDim / kWarpSize; // 2
static constexpr int64_t kLocalSize = kHeadDim / (kWarpSize * kVecSize);
static constexpr uint32_t kRopeSize = kRopeDim / kVecSize;
static_assert(kHeadDim % (kWarpSize * kVecSize) == 0);
static_assert(kRopeDim % kVecSize == 0);
static_assert(kRopeDim == kWarpSize * 2, "1 (real, imag) pair per lane");
};
template <int64_t kHeadDim, int64_t kRopeDim>
__global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_norm_rope_kernel(
const __grid_constant__ FusedQNormRopeParams params) {
using Traits = QKernelTraits<kHeadDim, kRopeDim>;
constexpr int64_t kVecSize = Traits::kVecSize;
constexpr int64_t kLocalSize = Traits::kLocalSize;
constexpr uint32_t kRopeSize = Traits::kRopeSize;
using Storage = AlignedVec<bf16_t, kVecSize>;
using Float2 = AlignedVec<float, 2>;
const auto warp_id = threadIdx.x / kWarpSize;
const auto lane_id = threadIdx.x % kWarpSize;
const auto work_id = blockIdx.x * kFusedQNumWarps + warp_id;
const uint32_t total_works = params.batch_size * params.num_q_heads;
if (work_id >= total_works) return;
const uint32_t batch_id = work_id / params.num_q_heads;
const uint32_t head_id = work_id % params.num_q_heads;
const auto input_ptr =
static_cast<const bf16_t*>(params.q_input) + batch_id * params.q_input_stride_batch + head_id * kHeadDim;
const auto output_ptr =
static_cast<bf16_t*>(params.q_output) + batch_id * params.q_output_stride_batch + head_id * kHeadDim;
const auto position = params.positions[batch_id];
__shared__ Storage s_rope[kFusedQNumWarps][kRopeSize];
// Prefetch freq pair.
Float2 freq;
freq.load(params.freqs_cis + position * kRopeDim, lane_id);
// Part 1: rmsnorm-self (no weight).
Storage input_vec[kLocalSize];
#pragma unroll
for (int i = 0; i < kLocalSize; ++i) {
input_vec[i].load(input_ptr, lane_id + i * kWarpSize);
}
float sum_of_squares = 0.0f;
#pragma unroll
for (int i = 0; i < kLocalSize; ++i) {
#pragma unroll
for (int j = 0; j < kVecSize; ++j) {
float x = bf16_to_float(input_vec[i][j]);
sum_of_squares += x * x;
}
}
sum_of_squares = warp_reduce_sum(sum_of_squares);
const float norm_factor = rsqrtf(sum_of_squares / static_cast<float>(kHeadDim) + params.eps);
#pragma unroll
for (int i = 0; i < kLocalSize; ++i) {
#pragma unroll
for (int j = 0; j < kVecSize; ++j) {
float x = bf16_to_float(input_vec[i][j]);
input_vec[i][j] = float_to_bf16(x * norm_factor);
}
}
// Stash rope tail into shared memory; write nope tiles to gmem.
const bool is_rope_lane = lane_id >= kWarpSize - kRopeSize;
#pragma unroll
for (int i = 0; i < kLocalSize; ++i) {
if (i == kLocalSize - 1 && is_rope_lane) {
const auto rope_id = lane_id - (kWarpSize - kRopeSize);
s_rope[warp_id][rope_id] = input_vec[i];
} else {
input_vec[i].store(output_ptr, lane_id + i * kWarpSize);
}
}
__syncwarp();
// Part 2: RoPE on all 32 lanes -- one (real, imag) bf16x2 pair per lane.
auto elem_ptr = reinterpret_cast<bf16x2_t*>(&s_rope[warp_id][0]);
bf16x2_t elem = elem_ptr[lane_id];
#ifndef USE_ROCM
float2 elem_f = __bfloat1622float2(elem);
float x_real = elem_f.x, x_imag = elem_f.y;
#else
float x_real = __bfloat162float(elem.x), x_imag = __bfloat162float(elem.y);
#endif
float freq_real = freq[0], freq_imag = freq[1];
float rot_real = x_real * freq_real - x_imag * freq_imag;
float rot_imag = x_real * freq_imag + x_imag * freq_real;
bf16x2_t rotated = __float22bfloat162_rn(make_float2(rot_real, rot_imag));
auto out_elem = reinterpret_cast<bf16x2_t*>(output_ptr + (kHeadDim - kRopeDim));
out_elem[lane_id] = rotated;
}
// ============================================================================
// Kernel 2: Fused K Norm + RoPE + FlashMLA Store
// block-per-token, rmsnorm (with kv_weight) + RoPE + FP8 quantized store.
// ============================================================================
struct FusedKNormRopeFlashMLAParams {
const void* __restrict__ kv;
const void* __restrict__ kv_weight;
const float* __restrict__ freqs_cis;
const int32_t* __restrict__ positions;
const int32_t* __restrict__ out_loc;
uint8_t* __restrict__ kvcache;
int64_t kv_stride_batch;
uint32_t batch_size;
float eps;
};
template <int64_t kHeadDim, int64_t kRopeDim, int32_t kPageBits>
__global__ __launch_bounds__(kFusedKBlockSize, 8) void fused_k_norm_rope_flashmla_kernel(
const __grid_constant__ FusedKNormRopeFlashMLAParams params) {
constexpr int64_t kVecSize = 2;
constexpr uint32_t kRopeWarp = kFusedKNumWarps - 1;
constexpr int64_t kPageBytes = ((584ll << kPageBits) + 575) / 576 * 576;
static_assert(kHeadDim == kFusedKBlockSize * kVecSize);
static_assert(kRopeDim == kWarpSize * kVecSize);
using Storage = AlignedVec<bf16_t, kVecSize>;
const auto tx = threadIdx.x;
const auto warp_id = tx / kWarpSize;
const auto lane_id = tx % kWarpSize;
const auto work_id = blockIdx.x;
if (work_id >= params.batch_size) return;
const auto input_ptr = static_cast<const bf16_t*>(params.kv) + work_id * params.kv_stride_batch;
const auto position = params.positions[work_id];
const auto out_loc = params.out_loc[work_id];
const auto freqs_cis = params.freqs_cis + position * kRopeDim;
AlignedVec<float, kVecSize> data, freq;
// Part 1: norm with block-wide reduction.
{
__shared__ float partial_sums[kFusedKNumWarps];
Storage input_vec, weight_vec;
input_vec.load(input_ptr, tx);
weight_vec.load(params.kv_weight, tx);
if (warp_id == kRopeWarp) freq.load(freqs_cis, lane_id);
float sum_of_squares = 0.0f;
#pragma unroll
for (int i = 0; i < kVecSize; ++i) {
float x = bf16_to_float(input_vec[i]);
sum_of_squares += x * x;
}
const float warp_sum = warp_reduce_sum(sum_of_squares);
if (lane_id == 0) partial_sums[warp_id] = warp_sum;
__syncthreads();
sum_of_squares = warp_reduce_sum<kFusedKNumWarps>(partial_sums[lane_id % kFusedKNumWarps]);
const float norm_factor = rsqrtf(sum_of_squares / static_cast<float>(kHeadDim) + params.eps);
#pragma unroll
for (int i = 0; i < kVecSize; ++i) {
float x = bf16_to_float(input_vec[i]);
float w = bf16_to_float(weight_vec[i]);
data[i] = x * norm_factor * w;
}
}
const int32_t page = out_loc >> kPageBits;
const int32_t offset = out_loc & ((1 << kPageBits) - 1);
const auto page_ptr = params.kvcache + page * kPageBytes;
const auto value_ptr = page_ptr + offset * 576;
// Part 2: rope on last warp (BF16 store), per-warp UE8M0 quant + store on others.
if (warp_id == kRopeWarp) {
float x_real = data[0], x_imag = data[1];
float freq_real = freq[0], freq_imag = freq[1];
float rot_real = x_real * freq_real - x_imag * freq_imag;
float rot_imag = x_real * freq_imag + x_imag * freq_real;
bf16x2_t result = __float22bfloat162_rn(make_float2(rot_real, rot_imag));
auto rope_ptr = value_ptr + 448;
reinterpret_cast<bf16x2_t*>(rope_ptr)[lane_id] = result;
} else {
float x = data[0], y = data[1];
float abs_max = warp_reduce_max(fmaxf(fabsf(x), fabsf(y)));
float scale_raw = fmaxf(1e-4f, abs_max) / kFP8Max;
int32_t scale_ue8m0 = cast_to_ue8m0(scale_raw);
float inv_scale = inv_scale_ue8m0(scale_ue8m0);
fp8x2_e4m3_t result = pack_fp8(x * inv_scale, y * inv_scale);
auto scale_ptr = page_ptr + (576ll << kPageBits) + offset * 8;
reinterpret_cast<fp8x2_e4m3_t*>(value_ptr)[tx] = result;
if (lane_id == 0) static_cast<uint8_t*>(scale_ptr)[warp_id] = static_cast<uint8_t>(scale_ue8m0);
}
}
// ============================================================================
// Kernel 3: Fused Q Indexer RoPE + Hadamard + FP8 Quantization
// Fused Q Indexer RoPE + Hadamard + FP8 Quantization
// warp-per-(token, head), no norm, RoPE + Hadamard + fp8 act-quant.
// ============================================================================
@@ -537,138 +284,6 @@ __global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_indexer_rope_had
// Host-side launchers (PyTorch C++ extension API)
// ============================================================================
void dsv4_fused_q_norm_rope(
const at::Tensor& q_input,
at::Tensor& q_output,
const at::Tensor& freqs_cis,
const at::Tensor& positions,
double eps) {
TORCH_CHECK(q_input.is_cuda(), "q_input must be a CUDA tensor");
TORCH_CHECK(q_output.is_cuda(), "q_output must be a CUDA tensor");
TORCH_CHECK(q_input.scalar_type() == at::ScalarType::BFloat16, "q_input must be bfloat16");
TORCH_CHECK(q_output.scalar_type() == at::ScalarType::BFloat16, "q_output must be bfloat16");
TORCH_CHECK(q_input.dim() == 3, "q_input must be 3D: (B, H, D)");
TORCH_CHECK(q_output.dim() == 3, "q_output must be 3D: (B, H, D)");
TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32");
const int64_t B = q_input.size(0);
const int64_t H = q_input.size(1);
const int64_t D = q_input.size(2);
TORCH_CHECK(
q_output.size(0) == B && q_output.size(1) == H && q_output.size(2) == D, "q_output shape must match q_input");
TORCH_CHECK(q_input.stride(2) == 1 && q_output.stride(2) == 1, "last dim must be contiguous");
TORCH_CHECK(q_input.stride(1) == D && q_output.stride(1) == D, "head dim must be contiguous");
if (B == 0) return;
const auto stream = at::cuda::getCurrentCUDAStream(q_input.get_device());
const auto params = FusedQNormRopeParams{
.q_input = q_input.data_ptr(),
.q_output = q_output.data_ptr(),
.freqs_cis = freqs_cis.data_ptr<float>(),
.positions = positions.data_ptr<int32_t>(),
.q_input_stride_batch = q_input.stride(0),
.q_output_stride_batch = q_output.stride(0),
.batch_size = static_cast<uint32_t>(B),
.num_q_heads = static_cast<uint32_t>(H),
.eps = static_cast<float>(eps),
};
const uint32_t total_works = static_cast<uint32_t>(B * H);
const uint32_t num_blocks = CEILDIV(total_works, kFusedQNumWarps);
// Dispatch on head_dim. DeepSeek V4 uses D=192 with kRopeDim=64.
constexpr int64_t kRopeDim = 64;
switch (D) {
case 128:
fused_q_norm_rope_kernel<128, kRopeDim><<<num_blocks, kFusedQBlockSize, 0, stream>>>(params);
break;
case 192:
fused_q_norm_rope_kernel<192, kRopeDim><<<num_blocks, kFusedQBlockSize, 0, stream>>>(params);
break;
default:
TORCH_CHECK(false, "Unsupported head_dim for dsv4_fused_q_norm_rope: ", D);
}
}
void dsv4_fused_k_norm_rope_flashmla(
const at::Tensor& kv,
const at::Tensor& kv_weight,
const at::Tensor& freqs_cis,
const at::Tensor& positions,
const at::Tensor& out_loc,
at::Tensor& kvcache,
double eps,
int64_t page_size) {
TORCH_CHECK(kv.is_cuda(), "kv must be a CUDA tensor");
TORCH_CHECK(kv.scalar_type() == at::ScalarType::BFloat16, "kv must be bfloat16");
TORCH_CHECK(kv.dim() == 2, "kv must be 2D: (B, D)");
TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32");
TORCH_CHECK(out_loc.scalar_type() == at::ScalarType::Int, "out_loc must be int32");
const int64_t B = kv.size(0);
const int64_t D = kv.size(1);
TORCH_CHECK(D == 512, "kv head_dim must be 512 for FlashMLA");
TORCH_CHECK(kv_weight.size(0) == D, "kv_weight size must match head_dim");
if (B == 0) return;
const auto stream = at::cuda::getCurrentCUDAStream(kv.get_device());
const auto params = FusedKNormRopeFlashMLAParams{
.kv = kv.data_ptr(),
.kv_weight = kv_weight.data_ptr(),
.freqs_cis = freqs_cis.data_ptr<float>(),
.positions = positions.data_ptr<int32_t>(),
.out_loc = out_loc.data_ptr<int32_t>(),
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
.kv_stride_batch = kv.stride(0),
.batch_size = static_cast<uint32_t>(B),
.eps = static_cast<float>(eps),
};
constexpr int64_t kHeadDim = 512;
constexpr int64_t kRopeDim = 64;
// Dispatch on page_size (must be power of 2).
TORCH_CHECK(page_size > 0 && (page_size & (page_size - 1)) == 0, "page_size must be a power of 2");
#define LAUNCH_K_KERNEL(PAGE_BITS) \
fused_k_norm_rope_flashmla_kernel<kHeadDim, kRopeDim, PAGE_BITS> \
<<<static_cast<uint32_t>(B), kFusedKBlockSize, 0, stream>>>(params)
switch (page_size) {
case 1:
LAUNCH_K_KERNEL(0);
break;
case 2:
LAUNCH_K_KERNEL(1);
break;
case 4:
LAUNCH_K_KERNEL(2);
break;
case 8:
LAUNCH_K_KERNEL(3);
break;
case 16:
LAUNCH_K_KERNEL(4);
break;
case 32:
LAUNCH_K_KERNEL(5);
break;
case 64:
LAUNCH_K_KERNEL(6);
break;
case 128:
LAUNCH_K_KERNEL(7);
break;
case 256:
LAUNCH_K_KERNEL(8);
break;
default:
TORCH_CHECK(false, "Unsupported page_size: ", page_size);
}
#undef LAUNCH_K_KERNEL
}
void dsv4_fused_q_indexer_rope_hadamard_quant(
const at::Tensor& q_input,
at::Tensor& q_fp8,
@@ -75,8 +75,13 @@ using Ws = CoopMbWorkspace<kHistBits, kTieCap>;
// which is the 32768/b1 line above -- 18% on a case the kernel already wins by 2x.
int row_split(int batch, int64_t row_len_hint) {
// Escape hatch for benchmarking the two paths against each other; 0 forces the
// one-block kernel. An empty value means unset, not zero.
if (const char* e = std::getenv("SGL_DSA_TOPK_ROW_SPLIT"); e != nullptr && *e != '\0') {
// one-block kernel. An empty value means unset, not zero. The legacy SGL_ name is
// read here too: without sglang's environ import nothing rewrites it to SGLANG_.
const char* e = std::getenv("SGLANG_DSA_TOPK_ROW_SPLIT");
if (e == nullptr || *e == '\0') {
e = std::getenv("SGL_DSA_TOPK_ROW_SPLIT");
}
if (e != nullptr && *e != '\0') {
const int v = std::atoi(e);
if (v >= 0) {
return v;
@@ -175,25 +175,8 @@ void deepseek_v4_topk_transform_512(
#endif
/*
* From csrc/elementwise (DeepSeek-V4 norm + rope)
* From csrc/elementwise (DeepSeek-V4 indexer rope + hadamard quant)
*/
void dsv4_fused_q_norm_rope(
const at::Tensor& q_input,
at::Tensor& q_output,
const at::Tensor& freqs_cis,
const at::Tensor& positions,
double eps);
void dsv4_fused_k_norm_rope_flashmla(
const at::Tensor& kv,
const at::Tensor& kv_weight,
const at::Tensor& freqs_cis,
const at::Tensor& positions,
const at::Tensor& out_loc,
at::Tensor& kvcache,
double eps,
int64_t page_size);
void dsv4_fused_q_indexer_rope_hadamard_quant(
const at::Tensor& q_input,
at::Tensor& q_fp8,
@@ -316,25 +299,8 @@ void apply_shuffle_mul_sum(
const std::optional<torch::Tensor>& factors);
/*
* From csrc/elementwise (DeepSeek-V4 norm + rope)
* From csrc/elementwise (DeepSeek-V4 indexer rope + hadamard quant)
*/
void dsv4_fused_q_norm_rope(
const at::Tensor& q_input,
at::Tensor& q_output,
const at::Tensor& freqs_cis,
const at::Tensor& positions,
double eps);
void dsv4_fused_k_norm_rope_flashmla(
const at::Tensor& kv,
const at::Tensor& kv_weight,
const at::Tensor& freqs_cis,
const at::Tensor& positions,
const at::Tensor& out_loc,
at::Tensor& kvcache,
double eps,
int64_t page_size);
void dsv4_fused_q_indexer_rope_hadamard_quant(
const at::Tensor& q_input,
at::Tensor& q_fp8,
@@ -32,9 +32,7 @@ else:
concat_mla_absorb_q,
concat_mla_k,
copy_to_gpu_no_ce,
dsv4_fused_k_norm_rope_flashmla,
dsv4_fused_q_indexer_rope_hadamard_quant,
dsv4_fused_q_norm_rope,
fused_add_rmsnorm,
gelu_and_mul,
gelu_tanh_and_mul,
@@ -155,9 +153,7 @@ else:
"concat_mla_absorb_q",
"concat_mla_k",
"copy_to_gpu_no_ce",
"dsv4_fused_k_norm_rope_flashmla",
"dsv4_fused_q_indexer_rope_hadamard_quant",
"dsv4_fused_q_norm_rope",
"es_fp8_blockwise_scaled_grouped_mm",
"es_sm100_mxfp8_blockscaled_grouped_mm",
"es_sm100_mxfp8_blockscaled_grouped_quant",
@@ -324,59 +324,6 @@ if torch.version.hip is not None:
return out
def dsv4_fused_q_norm_rope(
q_input: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
eps: float = 1e-6,
q_output: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""DeepSeek-V4 fused Q RMSNorm (no weight) + RoPE.
Parameters
----------
q_input : (B, num_q_heads, head_dim) bfloat16
freqs_cis: (max_pos, rope_dim) float32, re/im interleaved
positions: (B,) int32
eps : RMSNorm epsilon
q_output : optional pre-allocated output tensor
"""
if q_output is None:
q_output = torch.empty_like(q_input)
torch.ops.sgl_kernel.dsv4_fused_q_norm_rope.default(
q_input, q_output, freqs_cis, positions, eps
)
return q_output
def dsv4_fused_k_norm_rope_flashmla(
kv: torch.Tensor,
kv_weight: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
out_loc: torch.Tensor,
kvcache: torch.Tensor,
eps: float = 1e-6,
page_size: int = 1,
) -> None:
"""DeepSeek-V4 fused K RMSNorm + RoPE + FlashMLA FP8 store.
Parameters
----------
kv : (B, 512) bfloat16
kv_weight: (512,) bfloat16
freqs_cis: (max_pos, 64) float32
positions: (B,) int32
out_loc : (B,) int32 cache slot ids
kvcache : (npages, page_bytes) uint8
eps : RMSNorm epsilon
page_size: page size (power of 2)
"""
torch.ops.sgl_kernel.dsv4_fused_k_norm_rope_flashmla.default(
kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, page_size
)
def dsv4_fused_q_indexer_rope_hadamard_quant(
q_input: torch.Tensor,
q_fp8: torch.Tensor,
@@ -14,21 +14,9 @@
# ==============================================================================
import functools
from typing import Dict, Tuple
import torch
_cache_buf: Dict[Tuple[str, torch.device], torch.Tensor] = {}
def _get_cache_buf(name: str, bytes: int, device: torch.device) -> torch.Tensor:
key = (name, device)
buf = _cache_buf.get(key)
if buf is None:
buf = torch.empty(bytes, dtype=torch.uint8, device=device)
_cache_buf[key] = buf
return buf
def _to_tensor_scalar_tuple(x):
if isinstance(x, torch.Tensor):
@@ -1,94 +1,10 @@
"""Tests for DeepSeek-V4 fused norm + RoPE kernels."""
"""Tests for the DeepSeek-V4 fused indexer Q RoPE + Hadamard + FP8 quant kernel."""
import pytest
import sgl_kernel
import torch
def _ref_rmsnorm_self(x: torch.Tensor, eps: float) -> torch.Tensor:
"""Reference: RMSNorm without weight (identity weight)."""
rms = torch.sqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + eps)
return (x.float() / rms).to(x.dtype)
def _ref_rope_interleaved(
x: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, rope_dim: int
) -> torch.Tensor:
"""Reference: apply RoPE to the last `rope_dim` elements (interleaved re/im)."""
out = x.clone()
B = x.size(0)
head_dim = x.size(-1)
nope_dim = head_dim - rope_dim
for b in range(B):
pos = positions[b].item()
freq = freqs_cis[pos] # (rope_dim,) interleaved [re0, im0, re1, im1, ...]
rope_part = out[b, ..., nope_dim:].float()
# Reshape to pairs
pairs = rope_part.reshape(*rope_part.shape[:-1], rope_dim // 2, 2)
x_real = pairs[..., 0]
x_imag = pairs[..., 1]
freq_pairs = freq.reshape(rope_dim // 2, 2)
f_real = freq_pairs[:, 0]
f_imag = freq_pairs[:, 1]
rot_real = x_real * f_real - x_imag * f_imag
rot_imag = x_real * f_imag + x_imag * f_real
result = torch.stack([rot_real, rot_imag], dim=-1).reshape(rope_part.shape)
out[b, ..., nope_dim:] = result.to(x.dtype)
return out
@pytest.mark.parametrize("batch_size", [1, 4, 16])
@pytest.mark.parametrize("num_heads", [1, 8])
@pytest.mark.parametrize("head_dim", [128, 192])
def test_fused_q_norm_rope_correctness(batch_size, num_heads, head_dim):
"""Test Q norm + rope against reference."""
torch.manual_seed(42)
rope_dim = 64
max_pos = 512
eps = 1e-6
q_input = torch.randn(
batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda"
)
freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda")
positions = torch.randint(
0, max_pos, (batch_size,), dtype=torch.int32, device="cuda"
)
q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions, eps)
# Reference
normed = _ref_rmsnorm_self(q_input, eps)
expected = _ref_rope_interleaved(normed, freqs_cis, positions, rope_dim)
torch.testing.assert_close(q_output.float(), expected.float(), rtol=1e-2, atol=1e-2)
def test_fused_q_norm_rope_zero_batch():
"""Empty batch should not crash."""
q_input = torch.empty(0, 8, 192, dtype=torch.bfloat16, device="cuda")
freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda")
positions = torch.empty(0, dtype=torch.int32, device="cuda")
q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions)
assert q_output.shape == q_input.shape
def test_fused_q_norm_rope_preallocated_output():
"""Test with pre-allocated output tensor."""
torch.manual_seed(42)
B, H, D = 4, 8, 192
q_input = torch.randn(B, H, D, dtype=torch.bfloat16, device="cuda")
freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda")
positions = torch.randint(0, 512, (B,), dtype=torch.int32, device="cuda")
q_output = torch.empty_like(q_input)
result = sgl_kernel.dsv4_fused_q_norm_rope(
q_input, freqs_cis, positions, q_output=q_output
)
assert result is q_output
@pytest.mark.parametrize("batch_size", [1, 8])
def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size):
"""Basic launch coverage with finite output checks."""
@@ -8,21 +8,9 @@ from sglang.srt.server_args import (
ServerArgs,
set_global_server_args_for_scheduler,
)
from sglang.srt.utils import (
cpu_has_amx_support,
is_cpu,
is_cuda,
is_hip,
is_npu,
is_xpu,
)
from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_cpu = is_cpu()
_is_cpu_amx_available = cpu_has_amx_support()
_is_npu = is_npu()
_is_xpu = is_xpu()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -3,99 +3,9 @@ from typing import Optional
import pytest
import torch
import triton
import triton.language as tl
from sgl_kernel import merge_state_v2
@triton.jit
def merge_state_kernel(
output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_merged
output_lse, # [NUM_TOKENS, NUM_HEADS] s_merged
prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_a
prefix_lse, # [NUM_TOKENS, NUM_HEADS] s_a
suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_b
suffix_lse, # [NUM_TOKENS, NUM_HEADS] s_b
HEAD_SIZE: tl.constexpr,
PADDED_HEAD_SIZE: tl.constexpr,
OUTPUT_LSE: tl.constexpr,
):
token_idx = tl.program_id(0)
num_tokens = tl.num_programs(0)
head_idx = tl.program_id(1)
num_heads = tl.num_programs(1)
p_lse = tl.load(prefix_lse + token_idx * num_heads + head_idx)
s_lse = tl.load(suffix_lse + token_idx * num_heads + head_idx)
p_lse = float("-inf") if p_lse == float("inf") else p_lse
s_lse = float("-inf") if s_lse == float("inf") else s_lse
max_lse = tl.maximum(p_lse, s_lse)
p_lse = p_lse - max_lse
s_lse = s_lse - max_lse
out_se = tl.exp(p_lse) + tl.exp(s_lse)
if OUTPUT_LSE:
out_lse = tl.log(out_se) + max_lse
tl.store(output_lse + token_idx * num_heads + head_idx, out_lse)
head_arange = tl.arange(0, PADDED_HEAD_SIZE)
head_mask = head_arange < HEAD_SIZE
p_out = tl.load(
prefix_output
+ token_idx * num_heads * HEAD_SIZE
+ head_idx * HEAD_SIZE
+ head_arange,
mask=head_mask,
)
s_out = tl.load(
suffix_output
+ token_idx * num_heads * HEAD_SIZE
+ head_idx * HEAD_SIZE
+ head_arange,
mask=head_mask,
)
p_scale = tl.exp(p_lse) / out_se
s_scale = tl.exp(s_lse) / out_se
out = p_out * p_scale + s_out * s_scale
tl.store(
output + token_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE + head_arange,
out,
mask=head_mask,
)
def merge_state_triton(
prefix_output: torch.Tensor,
prefix_lse: torch.Tensor,
suffix_output: torch.Tensor,
suffix_lse: torch.Tensor,
output: Optional[torch.Tensor] = None,
output_lse: Optional[torch.Tensor] = None,
) -> None:
num_tokens = output.shape[0]
num_query_heads = output.shape[1]
head_size = output.shape[2]
padded_head_size = triton.next_power_of_2(head_size)
# Avoid creating new tensors if they are already provided
if output is None:
output = torch.empty_like(prefix_output)
if output_lse is None:
output_lse = torch.empty_like(prefix_lse)
merge_state_kernel[(num_tokens, num_query_heads)](
output,
output_lse,
prefix_output,
prefix_lse,
suffix_output,
suffix_lse,
head_size,
padded_head_size,
output_lse is not None,
)
return output, output_lse
from sglang.kernels.ops.attention.merge_state import merge_state_triton
# Naive PyTorch Implements of Merge Attn States
@@ -3,10 +3,10 @@ import sys
import pytest
import torch
import triton
import triton.language as tl
from sgl_kernel import moe_align_block_size, moe_sum
from sglang.test.kernels.moe_align import moe_align_block_size_triton
def is_hip() -> bool:
return torch.version.hip is not None
@@ -15,144 +15,14 @@ def is_hip() -> bool:
_is_hip = is_hip()
def ceil_div(a, b):
return (a + b - 1) // b
@triton.jit
def moe_align_block_size_stage1(
topk_ids_ptr,
tokens_cnts_ptr,
num_experts: tl.constexpr,
numel: tl.constexpr,
tokens_per_thread: tl.constexpr,
):
pid = tl.program_id(0)
start_idx = pid * tokens_per_thread
off_c = (pid + 1) * num_experts
for i in range(tokens_per_thread):
if start_idx + i < numel:
idx = tl.load(topk_ids_ptr + start_idx + i)
token_cnt = tl.load(tokens_cnts_ptr + off_c + idx)
tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1)
@triton.jit
def moe_align_block_size_stage2(
tokens_cnts_ptr,
num_experts: tl.constexpr,
):
pid = tl.program_id(0)
last_cnt = 0
for i in range(1, num_experts + 1):
token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid)
last_cnt = last_cnt + token_cnt
tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt)
@triton.jit
def moe_align_block_size_stage3(
total_tokens_post_pad_ptr,
tokens_cnts_ptr,
cumsum_ptr,
num_experts: tl.constexpr,
block_size: tl.constexpr,
):
last_cumsum = 0
off_cnt = num_experts * num_experts
for i in range(1, num_experts + 1):
token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1)
last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size
tl.store(cumsum_ptr + i, last_cumsum)
tl.store(total_tokens_post_pad_ptr, last_cumsum)
@triton.jit
def moe_align_block_size_stage4(
topk_ids_ptr,
sorted_token_ids_ptr,
expert_ids_ptr,
tokens_cnts_ptr,
cumsum_ptr,
num_experts: tl.constexpr,
block_size: tl.constexpr,
numel: tl.constexpr,
tokens_per_thread: tl.constexpr,
):
pid = tl.program_id(0)
start_idx = tl.load(cumsum_ptr + pid)
end_idx = tl.load(cumsum_ptr + pid + 1)
for i in range(start_idx, end_idx, block_size):
tl.store(expert_ids_ptr + i // block_size, pid)
start_idx = pid * tokens_per_thread
off_t = pid * num_experts
for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)):
expert_id = tl.load(topk_ids_ptr + i)
token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id)
rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id)
tl.store(sorted_token_ids_ptr + rank_post_pad, i)
tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1)
def moe_align_block_size_triton(
topk_ids: torch.Tensor,
num_experts: int,
block_size: int,
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
num_tokens_post_pad: torch.Tensor,
) -> None:
numel = topk_ids.numel()
grid = (num_experts,)
tokens_cnts = torch.zeros(
(num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device
)
cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device)
tokens_per_thread = ceil_div(numel, num_experts)
moe_align_block_size_stage1[grid](
topk_ids,
tokens_cnts,
num_experts,
numel,
tokens_per_thread,
)
moe_align_block_size_stage2[grid](
tokens_cnts,
num_experts,
)
moe_align_block_size_stage3[(1,)](
num_tokens_post_pad,
tokens_cnts,
cumsum,
num_experts,
block_size,
)
moe_align_block_size_stage4[grid](
topk_ids,
sorted_token_ids,
expert_ids,
tokens_cnts,
cumsum,
num_experts,
block_size,
numel,
tokens_per_thread,
)
@pytest.mark.parametrize(
"block_size,num_tokens,topk,num_experts,pad_sorted_token_ids",
list(
itertools.product(
[32, 64, 128, 256], # block_size
[1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096], # num_tokens
[1, 2, 4, 8, 16, 32, 64], # topk
[64, 160, 256, 257, 260, 264], # num_experts
[32, 256], # block_size
[1, 16, 4096], # num_tokens
[1, 8, 64], # topk
[64, 160, 257, 264], # num_experts
[True, False], # pad_sorted_token_ids
)
),
@@ -8,58 +8,68 @@ from sgl_kernel.test_utils import (
create_per_token_group_quant_test_data,
)
from sglang.kernels.ops.quantization.fp8_kernel import (
per_token_group_quant_8bit as triton_per_token_group_quant_8bit,
)
from sglang.kernels.ops.quantization.fp8_kernel import (
sglang_per_token_group_quant_8bit,
)
from sglang.srt.utils import is_hip
from sglang.test.kernels.per_token_group_quant import (
sglang_per_token_group_quant_8bit,
triton_per_token_group_quant_8bit,
)
_is_hip = is_hip()
fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn
configs = list(
itertools.product(
[1, 4, 16, 64, 127, 128, 512, 1024, 4096, 8192], # num_tokens
[128, 256, 384, 512, 1024, 1536, 1664, 2048, 4096, 7168, 16384], # hidden_dim
_ROW_MAJOR = dict(
column_major_scales=False,
scale_tma_aligned=False,
scale_ue8m0=False,
fuse_silu_and_mul=False,
masked_layout_mode=None,
)
_COLUMN_MAJOR_FLAGS = [
dict(
column_major_scales=True,
scale_tma_aligned=False,
scale_ue8m0=False,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
dict(
column_major_scales=True,
scale_tma_aligned=True,
scale_ue8m0=False,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
dict(
column_major_scales=True,
scale_tma_aligned=True,
scale_ue8m0=True,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
]
def _supported(group_size, dst_dtype, flags):
# int8 has no column-major scale layout; UE8M0 scales are only built for group 128.
if dst_dtype == torch.int8 and flags["column_major_scales"]:
return False
return not flags["scale_ue8m0"] or group_size == 128
configs = [
config
for config in itertools.product(
[1, 4, 127, 4096], # num_tokens
[128, 384, 1664, 7168], # hidden_dim
[16, 32, 64, 128], # group_size
[None], # num_ranks
[fp8_type_, torch.int8], # dtype
[
dict(
column_major_scales=False,
scale_tma_aligned=False,
scale_ue8m0=False,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
dict(
column_major_scales=True,
scale_tma_aligned=False,
scale_ue8m0=False,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
dict(
column_major_scales=True,
scale_tma_aligned=True,
scale_ue8m0=False,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
dict(
column_major_scales=True,
scale_tma_aligned=True,
scale_ue8m0=True,
fuse_silu_and_mul=False,
masked_layout_mode=None,
),
],
[_ROW_MAJOR, *_COLUMN_MAJOR_FLAGS],
)
) + list(
if _supported(config[2], config[4], config[5])
] + list(
itertools.product(
[1, 4, 1 * 8, 4 * 8, 64 * 8, 256 * 8, 768 * 8],
[1, 4 * 8, 768 * 8],
# TODO support more
[2048],
[128],
@@ -117,21 +127,11 @@ def test_per_token_group_quant_with_column_major(
arch_major, _ = torch.cuda.get_device_capability(torch.cuda.current_device())
if flags["scale_ue8m0"] and (arch_major <= 9):
pytest.skip("Only Blackwell need ue8m0 fusion")
return
if (flags["scale_ue8m0"] and (group_size != 128)) or (
(dst_dtype == torch.int8) and flags["column_major_scales"]
):
pytest.skip()
return
x, masked_m = create_per_token_group_quant_test_data(
num_tokens=num_tokens, hidden_dim=hidden_dim, num_ranks=num_ranks, flags=flags
)
# print("hack data!!!")
# x = torch.full_like(x, fill_value=100)
execute_kwargs = dict(
x=x,
masked_m=masked_m,
+3 -2
View File
@@ -539,10 +539,11 @@ def test_topk_variants_are_exact_on_variable_windows(
kind: str, force_one_block: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Cover both dispatches and all three APIs on adversarial score distributions."""
monkeypatch.delenv("SGL_DSA_TOPK_ROW_SPLIT", raising=False)
if force_one_block:
monkeypatch.setenv("SGL_DSA_TOPK_ROW_SPLIT", "0")
monkeypatch.setenv("SGLANG_DSA_TOPK_ROW_SPLIT", "0")
else:
monkeypatch.delenv("SGL_DSA_TOPK_ROW_SPLIT", raising=False)
monkeypatch.delenv("SGLANG_DSA_TOPK_ROW_SPLIT", raising=False)
bs, k = 4, 2048
score = _make_scores(kind, bs, MAX_SEQ_LEN, seed=1200 + force_one_block)
+13 -33
View File
@@ -35,7 +35,7 @@ Dispatch priority (highest first), resolved by :meth:`BaseFusedOp.forward`:
whose method is overridden, that is *declared* in :attr:`capabilities`,
and whose :class:`~sglang.kernels.spec.CapabilityRequirement` set matches
the detected platform. Ops may extend :meth:`backend_eligible` with
per-call shape/dtype gates; overriding it switches this step from a
per-call applicability gates; overriding it switches this step from a
statically cached choice to per-call selection.
5. **Platform-specific forward** — ``forward_cuda`` on CUDA, ``forward_hip``
(falling back to ``forward_cuda``) on ROCm, ``forward_musa`` on MUSA,
@@ -86,12 +86,11 @@ from torch import nn
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.kernels.registry import register_kernel
from sglang.kernels.selector import _platform
from sglang.kernels.spec import (
CapabilityRequirement,
FormatSignature,
KernelBackend,
KernelSpec,
PlatformInfo,
capabilities_satisfied,
)
@@ -159,11 +158,6 @@ _PLATFORM_METHODS: Dict[str, Tuple[str, ...]] = {
}
@functools.lru_cache(maxsize=1)
def _platform() -> PlatformInfo:
return PlatformInfo.detect()
@functools.lru_cache(maxsize=1)
def _platform_key() -> str:
"""In-tree platform dispatch key, or ``""`` for plain-native platforms.
@@ -207,13 +201,6 @@ def _oot_dispatch_key() -> Optional[str]:
return None
def clear_platform_caches() -> None:
"""Drop the cached platform detection (used by tests that mock platforms)."""
_platform.cache_clear()
_platform_key.cache_clear()
_oot_dispatch_key.cache_clear()
# --- global backend override ------------------------------------------------
# Sentinel distinguishing "not resolved yet" from "resolved to None (no force)".
@@ -350,7 +337,7 @@ class BaseFusedOp(nn.Module, ABC):
Class attributes
----------------
op:
Operator id, ``"<group>.<name>"`` (e.g. ``"layernorm.rmsnorm"``).
Operator id, ``"<group>.<name>"`` (e.g. ``"activation.silu_and_mul"``).
Required for :func:`register_fused_op`; layer-style subclasses that
are not registered in the kernel registry may leave it empty.
priority:
@@ -368,10 +355,6 @@ class BaseFusedOp(nn.Module, ABC):
the ``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.NPU`` shortcuts,
e.g. ``{KernelBackend.AOT: {CapabilityRequirement.CUDA,
CapabilityRequirement.HIP}}``.
format_signature:
Data-contract description shared by all backends of this op.
descriptions:
Optional per-backend one-liners for the registry inventory.
"""
op: ClassVar[str] = ""
@@ -379,8 +362,6 @@ class BaseFusedOp(nn.Module, ABC):
capabilities: ClassVar[
Mapping[KernelBackend, AbstractSet[CapabilityRequirement]]
] = {}
format_signature: ClassVar[FormatSignature] = FormatSignature()
descriptions: ClassVar[Mapping[KernelBackend, str]] = {}
# OOT forward registry shared by all fused ops: platform dispatch key ->
# {op class -> forward fn}. Populated by out-of-tree platform plugins.
@@ -438,6 +419,9 @@ class BaseFusedOp(nn.Module, ABC):
def forward_cute_dsl(self, *args, **kwargs):
raise NotImplementedError(f"{self._op_label()}: no cute_dsl backend")
def forward_flydsl(self, *args, **kwargs):
raise NotImplementedError(f"{self._op_label()}: no flydsl backend")
def forward_helion(self, *args, **kwargs):
raise NotImplementedError(f"{self._op_label()}: no helion backend")
@@ -480,10 +464,12 @@ class BaseFusedOp(nn.Module, ABC):
The base implementation checks the backend's
:class:`CapabilityRequirement` set (OR semantics) against the detected
platform. Subclasses may extend it with per-call shape/dtype gates so
auto-selection bounces to the next backend instead of raising;
overriding this method switches backend auto-selection from a cached
static choice to per-call resolution.
platform. Subclasses may extend it with per-call gates for operands the
kernel does not cover, where the next backend computes the same result;
operands that violate the op's contract are a caller bug the backend's
launcher must raise on, not a reason to bounce. Overriding this method
switches backend auto-selection from a cached static choice to per-call
resolution.
"""
return capabilities_satisfied(
self.capabilities.get(backend, frozenset()), _platform()
@@ -586,10 +572,6 @@ class BaseFusedOp(nn.Module, ABC):
_record_trace(self, _dispatch_label(method), args, kwargs)
return result
def dispatch_forward(self) -> Callable:
"""The static dispatch target for this op on the current platform."""
return self._resolve_forward_method()
# --- torch.compile mode ---
def _torch_compile_forward(self, num_tokens: int) -> Optional[Callable]:
@@ -684,7 +666,7 @@ def register_fused_op(instance: BaseFusedOp, module: str, attr: str) -> BaseFuse
``KernelSpec.load()`` can lazily resolve e.g.
``"<module>:<attr>.forward_aot"`` to the bound backend method. Returns
``instance`` so group packages can write
``_RMSNORM = register_fused_op(_RMSNormOp(), __name__, "_RMSNORM")``.
``_SILU_AND_MUL = register_fused_op(SiluAndMulOp(), __name__, "_SILU_AND_MUL")``.
"""
for backend in instance.available_backends():
register_kernel(
@@ -693,8 +675,6 @@ def register_fused_op(instance: BaseFusedOp, module: str, attr: str) -> BaseFuse
backend=backend,
target=f"{module}:{attr}.{BACKEND_METHODS[backend]}",
capabilities=frozenset(instance.capabilities.get(backend, ())),
format_signature=instance.format_signature,
description=instance.descriptions.get(backend, ""),
)
)
return instance
+1 -1
View File
@@ -5,7 +5,7 @@ from typing import Callable, List, Optional, Sequence, Tuple
import torch
import triton.testing
from sglang.kernels.ops.communication.mp import multigpu_launch
from sglang.test.kernels.multigpu import multigpu_launch
from sglang.utils import is_in_ci
@@ -1,522 +0,0 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/tile.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <sgl_kernel/warp.cuh>
#include <sgl_kernel/deepseek_v4/compress.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <tvm/ffi/object.h>
#include <cstdint>
namespace sglang {
using Plan128 = device::compress::PrefillPlan;
using IndiceT = int32_t;
/// \brief Each thread will handle this many elements (split along head_dim)
constexpr int32_t kTileElements = 2;
/// \brief Each warp will handle this many elements (split along 128)
constexpr int32_t kElementsPerWarp = 8;
constexpr uint32_t kNumWarps = 128 / kElementsPerWarp;
constexpr uint32_t kBlockSize = device::kWarpThreads * kNumWarps;
/// \brief Need to reduce register usage to increase occupancy
#define C128_KERNEL __global__ __launch_bounds__(kBlockSize, 2)
struct Compress128DecodeParams {
/**
* \brief Shape: `[num_indices, 128, head_dim * 2]` \n
* last dimension layout:
* | kv current | score current |
*/
void* __restrict__ kv_score_buffer;
/** \brief Shape: `[batch_size, head_dim * 2]` */
const void* __restrict__ kv_score_input;
/** \brief Shape: `[batch_size, head_dim]` */
void* __restrict__ kv_compressed_output;
/** \brief Shape: `[128, head_dim]` (called `ape`) */
const void* __restrict__ score_bias;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ indices;
/** \brief Shape: `[batch_size, ]` */
const IndiceT* __restrict__ seq_lens;
/** \NOTE: `batch_size` <= `num_indices` */
uint32_t batch_size;
};
struct Compress128PrefillParams {
/**
* \brief Shape: `[num_indices, 128, head_dim * 2]` \n
* last dimension layout:
* | kv current | score current |
*/
void* __restrict__ kv_score_buffer;
/** \brief Shape: `[batch_size, head_dim * 2]` */
const void* __restrict__ kv_score_input;
/** \brief Shape: `[batch_size, head_dim]` */
void* __restrict__ kv_compressed_output;
/** \brief Shape: `[128, head_dim]` (called `ape`) */
const void* __restrict__ score_bias;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ indices;
/** \brief Shape: `[batch_size, ]`*/
const int32_t* __restrict__ load_indices;
/** \brief The following part is plan info. */
const Plan128* __restrict__ compress_plan;
const Plan128* __restrict__ write_plan;
uint32_t num_compress;
uint32_t num_write;
};
struct Compress128SharedBuffer {
using Storage = device::AlignedVector<float, kTileElements>;
Storage data[kNumWarps][device::kWarpThreads + 1]; // padding to avoid bank conflict
SGL_DEVICE Storage& operator()(uint32_t warp_id, uint32_t lane_id) {
return data[warp_id][lane_id];
}
SGL_DEVICE float& operator()(uint32_t warp_id, uint32_t lane_id, uint32_t tile_id) {
return data[warp_id][lane_id][tile_id];
}
};
template <typename T>
SGL_DEVICE void c128_write(
T* kv_score_buf, //
const T* kv_score_src,
const int64_t head_dim,
const int32_t write_pos,
const uint32_t lane_id) {
using namespace device;
using Storage = AlignedVector<T, kTileElements>;
const auto element_size = head_dim * 2;
const auto gmem = tile::Memory<Storage>{lane_id, kWarpThreads};
kv_score_buf += write_pos * element_size;
/// NOTE: Layout | [0] = kv | [1] = score |
Storage kv_score[2];
#pragma unroll
for (int32_t i = 0; i < 2; ++i) {
kv_score[i] = gmem.load(kv_score_src + head_dim * i);
}
#pragma unroll
for (int32_t i = 0; i < 2; ++i) {
gmem.store(kv_score_buf + head_dim * i, kv_score[i]);
}
}
template <typename InFloat, typename OutFloat>
SGL_DEVICE void c128_forward(
const InFloat* kv_score_buf,
const InFloat* kv_score_src,
OutFloat* kv_out,
const InFloat* score_bias,
const int64_t head_dim,
const int32_t window_len,
const uint32_t warp_id,
const uint32_t lane_id) {
using namespace device;
const auto element_size = head_dim * 2;
const auto score_offset = head_dim;
/// NOTE: part 1: load kv + score
using StorageIn = AlignedVector<InFloat, kTileElements>;
const auto gmem_in = tile::Memory<StorageIn>{lane_id, kWarpThreads};
StorageIn kv[kElementsPerWarp];
StorageIn score[kElementsPerWarp];
StorageIn bias[kElementsPerWarp];
const int32_t warp_offset = warp_id * kElementsPerWarp;
#pragma unroll
for (int32_t i = 0; i < 8; ++i) {
const int32_t j = i + warp_offset;
bias[i] = gmem_in.load(score_bias + j * head_dim);
}
#pragma unroll
for (int32_t i = 0; i < kElementsPerWarp; ++i) {
const int32_t j = i + warp_offset;
const InFloat* src;
__builtin_assume(j < 128);
if (j < window_len) {
src = kv_score_buf + j * element_size;
} else {
/// NOTE: k in [-127, 0]. We'll load from the ragged `kv_score_src`
const int32_t k = j - 127;
src = kv_score_src + k * element_size;
}
kv[i] = gmem_in.load(src);
score[i] = gmem_in.load(src + score_offset);
}
/// NOTE: part 2: safe online softmax + weighted sum
using TmpStorage = typename Compress128SharedBuffer::Storage;
__shared__ Compress128SharedBuffer s_local_val_max;
__shared__ Compress128SharedBuffer s_local_exp_sum;
__shared__ Compress128SharedBuffer s_local_product;
TmpStorage tmp_val_max;
TmpStorage tmp_exp_sum;
TmpStorage tmp_product;
#pragma unroll
for (int32_t i = 0; i < kTileElements; ++i) {
float score_fp32[kElementsPerWarp];
#pragma unroll
for (int32_t j = 0; j < kElementsPerWarp; ++j) {
score_fp32[j] = cast<float>(score[j][i]) + cast<float>(bias[j][i]);
}
float max_value = score_fp32[0];
float sum_exp_value = 0.0f;
#pragma unroll
for (int32_t j = 1; j < kElementsPerWarp; ++j) {
const auto fp32_score = score_fp32[j];
max_value = fmaxf(max_value, fp32_score);
}
float sum_product = 0.0f;
#pragma unroll
for (int32_t j = 0; j < 8; ++j) {
const auto fp32_score = score_fp32[j];
const auto exp_score = expf(fp32_score - max_value);
sum_product += cast<float>(kv[j][i]) * exp_score;
sum_exp_value += exp_score;
}
tmp_val_max[i] = max_value;
tmp_exp_sum[i] = sum_exp_value;
tmp_product[i] = sum_product;
}
// naturally aligned, so no bank conflict
s_local_val_max(warp_id, lane_id) = tmp_val_max;
s_local_exp_sum(warp_id, lane_id) = tmp_exp_sum;
s_local_product(warp_id, lane_id) = tmp_product;
__syncthreads();
/// NOTE: part 3: online softmax
/// NOTE: We have `kTileElements * kWarpThreads * kNumWarps` values to reduce
/// each reduce will consume `kNumWarps` threads (use partial warp reduction)
constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps;
constexpr uint32_t kIteration = kReductionCount / kBlockSize;
#pragma unroll
for (uint32_t i = 0; i < kIteration; ++i) {
/// NOTE: Range `[0, kTileElements * kWarpThreads * kNumWarps)`
const uint32_t j = i * kBlockSize + warp_id * kWarpThreads + lane_id;
/// NOTE: Range `[0, kNumWarps)`
const uint32_t local_warp_id = j % kNumWarps;
/// NOTE: Range `[0, kTileElements * kWarpThreads)`
const uint32_t local_elem_id = j / kNumWarps;
/// NOTE: Range `[0, kTileElements)`
const uint32_t local_tile_id = local_elem_id % kTileElements;
/// NOTE: Range `[0, kWarpThreads)`
const uint32_t local_lane_id = local_elem_id / kTileElements;
/// NOTE: each warp will access the whole tile (all `kTileElements`)
/// and for different lanes, the memory access only differ in `local_warp_id`
/// so there's no bank conflict in shared memory access.
static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs");
const auto local_val_max = s_local_val_max(local_warp_id, local_lane_id, local_tile_id);
const auto local_exp_sum = s_local_exp_sum(local_warp_id, local_lane_id, local_tile_id);
const auto local_product = s_local_product(local_warp_id, local_lane_id, local_tile_id);
const auto global_val_max = warp::reduce_max<kNumWarps>(local_val_max);
const auto rescale = expf(local_val_max - global_val_max);
const auto global_exp_sum = warp::reduce_sum<kNumWarps>(local_exp_sum * rescale);
const auto final_scale = rescale / global_exp_sum;
const auto global_product = warp::reduce_sum<kNumWarps>(local_product * final_scale);
kv_out[local_elem_id] = cast<OutFloat>(global_product);
}
}
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kUsePDL>
C128_KERNEL void flash_c128_decode(const __grid_constant__ Compress128DecodeParams params) {
using namespace device;
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
constexpr int64_t kElementSize = kHeadDim * 2;
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
const auto& [
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
indices, seq_lens, batch_size // decode info
] = params;
const uint32_t warp_id = threadIdx.x / kWarpThreads;
const uint32_t lane_id = threadIdx.x % kWarpThreads;
const uint32_t global_bid = blockIdx.x / kNumSplit; // batch id
const uint32_t global_sid = blockIdx.x % kNumSplit; // split id
if (global_bid >= batch_size) return;
const int32_t index = indices[global_bid];
const int32_t seq_len = seq_lens[global_bid];
const int64_t split_offset = global_sid * kTileDim;
// kv score
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
const auto kv_buf = kv_score_buffer + index * (kElementSize * 128) + split_offset;
// kv input
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
const auto kv_src = kv_score_input + global_bid * kElementSize + split_offset;
// kv output
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
const auto kv_out = kv_compressed_output + global_bid * kHeadDim + split_offset;
// score bias (ape)
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
PDLWaitPrimary<kUsePDL>();
/// NOTE: the write must be visible to the subsequent c128_forward,
/// so only the last warp can write to HBM
/// In addition, `position` = `seq_len - 1`. To avoid underflow, we use `seq_len + 127`
if (warp_id == kNumWarps - 1) {
c128_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 127) % 128, lane_id);
}
if (seq_len % 128 == 0) {
c128_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, /*window_len=*/128, warp_id, lane_id);
}
PDLTriggerSecondary<kUsePDL>();
}
// compress kernel
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kWrite, bool kUsePDL>
C128_KERNEL void flash_c128_prefill(const __grid_constant__ Compress128PrefillParams params) {
using namespace device;
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
constexpr int64_t kElementSize = kHeadDim * 2;
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
const auto& [
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
indices, load_indices, compress_plan, write_plan, num_compress, num_write // prefill plan
] = params;
const uint32_t warp_id = threadIdx.x / kWarpThreads;
const uint32_t lane_id = threadIdx.x % kWarpThreads;
uint32_t global_id;
if constexpr (kWrite) {
// for write kernel, we use global warp_id to dispatch work
global_id = (blockIdx.x * blockDim.x + threadIdx.x) / kWarpThreads;
} else {
// for compress kernel, we use block id to dispatch work
global_id = blockIdx.x; // block id
}
const uint32_t global_pid = global_id / kNumSplit; // plan id
const uint32_t global_sid = global_id % kNumSplit; // split id
/// NOTE: compiler can optimize this if-else at compile time
const auto num_plans = kWrite ? num_write : num_compress;
const auto plan_ptr = kWrite ? write_plan : compress_plan;
if (global_pid >= num_plans) return;
const auto& [ragged_id, global_bid, position, window_len] = plan_ptr[global_pid];
const auto indices_ptr = kWrite ? indices : load_indices;
const int64_t split_offset = global_sid * kTileDim;
// kv input
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
const auto kv_src = kv_score_input + ragged_id * kElementSize + split_offset;
// kv output
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
const auto kv_out = kv_compressed_output + ragged_id * kHeadDim + split_offset;
// score bias (ape)
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
if (ragged_id == 0xFFFFFFFF) [[unlikely]]
return;
const int32_t index = indices_ptr[global_bid];
// kv score
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
const auto kv_buf = kv_score_buffer + index * (kElementSize * 128) + split_offset;
PDLWaitPrimary<kUsePDL>();
// only responsible for the compress part
if constexpr (kWrite) {
c128_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 128, lane_id);
} else {
c128_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, window_len, warp_id, lane_id);
}
PDLTriggerSecondary<kUsePDL>();
}
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kUsePDL>
struct FlashCompress128Kernel {
static constexpr auto decode_kernel = flash_c128_decode<kHeadDim, InFloat, OutFloat, kUsePDL>;
template <bool kWrite>
static constexpr auto prefill_kernel = flash_c128_prefill<kHeadDim, InFloat, OutFloat, kWrite, kUsePDL>;
static constexpr auto prefill_c_kernel = prefill_kernel</*kWrite=*/false>;
static constexpr auto prefill_w_kernel = prefill_kernel</*kWrite=*/true>;
static constexpr int64_t kTileDim = kTileElements * device::kWarpThreads; // 64
static constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
static constexpr uint32_t kWriteBlockSize = 128;
static constexpr uint32_t kWarpsPerWriteBlock = kWriteBlockSize / device::kWarpThreads;
static void run_decode(
const tvm::ffi::TensorView kv_score_buffer,
const tvm::ffi::TensorView kv_score_input,
const tvm::ffi::TensorView kv_compressed_output,
const tvm::ffi::TensorView ape,
const tvm::ffi::TensorView indices,
const tvm::ffi::TensorView seq_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> /* UNUSED */) {
using namespace host;
// this should not happen in practice
auto B = SymbolicSize{"batch_size"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({-1, 128, kHeadDim * 2}) // kv score
.with_dtype<InFloat>()
.with_device(device)
.verify(kv_score_buffer);
TensorMatcher({B, kHeadDim * 2}) // kv score input
.with_dtype<InFloat>()
.with_device(device)
.verify(kv_score_input);
TensorMatcher({B, kHeadDim}) // kv compressed output
.with_dtype<OutFloat>()
.with_device(device)
.verify(kv_compressed_output);
TensorMatcher({128, kHeadDim}) // ape
.with_dtype<InFloat>()
.with_device(device)
.verify(ape);
TensorMatcher({B}) // indices
.with_dtype<IndiceT>()
.with_device(device)
.verify(indices);
TensorMatcher({B}) // seq lens
.with_dtype<IndiceT>()
.with_device(device)
.verify(seq_lens);
const auto batch_size = static_cast<uint32_t>(B.unwrap());
const auto params = Compress128DecodeParams{
.kv_score_buffer = kv_score_buffer.data_ptr(),
.kv_score_input = kv_score_input.data_ptr(),
.kv_compressed_output = kv_compressed_output.data_ptr(),
.score_bias = ape.data_ptr(),
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
.seq_lens = static_cast<const IndiceT*>(seq_lens.data_ptr()),
.batch_size = batch_size,
};
const uint32_t num_blocks = batch_size * kNumSplit;
LaunchKernel(num_blocks, kBlockSize, device.unwrap()) //
.enable_pdl(kUsePDL)(decode_kernel, params);
}
static void run_prefill(
const tvm::ffi::TensorView kv_score_buffer,
const tvm::ffi::TensorView kv_score_input,
const tvm::ffi::TensorView kv_compressed_output,
const tvm::ffi::TensorView ape,
const tvm::ffi::TensorView indices,
const tvm::ffi::TensorView compress_plan,
const tvm::ffi::TensorView write_plan,
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
using namespace host;
auto B = SymbolicSize{"batch_size"};
auto N = SymbolicSize{"num_q_tokens"};
auto X = SymbolicSize{"compress_tokens"};
auto Y = SymbolicSize{"write_tokens"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
TensorMatcher({-1, 128, kHeadDim * 2}) // kv score
.with_dtype<InFloat>()
.with_device(device_)
.verify(kv_score_buffer);
TensorMatcher({N, kHeadDim * 2}) // kv score input
.with_dtype<InFloat>()
.with_device(device_)
.verify(kv_score_input);
TensorMatcher({N, kHeadDim}) // kv compressed output
.with_dtype<OutFloat>()
.with_device(device_)
.verify(kv_compressed_output);
TensorMatcher({128, kHeadDim}) // ape
.with_dtype<InFloat>()
.with_device(device_)
.verify(ape);
TensorMatcher({B}) // indices
.with_dtype<IndiceT>()
.with_device(device_)
.verify(indices);
TensorMatcher({X, compress::kPrefillPlanDim}) // compress plan
.with_dtype<compress::PrefillPlanTensorDtype>()
.with_device(device_)
.verify(compress_plan);
TensorMatcher({Y, compress::kPrefillPlanDim}) // write plan
.with_dtype<compress::PrefillPlanTensorDtype>()
.with_device(device_)
.verify(write_plan);
// might be needed for prefill write
const auto load_indices = extra.value_or(indices);
TensorMatcher({B}) // [read_positions]
.with_dtype<IndiceT>()
.with_device(device_)
.verify(load_indices);
const auto device = device_.unwrap();
const auto batch_size = static_cast<uint32_t>(B.unwrap());
const auto num_q_tokens = static_cast<uint32_t>(N.unwrap());
const auto num_c = static_cast<uint32_t>(X.unwrap());
const auto num_w = static_cast<uint32_t>(Y.unwrap());
const auto params = Compress128PrefillParams{
.kv_score_buffer = kv_score_buffer.data_ptr(),
.kv_score_input = kv_score_input.data_ptr(),
.kv_compressed_output = kv_compressed_output.data_ptr(),
.score_bias = ape.data_ptr(),
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
.load_indices = static_cast<const IndiceT*>(load_indices.data_ptr()),
.compress_plan = static_cast<const Plan128*>(compress_plan.data_ptr()),
.write_plan = static_cast<const Plan128*>(write_plan.data_ptr()),
.num_compress = num_c,
.num_write = num_w,
};
RuntimeCheck(num_q_tokens >= batch_size, "num_q_tokens must be >= batch_size");
RuntimeCheck(num_q_tokens >= std::max(num_c, num_w), "invalid prefill plan");
constexpr auto kBlockSize_C = kBlockSize;
constexpr auto kBlockSize_W = kWriteBlockSize;
if (const auto num_c_blocks = num_c * kNumSplit) {
LaunchKernel(num_c_blocks, kBlockSize_C, device) //
.enable_pdl(kUsePDL)(prefill_c_kernel, params);
}
if (const auto num_w_blocks = div_ceil(num_w * kNumSplit, kWarpsPerWriteBlock)) {
LaunchKernel(num_w_blocks, kBlockSize_W, device) //
.enable_pdl(kUsePDL)(prefill_w_kernel, params);
}
}
};
} // namespace sglang
@@ -1,722 +0,0 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/tile.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <sgl_kernel/warp.cuh>
#include <sgl_kernel/deepseek_v4/compress.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <tvm/ffi/container/tuple.h>
#include <tvm/ffi/object.h>
#include <algorithm>
#include <cfloat>
#include <cstdint>
namespace sglang {
namespace device::compress {
/// \brief Plan entry for online compress 128 prefill.
/// Each entry describes a contiguous segment of tokens that lies inside a
/// single 128-chunk. Multiple segments can map to the same batch id when the
/// extend tokens span chunk boundaries.
///
/// **Layout compatibility:** the field order/types match `PrefillPlan` so that
/// downstream kernels (e.g. `fused_norm_rope` in `CompressExtend` mode) can
/// consume the compress_plan tensor as-if it were a `PrefillPlan` tensor --
/// they only read `ragged_id` and `position`, both of which carry identical
/// semantics here (the LAST token of the segment in q-ragged and global
/// coordinates respectively).
///
/// Note that `window_len` here means "number of real tokens in this segment"
/// (1..128), which differs from `PrefillPlan::window_len`. Downstream kernels
/// that share the tensor MUST NOT read it under that name.
struct alignas(16) OnlinePrefillPlan {
/// \brief Ragged-q position of the LAST token in this segment.
/// Equal to `segment_start_ragged + window_len - 1`.
uint32_t ragged_id;
/// \brief Index into the `indices` / `load_indices` arrays.
uint32_t batch_id;
/// \brief Global position of the LAST token in this segment.
/// For compress plans, `position % 128 == 127` (chunk-closing); for write
/// plans, `position % 128 < 127`.
uint32_t position;
/// \brief Number of real tokens in this segment (1..128).
/// The first segment token sits at `position - window_len + 1` (global) and
/// at `ragged_id - window_len + 1` (ragged).
uint32_t window_len;
};
static_assert(alignof(OnlinePrefillPlan) == alignof(PrefillPlan));
static_assert(sizeof(OnlinePrefillPlan) == sizeof(PrefillPlan));
} // namespace device::compress
namespace host::compress {
using device::compress::OnlinePrefillPlan;
using OnlinePrefillPlanTensorDtype = uint8_t;
inline constexpr int64_t kOnlinePrefillPlanDim = 16;
static_assert(alignof(OnlinePrefillPlan) == sizeof(OnlinePrefillPlan));
static_assert(sizeof(OnlinePrefillPlan) == kOnlinePrefillPlanDim * sizeof(OnlinePrefillPlanTensorDtype));
} // namespace host::compress
using OnlinePlan = device::compress::OnlinePrefillPlan;
using IndiceT = int32_t;
/// \brief Need to reduce register usage to increase occupancy
struct Compress128OnlineDecodeParams {
/** \brief Shape: `[num_indices, 1, head_dim * 3 (max, sum, kv) ]` \n */
void* __restrict__ kv_score_buffer;
/** \brief Shape: `[batch_size, head_dim * 2]` */
const void* __restrict__ kv_score_input;
/** \brief Shape: `[batch_size, head_dim]` */
void* __restrict__ kv_compressed_output;
/** \brief Shape: `[128, head_dim]` (called `ape`) */
const void* __restrict__ score_bias;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ indices;
/** \brief Shape: `[batch_size, ]` */
const IndiceT* __restrict__ seq_lens;
/** \NOTE: `batch_size` <= `num_indices` */
uint32_t batch_size;
};
/// \brief Need to reduce register usage to increase occupancy
struct Compress128OnlinePrefillParams {
/** \brief Shape: `[num_indices, 1, head_dim * 3 (max, sum, kv) ]` \n */
void* __restrict__ kv_score_buffer;
/** \brief Shape: `[num_q_tokens, head_dim * 2]` */
const void* __restrict__ kv_score_input;
/** \brief Shape: `[num_q_tokens, head_dim]` */
void* __restrict__ kv_compressed_output;
/** \brief Shape: `[128, head_dim]` (called `ape`) */
const void* __restrict__ score_bias;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ indices;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ load_indices;
/// \brief Plan for segments that close a chunk (write to `kv_compressed_output`).
/// Shape: `[num_compress, 16]` (uint8).
const OnlinePlan* __restrict__ compress_plan;
/// \brief Plan for the trailing partial segment of each batch (write back to
/// `kv_score_buffer`). Shape: `[num_write, 16]` (uint8).
const OnlinePlan* __restrict__ write_plan;
uint32_t num_compress;
uint32_t num_write;
};
// 4 elements per thread, kHeadDim / 4 threads per block
template <int64_t kHeadDim, bool kUsePDL>
__global__ void flash_c128_online_decode(const __grid_constant__ Compress128OnlineDecodeParams params) {
using namespace device;
constexpr uint32_t kVecSize = 4;
constexpr uint32_t kBlockSize = kHeadDim / kVecSize;
using Vec = AlignedVector<float, kVecSize>;
const auto gmem = tile::Memory<Vec>::cta(kBlockSize);
const auto batch_id = blockIdx.x;
const auto index = params.indices[batch_id];
const auto seq_len = params.seq_lens[batch_id];
const auto kv_score_buffer = static_cast<float*>(params.kv_score_buffer);
const auto kv_buf = kv_score_buffer + index * (kHeadDim * 3);
const auto kv_score_input = static_cast<const float*>(params.kv_score_input);
const auto kv_src = kv_score_input + batch_id * (kHeadDim * 2);
/// NOTE: kv_score_buffer layout is [max, sum, kv] (slot 0 / 1 / 2). Reads,
/// writes, and the prefill kernel must all agree on this order.
const auto max_score_vec = gmem.load(kv_buf, 0);
const auto sum_score_vec = gmem.load(kv_buf, 1);
const auto old_kv_vec = gmem.load(kv_buf, 2);
/// NOTE: kv_score_input layout is | kv | score | (head_dim each), matching
/// the offline c128 kernel and the online prefill kernel.
const auto new_kv_vec = gmem.load(kv_src, 0);
const auto new_score_raw_vec = gmem.load(kv_src, 1);
/// NOTE: the new token sits at global position `seq_len - 1`, so its
/// position inside the 128-chunk is `(seq_len - 1) % 128`. The previous
/// `seq_len % 128` was off by one (`bias[127]` vs `bias[0]`, etc.).
const auto pos_in_chunk = (seq_len - 1) % 128;
const auto bias_vec = gmem.load(params.score_bias, pos_in_chunk);
Vec out_kv_vec;
Vec out_max_vec;
Vec out_sum_vec;
if (pos_in_chunk != 0) {
// Mid-chunk: combine prior partial state with the new token via online softmax.
#pragma unroll
for (uint32_t i = 0; i < 4; ++i) {
const auto old_max = max_score_vec[i];
const auto old_kv = old_kv_vec[i];
const auto new_score = new_score_raw_vec[i] + bias_vec[i];
const auto new_kv = new_kv_vec[i];
const auto new_max = fmax(old_max, new_score);
const auto old_sum = sum_score_vec[i] * expf(old_max - new_max);
const auto new_exp = expf(new_score - new_max);
const auto new_sum = old_sum + new_exp;
out_kv_vec[i] = (old_kv * old_sum + new_kv * new_exp) / new_sum;
out_max_vec[i] = new_max;
out_sum_vec[i] = new_sum;
}
} else {
// First token of a new 128-chunk: initialize state with this token alone.
#pragma unroll
for (uint32_t i = 0; i < 4; ++i) {
out_kv_vec[i] = new_kv_vec[i];
out_max_vec[i] = new_score_raw_vec[i] + bias_vec[i];
out_sum_vec[i] = 1.0f; // exp(score - max) with max == score
}
}
if (pos_in_chunk == 127) {
// Chunk just closed: emit the compressed kv. No need to update the buffer
// -- the next chunk's first token will overwrite it.
const auto kv_out = static_cast<float*>(params.kv_compressed_output) + batch_id * kHeadDim;
gmem.store(kv_out, out_kv_vec);
} else {
// Otherwise persist the running [max, sum, kv] state for the next step.
gmem.store(kv_buf, out_max_vec, 0);
gmem.store(kv_buf, out_sum_vec, 1);
gmem.store(kv_buf, out_kv_vec, 2);
}
}
constexpr int32_t kTileElements = 2; // split (along head-dim)
/// \brief Each warp will handle this many elements (split along softmax-128)
constexpr int32_t kElementsPerWarp = 8;
constexpr uint32_t kNumWarps = 128 / kElementsPerWarp;
constexpr uint32_t kPrefillBlockSize = device::kWarpThreads * kNumWarps;
using PrefillStorage = device::AlignedVector<float, kTileElements>;
struct Compress128SharedBuffer {
using Storage = device::AlignedVector<float, 4>;
Storage data[kNumWarps][device::kWarpThreads + 1]; // padding to avoid bank conflict
SGL_DEVICE Storage& operator()(uint32_t warp_id, uint32_t lane_id) {
return data[warp_id][lane_id];
}
SGL_DEVICE float& operator()(uint32_t warp_id, uint32_t lane_id, uint32_t tile_id) {
return data[warp_id][lane_id][tile_id];
}
};
template <bool kNeedData>
SGL_DEVICE void c128_prefill_forward(
const PrefillStorage (&kv)[kElementsPerWarp],
const PrefillStorage (&score)[kElementsPerWarp],
float* kv_out,
float* max_out,
float* sum_out,
const uint32_t warp_id,
const uint32_t lane_id) {
using namespace device;
/// NOTE: part 2: safe online softmax + weighted sum
using TmpStorage = typename Compress128SharedBuffer::Storage;
__shared__ Compress128SharedBuffer s_local_val_max;
__shared__ Compress128SharedBuffer s_local_exp_sum;
__shared__ Compress128SharedBuffer s_local_product;
TmpStorage tmp_val_max;
TmpStorage tmp_exp_sum;
TmpStorage tmp_product;
#pragma unroll
for (int32_t i = 0; i < kTileElements; ++i) {
float score_fp32[kElementsPerWarp];
#pragma unroll
for (int32_t j = 0; j < kElementsPerWarp; ++j) {
score_fp32[j] = score[j][i];
}
float max_value = score_fp32[0];
float sum_exp_value = 0.0f;
#pragma unroll
for (int32_t j = 1; j < kElementsPerWarp; ++j) {
const auto fp32_score = score_fp32[j];
max_value = fmaxf(max_value, fp32_score);
}
float sum_product = 0.0f;
#pragma unroll
for (int32_t j = 0; j < 8; ++j) {
const auto fp32_score = score_fp32[j];
const auto exp_score = expf(fp32_score - max_value);
sum_product += cast<float>(kv[j][i]) * exp_score;
sum_exp_value += exp_score;
}
tmp_val_max[i] = max_value;
tmp_exp_sum[i] = sum_exp_value;
tmp_product[i] = sum_product;
}
// naturally aligned, so no bank conflict
s_local_val_max(warp_id, lane_id) = tmp_val_max;
s_local_exp_sum(warp_id, lane_id) = tmp_exp_sum;
s_local_product(warp_id, lane_id) = tmp_product;
__syncthreads();
/// NOTE: part 3: online softmax
/// NOTE: We have `kTileElements * kWarpThreads * kNumWarps` values to reduce
/// each reduce will consume `kNumWarps` threads (use partial warp reduction)
constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps;
constexpr uint32_t kIteration = kReductionCount / kPrefillBlockSize;
#pragma unroll
for (uint32_t i = 0; i < kIteration; ++i) {
/// NOTE: Range `[0, kTileElements * kWarpThreads * kNumWarps)`
const uint32_t j = i * kPrefillBlockSize + warp_id * kWarpThreads + lane_id;
/// NOTE: Range `[0, kNumWarps)`
const uint32_t local_warp_id = j % kNumWarps;
/// NOTE: Range `[0, kTileElements * kWarpThreads)`
const uint32_t local_elem_id = j / kNumWarps;
/// NOTE: Range `[0, kTileElements)`
const uint32_t local_tile_id = local_elem_id % kTileElements;
/// NOTE: Range `[0, kWarpThreads)`
const uint32_t local_lane_id = local_elem_id / kTileElements;
/// NOTE: each warp will access the whole tile (all `kTileElements`)
/// and for different lanes, the memory access only differ in `local_warp_id`
/// so there's no bank conflict in shared memory access.
static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs");
const auto local_val_max = s_local_val_max(local_warp_id, local_lane_id, local_tile_id);
const auto local_exp_sum = s_local_exp_sum(local_warp_id, local_lane_id, local_tile_id);
const auto local_product = s_local_product(local_warp_id, local_lane_id, local_tile_id);
const auto global_val_max = warp::reduce_max<kNumWarps>(local_val_max);
const auto rescale = expf(local_val_max - global_val_max);
const auto global_exp_sum = warp::reduce_sum<kNumWarps>(local_exp_sum * rescale);
const auto final_scale = rescale / global_exp_sum;
const auto global_product = warp::reduce_sum<kNumWarps>(local_product * final_scale);
kv_out[local_elem_id] = global_product;
if constexpr (kNeedData) {
max_out[local_elem_id] = global_val_max;
sum_out[local_elem_id] = global_exp_sum;
}
}
if constexpr (kNeedData) __syncthreads();
}
/// \brief Sentinel score for padded positions in a 128-segment.
/// Must be finite so that `score - max` never produces NaN even when an
/// entire warp has only padded positions.
constexpr float kPadScore = -FLT_MAX;
/// \brief Online compress 128 prefill. Two passes share this body:
/// - `kWrite=false` (compress pass): handles segments that close a chunk.
/// May load prior partial state from the buffer, but never writes to it,
/// so concurrent blocks can read the same slot without racing.
/// - `kWrite=true` (write pass): handles the trailing partial segment of each
/// batch. Each batch contributes at most one such plan, so concurrent blocks
/// touch disjoint buffer slots.
///
/// The two passes MUST run as separate kernel launches (in stream order) so
/// that all reads in pass 1 finish before any writes in pass 2 start.
template <int64_t kHeadDim, bool kWrite, bool kUsePDL>
__global__ __launch_bounds__(kPrefillBlockSize, 2) //
void flash_c128_online_prefill(const __grid_constant__ Compress128OnlinePrefillParams params) {
using namespace device;
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
/// NOTE: the compiler folds the if-else at compile time.
const auto num_plans = kWrite ? params.num_write : params.num_compress;
const auto plan_ptr = kWrite ? params.write_plan : params.compress_plan;
const uint32_t global_id = blockIdx.x;
const uint32_t global_pid = global_id / kNumSplit; // plan id
const uint32_t global_sid = global_id % kNumSplit; // split id
if (global_pid >= num_plans) return;
const auto [ragged_id, batch_id, position, window_len] = plan_ptr[global_pid];
if (ragged_id == 0xFFFFFFFFu) [[unlikely]]
return;
const uint32_t warp_id = threadIdx.x / kWarpThreads;
const uint32_t lane_id = threadIdx.x % kWarpThreads;
const int32_t split_offset = global_sid * kTileDim; // int32 is enough
const auto kv_score_buffer = static_cast<float*>(params.kv_score_buffer);
const auto kv_score_input = static_cast<const float*>(params.kv_score_input);
const auto kv_compressed_output = static_cast<float*>(params.kv_compressed_output);
const auto score_bias_base = static_cast<const float*>(params.score_bias);
constexpr int64_t kElementSize = kHeadDim * 2; // | kv | score |
const uint32_t chunk_offset = (position % 128u) + 1u - window_len;
const uint32_t window_end = chunk_offset + window_len; // exclusive, in [1, 128]
const int32_t segment_start = ragged_id - (position % 128u); // can be negative, but safe
const int32_t load_index = chunk_offset != 0 ? params.load_indices[batch_id] : -1;
const int32_t store_index = kWrite ? params.indices[batch_id] : -1;
PDLWaitPrimary<kUsePDL>();
// 2 * 8 = 16 register per elem. in theory we should consume 48 register here
PrefillStorage kv[kElementsPerWarp];
PrefillStorage score[kElementsPerWarp];
PrefillStorage bias[kElementsPerWarp];
const auto warp_offset = warp_id * kElementsPerWarp;
#pragma unroll
for (uint32_t i = 0; i < kElementsPerWarp; ++i) {
const uint32_t j = i + warp_offset;
if (j >= chunk_offset && j < window_end) {
const auto kv_src_ptr = kv_score_input + (segment_start + j) * kElementSize + split_offset;
const auto score_src_ptr = kv_src_ptr + kHeadDim;
const auto bias_src_ptr = score_bias_base + j * kHeadDim + split_offset;
kv[i].load(kv_src_ptr, lane_id);
score[i].load(score_src_ptr, lane_id);
bias[i].load(bias_src_ptr, lane_id);
}
}
#pragma unroll
for (uint32_t i = 0; i < kElementsPerWarp; ++i) {
const uint32_t j = i + warp_offset;
const bool is_valid = (j >= chunk_offset && j < window_end);
#pragma unroll
for (uint32_t ii = 0; ii < kTileElements; ++ii) {
score[i][ii] = is_valid ? score[i][ii] + bias[i][ii] : kPadScore;
/// NOTE: must zero out kv on padded slots -- `c128_prefill_forward`
/// computes `kv * exp_score` where `exp_score = expf(-FLT_MAX - max) ??? 0`,
/// and IEEE-754 makes `NaN * 0 = NaN` / `+-inf * 0 = NaN`. An
/// uninitialized register can hold a NaN/inf bit pattern, so without
/// this reset a single padded warp can poison the whole softmax.
kv[i][ii] = is_valid ? kv[i][ii] : 0.0f;
}
}
__shared__ alignas(16) float seg_kv[kTileDim];
__shared__ alignas(16) float seg_max[kTileDim];
__shared__ alignas(16) float seg_sum[kTileDim];
c128_prefill_forward<true>(kv, score, seg_kv, seg_max, seg_sum, warp_id, lane_id);
PDLTriggerSecondary<kUsePDL>();
if (warp_id == 0) {
PrefillStorage out_kv_vec, out_max_vec, out_sum_vec;
out_kv_vec.load(seg_kv, lane_id);
out_max_vec.load(seg_max, lane_id);
out_sum_vec.load(seg_sum, lane_id);
if (chunk_offset != 0) {
/// NOTE: load (max, sum, kv) of the in-progress chunk for this index.
/// `load_indices` may differ from `indices` when the prior partial state
/// lives on a different slot than the slot we ultimately write to.
const auto buf_load = kv_score_buffer + load_index * (kHeadDim * 3) + split_offset;
PrefillStorage buf_max_vec, buf_sum_vec, buf_kv_vec;
buf_max_vec.load(buf_load + 0 * kHeadDim, lane_id);
buf_sum_vec.load(buf_load + 1 * kHeadDim, lane_id);
buf_kv_vec.load(buf_load + 2 * kHeadDim, lane_id);
#pragma unroll
for (uint32_t ii = 0; ii < kTileElements; ++ii) {
const float m1 = buf_max_vec[ii];
const float s1 = buf_sum_vec[ii];
const float k1 = buf_kv_vec[ii];
const float m2 = out_max_vec[ii];
const float s2 = out_sum_vec[ii];
const float k2 = out_kv_vec[ii];
const float new_max = fmaxf(m1, m2);
const float new_s1 = s1 * expf(m1 - new_max);
const float new_s2 = s2 * expf(m2 - new_max);
const float new_sum = new_s1 + new_s2;
const float new_kv = (k1 * new_s1 + k2 * new_s2) / new_sum;
out_max_vec[ii] = new_max;
out_sum_vec[ii] = new_sum;
out_kv_vec[ii] = new_kv;
}
}
if constexpr (kWrite) {
const auto buf_store = kv_score_buffer + store_index * (kHeadDim * 3) + split_offset;
reinterpret_cast<PrefillStorage*>(buf_store + 0 * kHeadDim)[lane_id] = out_max_vec;
reinterpret_cast<PrefillStorage*>(buf_store + 1 * kHeadDim)[lane_id] = out_sum_vec;
reinterpret_cast<PrefillStorage*>(buf_store + 2 * kHeadDim)[lane_id] = out_kv_vec;
} else {
const auto out_ptr = kv_compressed_output + ragged_id * kHeadDim + split_offset;
reinterpret_cast<PrefillStorage*>(out_ptr)[lane_id] = out_kv_vec;
}
}
}
template <int64_t kHeadDim, bool kUsePDL>
struct FlashCompress128OnlineKernel {
static constexpr auto decode_kernel = flash_c128_online_decode<kHeadDim, kUsePDL>;
template <bool kWrite>
static constexpr auto prefill_kernel = flash_c128_online_prefill<kHeadDim, kWrite, kUsePDL>;
static constexpr auto prefill_c_kernel = prefill_kernel</*kWrite=*/false>;
static constexpr auto prefill_w_kernel = prefill_kernel</*kWrite=*/true>;
static constexpr int64_t kTileDim = kTileElements * device::kWarpThreads; // 64
static constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
static constexpr uint32_t kDecodeBlockSize = kHeadDim / 4;
static void run_decode(
const tvm::ffi::TensorView kv_score_buffer,
const tvm::ffi::TensorView kv_score_input,
const tvm::ffi::TensorView kv_compressed_output,
const tvm::ffi::TensorView ape,
const tvm::ffi::TensorView indices,
const tvm::ffi::TensorView seq_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> /* UNUSED */) {
using namespace host;
auto B = SymbolicSize{"batch_size"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({-1, 1, kHeadDim * 3}) // kv score buffer (max, sum, kv)
.with_dtype<float>()
.with_device(device)
.verify(kv_score_buffer);
TensorMatcher({B, kHeadDim * 2}) // kv score input
.with_dtype<float>()
.with_device(device)
.verify(kv_score_input);
TensorMatcher({B, kHeadDim}) // kv compressed output
.with_dtype<float>()
.with_device(device)
.verify(kv_compressed_output);
TensorMatcher({128, kHeadDim}) // ape
.with_dtype<float>()
.with_device(device)
.verify(ape);
TensorMatcher({B}).with_dtype<IndiceT>().with_device(device).verify(indices);
TensorMatcher({B}).with_dtype<IndiceT>().with_device(device).verify(seq_lens);
const auto batch_size = static_cast<uint32_t>(B.unwrap());
const auto params = Compress128OnlineDecodeParams{
.kv_score_buffer = kv_score_buffer.data_ptr(),
.kv_score_input = kv_score_input.data_ptr(),
.kv_compressed_output = kv_compressed_output.data_ptr(),
.score_bias = ape.data_ptr(),
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
.seq_lens = static_cast<const IndiceT*>(seq_lens.data_ptr()),
.batch_size = batch_size,
};
LaunchKernel(batch_size, kDecodeBlockSize, device.unwrap()) //
.enable_pdl(kUsePDL)(decode_kernel, params);
}
static void run_prefill(
const tvm::ffi::TensorView kv_score_buffer,
const tvm::ffi::TensorView kv_score_input,
const tvm::ffi::TensorView kv_compressed_output,
const tvm::ffi::TensorView ape,
const tvm::ffi::TensorView indices,
const tvm::ffi::TensorView compress_plan,
const tvm::ffi::TensorView write_plan,
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
using namespace host;
using host::compress::kOnlinePrefillPlanDim;
using host::compress::OnlinePrefillPlanTensorDtype;
auto B = SymbolicSize{"batch_size"};
auto N = SymbolicSize{"num_q_tokens"};
auto X = SymbolicSize{"compress_tokens"};
auto Y = SymbolicSize{"write_tokens"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
TensorMatcher({-1, 1, kHeadDim * 3}) // kv score buffer (max, sum, kv) ??? 2D
.with_dtype<float>()
.with_device(device_)
.verify(kv_score_buffer);
TensorMatcher({N, kHeadDim * 2}) // kv score input
.with_dtype<float>()
.with_device(device_)
.verify(kv_score_input);
TensorMatcher({N, kHeadDim}) // kv compressed output
.with_dtype<float>()
.with_device(device_)
.verify(kv_compressed_output);
TensorMatcher({128, kHeadDim}) // ape
.with_dtype<float>()
.with_device(device_)
.verify(ape);
TensorMatcher({B}) // indices
.with_dtype<IndiceT>()
.with_device(device_)
.verify(indices);
TensorMatcher({X, kOnlinePrefillPlanDim}) // compress plan
.with_dtype<OnlinePrefillPlanTensorDtype>()
.with_device(device_)
.verify(compress_plan);
TensorMatcher({Y, kOnlinePrefillPlanDim}) // write plan
.with_dtype<OnlinePrefillPlanTensorDtype>()
.with_device(device_)
.verify(write_plan);
/// NOTE: `extra` is `load_indices`. When the previous partial state lives
/// on a slot different from the destination slot (e.g. paged buffers), the
/// caller must supply this; otherwise it defaults to `indices`.
const auto load_indices = extra.value_or(indices);
TensorMatcher({B}).with_dtype<IndiceT>().with_device(device_).verify(load_indices);
const auto device = device_.unwrap();
const auto num_c = static_cast<uint32_t>(X.unwrap());
const auto num_w = static_cast<uint32_t>(Y.unwrap());
const auto params = Compress128OnlinePrefillParams{
.kv_score_buffer = kv_score_buffer.data_ptr(),
.kv_score_input = kv_score_input.data_ptr(),
.kv_compressed_output = kv_compressed_output.data_ptr(),
.score_bias = ape.data_ptr(),
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
.load_indices = static_cast<const IndiceT*>(load_indices.data_ptr()),
.compress_plan = static_cast<const OnlinePlan*>(compress_plan.data_ptr()),
.write_plan = static_cast<const OnlinePlan*>(write_plan.data_ptr()),
.num_compress = num_c,
.num_write = num_w,
};
/// NOTE: pass 1 reads the buffer (for the first segment of each batch
/// that started mid-chunk) and writes only to `kv_compressed_output`.
/// Pass 2 then writes the trailing partial state of each batch back to
/// the buffer. Stream serialization between the two launches enforces
/// read-before-write on shared buffer slots.
if (const auto num_c_blocks = num_c * kNumSplit) {
LaunchKernel(num_c_blocks, kPrefillBlockSize, device) //
.enable_pdl(kUsePDL)(prefill_c_kernel, params);
}
if (const auto num_w_blocks = num_w * kNumSplit) {
LaunchKernel(num_w_blocks, kPrefillBlockSize, device) //
.enable_pdl(kUsePDL)(prefill_w_kernel, params);
}
}
};
namespace host::compress {
using OnlinePlanResult = tvm::ffi::Tuple<uint32_t, uint32_t>;
struct OnlinePrefillCompressParams {
OnlinePrefillPlan* __restrict__ compress_plan;
OnlinePrefillPlan* __restrict__ write_plan;
const int64_t* __restrict__ seq_lens;
const int64_t* __restrict__ extend_lens;
uint32_t batch_size;
uint32_t num_tokens;
};
/// \brief Build the compress + write plans for online compress 128 prefill.
///
/// Each batch's `[prefix_len, prefix_len + extend_len)` range is split at
/// 128-aligned boundaries. Every resulting segment falls into one of:
/// - **compress**: closes a 128-chunk (`chunk_offset + window_len == 128`).
/// These plans only read the buffer (when starting mid-chunk) and write the
/// compressed kv to `kv_compressed_output`.
/// - **write**: trailing partial of the batch (`chunk_offset + window_len < 128`).
/// May read the buffer and always writes the new partial state back to it.
/// Each batch produces at most one such plan.
///
/// The two plans MUST be dispatched as separate kernel launches in stream
/// order so that pass-1 reads of a buffer slot complete before any pass-2
/// write of the same slot.
inline OnlinePlanResult plan_online_prefill_host(const OnlinePrefillCompressParams& params, const bool use_cuda_graph) {
const auto& [compress_plan, write_plan, seq_lens, extend_lens, batch_size, num_tokens] = params;
uint32_t counter = 0;
uint32_t compress_count = 0;
uint32_t write_count = 0;
for (const auto i : irange(batch_size)) {
const uint32_t seq_len = static_cast<uint32_t>(seq_lens[i]);
const uint32_t extend_len = static_cast<uint32_t>(extend_lens[i]);
RuntimeCheck(0 < extend_len && extend_len <= seq_len);
const uint32_t prefix_len = seq_len - extend_len;
const uint32_t end_pos = prefix_len + extend_len;
/// NOTE: split the extend range into per-128-chunk segments. Each segment
/// stays inside one chunk, so the kernel can decide load/store from
/// `chunk_offset` and `window_len` alone.
uint32_t pos = prefix_len;
while (pos < end_pos) {
const uint32_t chunk_start = (pos / 128u) * 128u;
const uint32_t seg_end = std::min(end_pos, chunk_start + 128u); // exclusive
const uint32_t seg_len = seg_end - pos;
const uint32_t chunk_off = pos - chunk_start;
/// NOTE: store last-token coordinates so that downstream consumers
/// (e.g. `fused_norm_rope`) can read `ragged_id` and `position` with the
/// same semantics as `PrefillPlan`. The segment start is recoverable as
/// `ragged_id - window_len + 1` and `position - window_len + 1`.
const uint32_t last_pos = seg_end - 1;
const uint32_t last_ragged = counter + (last_pos - prefix_len);
const auto plan = OnlinePrefillPlan{
.ragged_id = last_ragged,
.batch_id = i,
.position = last_pos,
.window_len = seg_len,
};
if (chunk_off + seg_len == 128u) {
// full chunk, must be complete, maybe read the buffer, no write
RuntimeCheck(compress_count < num_tokens);
compress_plan[compress_count++] = plan;
} else {
// last chunk, must be incomplete, maybe read the buffer, must write
RuntimeCheck(write_count < num_tokens);
write_plan[write_count++] = plan;
}
pos = seg_end;
}
counter += extend_len;
}
RuntimeCheck(counter == num_tokens, "input size ", counter, " != num_q_tokens ", num_tokens);
if (!use_cuda_graph) return OnlinePlanResult{compress_count, write_count};
/// NOTE: pad both plans with sentinel entries so cuda-graph runs always see
/// the same number of blocks. The kernel skips plans whose `ragged_id` is -1.
constexpr auto kInvalid = static_cast<uint32_t>(-1);
constexpr auto kInvalidPlan = OnlinePrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid};
for (const auto i : irange(compress_count, num_tokens)) {
compress_plan[i] = kInvalidPlan;
}
for (const auto i : irange(write_count, num_tokens)) {
write_plan[i] = kInvalidPlan;
}
return OnlinePlanResult{num_tokens, num_tokens};
}
inline OnlinePlanResult plan_online_prefill(
const tvm::ffi::TensorView extend_lens,
const tvm::ffi::TensorView seq_lens,
const tvm::ffi::TensorView compress_plan,
const tvm::ffi::TensorView write_plan,
const bool use_cuda_graph) {
auto N = SymbolicSize{"batch_size"};
auto M = SymbolicSize{"num_tokens"};
auto device = SymbolicDevice{};
/// NOTE: only host (CPU/cuda-host) planning is implemented for now. The
device.set_options<kDLCPU, kDLCUDAHost>();
TensorMatcher({N}) //
.with_dtype<int64_t>()
.with_device(device)
.verify(extend_lens)
.verify(seq_lens);
TensorMatcher({M, kOnlinePrefillPlanDim}) //
.with_dtype<OnlinePrefillPlanTensorDtype>()
.with_device(device)
.verify(compress_plan)
.verify(write_plan);
const auto params = OnlinePrefillCompressParams{
.compress_plan = static_cast<OnlinePrefillPlan*>(compress_plan.data_ptr()),
.write_plan = static_cast<OnlinePrefillPlan*>(write_plan.data_ptr()),
.seq_lens = static_cast<const int64_t*>(seq_lens.data_ptr()),
.extend_lens = static_cast<const int64_t*>(extend_lens.data_ptr()),
.batch_size = static_cast<uint32_t>(N.unwrap()),
.num_tokens = static_cast<uint32_t>(M.unwrap()),
};
return plan_online_prefill_host(params, use_cuda_graph);
}
} // namespace host::compress
[[maybe_unused]]
constexpr auto& plan_compress_online_prefill = host::compress::plan_online_prefill;
} // namespace sglang
@@ -233,8 +233,8 @@ constexpr float kPadScore = -FLT_MAX;
__syncthreads();
// Cross-warp reduction. Same recipe as c128_online.cuh: each block-thread
// pair reduces a (tile_id, lane_id) slot using a kNumWarps-wide warp shuffle.
// Cross-warp reduction: each block-thread pair reduces a (tile_id, lane_id)
// slot using a kNumWarps-wide warp shuffle.
constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps;
constexpr uint32_t kIteration = kReductionCount / kPrefillBlockSize;
static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs");
@@ -1,549 +0,0 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/tile.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <sgl_kernel/warp.cuh>
#include <sgl_kernel/deepseek_v4/compress.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <tvm/ffi/object.h>
#include <cstdint>
namespace sglang {
using Plan4 = device::compress::PrefillPlan;
using IndiceT = int32_t;
/// \brief Each thread will handle this many elements (split along head_dim)
constexpr int kTileElements = 4;
/// \brief Need to improve register usage to reduce latency
#define C4_KERNEL __global__ __launch_bounds__(128, 4)
enum class PageMode {
RingBuffer = 8,
Page4Align = 4,
};
struct alignas(16) C4IndexBundle {
int32_t load_first_page;
int32_t load_second_page;
int32_t write_first_page;
int32_t last_position;
};
struct Compress4DecodeParams {
/**
* \brief Shape: `[num_indices, 8, head_dim * 4]` \n
* last dimension layout:
* | kv overlap | kv | score overlap | score |
*/
void* __restrict__ kv_score_buffer;
/** \brief Shape: `[batch_size, head_dim * 4]` */
const void* __restrict__ kv_score_input;
/** \brief Shape: `[batch_size, head_dim]` */
void* __restrict__ kv_compressed_output;
/** \brief Shape: `[8, head_dim]` (called `ape`) */
const void* __restrict__ score_bias;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ indices;
/** \brief Shape: `[batch_size, ]` */
const IndiceT* __restrict__ seq_lens;
/** \brief Shape: `[batch_size, 1]` */
const int32_t* __restrict__ extra;
/** \NOTE: `batch_size` <= `num_indices` */
uint32_t batch_size;
};
struct Compress4PrefillParams {
/**
* \brief Shape: `[num_indices, 8, head_dim * 4]` \n
* last dimension layout:
* | kv overlap | kv | score overlap | score |
*/
void* __restrict__ kv_score_buffer;
/** \brief Shape: `[num_q_tokens, head_dim * 4]` */
const void* __restrict__ kv_score_input;
/** \brief Shape: `[num_q_tokens, head_dim]` */
void* __restrict__ kv_compressed_output;
/** \brief Shape: `[8, head_dim]` (called `ape`) */
const void* __restrict__ score_bias;
/** \brief Shape: `[batch_size, ]`*/
const IndiceT* __restrict__ indices;
/** \brief Shape: `[batch_size, 4]` */
const C4IndexBundle* __restrict__ extra;
/** \brief The following part is plan info. */
const Plan4* __restrict__ compress_plan;
const Plan4* __restrict__ write_plan;
uint32_t num_compress;
uint32_t num_write;
};
template <typename T>
SGL_DEVICE void c4_write(
T* kv_score_buf, //
const T* kv_score_src,
const int64_t head_dim,
const int32_t write_pos) {
using namespace device;
using Storage = AlignedVector<T, kTileElements>;
const auto element_size = head_dim * 4;
const auto gmem = tile::Memory<Storage>::warp();
kv_score_buf += write_pos * element_size;
/// NOTE: Layout | [0] = kv overlap | [1] = kv | [2] = score overlap | [3] = score |
Storage kv_score[4];
#pragma unroll
for (int32_t i = 0; i < 4; ++i) {
kv_score[i] = gmem.load(kv_score_src + head_dim * i);
}
#pragma unroll
for (int32_t i = 0; i < 4; ++i) {
gmem.store(kv_score_buf + head_dim * i, kv_score[i]);
}
}
template <bool kPaged, typename InFloat, typename OutFloat>
SGL_DEVICE void c4_forward(
const InFloat* kv_score_buf,
const InFloat* kv_score_src,
OutFloat* kv_out,
const InFloat* score_bias,
const int64_t head_dim,
const int32_t seq_len,
const int32_t window_len,
[[maybe_unused]] const InFloat* kv_score_overlap_buf = nullptr) {
using namespace device;
const auto element_size = head_dim * 4;
const auto score_offset = head_dim * 2;
const auto overlap_stride = head_dim;
/// NOTE: part 1: load kv + score
using StorageIn = AlignedVector<InFloat, kTileElements>;
const auto gmem_in = tile::Memory<StorageIn>::warp();
StorageIn kv[8];
StorageIn score[8];
StorageIn bias[8];
#pragma unroll
for (int32_t i = 0; i < 8; ++i) {
bias[i] = gmem_in.load(score_bias + i * head_dim);
}
#pragma unroll
for (int32_t i = 0; i < 8; ++i) {
const bool is_overlap = i < 4;
const InFloat* src;
if (i < window_len) {
/// NOTE: `seq_len` must be a multiple of 4 here
if constexpr (kPaged) {
const auto kv_score_ptr = is_overlap ? kv_score_overlap_buf : kv_score_buf;
const int32_t k = i % 4;
src = kv_score_ptr + k * element_size;
} else {
const int32_t k = (seq_len + i) % 8;
src = kv_score_buf + k * element_size;
}
} else {
/// NOTE: k in [-7, 0]. We'll load from the ragged `kv_score_src`
const int32_t k = i - 7;
src = kv_score_src + k * element_size;
}
src += (is_overlap ? 0 : overlap_stride);
kv[i] = gmem_in.load(src);
score[i] = gmem_in.load(src + score_offset);
}
if (seq_len == 4) {
[[unlikely]];
constexpr float kFloatNegInf = -1e9f;
#pragma unroll
for (int32_t i = 0; i < 4; ++i) {
kv[i].fill(cast<InFloat>(0.0f));
score[i].fill(cast<InFloat>(kFloatNegInf));
}
}
/// NOTE: part 2: safe online softmax + weighted sum
using StorageOut = AlignedVector<OutFloat, kTileElements>;
const auto gmem_out = tile::Memory<StorageOut>::warp();
StorageOut result;
#pragma unroll
for (int32_t i = 0; i < kTileElements; ++i) {
float score_fp32[8];
#pragma unroll
for (int32_t j = 0; j < 8; ++j) {
score_fp32[j] = cast<float>(score[j][i]) + cast<float>(bias[j][i]);
}
float max_value = score_fp32[0];
float sum_exp_value = 0.0f;
#pragma unroll
for (int32_t j = 1; j < 8; ++j) {
const auto fp32_score = score_fp32[j];
max_value = fmaxf(max_value, fp32_score);
}
float sum_product = 0.0f;
#pragma unroll
for (int32_t j = 0; j < 8; ++j) {
const auto fp32_score = score_fp32[j];
const auto exp_score = expf(fp32_score - max_value);
sum_product += cast<float>(kv[j][i]) * exp_score;
sum_exp_value += exp_score;
}
result[i] = cast<OutFloat>(sum_product / sum_exp_value);
}
gmem_out.store(kv_out, result);
}
template <int64_t kHeadDim, typename InFloat, typename OutFloat, PageMode kMode, bool kUsePDL>
C4_KERNEL void flash_c4_decode(const __grid_constant__ Compress4DecodeParams params) {
using namespace device;
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 128
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
constexpr int64_t kElementSize = kHeadDim * 4; // `* 4` due to overlap transform + score
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
const auto& [
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
indices, seq_lens, extra, batch_size // decode info
] = params;
const uint32_t global_tid = blockIdx.x * blockDim.x + threadIdx.x;
const uint32_t global_wid = global_tid / kWarpThreads; // warp id
const uint32_t global_bid = global_wid / kNumSplit; // batch id
const uint32_t global_sid = global_wid % kNumSplit; // split id
if (global_bid >= batch_size) return;
const int32_t index = indices[global_bid];
const int32_t seq_len = seq_lens[global_bid];
const int64_t split_offset = global_sid * kTileDim;
// kv score
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
// kv input
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
const auto kv_src = kv_score_input + global_bid * kElementSize + split_offset;
// kv output
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
const auto kv_out = kv_compressed_output + global_bid * kHeadDim + split_offset;
// score bias (ape)
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
PDLWaitPrimary<kUsePDL>();
/// NOTE: `position` = `seq_len - 1`. To avoid underflow, we use `seq_len + page_size - 1`
if constexpr (kMode == PageMode::Page4Align) {
const auto index_prev = extra[global_bid];
const auto kv_buf = kv_score_buffer + index * (kElementSize * 4) + split_offset;
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 3) % 4);
if (seq_len % 4 == 0) {
const auto kv_overlap = kv_buf + (index_prev - index) * (kElementSize * 4);
c4_forward<true>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, 8, kv_overlap);
}
} else {
static_assert(kMode == PageMode::RingBuffer, "Unsupported PageMode");
const auto kv_buf = kv_score_buffer + index * (kElementSize * 8) + split_offset;
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 7) % 8);
if (seq_len % 4 == 0) {
c4_forward<false>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, /*window_size=*/8);
}
}
PDLTriggerSecondary<kUsePDL>();
}
template <int64_t kHeadDim, typename InFloat, typename OutFloat, PageMode kMode, bool kWrite, bool kUsePDL>
C4_KERNEL void flash_c4_prefill(const __grid_constant__ Compress4PrefillParams params) {
using namespace device;
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 128
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
constexpr int64_t kElementSize = kHeadDim * 4; // `* 4` due to overlap transform + score
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
const auto& [
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
indices, extra, compress_plan, write_plan, num_compress, num_write // prefill plan
] = params;
const uint32_t global_tid = blockIdx.x * blockDim.x + threadIdx.x;
const uint32_t global_wid = global_tid / kWarpThreads; // warp id
const uint32_t global_pid = global_wid / kNumSplit; // plan id
const uint32_t global_sid = global_wid % kNumSplit; // split id
/// NOTE: compiler can optimize this if-else at compile time
const auto num_plans = kWrite ? num_write : num_compress;
const auto plan_ptr = kWrite ? write_plan : compress_plan;
if (global_pid >= num_plans) return;
const auto& [ragged_id, global_bid, position, window_len] = plan_ptr[global_pid];
const int64_t split_offset = global_sid * kTileDim;
// kv score
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
// kv input
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
const auto kv_src = kv_score_input + ragged_id * kElementSize + split_offset;
// kv output
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
const auto kv_out = kv_compressed_output + ragged_id * kHeadDim + split_offset;
if (ragged_id == 0xFFFFFFFF) [[unlikely]]
return;
// score bias (ape)
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
const auto seq_len = position + 1;
const int32_t index = indices[global_bid];
PDLWaitPrimary<kUsePDL>();
if constexpr (kMode == PageMode::Page4Align) {
const auto write_second_page = index;
const auto [load_first_page, load_second_page, write_first_page, last_pos] = extra[global_bid];
if constexpr (kWrite) {
int32_t index;
if (position < static_cast<uint32_t>(last_pos)) {
index = write_first_page;
} else {
index = write_second_page;
}
const auto kv_buf = kv_score_buffer + index * (kElementSize * 4) + split_offset;
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 4);
} else {
int32_t index_overlap, index_normal;
if (window_len <= 4) {
index_overlap = load_second_page;
index_normal = load_second_page; // not used
} else {
index_overlap = load_first_page;
index_normal = load_second_page;
}
const auto kv_buf = kv_score_buffer + index_normal * (kElementSize * 4) + split_offset;
const auto kv_overlap = kv_score_buffer + index_overlap * (kElementSize * 4) + split_offset;
c4_forward<true>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, window_len, kv_overlap);
}
} else {
static_assert(kMode == PageMode::RingBuffer, "Unsupported PageMode");
const auto kv_buf = kv_score_buffer + index * (kElementSize * 8) + split_offset;
if constexpr (kWrite) {
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 8);
} else {
c4_forward<false>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, window_len);
}
}
PDLTriggerSecondary<kUsePDL>();
}
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kUsePDL>
struct FlashCompress4Kernel {
template <PageMode kMode>
static constexpr auto decode_kernel = flash_c4_decode<kHeadDim, InFloat, OutFloat, kMode, kUsePDL>;
template <PageMode kMode, bool kWrite>
static constexpr auto prefill_kernel = flash_c4_prefill<kHeadDim, InFloat, OutFloat, kMode, kWrite, kUsePDL>;
template <PageMode kMode>
static constexpr auto prefill_c_kernel = prefill_kernel<kMode, /*kWrite=*/false>;
template <PageMode kMode>
static constexpr auto prefill_w_kernel = prefill_kernel<kMode, /*kWrite=*/true>;
static constexpr uint32_t kBlockSize = 128;
static constexpr uint32_t kTileDim = kTileElements * device::kWarpThreads;
static constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
static constexpr uint32_t kWarpsPerBlock = kBlockSize / device::kWarpThreads;
using Self = FlashCompress4Kernel;
static void run_decode(
const tvm::ffi::TensorView kv_score_buffer,
const tvm::ffi::TensorView kv_score_input,
const tvm::ffi::TensorView kv_compressed_output,
const tvm::ffi::TensorView ape,
const tvm::ffi::TensorView indices,
const tvm::ffi::TensorView seq_lens,
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
using namespace host;
// this should not happen in practice
auto B = SymbolicSize{"batch_size"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
const auto extra_ptr = _get_extra_pointer(B, device_, extra);
const auto page_size = extra_ptr != nullptr ? 4 : 8;
TensorMatcher({-1, page_size, kHeadDim * 4}) // kv score
.with_dtype<InFloat>()
.with_device(device_)
.verify(kv_score_buffer);
TensorMatcher({B, kHeadDim * 4}) // kv score input
.with_dtype<InFloat>()
.with_device(device_)
.verify(kv_score_input);
TensorMatcher({B, kHeadDim}) // kv compressed output
.with_dtype<OutFloat>()
.with_device(device_)
.verify(kv_compressed_output);
TensorMatcher({8, kHeadDim}) // ape
.with_dtype<InFloat>()
.with_device(device_)
.verify(ape);
TensorMatcher({B}) // indices
.with_dtype<IndiceT>()
.with_device(device_)
.verify(indices);
TensorMatcher({B}) // seq lens
.with_dtype<IndiceT>()
.with_device(device_)
.verify(seq_lens);
const auto device = device_.unwrap();
const auto batch_size = static_cast<uint32_t>(B.unwrap());
const auto params = Compress4DecodeParams{
.kv_score_buffer = kv_score_buffer.data_ptr(),
.kv_score_input = kv_score_input.data_ptr(),
.kv_compressed_output = kv_compressed_output.data_ptr(),
.score_bias = ape.data_ptr(),
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
.seq_lens = static_cast<const IndiceT*>(seq_lens.data_ptr()),
.extra = static_cast<const int32_t*>(extra_ptr),
.batch_size = batch_size,
};
const auto kernel = extra_ptr != nullptr ? decode_kernel<PageMode::Page4Align> //
: decode_kernel<PageMode::RingBuffer>;
const uint32_t num_blocks = div_ceil(batch_size * kNumSplit, kWarpsPerBlock);
LaunchKernel(num_blocks, kBlockSize, device) //
.enable_pdl(kUsePDL)(kernel, params);
}
static void run_prefill(
const tvm::ffi::TensorView kv_score_buffer,
const tvm::ffi::TensorView kv_score_input,
const tvm::ffi::TensorView kv_compressed_output,
const tvm::ffi::TensorView ape,
const tvm::ffi::TensorView indices,
const tvm::ffi::TensorView compress_plan,
const tvm::ffi::TensorView write_plan,
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
using namespace host;
auto B = SymbolicSize{"batch_size"};
auto N = SymbolicSize{"num_q_tokens"};
auto X = SymbolicSize{"compress_tokens"};
auto Y = SymbolicSize{"write_tokens"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
const auto extra_ptr = _get_extra_pointer(B, device_, extra, /*is_prefill=*/true);
const auto page_size = extra_ptr != nullptr ? 4 : 8;
TensorMatcher({-1, page_size, kHeadDim * 4}) // kv score
.with_dtype<InFloat>()
.with_device(device_)
.verify(kv_score_buffer);
TensorMatcher({N, kHeadDim * 4}) // kv score input
.with_dtype<InFloat>()
.with_device(device_)
.verify(kv_score_input);
TensorMatcher({N, kHeadDim}) // kv compressed output
.with_dtype<OutFloat>()
.with_device(device_)
.verify(kv_compressed_output);
TensorMatcher({8, kHeadDim}) // ape
.with_dtype<InFloat>()
.with_device(device_)
.verify(ape);
TensorMatcher({B}) // indices
.with_dtype<IndiceT>()
.with_device(device_)
.verify(indices);
TensorMatcher({X, compress::kPrefillPlanDim}) // compress plan
.with_dtype<compress::PrefillPlanTensorDtype>()
.with_device(device_)
.verify(compress_plan);
TensorMatcher({Y, compress::kPrefillPlanDim}) // write plan
.with_dtype<compress::PrefillPlanTensorDtype>()
.with_device(device_)
.verify(write_plan);
const auto device = device_.unwrap();
const auto batch_size = static_cast<uint32_t>(B.unwrap());
const auto num_q_tokens = static_cast<uint32_t>(N.unwrap());
const auto num_c = static_cast<uint32_t>(X.unwrap());
const auto num_w = static_cast<uint32_t>(Y.unwrap());
const auto params = Compress4PrefillParams{
.kv_score_buffer = kv_score_buffer.data_ptr(),
.kv_score_input = kv_score_input.data_ptr(),
.kv_compressed_output = kv_compressed_output.data_ptr(),
.score_bias = ape.data_ptr(),
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
.extra = static_cast<const C4IndexBundle*>(extra_ptr),
.compress_plan = static_cast<const Plan4*>(compress_plan.data_ptr()),
.write_plan = static_cast<const Plan4*>(write_plan.data_ptr()),
.num_compress = num_c,
.num_write = num_w,
};
RuntimeCheck(num_q_tokens >= batch_size, "num_q_tokens must be >= batch_size");
RuntimeCheck(num_q_tokens >= std::max(num_c, num_w), "invalid prefill plan");
if (const auto num_c_blocks = div_ceil(num_c * kNumSplit, kWarpsPerBlock)) {
const auto c_kernel = extra_ptr != nullptr ? prefill_c_kernel<PageMode::Page4Align> //
: prefill_c_kernel<PageMode::RingBuffer>;
LaunchKernel(num_c_blocks, kBlockSize, device) //
.enable_pdl(kUsePDL)(c_kernel, params);
}
if (const auto num_w_blocks = div_ceil(num_w * kNumSplit, kWarpsPerBlock)) {
const auto w_kernel = extra_ptr != nullptr ? prefill_w_kernel<PageMode::Page4Align> //
: prefill_w_kernel<PageMode::RingBuffer>;
LaunchKernel(num_w_blocks, kBlockSize, device) //
.enable_pdl(kUsePDL)(w_kernel, params);
}
}
// some auxiliary functions
private:
static const void* _get_extra_pointer(
host::SymbolicSize& B, // batch_size
host::SymbolicDevice& device,
const tvm::ffi::Optional<tvm::ffi::TensorView>& extra,
bool is_prefill = false) {
// only have value when using page-aligned mode
if (!extra.has_value()) return nullptr;
const auto& extra_tensor = extra.value();
/// NOTE: the metadata layout is different for prefill and decode:
/// for prefill, last 4 are:
/// load overlap | load normal | write overlap | last written page
/// for decode, last 1 is the write (also load) overlap
host::TensorMatcher({B, is_prefill ? 4 : 1}) // extra tensor
.with_dtype<int32_t>()
.with_device(device)
.verify(extra_tensor);
const auto data_ptr = extra_tensor.data_ptr();
host::RuntimeCheck(data_ptr != nullptr, "extra tensor data ptr is null");
if (is_prefill) {
static_assert(alignof(C4IndexBundle) == 16);
host::RuntimeCheck(std::bit_cast<uintptr_t>(data_ptr) % 16 == 0, "extra tensor is not properly aligned");
}
return data_ptr;
}
};
} // namespace sglang
@@ -1,208 +0,0 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/deepseek_v4/compress.cuh>
#include <dlpack/dlpack.h>
namespace sglang {
namespace host::compress {
using PlanResult = tvm::ffi::Tuple<uint32_t, uint32_t>;
struct CompressParams {
PrefillPlan* __restrict__ compress_plan;
PrefillPlan* __restrict__ write_plan;
const int64_t* __restrict__ seq_lens;
const int64_t* __restrict__ extend_lens;
uint32_t batch_size;
uint32_t num_tokens;
uint32_t compress_ratio;
bool is_overlap;
};
inline constexpr uint32_t kBlockSize = 1024;
#define PLAN_KERNEL __global__ __launch_bounds__(kBlockSize, 1) inline
PLAN_KERNEL void plan_prefill_cuda(const __grid_constant__ CompressParams params) {
const auto &[
compress_plan, write_plan, seq_lens, extend_lens, // pointers
batch_size, num_tokens, compress_ratio, is_overlap // values
] = params;
__shared__ uint32_t compress_counter;
__shared__ uint32_t write_counter;
uint32_t batch_id = 0;
uint32_t counter = 0;
uint32_t extend_len = extend_lens[0];
const auto tid = threadIdx.x;
if (tid == 0) {
compress_counter = 0;
write_counter = 0;
}
__syncthreads();
for (uint32_t i = tid; i < num_tokens; i += blockDim.x) {
const uint32_t ragged_id = i;
uint32_t j = ragged_id - counter;
while (j >= extend_len) {
j -= extend_len;
batch_id += 1;
if (batch_id >= batch_size) [[unlikely]]
break;
counter += extend_len;
extend_len = extend_lens[batch_id];
}
if (batch_id >= batch_size) [[unlikely]]
break;
const uint32_t seq_len = seq_lens[batch_id];
const uint32_t extend_len = extend_lens[batch_id];
const uint32_t prefix_len = seq_len - extend_len;
const uint32_t ratio = compress_ratio * (1 + is_overlap);
const uint32_t window_len = j + 1 < ratio ? ratio - (j + 1) : 0;
const uint32_t position = prefix_len + j;
const auto plan = PrefillPlan{
.ragged_id = ragged_id,
.batch_id = batch_id,
.position = position,
.window_len = window_len,
};
const uint32_t start_write_pos = [seq_len, compress_ratio, is_overlap] {
const uint32_t pos = seq_len / compress_ratio * compress_ratio;
if (!is_overlap) return pos;
return pos >= compress_ratio ? pos - compress_ratio : 0;
}();
if ((position + 1) % compress_ratio == 0) {
const auto write_pos = atomicAdd(&compress_counter, 1);
compress_plan[write_pos] = plan;
}
if (position >= start_write_pos) {
const auto write_pos = atomicAdd(&write_counter, 1);
write_plan[write_pos] = plan;
}
}
__syncthreads();
constexpr auto kInvalid = static_cast<uint32_t>(-1);
const auto kInvalidPlan = PrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid};
const auto compress_count = compress_counter;
const auto write_count = write_counter;
for (uint32_t i = compress_count + tid; i < num_tokens; i += blockDim.x) {
compress_plan[i] = kInvalidPlan;
}
for (uint32_t i = write_count + tid; i < num_tokens; i += blockDim.x) {
write_plan[i] = kInvalidPlan;
}
}
inline PlanResult plan_prefill_host(const CompressParams& params, const bool use_cuda_graph) {
const auto &[
compress_ptr, write_ptr, seq_lens_ptr, extend_lens_ptr, // pointers
batch_size, num_tokens, compress_ratio, is_overlap // values
] = params;
uint32_t counter = 0;
uint32_t compress_counter = 0;
uint32_t write_counter = 0;
const auto ratio = compress_ratio * (1 + is_overlap);
for (const auto i : irange(batch_size)) {
const uint32_t seq_len = seq_lens_ptr[i];
const uint32_t extend_len = extend_lens_ptr[i];
const uint32_t prefix_len = seq_len - extend_len;
RuntimeCheck(0 < extend_len && extend_len <= seq_len);
/// NOTE: `start_write_pos` must be a multiple of `compress_ratio`
const uint32_t start_write_pos = [seq_len, compress_ratio, is_overlap] {
const uint32_t pos = seq_len / compress_ratio * compress_ratio;
if (!is_overlap) return pos;
/// NOTE: to avoid unsigned integer underflow, don't use `pos - compress_ratio`
return pos >= compress_ratio ? pos - compress_ratio : 0;
}();
/// NOTE: `position` is within [prefix_len, seq_len)
for (const auto j : irange(extend_len)) {
const uint32_t position = prefix_len + j;
const auto plan = PrefillPlan{
.ragged_id = counter + j,
.batch_id = i,
.position = position,
.window_len = ratio - std::min(j + 1, ratio),
};
RuntimeCheck(plan.is_valid(compress_ratio, is_overlap), "Internal error!");
if ((position + 1) % compress_ratio == 0) {
compress_ptr[compress_counter++] = plan;
}
if (position >= start_write_pos) {
write_ptr[write_counter++] = plan;
}
}
counter += extend_len;
}
RuntimeCheck(counter == num_tokens, "input size ", counter, " != num_q_tokens ", num_tokens);
if (!use_cuda_graph) return PlanResult{compress_counter, write_counter};
constexpr auto kInvalid = static_cast<uint32_t>(-1);
constexpr auto kInvalidPlan = PrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid};
for (const auto i : irange(compress_counter, num_tokens)) {
compress_ptr[i] = kInvalidPlan;
}
for (const auto i : irange(write_counter, num_tokens)) {
write_ptr[i] = kInvalidPlan;
}
return PlanResult{num_tokens, num_tokens};
}
inline PlanResult plan_prefill(
const tvm::ffi::TensorView extend_lens,
const tvm::ffi::TensorView seq_lens,
const tvm::ffi::TensorView compress_plan,
const tvm::ffi::TensorView write_plan,
const uint32_t compress_ratio,
const bool is_overlap, // for overlap transform, we have to keep 1 more extra window
const bool use_cuda_graph) {
auto N = SymbolicSize{"batch_size"};
auto M = SymbolicSize{"num_tokens"};
auto device = SymbolicDevice{};
const bool is_cuda = [&] {
if (extend_lens.device().device_type == kDLCUDA) {
device.set_options<kDLCUDA>();
return true;
} else {
device.set_options<kDLCPU, kDLCUDAHost>();
return false;
}
}();
TensorMatcher({N}) // extend_lens and seq_lens
.with_dtype<int64_t>()
.with_device(device)
.verify(extend_lens)
.verify(seq_lens);
TensorMatcher({M, kPrefillPlanDim}) // compress_plan and write_plan
.with_dtype<PrefillPlanTensorDtype>()
.with_device(device)
.verify(compress_plan)
.verify(write_plan);
const auto params = CompressParams{
.compress_plan = static_cast<PrefillPlan*>(compress_plan.data_ptr()),
.write_plan = static_cast<PrefillPlan*>(write_plan.data_ptr()),
.seq_lens = static_cast<const int64_t*>(seq_lens.data_ptr()),
.extend_lens = static_cast<const int64_t*>(extend_lens.data_ptr()),
.batch_size = static_cast<uint32_t>(N.unwrap()),
.num_tokens = static_cast<uint32_t>(M.unwrap()),
.compress_ratio = compress_ratio,
.is_overlap = is_overlap,
};
if (!is_cuda) return plan_prefill_host(params, use_cuda_graph);
/// NOTE: cuda kernel plan is naturally compatible with cuda graph
LaunchKernel(1, kBlockSize, device.unwrap())(plan_prefill_cuda, params);
return PlanResult{params.num_tokens, params.num_tokens};
}
} // namespace host::compress
[[maybe_unused]]
constexpr auto& plan_compress_prefill = host::compress::plan_prefill;
} // namespace sglang
@@ -43,7 +43,7 @@ struct MegaMoEPreDispatchParams {
uint32_t top_k;
};
// kGroupSize must match sglang_per_token_group_quant_fp8_ue8m0(group_size=).
// kGroupSize must match the consuming MoE's activation scale group (quant_group_size).
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
// parameter rather than a runtime check because this kernel is issue-bound: the
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
@@ -583,34 +583,12 @@ struct NVLinkComm {
// specialized for each world size
template <uint32_t kWorldSize>
static void
all_reduce_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional<TensorView> residual) {
return run_push<Primitive::AR, kWorldSize>(comm->get_push_obj(), in, out, residual);
}
template <uint32_t kWorldSize>
static void
all_gather_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional<TensorView> residual) {
return run_push<Primitive::AG, kWorldSize>(comm->get_push_obj(), in, out, residual);
}
template <uint32_t kWorldSize>
static void
reduce_scatter_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional<TensorView> residual) {
return run_push<Primitive::RS, kWorldSize>(comm->get_push_obj(), in, out, residual);
}
// only compile once for each world size
template <uint32_t kPullUnroll>
static void all_reduce_pull(
CommunicatorRef comm, // only pull is needed
TensorView in,
TensorView out,
tvm::ffi::Optional<TensorView> residual,
int64_t in_mc_ptr,
int64_t out_mc_ptr,
uint32_t num_blocks_hint) {
return run_pull<Primitive::AR, kPullUnroll>(
comm->get_pull_obj(), in, out, residual, in_mc_ptr, out_mc_ptr, num_blocks_hint);
}
template <uint32_t kPullUnroll>
static void all_gather_pull(
CommunicatorRef comm, // only pull is needed
TensorView in,
@@ -621,16 +599,6 @@ struct NVLinkComm {
return run_pull<Primitive::AG, kPullUnroll>(
comm->get_pull_obj(), in, out, residual, 0, out_mc_ptr, num_blocks_hint);
}
template <uint32_t kPullUnroll>
static void reduce_scatter_pull(
CommunicatorRef comm, // only pull is needed
TensorView in,
TensorView out,
tvm::ffi::Optional<TensorView> residual,
int64_t in_mc_ptr,
uint32_t num_blocks_hint) {
return run_pull<Primitive::RS, kPullUnroll>(comm->get_pull_obj(), in, out, residual, in_mc_ptr, 0, num_blocks_hint);
}
};
} // namespace sglang
@@ -1,9 +1,7 @@
// Adler-32 checksum CUDA kernels for tensor verification.
//
// Two modes:
// 1. Whole-tensor: checksum all bytes of a contiguous tensor
// 2. Strided: checksum selected items across multiple tensors,
// given precomputed byte pointers and per-item lengths
// Checksums selected items across multiple tensors, given precomputed byte
// pointers and per-item lengths.
//
// Adler-32:
// A = 1 + sum(bytes) (mod 65521)
@@ -164,75 +162,7 @@ __device__ Adler32State adler32_block_reduce(Adler32State state, Adler32State* s
return state;
}
// --- Kernel 1: Whole tensor ---
__global__ void adler32_whole_kernel(
const uint8_t* __restrict__ data,
uint32_t* __restrict__ out_a,
uint32_t* __restrict__ out_b,
uint64_t* __restrict__ out_len,
uint64_t num_bytes,
uint64_t chunk_size) {
extern __shared__ Adler32State smem[];
uint64_t global_tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
uint64_t start = global_tid * chunk_size;
Adler32State state;
if (start < num_bytes) {
uint64_t end = start + chunk_size;
if (end > num_bytes) end = num_bytes;
state = adler32_compute(data + start, end - start);
} else {
state = adler32_identity();
}
uint32_t threads_in_block = blockDim.x;
uint64_t block_start = (uint64_t)blockIdx.x * blockDim.x;
uint64_t total_threads = (num_bytes + chunk_size - 1) / chunk_size;
if (block_start + threads_in_block > total_threads) {
threads_in_block = (uint32_t)(total_threads - block_start);
}
Adler32State result = adler32_block_reduce(state, smem, threads_in_block);
if (threadIdx.x == 0) {
out_a[blockIdx.x] = result.a;
out_b[blockIdx.x] = result.b;
out_len[blockIdx.x] = result.len;
}
}
struct Adler32WholeKernel {
static void run(tvm::ffi::TensorView data, tvm::ffi::TensorView states, int64_t num_bytes, int64_t chunk_size) {
using namespace host;
const uint8_t* data_ptr = static_cast<const uint8_t*>(data.data_ptr());
int64_t total_threads = (num_bytes + chunk_size - 1) / chunk_size;
int64_t num_blocks = (total_threads + kBlockSize - 1) / kBlockSize;
uint32_t* out_a = static_cast<uint32_t*>(states.data_ptr());
uint32_t* out_b = out_a + num_blocks;
uint64_t* out_len = reinterpret_cast<uint64_t*>(out_b + num_blocks);
DLDevice device;
device.device_type = kDLCUDA;
device.device_id = data.device().device_id;
size_t smem_bytes = (kBlockSize / 32) * sizeof(Adler32State);
LaunchKernel(num_blocks, kBlockSize, device, smem_bytes)(
adler32_whole_kernel,
data_ptr,
out_a,
out_b,
out_len,
static_cast<uint64_t>(num_bytes),
static_cast<uint64_t>(chunk_size));
}
};
// --- Kernel 2: Strided items ---
// --- Kernel 1: Strided items ---
__global__ void adler32_strided_kernel(
const int64_t* __restrict__ ptrs,
@@ -293,7 +223,7 @@ struct Adler32StridedKernel {
}
};
// --- Kernel 3: Final reduction ---
// --- Kernel 2: Final reduction ---
__global__ void adler32_reduce_kernel(
const uint32_t* __restrict__ in_a,
@@ -1,179 +0,0 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/tile.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <cooperative_groups/reduce.h>
#include <tvm/ffi/container/tensor.h>
#include <cooperative_groups.h>
#include <type_traits>
namespace sglang {
template <typename T, int VEC_SIZE_IN_BYTE>
struct VecTypeTrait;
template <>
struct VecTypeTrait<bf16_t, 16> {
using packed_t = packed_t<bf16_t>;
using vec_t = device::AlignedVector<packed_t, 4>;
};
template <>
struct VecTypeTrait<fp16_t, 16> {
using packed_t = packed_t<fp16_t>;
using vec_t = device::AlignedVector<packed_t, 4>;
};
template <>
struct VecTypeTrait<bf16_t, 32> {
using packed_t = packed_t<bf16_t>;
using vec_t = device::AlignedVector<packed_t, 8>;
};
template <>
struct VecTypeTrait<fp16_t, 32> {
using packed_t = packed_t<fp16_t>;
using vec_t = device::AlignedVector<packed_t, 8>;
};
template <typename packed_t>
SGL_DEVICE packed_t rms(const packed_t& val, const packed_t& weight, float rsqrt_square_sum) {
float2 valf = device::cast<fp32x2_t, packed_t>(val);
float2 weightf = device::cast<fp32x2_t, packed_t>(weight);
return device::cast<packed_t, fp32x2_t>(
make_float2(valf.x * weightf.x * rsqrt_square_sum, valf.y * weightf.y * rsqrt_square_sum));
}
template <typename T, int VEC_SIZE_IN_BYTE>
__global__ void qknorm_across_heads_reg_kernel(
T* __restrict__ q,
T* __restrict__ k,
const T* __restrict__ q_weight,
const T* __restrict__ k_weight,
int vec_hidden_size,
float eps) {
constexpr int inner_loop = VEC_SIZE_IN_BYTE == 16 ? 4 : 8;
__shared__ float shared_memory[32];
using vec_t = typename VecTypeTrait<T, VEC_SIZE_IN_BYTE>::vec_t;
using packed_t = typename VecTypeTrait<T, VEC_SIZE_IN_BYTE>::packed_t;
vec_t v_data;
vec_t v_weight;
const int warp_id = threadIdx.x >> 5;
const int lane_id = threadIdx.x & 31;
const int warp_count = (blockDim.x + 31) >> 5;
const float inv_hidden_size = 1.0f / static_cast<float>(vec_hidden_size * (VEC_SIZE_IN_BYTE / sizeof(T)));
const bool is_q = blockIdx.y == 0;
const auto token_id = blockIdx.x;
float2 acc_square = make_float2(0.0f, 0.0f);
vec_t* data = reinterpret_cast<vec_t*>(is_q ? q : k) + token_id * vec_hidden_size;
const vec_t* weight = reinterpret_cast<const vec_t*>(is_q ? q_weight : k_weight);
if (threadIdx.x < vec_hidden_size) {
v_data = data[threadIdx.x];
v_weight = weight[threadIdx.x];
for (int i = 0; i < inner_loop; i++) {
float2 val = device::cast<fp32x2_t, packed_t>(v_data[i]);
acc_square.x += val.x * val.x;
acc_square.y += val.y * val.y;
}
}
auto cg_warp = cooperative_groups::tiled_partition<32>(cooperative_groups::this_thread_block());
float* buffer = shared_memory;
float warp_sum = cooperative_groups::reduce(cg_warp, acc_square.x + acc_square.y, cooperative_groups::plus<float>());
if (lane_id == 0) {
buffer[warp_id] = warp_sum;
}
__syncthreads();
if (threadIdx.x < 32) {
float cta_sum = cooperative_groups::reduce(
cg_warp, (threadIdx.x < warp_count) ? buffer[threadIdx.x] : 0.0f, cooperative_groups::plus<float>());
if (threadIdx.x == 0) {
buffer[0] = rsqrtf(eps + cta_sum * inv_hidden_size);
}
}
__syncthreads();
if (threadIdx.x < vec_hidden_size) {
float rsqrt_val = buffer[0];
for (int i = 0; i < inner_loop; i++) {
v_data[i] = rms(v_data[i], v_weight[i], rsqrt_val);
}
data[threadIdx.x] = v_data;
}
}
template <typename DType>
struct QKNormAcrossHeadsKernel {
static void
run(const tvm::ffi::TensorView q,
const tvm::ffi::TensorView k,
const tvm::ffi::TensorView q_weight,
const tvm::ffi::TensorView k_weight,
float eps) {
using namespace host;
auto N = SymbolicSize{"num_tokens"};
auto D = SymbolicSize{"hidden_size"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({N, D}) // q
.with_strides({D, 1})
.with_dtype<DType>()
.with_device(device)
.verify(q);
TensorMatcher({N, D}) // k
.with_strides({D, 1})
.with_dtype<DType>()
.with_device(device)
.verify(k);
TensorMatcher({D}) // q_weight
.with_dtype<DType>()
.with_device(device)
.verify(q_weight);
TensorMatcher({D}) // k_weight
.with_dtype<DType>()
.with_device(device)
.verify(k_weight);
int hidden_size = static_cast<int>(D.unwrap());
if (hidden_size <= (device::kMaxVecBytes == 32 ? 12288 : 8192)) {
int elements_in_vec = device::kMaxVecBytes / sizeof(DType);
int vec_hidden_size = hidden_size / elements_in_vec;
uint threads = (vec_hidden_size + 31) / 32 * 32;
// Runtime check
host::RuntimeCheck(
hidden_size % elements_in_vec == 0,
"hidden_size",
hidden_size,
" can not align to elements_in_vec ",
elements_in_vec);
auto kernel = qknorm_across_heads_reg_kernel<DType, device::kMaxVecBytes>;
LaunchKernel(dim3(static_cast<uint>(N.unwrap()), 2), threads, device.unwrap())
.enable_pdl(false)(
kernel,
reinterpret_cast<DType*>(q.data_ptr()),
reinterpret_cast<DType*>(k.data_ptr()),
reinterpret_cast<DType*>(q_weight.data_ptr()),
reinterpret_cast<DType*>(k_weight.data_ptr()),
vec_hidden_size,
eps);
} else {
host::RuntimeCheck(false, "Large hidden_sizes are not supported for now.");
}
}
};
} // namespace sglang
@@ -12,16 +12,13 @@
// slice S_r (in any buffer), so there is no write-write conflict, and each
// per-element load completes before its store (data dependency).
//
// Two variants:
// * ..._kernel (v1): no in-kernel sync; the caller fences with the symm-mem
// handle's barrier() on each side (3 launches total).
// * ..._fused_kernel (v2): an in-kernel per-block system barrier (entry:
// producers done + visible; exit: broadcasts done + visible), so the whole
// all-reduce is a single launch. The barrier uses a DEDICATED symmetric
// flags buffer (independent of torch's signal pad, so no interference with
// multimem) and a device-resident monotonic epoch counter per block, which
// keeps advancing across launches -- including CUDA-graph replays -- so
// flags never go stale (spin is `flag < epoch`, epoch strictly increasing).
// ..._fused_kernel (v2) uses an in-kernel per-block system barrier (entry:
// producers done + visible; exit: broadcasts done + visible), so the whole
// all-reduce is a single launch. The barrier uses a DEDICATED symmetric
// flags buffer (independent of torch's signal pad, so no interference with
// multimem) and a device-resident monotonic epoch counter per block, which
// keeps advancing across launches -- including CUDA-graph replays -- so
// flags never go stale (spin is `flag < epoch`, epoch strictly increasing).
//
// Fusion seam: the reduced `result` Storage below is where an epilogue (RMSNorm
// / short-conv / bias) plugs in -- applied in registers before the broadcast
@@ -162,15 +159,6 @@ __device__ __forceinline__ void two_shot_reduce_local(DType* (&input)[kNumGPU],
}
}
// v1: no in-kernel barrier (caller fences via hdl.barrier()).
template <typename DType, uint32_t kNumGPU>
__global__ __launch_bounds__(1024, 1) void inkling_two_shot_all_reduce_kernel(
void* const* __restrict__ peer_ptrs, const uint32_t rank, const uint32_t num_items) {
DType* input[kNumGPU];
const uint32_t local_vecs = slice_setup<DType, kNumGPU>(input, peer_ptrs, rank, num_items);
two_shot_reduce_local<DType, kNumGPU>(input, local_vecs);
}
// v2: single-launch, fused entry + exit system barrier. `shared` (optional):
// this rank's LOCAL shared-expert partials, folded into its own buffer before
// the entry barrier (which then must publish -- see fold_shared_local).
@@ -408,7 +396,7 @@ __global__ __launch_bounds__(1024, 1) void inkling_multimem_full_oneshot_kernel(
// NO exit barrier: result is local & complete; input reuse is caller-managed.
}
// Blocks needed to cover this rank's two-shot slice (v1/v2/v3 partition).
// Blocks needed to cover this rank's two-shot slice (v2/v3 partition).
template <typename DType, uint32_t kNumGPU>
uint32_t work_num_blocks(uint32_t n, uint32_t block_size) {
using Trait = InklingAllReduceTrait<DType, kNumGPU>;
@@ -483,22 +471,6 @@ void validate(tvm::ffi::TensorView buf, int64_t peer_ptrs_dev, int64_t rank, int
RuntimeCheck(rank >= 0 && rank < kNumGPU, "rank out of range");
}
template <typename DType, uint32_t kNumGPU>
void inkling_two_shot_all_reduce(
tvm::ffi::TensorView local_buffer, int64_t peer_ptrs_dev, int64_t rank, int64_t num_items) {
using namespace host;
uint32_t n;
validate<DType, kNumGPU>(local_buffer, peer_ptrs_dev, rank, num_items, n);
const auto device = local_buffer.device();
const uint32_t num_blocks = work_num_blocks<DType, kNumGPU>(n, 1024u); // no in-kernel barrier -> uncapped
const auto stream = LaunchKernel::resolve_device(device);
LaunchKernel(num_blocks, 1024u, stream)(
inkling_two_shot_all_reduce_kernel<DType, kNumGPU>,
reinterpret_cast<void* const*>(peer_ptrs_dev),
static_cast<uint32_t>(rank),
n);
}
template <typename DType, uint32_t kNumGPU>
void inkling_two_shot_all_reduce_fused(
tvm::ffi::TensorView local_buffer,
@@ -19,81 +19,12 @@ namespace sglang {
using deepseek_v4::fp8::cast_to_ue8m0;
using deepseek_v4::fp8::pack_fp8;
// Per-token group quant to FP8-e4m3 with a fused UE8M0 scale. Each group of
// kGroupSize columns gets one UE8M0 exponent byte written contiguously in
// row-major order into ``x_sf`` (int32 [num_tokens, num_groups/4], 4 group
// bytes per int32). This is the deep_gemm "transform_sf" pack done inline in
// the quant, reusing the dsv4 ``cast_to_ue8m0``/``pack_fp8`` primitives -- it
// is byte-identical to ``per_token_group_quant_fp8(scale_ue8m0=True)`` followed
// by ``transform_sf_into_required_layout`` (both round via ceil(log2(absmax/
// FP8_MAX))), but emits no separate transpose/pack kernel.
struct PerTokenQuantUe8m0Params {
const bf16_t* __restrict__ x; // [num_tokens, hidden]
fp8_e4m3_t* __restrict__ x_q; // [num_tokens, hidden]
int32_t* __restrict__ x_sf; // [num_tokens, num_groups/4]; written as bytes
uint32_t num_tokens;
uint32_t hidden;
uint32_t num_groups; // hidden / kGroupSize
};
template <uint32_t kGroupSize, bool kUsePDL>
__global__ __launch_bounds__(1024, 2) void //
per_token_quant_ue8m0_kernel(const PerTokenQuantUe8m0Params __grid_constant__ params) {
using namespace device;
constexpr uint32_t kVecElems = 8; // 8 bf16 = 16B load per thread
static_assert(kGroupSize % kVecElems == 0, "group_size must be a multiple of 8");
constexpr uint32_t kThreadsPerGroup = kGroupSize / kVecElems;
using InputVec = AlignedVector<bf16x2_t, kVecElems / 2>;
using OutputVec = AlignedVector<fp8x2_e4m3_t, kVecElems / 2>;
const uint32_t token_id = blockIdx.x;
const uint32_t tid = threadIdx.x;
PDLWaitPrimary<kUsePDL>();
const auto token_in = params.x + static_cast<uint64_t>(token_id) * params.hidden;
const auto token_out = params.x_q + static_cast<uint64_t>(token_id) * params.hidden;
InputVec in_vec;
in_vec.load(token_in, tid);
float local_max = 0.0f;
float vals[kVecElems];
#pragma unroll
for (uint32_t i = 0; i < kVecElems / 2; ++i) {
const auto [v0, v1] = cast<fp32x2_t>(in_vec[i]);
vals[2 * i + 0] = v0;
vals[2 * i + 1] = v1;
local_max = fmaxf(local_max, fmaxf(fabsf(v0), fabsf(v1)));
}
// Absmax across the kThreadsPerGroup threads that cover one group.
local_max = warp::reduce_max<kThreadsPerGroup>(local_max);
const float absmax = fmaxf(local_max, 1e-10f);
const float raw_scale = absmax / math::FP8_E4M3_MAX;
const uint32_t ue8m0_exp = cast_to_ue8m0(raw_scale);
const float inv_scale = __uint_as_float((127u + 127u - ue8m0_exp) << 23);
OutputVec out_vec;
#pragma unroll
for (uint32_t i = 0; i < kVecElems / 2; ++i) {
out_vec[i] = pack_fp8(vals[2 * i + 0] * inv_scale, vals[2 * i + 1] * inv_scale);
}
out_vec.store(token_out, tid);
const uint32_t group_id = tid / kThreadsPerGroup;
const uint32_t within_group_id = tid % kThreadsPerGroup;
if (within_group_id == 0 && group_id < params.num_groups) {
const uint32_t byte_off = token_id * params.num_groups + group_id;
reinterpret_cast<uint8_t*>(params.x_sf)[byte_off] = static_cast<uint8_t>(ue8m0_exp);
}
PDLTriggerSecondary<kUsePDL>();
}
// Fused quant + scatter: like per_token_quant_ue8m0_kernel, but instead of
// writing the fp8/scale for the single source token, it scatters them straight
// into the permuted grouped-GEMM input -- replicating each token to its ``topk``
// destination rows -- so the separate fill_gateup_input_triton_kernel launch (and
// the intermediate x_q/x_sf buffers) are eliminated. The fp8 value + UE8M0 scale
// are computed exactly once per token (identical to the non-fused kernel); only
// the stores differ.
// Fused per-token group quant to FP8-e4m3 with a UE8M0 scale + scatter: each
// group of kGroupSize columns gets one UE8M0 exponent (ceil(log2(absmax /
// FP8_MAX)), via the dsv4 ``cast_to_ue8m0``/``pack_fp8`` primitives), and the
// fp8 row + scale bytes are written straight into the permuted grouped-GEMM
// input -- replicating each token to its ``topk`` destination rows -- so no
// separate fill_gateup_input_triton_kernel launch or x_q/x_sf buffers are needed.
struct PerTokenQuantUe8m0ScatterParams {
const bf16_t* __restrict__ x; // [num_tokens, hidden]
fp8_e4m3_t* __restrict__ gateup_input; // [E, m_max, hidden]
@@ -242,38 +173,4 @@ void per_token_quant_ue8m0_scatter(
.enable_pdl(kUsePDL)(kernel, params);
}
template <int64_t kGroupSize, bool kUsePDL>
void per_token_quant_ue8m0(tvm::ffi::TensorView x, tvm::ffi::TensorView x_q, tvm::ffi::TensorView x_sf) {
using namespace host;
auto device = SymbolicDevice{};
auto M = SymbolicSize{"num_tokens"};
auto H = SymbolicSize{"hidden"};
auto G4 = SymbolicSize{"num_groups_div_4"};
device.set_options<kDLCUDA>();
TensorMatcher({M, H}).with_dtype<bf16_t>().with_device(device).verify(x);
TensorMatcher({M, H}).with_dtype<fp8_e4m3_t>().with_device(device).verify(x_q);
TensorMatcher({M, G4}).with_dtype<int32_t>().with_device(device).verify(x_sf);
const uint32_t num_tokens = static_cast<uint32_t>(M.unwrap());
const uint32_t hidden = static_cast<uint32_t>(H.unwrap());
RuntimeCheck(hidden % kGroupSize == 0, "hidden ", hidden, " not divisible by group_size ", kGroupSize);
const uint32_t num_groups = hidden / static_cast<uint32_t>(kGroupSize);
RuntimeCheck(static_cast<uint32_t>(G4.unwrap()) * 4 == num_groups);
const uint32_t threads = hidden / 8; // kVecElems
RuntimeCheck(threads <= 1024, "hidden/8 must be <= 1024, got ", threads);
const auto params = PerTokenQuantUe8m0Params{
.x = static_cast<const bf16_t*>(x.data_ptr()),
.x_q = static_cast<fp8_e4m3_t*>(x_q.data_ptr()),
.x_sf = static_cast<int32_t*>(x_sf.data_ptr()),
.num_tokens = num_tokens,
.hidden = hidden,
.num_groups = num_groups,
};
if (num_tokens == 0) return;
constexpr auto kernel = per_token_quant_ue8m0_kernel<kGroupSize, kUsePDL>;
LaunchKernel(num_tokens, threads, device.unwrap()) //
.enable_pdl(kUsePDL)(kernel, params);
}
} // namespace sglang
@@ -18,7 +18,6 @@ static constexpr int kInklingRoutedExperts = 256;
static constexpr int kInklingSharedExperts = 2;
static constexpr int kInklingTotalExperts = kInklingRoutedExperts + kInklingSharedExperts;
static constexpr int kInklingTopK = 6;
static constexpr int kInklingTopPow2 = 8;
static constexpr int kInklingWarpSize = 32;
static constexpr int kInklingValuesPerLane = kInklingRoutedExperts / kInklingWarpSize;
@@ -26,10 +25,6 @@ __device__ __forceinline__ float inkling_sigmoid(float x) {
return 1.0f / (1.0f + __expf(-x));
}
__device__ __forceinline__ bool inkling_score_better(float score, int idx, float best_score, int best_idx) {
return score > best_score || (score == best_score && idx < best_idx);
}
// Order-preserving uint32 key of an fp32 (same total order as the triton
// kernel's fpval_to_key): flip sign bit for positives, all bits for negatives.
__device__ __forceinline__ uint32_t inkling_fp_key(float f) {
@@ -44,158 +39,6 @@ __device__ __forceinline__ int32_t inkling_pack_routed(int32_t id, float w) {
return static_cast<int32_t>((static_cast<uint32_t>(id) << 16) | wbits);
}
template <int WarpsPerBlock, bool ReturnPacked>
__launch_bounds__(kInklingWarpSize* WarpsPerBlock) __global__ void inkling_gate_topk_renorm_kernel(
const float* __restrict__ logits,
const float* __restrict__ bias,
const float* __restrict__ global_scale,
float* __restrict__ routed_w,
float* __restrict__ shared_w,
int64_t* __restrict__ indices,
int32_t* __restrict__ packed,
int64_t M,
int64_t logits_stride_m,
float route_scale) {
const int lane = threadIdx.x;
const int warp_in_block = threadIdx.y;
const int64_t row = static_cast<int64_t>(blockIdx.x) * WarpsPerBlock + warp_in_block;
if (row >= M) {
return;
}
const int64_t row_base = row * logits_stride_m;
float local_scores[kInklingValuesPerLane];
#pragma unroll
for (int i = 0; i < kInklingValuesPerLane; ++i) {
const int expert = lane + i * kInklingWarpSize;
const float raw = logits[row_base + expert];
local_scores[i] = inkling_sigmoid(raw) + bias[expert];
}
int selected_idx[kInklingTopK];
#pragma unroll
for (int k = 0; k < kInklingTopK; ++k) {
float best_score = -FLT_MAX;
int best_idx = INT_MAX;
#pragma unroll
for (int i = 0; i < kInklingValuesPerLane; ++i) {
const int expert = lane + i * kInklingWarpSize;
const float score = local_scores[i];
if (inkling_score_better(score, expert, best_score, best_idx)) {
best_score = score;
best_idx = expert;
}
}
#pragma unroll
for (int offset = kInklingWarpSize / 2; offset > 0; offset >>= 1) {
const float other_score = __shfl_xor_sync(0xffffffff, best_score, offset);
const int other_idx = __shfl_xor_sync(0xffffffff, best_idx, offset);
if (inkling_score_better(other_score, other_idx, best_score, best_idx)) {
best_score = other_score;
best_idx = other_idx;
}
}
selected_idx[k] = best_idx;
if (best_idx % kInklingWarpSize == lane) {
local_scores[best_idx / kInklingWarpSize] = -FLT_MAX;
}
__syncwarp();
}
if (lane != 0) {
return;
}
float active[kInklingTopPow2];
#pragma unroll
for (int i = 0; i < kInklingTopK; ++i) {
active[i] = inkling_sigmoid(logits[row_base + selected_idx[i]]);
}
#pragma unroll
for (int i = 0; i < kInklingSharedExperts; ++i) {
active[kInklingTopK + i] = inkling_sigmoid(logits[row_base + kInklingRoutedExperts + i]);
}
float sum = 0.0f;
#pragma unroll
for (int i = 0; i < kInklingTopPow2; ++i) {
sum += active[i];
}
const float scale = route_scale * global_scale[0] / sum;
#pragma unroll
for (int i = 0; i < kInklingTopK; ++i) {
const float w = active[i] * scale;
if constexpr (ReturnPacked) {
packed[row * kInklingTopK + i] = inkling_pack_routed(selected_idx[i], w);
} else {
routed_w[row * kInklingTopK + i] = w;
indices[row * kInklingTopK + i] = static_cast<int64_t>(selected_idx[i]);
}
}
#pragma unroll
for (int i = 0; i < kInklingSharedExperts; ++i) {
shared_w[row * kInklingSharedExperts + i] = active[kInklingTopK + i] * scale;
}
}
template <int WarpsPerBlock, bool ReturnPacked>
void launch_inkling_gate_topk_renorm(
const float* logits,
const float* bias,
const float* global_scale,
float* routed_w,
float* shared_w,
int64_t* indices,
int32_t* packed,
int64_t tokens,
int64_t logits_stride_m,
float route_scale,
DLDevice device) {
using namespace host;
const dim3 block(kInklingWarpSize, WarpsPerBlock);
const dim3 grid(static_cast<unsigned int>(div_ceil(tokens, static_cast<int64_t>(WarpsPerBlock))));
LaunchKernel(grid, block, device)(
inkling_gate_topk_renorm_kernel<WarpsPerBlock, ReturnPacked>,
logits,
bias,
global_scale,
routed_w,
shared_w,
indices,
packed,
tokens,
logits_stride_m,
route_scale);
}
template <bool ReturnPacked>
void dispatch_inkling_gate_topk_renorm(
const float* logits,
const float* bias,
const float* global_scale,
float* routed_w,
float* shared_w,
int64_t* indices,
int32_t* packed,
int64_t tokens,
int64_t logits_stride_m,
float route_scale,
DLDevice device) {
if (tokens <= 64) {
launch_inkling_gate_topk_renorm<1, ReturnPacked>(
logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device);
} else if (tokens <= 1024) {
launch_inkling_gate_topk_renorm<4, ReturnPacked>(
logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device);
} else {
launch_inkling_gate_topk_renorm<8, ReturnPacked>(
logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device);
}
}
// Warp-per-row gate kernel with optional expert-per-block GEMV fusion.
// Inkling gate GEMM shape: x [M, 6144] bf16 x W [264 (row-padded), 6144] bf16.
@@ -736,122 +579,6 @@ void check_gate_row_alignment(const void* ptr, int64_t stride_m) {
"logits rows must be device::kMaxVecBytes-aligned (production pitch is 264 floats)");
}
void inkling_gate_topk_renorm(
tvm::ffi::TensorView logits,
tvm::ffi::TensorView bias,
tvm::ffi::TensorView global_scale,
tvm::ffi::TensorView routed_w,
tvm::ffi::TensorView shared_w,
tvm::ffi::TensorView indices,
double route_scale) {
using namespace host;
SymbolicSize M{"tokens"};
SymbolicDevice device_;
device_.set_options<kDLCUDA>();
TensorMatcher({M, kInklingTotalExperts})
.with_strides({-1, 1})
.with_dtype<fp32_t>()
.with_device<kDLCUDA>(device_)
.verify(logits);
TensorMatcher({kInklingRoutedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(bias);
TensorMatcher({1}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(global_scale);
TensorMatcher({M, kInklingTopK}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(routed_w);
TensorMatcher({M, kInklingSharedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(shared_w);
TensorMatcher({M, kInklingTopK}).with_dtype<int64_t>().with_device<kDLCUDA>(device_).verify(indices);
RuntimeCheck(logits.stride(1) == 1, "logits must be contiguous along the expert dimension");
RuntimeCheck(bias.stride(0) == 1, "bias must be contiguous");
RuntimeCheck(routed_w.stride(1) == 1, "routed_w must be contiguous along top-k dimension");
RuntimeCheck(shared_w.stride(1) == 1, "shared_w must be contiguous along shared dimension");
RuntimeCheck(indices.stride(1) == 1, "indices must be contiguous along top-k dimension");
const int64_t tokens = M.unwrap();
if (tokens == 0) {
return;
}
const auto* logits_ptr = static_cast<const float*>(logits.data_ptr());
const auto* bias_ptr = static_cast<const float*>(bias.data_ptr());
const auto* global_scale_ptr = static_cast<const float*>(global_scale.data_ptr());
auto* routed_w_ptr = static_cast<float*>(routed_w.data_ptr());
auto* shared_w_ptr = static_cast<float*>(shared_w.data_ptr());
auto* indices_ptr = static_cast<int64_t*>(indices.data_ptr());
const float route_scale_f = static_cast<float>(route_scale);
const DLDevice device = device_.unwrap();
dispatch_inkling_gate_topk_renorm<false>(
logits_ptr,
bias_ptr,
global_scale_ptr,
routed_w_ptr,
shared_w_ptr,
indices_ptr,
nullptr,
tokens,
logits.stride(0),
route_scale_f,
device);
}
// Packed variant: emits packed[M, kInklingTopK] int32 ((id<<16)|bf16 weight) instead of
// the routed_w + indices pair; shared_w still written.
void inkling_gate_topk_renorm_packed(
tvm::ffi::TensorView logits,
tvm::ffi::TensorView bias,
tvm::ffi::TensorView global_scale,
tvm::ffi::TensorView packed,
tvm::ffi::TensorView shared_w,
double route_scale) {
using namespace host;
SymbolicSize M{"tokens"};
SymbolicDevice device_;
device_.set_options<kDLCUDA>();
TensorMatcher({M, kInklingTotalExperts})
.with_strides({-1, 1})
.with_dtype<fp32_t>()
.with_device<kDLCUDA>(device_)
.verify(logits);
TensorMatcher({kInklingRoutedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(bias);
TensorMatcher({1}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(global_scale);
TensorMatcher({M, kInklingTopK}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(packed);
TensorMatcher({M, kInklingSharedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(shared_w);
RuntimeCheck(logits.stride(1) == 1, "logits must be contiguous along the expert dimension");
RuntimeCheck(bias.stride(0) == 1, "bias must be contiguous");
RuntimeCheck(packed.stride(1) == 1, "packed must be contiguous along top-k dimension");
RuntimeCheck(shared_w.stride(1) == 1, "shared_w must be contiguous along shared dimension");
const int64_t tokens = M.unwrap();
if (tokens == 0) {
return;
}
const auto* logits_ptr = static_cast<const float*>(logits.data_ptr());
const auto* bias_ptr = static_cast<const float*>(bias.data_ptr());
const auto* global_scale_ptr = static_cast<const float*>(global_scale.data_ptr());
auto* packed_ptr = static_cast<int32_t*>(packed.data_ptr());
auto* shared_w_ptr = static_cast<float*>(shared_w.data_ptr());
const float route_scale_f = static_cast<float>(route_scale);
const DLDevice device = device_.unwrap();
dispatch_inkling_gate_topk_renorm<true>(
logits_ptr,
bias_ptr,
global_scale_ptr,
nullptr,
shared_w_ptr,
nullptr,
packed_ptr,
tokens,
logits.stride(0),
route_scale_f,
device);
}
// Uses int32 indices for MoeRunner, optional PDL, and tunable warps (0 = auto).
// Logit rows must be device::kMaxVecBytes-aligned (the standard pitch is 264).
void inkling_gate_topk_renorm_v2(
@@ -1,376 +0,0 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/warp.cuh>
#include <tvm/ffi/container/tensor.h>
#include <cfloat>
#include <cstdint>
// gfx1250 needs a 64-bit __shfl_*_sync mask (static_assert sizeof == 8); it's
// masked to wave32 internally, so 64-bit is fine everywhere. __gfx1250__ is
// device-pass only, so widen in the host pass too or the launch stub won't build.
#if defined(__gfx1250__) || (defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_DEVICE_COMPILE__))
#define SGL_WARP_SYNC_MASK 0xFFFFFFFFFFFFFFFFULL
#else
#define SGL_WARP_SYNC_MASK 0xFFFFFFFF
#endif
namespace sglang {
constexpr uint32_t kWarpSize = 32;
constexpr uint32_t kWarpsPerCTA = 6;
constexpr uint32_t kSmallTokenThreshold = 512;
constexpr uint32_t kMaxExperts = 512;
constexpr uint32_t kMaxTopK = 16;
enum class ScoringFunc : uint32_t {
kSigmoid = 0,
kSqrtSoftplus = 1,
};
struct MoEFusedGateParams {
const float* __restrict__ input;
const float* __restrict__ bias;
float* __restrict__ output;
int32_t* __restrict__ indices;
uint32_t num_rows;
uint32_t num_experts;
uint32_t topk;
uint32_t num_fused_shared_experts;
bool renormalize;
float routed_scaling_factor;
bool apply_routed_scaling_factor_on_output;
};
template <ScoringFunc kScoringFunc>
__device__ __forceinline__ float compute_score(float x) {
if constexpr (kScoringFunc == ScoringFunc::kSigmoid) {
// sigmoid(x) = 1 / (1 + exp(-x))
return 1.0f / (1.0f + expf(-x));
} else {
// sqrt(softplus(x)); sign folded out because expf overflows above 88.7.
const float softplus = fmaxf(x, 0.0f) + log1pf(expf(-fabsf(x)));
return sqrtf(softplus);
}
}
template <uint32_t kWarpsPerToken, ScoringFunc kScoringFunc>
__global__ void moe_fused_gate_kernel_small_token(const MoEFusedGateParams __grid_constant__ params) {
const auto& [input, bias, output, indices, num_rows, num_experts, topk, num_fused_shared_experts, renormalize, routed_scaling_factor, apply_routed_scaling_factor_on_output] =
params;
uint32_t row_idx = blockIdx.x;
if (row_idx >= num_rows) return;
// number of routed experts to select (excluding fused shared experts)
const uint32_t topk_routed = topk - num_fused_shared_experts;
uint32_t tid = threadIdx.x;
uint32_t warp_id = tid / kWarpSize;
uint32_t lane_id = tid % kWarpSize;
// Actual warps launched (<= kWarpsPerToken). num_experts that need fewer than
// kWarpsPerToken warps leave the upper warp_maxs/warp_experts slots unwritten,
// so the cross-warp reduction below must only read the launched warps.
const uint32_t num_warps = blockDim.x / kWarpSize;
extern __shared__ float shared_mem[];
float* shared_scores = shared_mem;
float* shared_original_scores = shared_mem + num_experts;
// For warp-level reduction
__shared__ float warp_maxs[kWarpsPerToken];
__shared__ int warp_experts[kWarpsPerToken];
__shared__ int selected_experts[kMaxTopK];
for (uint32_t e = tid; e < num_experts; e += blockDim.x) {
float input_val = input[row_idx * num_experts + e];
float bias_val = bias[e];
float score_val = compute_score<kScoringFunc>(input_val);
float biased_val = score_val + bias_val;
shared_scores[e] = biased_val;
shared_original_scores[e] = score_val;
}
__syncthreads();
// only select topk_routed experts (excluding shared experts)
for (uint32_t k = 0; k < topk_routed; k++) {
float my_val = -FLT_MAX;
int my_expert = -1;
for (uint32_t e = tid; e < num_experts; e += blockDim.x) {
if (shared_scores[e] > my_val) {
my_val = shared_scores[e];
my_expert = e;
}
}
float warp_max_val = my_val;
int warp_max_expert = my_expert;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, warp_max_val, offset);
int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, warp_max_expert, offset);
if (other_val > warp_max_val) {
warp_max_val = other_val;
warp_max_expert = other_expert;
}
}
if (lane_id == 0 && warp_id < kWarpsPerToken) {
warp_maxs[warp_id] = warp_max_val;
warp_experts[warp_id] = warp_max_expert;
}
__syncthreads();
if (warp_id == 0) {
float final_max = (lane_id < num_warps) ? warp_maxs[lane_id] : -FLT_MAX;
int final_expert = (lane_id < num_warps) ? warp_experts[lane_id] : -1;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, final_max, offset);
int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, final_expert, offset);
if (other_val > final_max) {
final_max = other_val;
final_expert = other_expert;
}
}
if (lane_id == 0) {
selected_experts[k] = final_expert;
}
}
__syncthreads();
int selected = selected_experts[k];
if (selected >= 0 && tid == 0) {
shared_scores[selected] = -FLT_MAX;
}
__syncthreads();
}
static_assert(kMaxTopK <= device::kWarpThreads);
if (tid >= device::kWarpThreads) return;
// only use the first warp to perform write to global operation
float routed_weight = 0.0f;
int32_t selected_expert = 0;
if (tid < topk_routed) {
int expert_id = selected_experts[tid];
float score = shared_original_scores[expert_id];
if (expert_id >= 0 && expert_id < static_cast<int>(num_experts)) {
routed_weight = score;
selected_expert = expert_id;
}
}
const auto routed_sum = device::warp::reduce_sum<kMaxTopK>(routed_weight);
if (tid < topk) {
const bool is_shared = tid >= topk_routed;
const auto output_offset = row_idx * topk + tid;
const auto weight = is_shared ? (routed_sum / routed_scaling_factor) : routed_weight;
const auto expert_id = is_shared ? (num_experts + tid - topk_routed) : selected_expert;
const auto scale = apply_routed_scaling_factor_on_output ? routed_scaling_factor : 1.0f;
const auto norm = renormalize && routed_sum > 0.0f ? routed_sum : 1.0f;
output[output_offset] = weight / norm * scale;
indices[output_offset] = expert_id;
}
}
template <ScoringFunc kScoringFunc>
__global__ void moe_fused_gate_kernel(const MoEFusedGateParams __grid_constant__ params) {
const auto& [input, bias, output, indices, num_rows, num_experts, topk, num_fused_shared_experts, renormalize, routed_scaling_factor, apply_routed_scaling_factor_on_output] =
params;
uint32_t row_idx = blockIdx.x * kWarpsPerCTA + threadIdx.y;
if (row_idx >= num_rows) return;
// number of routed experts to select (excluding fused shared experts)
const uint32_t topk_routed = topk - num_fused_shared_experts;
uint32_t lane_id = threadIdx.x;
uint32_t warp_id = threadIdx.y;
extern __shared__ float shared_mem[];
float* shared_scores = shared_mem + warp_id * num_experts * 2;
float* shared_original_scores = shared_scores + num_experts;
__shared__ int selected_experts[kWarpsPerCTA][kMaxTopK];
int* warp_selected_experts = selected_experts[warp_id];
for (uint32_t e = lane_id; e < num_experts; e += kWarpSize) {
float input_val = input[row_idx * num_experts + e];
float bias_val = bias[e];
float score_val = compute_score<kScoringFunc>(input_val);
float biased_val = score_val + bias_val;
shared_scores[e] = biased_val;
shared_original_scores[e] = score_val;
}
__syncwarp();
// only select topk_routed experts
for (uint32_t k = 0; k < topk_routed; k++) {
float max_val = -FLT_MAX;
int max_expert = -1;
for (uint32_t expert = lane_id; expert < num_experts; expert += kWarpSize) {
if (shared_scores[expert] > max_val) {
max_val = shared_scores[expert];
max_expert = expert;
}
}
for (int offset = kWarpSize / 2; offset > 0; offset /= 2) {
float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, max_val, offset);
int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, max_expert, offset);
if (other_val > max_val || (other_val == max_val && other_expert < max_expert)) {
max_val = other_val;
max_expert = other_expert;
}
}
if (lane_id == 0) {
warp_selected_experts[k] = max_expert;
if (max_expert != -1) {
shared_scores[max_expert] = -FLT_MAX;
}
}
__syncwarp();
}
static_assert(kMaxTopK <= device::kWarpThreads);
float routed_weight = 0.0f;
int32_t selected_expert = 0;
if (lane_id < topk_routed) {
int expert_id = warp_selected_experts[lane_id];
if (expert_id >= 0 && expert_id < static_cast<int>(num_experts)) {
routed_weight = shared_original_scores[expert_id];
selected_expert = expert_id;
}
}
const auto routed_sum = device::warp::reduce_sum<kMaxTopK>(routed_weight);
if (lane_id < topk) {
const bool is_shared = lane_id >= topk_routed;
const auto output_idx = row_idx * topk + lane_id;
const auto weight = is_shared ? (routed_sum / routed_scaling_factor) : routed_weight;
const auto expert_id = is_shared ? (num_experts + lane_id - topk_routed) : selected_expert;
const auto scale = apply_routed_scaling_factor_on_output ? routed_scaling_factor : 1.0f;
const auto norm = renormalize && routed_sum > 0.0f ? routed_sum : 1.0f;
output[output_idx] = weight / norm * scale;
indices[output_idx] = expert_id;
}
}
template <ScoringFunc kScoringFunc>
void dispatch_small_token_kernel(
uint32_t num_rows,
uint32_t threads_per_block,
uint32_t warps_per_token,
DLDevice device,
size_t smem_per_row,
const MoEFusedGateParams& params) {
using namespace host;
if (warps_per_token <= 8) {
LaunchKernel(num_rows, threads_per_block, device, smem_per_row)(
moe_fused_gate_kernel_small_token<8, kScoringFunc>, params);
} else if (warps_per_token <= 12) {
LaunchKernel(num_rows, threads_per_block, device, smem_per_row)(
moe_fused_gate_kernel_small_token<12, kScoringFunc>, params);
} else {
LaunchKernel(num_rows, threads_per_block, device, smem_per_row)(
moe_fused_gate_kernel_small_token<16, kScoringFunc>, params);
}
}
struct MoEFusedGateKernel {
static void
run(const tvm::ffi::TensorView input,
const tvm::ffi::TensorView bias,
const tvm::ffi::TensorView output,
const tvm::ffi::TensorView indices,
uint32_t topk,
uint32_t scoring_func, // 0 = sigmoid, 1 = sqrtsoftplus
uint32_t num_fused_shared_experts,
bool renormalize,
float routed_scaling_factor,
bool apply_routed_scaling_factor_on_output) {
using namespace host;
auto N = SymbolicSize{"num_rows"};
auto E = SymbolicSize{"num_experts"};
auto K = SymbolicSize{"topk"};
auto device = SymbolicDevice{};
K.set_value(topk);
device.set_options<kDLCUDA>();
TensorMatcher({N, E}).with_dtype<float>().with_device(device).verify(input);
TensorMatcher({E}).with_dtype<float>().with_device(device).verify(bias);
TensorMatcher({N, K}).with_dtype<float>().with_device(device).verify(output);
TensorMatcher({N, K}).with_dtype<int32_t>().with_device(device).verify(indices);
const auto num_rows = static_cast<uint32_t>(N.unwrap());
const auto num_experts = static_cast<uint32_t>(E.unwrap());
RuntimeCheck(num_experts <= kMaxExperts, "num_experts exceeds maximum supported value");
RuntimeCheck(scoring_func <= 1, "scoring_func must be 0 (sigmoid) or 1 (sqrtsoftplus)");
RuntimeCheck(topk > num_fused_shared_experts, "topk must be greater than num_fused_shared_experts");
const auto params = MoEFusedGateParams{
.input = static_cast<const float*>(input.data_ptr()),
.bias = static_cast<const float*>(bias.data_ptr()),
.output = static_cast<float*>(output.data_ptr()),
.indices = static_cast<int32_t*>(indices.data_ptr()),
.num_rows = num_rows,
.num_experts = num_experts,
.topk = topk,
.num_fused_shared_experts = num_fused_shared_experts,
.renormalize = renormalize,
.routed_scaling_factor = routed_scaling_factor,
.apply_routed_scaling_factor_on_output = apply_routed_scaling_factor_on_output,
};
const size_t smem_per_row = 2 * num_experts * sizeof(float);
bool use_small_token_kernel = num_rows <= kSmallTokenThreshold;
if (use_small_token_kernel) {
// 1 token per block
uint32_t warps_per_token = div_ceil(num_experts, kWarpSize);
warps_per_token = std::min(warps_per_token, 16u);
uint32_t threads_per_block = warps_per_token * kWarpSize;
if (scoring_func == 0) {
dispatch_small_token_kernel<ScoringFunc::kSigmoid>(
num_rows, threads_per_block, warps_per_token, device.unwrap(), smem_per_row, params);
} else {
dispatch_small_token_kernel<ScoringFunc::kSqrtSoftplus>(
num_rows, threads_per_block, warps_per_token, device.unwrap(), smem_per_row, params);
}
} else {
// multiple tokens per block
uint32_t num_blocks = div_ceil(num_rows, kWarpsPerCTA);
dim3 block_dim(kWarpSize, kWarpsPerCTA);
size_t large_smem = smem_per_row * kWarpsPerCTA;
if (scoring_func == 0) {
LaunchKernel(num_blocks, block_dim, device.unwrap(), large_smem)(
moe_fused_gate_kernel<ScoringFunc::kSigmoid>, params);
} else {
LaunchKernel(num_blocks, block_dim, device.unwrap(), large_smem)(
moe_fused_gate_kernel<ScoringFunc::kSqrtSoftplus>, params);
}
}
}
};
} // namespace sglang
@@ -29,7 +29,7 @@
// The reader requires the rows of a page to be contiguous and the page stride
// to be a multiple of kPageAlign (its TMA row stride), which is what
// kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is
// `sglang.kernels.ops.attention.dsv4.torch_quant`.
// `sglang.test.kernels.deepseek_v4.k_cache_reference`.
namespace sglang {
+52 -265
View File
@@ -1,10 +1,11 @@
"""Fused gated-activation kernels (``act(x[:h]) * x[h:]``).
"""Activation kernels.
Each operator is a :class:`~sglang.kernels.fused_op.BaseFusedOp` with a
pure-``torch`` reference (``forward_native``) plus AOT (``sgl_kernel``) and
JIT CUDA backends behind one ``(input, out)`` signature. The JIT backend
additionally accepts ``expert_ids`` / ``expert_step`` — call
``forward_jit`` directly when those are needed.
``silu_and_mul`` (``silu(x[..., :d]) * x[..., d:]``) is a
:class:`~sglang.kernels.fused_op.BaseFusedOp` with a pure-``torch`` reference
(``forward_native``) plus AOT (``sgl_kernel``), JIT CUDA and aiter backends
behind one ``(input, out)`` signature. The JIT backend additionally accepts
``expert_ids`` / ``expert_step`` — call ``forward_jit`` directly when those are
needed.
"""
from __future__ import annotations
@@ -14,8 +15,8 @@ from typing import TYPE_CHECKING, Optional
from sglang.kernels.fused_op import BaseFusedOp, register_fused_op
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import (
CUDA_ONLY,
CapabilityRequirement,
FormatSignature,
KernelBackend,
KernelSpec,
)
@@ -23,85 +24,11 @@ from sglang.kernels.spec import (
if TYPE_CHECKING:
import torch
_ACT_DTYPES = ("float16", "bfloat16")
_CUDA = frozenset({CapabilityRequirement.CUDA})
_HIP = frozenset({CapabilityRequirement.HIP})
# sgl_kernel's gated-activation ops build for CUDA *and* ROCm (production
# imports them from sgl_kernel on both), so the AOT backend spans both devices
# — the canonical OR-semantics case that a device-baked backend name couldn't.
_CUDA_HIP = frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP})
# JIT before AOT to match the production path (srt/layers/activation.py imports
# from sglang.kernels.ops.activation.activation on CUDA); auto-selection must not invert it.
_ACT_PRIORITY = (
KernelBackend.JIT,
KernelBackend.AOT,
KernelBackend.TORCH,
)
class _GatedActivationOp(BaseFusedOp):
"""Shared structure for ``act(x[..., :d]) * x[..., d:]`` operators."""
# Set by subclasses: sgl_kernel / jit_kernel attr name (same for both).
kernel_attr: str
priority = _ACT_PRIORITY
capabilities = {
KernelBackend.AOT: _CUDA_HIP,
KernelBackend.JIT: _CUDA,
}
format_signature = FormatSignature(
supported_dtypes=_ACT_DTYPES,
description="gated activation; returns tensor",
)
def _act(self, gate: torch.Tensor) -> torch.Tensor:
raise NotImplementedError
def forward_native(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
d = input.shape[-1] // 2
result = self._act(input[..., :d]) * input[..., d:]
if out is None:
return result
out.copy_(result)
return out
def forward_aot(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
import sgl_kernel
return getattr(sgl_kernel, self.kernel_attr)(input, out)
def forward_jit(
self,
input: torch.Tensor,
out: Optional[torch.Tensor] = None,
expert_ids: Optional[torch.Tensor] = None,
expert_step: int = 1,
) -> torch.Tensor:
import sglang.kernels.ops.activation.activation as jit_activation
return getattr(jit_activation, self.kernel_attr)(
input, out, expert_ids, expert_step
)
class SiluAndMulOp(_GatedActivationOp):
"""``out = silu(input[..., :d]) * input[..., d:]`` with ``d = input.shape[-1] // 2``.
Adds an ``AITER`` backend on ``device=HIP``: on ROCm this op has a native
``aiter`` kernel (``srt/layers/activation.py`` uses it in production). Note
the sibling gelu ops below deliberately do *not* register AITER — ROCm/aiter
coverage is a per-``(op, backend)`` subset, which the decoupled backend/device
model expresses directly (a device-agnostic ``KernelBackend`` name plus a
per-backend ``CapabilityRequirement``).
"""
class SiluAndMulOp(BaseFusedOp):
"""``out = silu(input[..., :d]) * input[..., d:]`` with ``d = input.shape[-1] // 2``."""
op = "activation.silu_and_mul"
kernel_attr = "silu_and_mul"
# AOT spans CUDA+HIP; JIT is CUDA; AITER is an opt-in HIP path. By priority,
# CUDA resolves to JIT and HIP resolves to AOT (matching production
# defaults); AITER is registered and HIP-eligible but sits below AOT, so it
@@ -113,21 +40,44 @@ class SiluAndMulOp(_GatedActivationOp):
KernelBackend.TORCH,
)
capabilities = {
KernelBackend.AOT: _CUDA_HIP,
KernelBackend.JIT: _CUDA,
KernelBackend.AITER: _HIP,
}
descriptions = {
KernelBackend.AOT: "silu_and_mul (sgl_kernel wheel).",
KernelBackend.JIT: "silu_and_mul (sglang.kernels.jit).",
KernelBackend.AITER: "silu_and_mul (aiter, ROCm).",
KernelBackend.TORCH: "silu_and_mul (pure-torch reference).",
KernelBackend.AOT: frozenset(
{CapabilityRequirement.CUDA, CapabilityRequirement.HIP}
),
KernelBackend.JIT: CUDA_ONLY,
KernelBackend.AITER: frozenset({CapabilityRequirement.HIP}),
}
def _act(self, gate: torch.Tensor) -> torch.Tensor:
def forward_native(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
import torch.nn.functional as F
return F.silu(gate)
d = input.shape[-1] // 2
result = F.silu(input[..., :d]) * input[..., d:]
if out is None:
return result
out.copy_(result)
return out
def forward_aot(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
from sgl_kernel import silu_and_mul as aot_silu_and_mul
return aot_silu_and_mul(input, out)
def forward_jit(
self,
input: torch.Tensor,
out: Optional[torch.Tensor] = None,
expert_ids: Optional[torch.Tensor] = None,
expert_step: int = 1,
) -> torch.Tensor:
from sglang.kernels.ops.activation.activation import (
silu_and_mul as jit_silu_and_mul,
)
return jit_silu_and_mul(input, out, expert_ids, expert_step)
def forward_aiter(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
@@ -146,132 +96,7 @@ class SiluAndMulOp(_GatedActivationOp):
return out
class GeluAndMulOp(_GatedActivationOp):
"""``out = gelu(input[..., :d]) * input[..., d:]`` (erf-based GELU)."""
op = "activation.gelu_and_mul"
kernel_attr = "gelu_and_mul"
descriptions = {
KernelBackend.AOT: "gelu_and_mul (sgl_kernel wheel).",
KernelBackend.JIT: "gelu_and_mul (sglang.kernels.jit).",
KernelBackend.TORCH: "gelu_and_mul (pure-torch reference).",
}
def _act(self, gate: torch.Tensor) -> torch.Tensor:
import torch.nn.functional as F
return F.gelu(gate, approximate="none")
class GeluTanhAndMulOp(_GatedActivationOp):
"""``out = gelu_tanh(input[..., :d]) * input[..., d:]`` (tanh-approximated GELU)."""
op = "activation.gelu_tanh_and_mul"
kernel_attr = "gelu_tanh_and_mul"
descriptions = {
KernelBackend.AOT: "gelu_tanh_and_mul (sgl_kernel wheel).",
KernelBackend.JIT: "gelu_tanh_and_mul (sglang.kernels.jit).",
KernelBackend.TORCH: "gelu_tanh_and_mul (pure-torch reference).",
}
def _act(self, gate: torch.Tensor) -> torch.Tensor:
import torch.nn.functional as F
return F.gelu(gate, approximate="tanh")
class ReLU2Op(BaseFusedOp):
"""``out = relu(input) ** 2`` (single-input, not gated).
The real kernel is the CUDA JIT path (``sglang.kernels.ops.activation.activation.relu2``,
used in production on CUDA); elsewhere the torch reference runs.
"""
op = "activation.relu2"
priority = (KernelBackend.JIT, KernelBackend.TORCH)
capabilities = {KernelBackend.JIT: _CUDA}
format_signature = FormatSignature(
supported_dtypes=_ACT_DTYPES,
description="relu(x) ** 2; returns tensor",
)
descriptions = {
KernelBackend.JIT: "relu(x)**2 (sglang.kernels.jit).",
KernelBackend.TORCH: "relu(x)**2 (pure-torch reference).",
}
def forward_native(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
import torch.nn.functional as F
x = F.relu(input)
result = x * x
if out is None:
return result
out.copy_(result)
return out
def forward_jit(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
from sglang.kernels.ops.activation.activation import relu2
result = relu2(input)
if out is None:
return result
out.copy_(result)
return out
class QuickGELUOp(BaseFusedOp):
"""``out = input * sigmoid(1.702 * input)`` (single-input, not gated).
Only ROCm has a native kernel (``sgl_kernel.gelu_quick``, AOT on HIP); CUDA
uses the torch reference in production, so no CUDA backend is registered.
"""
op = "activation.gelu_quick"
priority = (KernelBackend.AOT, KernelBackend.TORCH)
capabilities = {KernelBackend.AOT: _HIP}
format_signature = FormatSignature(
supported_dtypes=_ACT_DTYPES,
description="x * sigmoid(1.702 * x); returns tensor",
)
descriptions = {
KernelBackend.AOT: "quick GELU (sgl_kernel wheel, ROCm).",
KernelBackend.TORCH: "quick GELU (pure-torch reference).",
}
def forward_native(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
import torch
result = input * torch.sigmoid(1.702 * input)
if out is None:
return result
out.copy_(result)
return out
def forward_aot(
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
import torch
from sgl_kernel import gelu_quick
if out is None:
out = torch.empty(input.shape, dtype=input.dtype, device=input.device)
gelu_quick(input, out)
return out
_SILU_AND_MUL = register_fused_op(SiluAndMulOp(), __name__, "_SILU_AND_MUL")
_GELU_AND_MUL = register_fused_op(GeluAndMulOp(), __name__, "_GELU_AND_MUL")
_GELU_TANH_AND_MUL = register_fused_op(
GeluTanhAndMulOp(), __name__, "_GELU_TANH_AND_MUL"
)
_RELU2 = register_fused_op(ReLU2Op(), __name__, "_RELU2")
_GELU_QUICK = register_fused_op(QuickGELUOp(), __name__, "_GELU_QUICK")
def silu_and_mul(
@@ -281,30 +106,6 @@ def silu_and_mul(
return _SILU_AND_MUL(input, out)
def gelu_and_mul(
input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
"""``out = gelu(input[..., :d]) * input[..., d:]``."""
return _GELU_AND_MUL(input, out)
def gelu_tanh_and_mul(
input: torch.Tensor, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
"""``out = gelu_tanh(input[..., :d]) * input[..., d:]``."""
return _GELU_TANH_AND_MUL(input, out)
def relu2(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
"""``out = relu(input) ** 2``."""
return _RELU2(input, out)
def gelu_quick(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
"""``out = input * sigmoid(1.702 * input)``."""
return _GELU_QUICK(input, out)
def situ_and_mul(
input: torch.Tensor,
out: Optional[torch.Tensor],
@@ -319,29 +120,19 @@ def situ_and_mul(
__all__ = [
"situ_and_mul",
"SiluAndMulOp",
"GeluAndMulOp",
"GeluTanhAndMulOp",
"ReLU2Op",
"QuickGELUOp",
"silu_and_mul",
"gelu_and_mul",
"gelu_tanh_and_mul",
"relu2",
"gelu_quick",
]
# Triton kernel migrated into this group (from layers/triton_ops/softcap);
# registered for inventory. Import it from its module.
for _fn in ("softcap_out", "softcap_inplace_logits"):
register_kernel(
KernelSpec(
op=f"activation.{_fn}",
backend=KernelBackend.TRITON,
target=f"sglang.kernels.ops.activation.softcap:{_fn}",
)
register_kernel(
KernelSpec(
op="activation.softcap_inplace_logits",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.activation.softcap:softcap_inplace_logits",
)
del _fn
)
register_kernel(
@@ -349,11 +140,7 @@ register_kernel(
op="activation.softcap_to_float32_logits",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.activation.softcap:softcap_to_float32_logits",
capabilities=_CUDA,
format_signature=FormatSignature(
supported_dtypes=("float16", "bfloat16", "float32"),
description="Softcap logits into a caller-provided FP32 output.",
),
capabilities=CUDA_ONLY,
)
)
@@ -364,6 +151,6 @@ register_kernel(
op="activation.situ_and_mul",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
@@ -3,70 +3,6 @@ import triton
import triton.language as tl
from triton.language.extra import libdevice
softcap_out_autotune = triton.autotune(
configs=[
triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=4),
triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=8),
triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=16),
triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=4),
triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=8),
triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=4),
triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=8),
triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=16),
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4),
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8),
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16),
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=32),
triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=32),
triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=32),
triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32),
triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32),
triton.Config(kwargs={"BLOCK_SIZE": 32768}, num_warps=32),
],
key=["n_ele"],
)
@triton.jit
def softcap_out_kernel(
output_ptr,
input_ptr,
n_ele,
softcap_const: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(axis=0)
block_start = pid * BLOCK_SIZE
offsets = block_start + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_ele
x = tl.load(input_ptr + offsets, mask=mask)
fx = x.to(tl.float32)
fxs = fx / softcap_const
exped = tl.exp(2 * fxs)
top = exped - 1
bottom = exped + 1
output = top / bottom * softcap_const
tl.store(output_ptr + offsets, output, mask=mask)
softcap_out_kernel_autotuned = softcap_out_autotune(softcap_out_kernel)
def softcap_out(x, softcap_const, autotune=False):
output = torch.empty_like(x, dtype=torch.float32)
n_elements = output.numel()
if autotune:
def grid(meta):
return (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)
softcap_out_kernel_autotuned[grid](output, x, n_elements, softcap_const)
else:
softcap_out_kernel[(triton.cdiv(n_elements, 128),)](
output, x, n_elements, softcap_const, BLOCK_SIZE=128, num_warps=8
)
return output
@triton.jit
def softcap_inplace_logits_kernel(
+10 -35
View File
@@ -13,8 +13,8 @@ from typing import TYPE_CHECKING
from sglang.kernels.registry import register_kernel
from sglang.kernels.selector import get_kernel
from sglang.kernels.spec import (
CUDA_ONLY,
CapabilityRequirement,
FormatSignature,
KernelBackend,
KernelSpec,
)
@@ -66,15 +66,6 @@ register_kernel(
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(12, 1), max_sm=(12, 1))}
),
format_signature=FormatSignature(
supported_dtypes=("bfloat16",),
description=(
"Qwen3.8 packed QSA decode: D=256, 12:1 GQA, 1 <= q_rows <= 128"
),
),
description=(
"SM121 Qwen3.8 QSA decode optimized by Codex/Kimi K3 through KDA-1.5."
),
)
)
@@ -265,12 +256,6 @@ for _mod, _fn, _backend, _device in [
KernelBackend.CUTE_DSL,
CapabilityRequirement.CUDA,
),
(
"kda_flydsl.kimi_k3_kda_decode",
"flydsl_kimi_k3_kda_decode",
KernelBackend.FLYDSL,
CapabilityRequirement.HIP,
),
(
"kda_flydsl.kimi_k3_kda_decode",
"flydsl_kimi_k3_kda_decode_with_f_b",
@@ -308,7 +293,7 @@ register_kernel(
op="attention.get_block_table",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.minicpm_sala.get_block_table:get_block_table",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -316,7 +301,7 @@ register_kernel(
op="attention.fast_kpool_topk_transform_fused",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsa.kpool_topk_transform:fast_kpool_topk_transform_fused",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -324,7 +309,7 @@ register_kernel(
op="attention.fast_topk",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.fast_topk:fast_topk",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -344,7 +329,7 @@ register_kernel(
op="attention.fused_k_indexer_norm_rope",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -352,7 +337,7 @@ register_kernel(
op="attention.fused_k_indexer_norm_rope_store",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope_store",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -370,7 +355,7 @@ register_kernel(
op="attention.fp4_index_logits_decode",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.attention.dsv4.fp4_indexer:fp4_index_logits_decode",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
@@ -379,8 +364,7 @@ register_kernel(
op="attention.pack_prefix_current",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.attention.dllm_kv_pack:pack_prefix_current",
capabilities=frozenset({CapabilityRequirement.CUDA}),
description="Pack CSR prefix and current K/V into caller-provided dense buffers.",
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -391,10 +375,6 @@ register_kernel(
capabilities=frozenset(
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(11, 9))}
),
format_signature=FormatSignature(
supported_dtypes=("float16", "bfloat16"),
description="SM100/SM110 D512 GQA with separate K/V and optional page tables.",
),
)
)
register_kernel(
@@ -402,12 +382,7 @@ register_kernel(
op="attention.gemma_qkv_norm_rope",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.attention.gemma_qkv_norm_rope:gemma_qkv_norm_rope",
capabilities=frozenset({CapabilityRequirement.CUDA}),
format_signature=FormatSignature(
supported_dtypes=("float16", "bfloat16"),
in_place=True,
description="Strided QKV normalization and partial NeoX RoPE.",
),
capabilities=CUDA_ONLY,
)
)
@@ -417,7 +392,7 @@ for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"):
op=f"attention.{_fn}",
backend=KernelBackend.TRITON,
target=f"sglang.kernels.ops.attention.dsv4.fp4_indexer:{_fn}",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
del _fn
@@ -6,11 +6,7 @@ import torch
from sglang.kernels.fused_op import BaseFusedOp, register_fused_op
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.kernels.spec import (
CapabilityRequirement,
FormatSignature,
KernelBackend,
)
from sglang.kernels.spec import CapabilityRequirement, KernelBackend
if TYPE_CHECKING:
from tvm_ffi.module import Module
@@ -51,13 +47,6 @@ class ClampPositionOp(BaseFusedOp):
{CapabilityRequirement.CUDA, CapabilityRequirement.HIP}
)
}
format_signature = FormatSignature(
supported_dtypes=("int32", "int64"),
description="clamp(seq_lens - 1, min=0); returns int64 positions",
)
descriptions = {
KernelBackend.JIT: "Fused clamp-position kernel (sglang.kernels.jit).",
}
def forward_native(self, seq_lens: torch.Tensor) -> torch.Tensor:
return torch.clamp(seq_lens - 1, min=0).to(torch.int64)
@@ -1,9 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/cute_utils/cvt.py
from cutlass import Constexpr, Float32, Uint32, cute
from cutlass._mlir import ir
from cutlass._mlir.dialects import llvm, vector
from cutlass import Float32, Uint32, cute
from cutlass._mlir.dialects import llvm
from cutlass.cutlass_dsl import T, dsl_user_op
@@ -48,99 +47,3 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]:
else:
raise ValueError(f"Unsupported type {type(data)}")
@dsl_user_op
def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
# there is only fp8->fp16 conversion, hence we need to go
# round trip through fp16.
out = llvm.inline_asm(
llvm.StructType.get_literal([T.i32()] * 2),
[x.ir_value(loc=loc, ip=ip)],
"{\n\t"
".reg .b16 x0, x1;\n\t"
".reg .b16 t00, t01, t10, t11;\n\t"
"mov.b32 {x0, x1}, $2;\n\t"
"cvt.rn.f16x2.e4m3x2 $0, x0;\n\t"
"cvt.rn.f16x2.e4m3x2 $1, x1;\n\t"
"mov.b32 {t00, t01}, $0;\n\t"
"mov.b32 {t10, t11}, $1;\n\t"
"cvt.rn.bf16.f16 t00, t00;\n\t"
"cvt.rn.bf16.f16 t01, t01;\n\t"
"cvt.rn.bf16.f16 t10, t10;\n\t"
"cvt.rn.bf16.f16 t11, t11;\n\t"
"mov.b32 $0, {t00, t01};\n\t"
"mov.b32 $1, {t10, t11};\n\t"
"}\n",
"=r,=r,r",
has_side_effects=False,
is_align_stack=False,
)
vec = vector.from_elements(
ir.VectorType.get([2], T.i32(), loc=loc),
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)],
loc=loc,
ip=ip,
)
return cute.TensorSSA(vec, 2, Uint32)
@dsl_user_op
def fp32x4_to_fp8x4(
a0: Float32,
a1: Float32,
a2: Float32,
a3: Float32,
*,
loc=None,
ip=None,
) -> Uint32:
# Pack four FP32 values into one b32 of four e4m3 bytes, byte order
# {a0, a1, a2, a3} from low to high address.
out = llvm.inline_asm(
T.i32(),
[
a0.ir_value(loc=loc, ip=ip),
a1.ir_value(loc=loc, ip=ip),
a2.ir_value(loc=loc, ip=ip),
a3.ir_value(loc=loc, ip=ip),
],
"{\n\t"
".reg .b16 t0, t1;\n\t"
"cvt.rn.satfinite.e4m3x2.f32 t0, $2, $1;\n\t"
"cvt.rn.satfinite.e4m3x2.f32 t1, $4, $3;\n\t"
"mov.b32 $0, {t0, t1};\n\t"
"}\n",
"=r,f,f,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint32(out)
@dsl_user_op
def fp32x8_to_fp4x8(
vals: cute.Tensor,
offset: Constexpr[int],
*,
loc=None,
ip=None,
) -> Uint32:
# Pack eight scaled FP32 values into four E2M1x2 bytes, returned as one b32.
assert vals.element_type is Float32
out = llvm.inline_asm(
T.i32(),
[vals[offset + i].ir_value(loc=loc, ip=ip) for i in range(8)],
"{\n\t"
".reg .b8 x0, x1, x2, x3;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x0, $2, $1;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x1, $4, $3;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x2, $6, $5;\n\t"
"cvt.rn.satfinite.e2m1x2.f32 x3, $8, $7;\n\t"
"mov.b32 $0, {x0, x1, x2, x3};\n\t"
"}\n",
"=r,f,f,f,f,f,f,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint32(out)
File diff suppressed because it is too large Load Diff
@@ -671,46 +671,3 @@ def dcp_lse_combine_triton(
RETURN_LSE=return_lse,
)
return out, (out_lse if return_lse else None)
def _lse_weighted_combine_cpu(
partial_outputs: torch.Tensor,
partial_lses: torch.Tensor,
is_lse_base_on_e: bool = True,
) -> torch.Tensor:
"""CPU reference: combine N partial attention outputs using LSE weights.
Args:
partial_outputs: [N, B, H_local, D]
partial_lses: [N, B, H_local]
is_lse_base_on_e: base-e (True) or base-2 (False)
Returns:
[B, H_local, D] combined output
"""
N, B, H_local, D = partial_outputs.shape
partial_outputs = partial_outputs.float()
partial_lses = partial_lses.float()
# Sanitize
partial_lses = torch.where(
torch.isnan(partial_lses) | torch.isinf(partial_lses),
torch.full_like(partial_lses, float("-inf")),
partial_lses,
)
# max LSE for numerical stability
lse_max, _ = partial_lses.max(dim=0)
lse_max = torch.where(lse_max == float("-inf"), torch.zeros_like(lse_max), lse_max)
centered = partial_lses - lse_max.unsqueeze(0)
if is_lse_base_on_e:
weights = torch.exp(centered)
else:
weights = torch.pow(2.0, centered)
weight_sum = weights.sum(dim=0, keepdim=True)
weights = weights / weight_sum.clamp_min(1.0)
combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0)
return combined
@@ -519,92 +519,6 @@ def _fused_norm_rope_kernel(
)
@triton.jit
def _fused_softmax_pool_kernel(
kv_score_ptr,
out_ptr,
stride_bs: tl.constexpr,
stride_k: tl.constexpr,
K: tl.constexpr,
HEAD_DIM: tl.constexpr,
HEAD_BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
base = pid * stride_bs
offs = tl.arange(0, HEAD_BLOCK)
mask = offs < HEAD_DIM
max_val = tl.full([HEAD_BLOCK], float("-inf"), dtype=tl.float32)
for k in range(K):
s = tl.load(
kv_score_ptr + base + k * stride_k + HEAD_DIM + offs,
mask=mask,
other=float("-inf"),
).to(tl.float32)
max_val = tl.maximum(max_val, s)
sum_exp = tl.zeros([HEAD_BLOCK], dtype=tl.float32)
weighted = tl.zeros([HEAD_BLOCK], dtype=tl.float32)
for k in range(K):
s = tl.load(
kv_score_ptr + base + k * stride_k + HEAD_DIM + offs,
mask=mask,
other=float("-inf"),
).to(tl.float32)
v = tl.load(
kv_score_ptr + base + k * stride_k + offs,
mask=mask,
other=0.0,
).to(tl.float32)
w = tl.exp(s - max_val)
sum_exp += w
weighted += v * w
result = weighted / sum_exp
tl.store(
out_ptr + pid * HEAD_DIM + offs, result.to(out_ptr.dtype.element_ty), mask=mask
)
def fused_softmax_pool_triton(
kv_score: torch.Tensor,
head_dim: int,
) -> torch.Tensor:
"""Fused softmax-weighted-sum: out = (kv * softmax(score, dim=1)).sum(dim=1).
Replaces the generic cunn_SpatialSoftMaxForward + elementwise multiply + sum
with a single Triton kernel.
Args:
kv_score: [bs, K, 2 * head_dim] where first head_dim is kv, second is score.
head_dim: dimension of each of kv and score.
Returns:
output: [bs, head_dim]
"""
assert kv_score.dim() == 3
bs, K, last = kv_score.shape
assert last == 2 * head_dim
assert kv_score.is_contiguous()
out = torch.empty(bs, head_dim, dtype=kv_score.dtype, device=kv_score.device)
if bs == 0:
return out
HEAD_BLOCK = triton.next_power_of_2(head_dim)
grid = (bs,)
_fused_softmax_pool_kernel[grid](
kv_score,
out,
stride_bs=kv_score.stride(0),
stride_k=kv_score.stride(1),
K=K,
HEAD_DIM=head_dim,
HEAD_BLOCK=HEAD_BLOCK,
)
return out
def fused_norm_rope_inplace_triton(
kv: torch.Tensor,
weight: Optional[torch.Tensor],
@@ -9,12 +9,11 @@ from sglang.srt.layers.attention.dsa.utils import (
INDEXER_K_CACHE_PRESHUFFLE_TILE,
aiter_can_use_preshuffle_paged_mqa,
)
from sglang.srt.utils import get_bool_env_var, is_hip, is_xpu
from sglang.srt.utils import is_hip, is_xpu
_is_hip = is_hip()
_is_xpu = is_xpu()
_is_fp8_fnuz = is_fp8_fnuz()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
# aiter cp_gather kernel with preshuffle=True is only valid when the indexer
# uses the page_size=64 preshuffle layout (i.e. when the matching MQA gluon path
# is also enabled).
@@ -32,149 +31,6 @@ s: scale, 1 item per token, fp32
"""
class GetK:
@classmethod
def execute(cls, *args, **kwargs):
return cls.triton(*args, **kwargs)
@classmethod
def slow(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
page_size = pool.index_page_size
num_pages = (seq_len + page_size - 1) // page_size
seq_len_ = num_pages * page_size
index_k_fp8 = torch.empty(
(seq_len_, pool.index_head_dim),
dtype=torch.uint8,
device=pool.device,
)
for i in range(num_pages):
page_index = page_indices[i]
index_k_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
: page_size * pool.index_head_dim
].view(-1, pool.index_head_dim)
return index_k_fp8[:seq_len]
@classmethod
def torch_fast(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
"""
:param page_indices: (num_pages,), int32
:return: (seq_len, index_head_dim), uint8
"""
# can handle per 128B instead of per element
# page_indices: (num_pages,), element := a page index
buf_numel_per_page = buf.shape[1]
num_k_bytes_per_page = pool.index_page_size * pool.index_head_dim
num_k_bytes_per_token = pool.index_head_dim
# buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8
# flat_buf: (whatever,), uint8
flat_buf = buf.flatten()
# flat_indices: (num_pages, num_k_bytes_per_page), int32, element := an index into flat_buf that we want to access
flat_indices = (page_indices * buf_numel_per_page)[:, None] + torch.arange(
num_k_bytes_per_page, dtype=torch.int32, device="cuda"
)[None, :]
flat_indices = flat_indices.flatten()[: seq_len * num_k_bytes_per_token]
out = flat_buf[flat_indices]
return out.view(-1, 128)
@classmethod
def triton(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
"""
Triton implementation for gathering K data from paged buffer.
:param page_indices: (num_pages,), int32/int64
:return: (seq_len, index_head_dim), uint8
"""
return _get_k_triton(
buf=buf,
page_indices=page_indices,
seq_len=seq_len,
page_size=pool.index_page_size,
index_head_dim=pool.index_head_dim,
)
class GetS:
@classmethod
def execute(cls, *args, **kwargs):
return cls.triton(*args, **kwargs)
@classmethod
def slow(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
page_size = pool.index_page_size
num_pages = (seq_len + page_size - 1) // page_size
seq_len_ = num_pages * page_size
assert pool.index_head_dim // pool.quant_block_size == 1
index_k_scale_fp8 = torch.empty(
(seq_len_, 4),
dtype=torch.uint8,
device=pool.device,
)
for i in range(num_pages):
page_index = page_indices[i]
index_k_scale_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
page_size * pool.index_head_dim :
].view(-1, 4)
return index_k_scale_fp8[:seq_len]
@classmethod
def torch_fast(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
"""
:param page_indices: (num_pages,), int32
:return: (seq_len, index_head_dim // quant_block_size), uint8
"""
buf_numel_per_page = buf.shape[1]
s_offset_in_page = pool.index_page_size * pool.index_head_dim
num_s_bytes_per_page = buf.shape[1] - s_offset_in_page
num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4
flat_buf = buf.flatten()
flat_indices = (
(page_indices * buf_numel_per_page)[:, None]
+ torch.arange(num_s_bytes_per_page, dtype=torch.int32, device="cuda")[
None, :
]
+ s_offset_in_page
)
flat_indices = flat_indices.flatten()[: seq_len * num_s_bytes_per_token]
out = flat_buf[flat_indices]
return out.view(-1, 4)
@classmethod
def triton(
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
):
"""
Triton implementation for gathering S (scale) data from paged buffer.
:param page_indices: (num_pages,), int32/int64
:return: (seq_len, 4), uint8
"""
return _get_s_triton(
buf=buf,
page_indices=page_indices,
seq_len=seq_len,
page_size=pool.index_page_size,
index_head_dim=pool.index_head_dim,
)
class GetKAndS:
@classmethod
def execute(cls, *args, **kwargs):
@@ -415,161 +271,6 @@ def _set_k_and_s_triton_kernel(
tl.store(buf_fp32_ptr + out_s_offset, k_scale)
def _get_k_triton(
buf: torch.Tensor,
page_indices: torch.Tensor,
seq_len: int,
page_size: int,
index_head_dim: int,
):
"""
Gather K (key) data from paged buffer using Triton.
:param buf: (num_pages, page_size * 128 + page_size * 4), uint8
:param page_indices: (num_pages,), int32/int64
:param seq_len: int, number of tokens to gather
:param page_size: int, typically 64
:param index_head_dim: int, typically 128
:return: (seq_len, index_head_dim), uint8
"""
num_pages, buf_numel_per_page = buf.shape
# Allocate output
out = torch.empty((seq_len, index_head_dim), dtype=torch.uint8, device=buf.device)
# Launch kernel with one thread per token
grid = (seq_len,)
_get_k_triton_kernel[grid](
buf,
page_indices,
out,
seq_len,
page_size,
buf_numel_per_page,
index_head_dim,
BLOCK_SIZE=128,
)
return out
@triton.jit
def _get_k_triton_kernel(
buf_ptr,
page_indices_ptr,
out_ptr,
seq_len: tl.constexpr,
page_size: tl.constexpr,
buf_numel_per_page: tl.constexpr,
index_head_dim: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""
Each program handles one token (seq_len tokens total).
Loads 128 bytes from the appropriate page.
"""
token_id = tl.program_id(0)
# Calculate which page and offset within page
page_idx = token_id // page_size
token_offset_in_page = token_id % page_size
# Load the page index from page_indices
page_index = tl.load(page_indices_ptr + page_idx)
# Calculate source offset in buf
# buf[page_index, token_offset_in_page * index_head_dim : ...]
src_base_offset = (
page_index * buf_numel_per_page + token_offset_in_page * index_head_dim
)
# Load 128 bytes (index_head_dim elements)
offsets = tl.arange(0, BLOCK_SIZE)
mask = offsets < index_head_dim
data = tl.load(buf_ptr + src_base_offset + offsets, mask=mask)
# Store to output
dst_offset = token_id * index_head_dim
tl.store(out_ptr + dst_offset + offsets, data, mask=mask)
def _get_s_triton(
buf: torch.Tensor,
page_indices: torch.Tensor,
seq_len: int,
page_size: int,
index_head_dim: int,
):
"""
Gather S (scale) data from paged buffer using Triton.
:param buf: (num_pages, page_size * 128 + page_size * 4), uint8
:param page_indices: (num_pages,), int32/int64
:param seq_len: int, number of tokens to gather
:param page_size: int, typically 64
:param index_head_dim: int, typically 128
:return: (seq_len, 4), uint8 (representing fp32 scale)
"""
num_pages, buf_numel_per_page = buf.shape
s_offset_in_page = page_size * index_head_dim # Scales start after K data
# Allocate output
out = torch.empty((seq_len, 4), dtype=torch.uint8, device=buf.device)
# Launch kernel with one thread per token
grid = (seq_len,)
_get_s_triton_kernel[grid](
buf,
page_indices,
out,
seq_len,
page_size,
buf_numel_per_page,
s_offset_in_page,
)
return out
@triton.jit
def _get_s_triton_kernel(
buf_ptr,
page_indices_ptr,
out_ptr,
seq_len: tl.constexpr,
page_size: tl.constexpr,
buf_numel_per_page: tl.constexpr,
s_offset_in_page: tl.constexpr,
):
"""
Each program handles one token (seq_len tokens total).
Loads 4 bytes (fp32 scale) from the appropriate page.
"""
token_id = tl.program_id(0)
# Calculate which page and offset within page
page_idx = token_id // page_size
token_offset_in_page = token_id % page_size
# Load the page index from page_indices
page_index = tl.load(page_indices_ptr + page_idx)
# Calculate source offset in buf
# Scales are stored after K data: page_size * index_head_dim offset
# buf[page_index, s_offset_in_page + token_offset_in_page * 4 : ...]
src_base_offset = (
page_index * buf_numel_per_page + s_offset_in_page + token_offset_in_page * 4
)
# Load 4 bytes (fp32 scale)
offsets = tl.arange(0, 4)
data = tl.load(buf_ptr + src_base_offset + offsets)
# Store to output
dst_offset = token_id * 4
tl.store(out_ptr + dst_offset + offsets, data)
def _get_k_and_s_triton(
buf: torch.Tensor,
page_indices: torch.Tensor,
@@ -581,7 +282,6 @@ def _get_k_and_s_triton(
):
"""
Fused gather of both K (key) and S (scale) data from paged buffer using Triton.
This is more efficient than calling GetK and GetS separately.
:param buf: (num_pages, page_size * 128 + page_size * 4), uint8
:param page_indices: (num_pages,), int32/int64
@@ -3,57 +3,6 @@ import triton
import triton.language as tl
def gather_dsa_kv_scales(
scale_src,
scale_dst,
kv_indices,
kv_indptr,
kv_indptr_idx,
):
_gather_dsa_kv_scales[(32,)](
scale_src,
scale_dst,
kv_indices,
kv_indptr,
scale_src.stride(0),
KV_INDPTR_IDX=kv_indptr_idx,
NUM_TILES=scale_src.shape[-1],
BLOCK=256,
)
@triton.jit
def _gather_dsa_kv_scales(
scale_src,
scale_dst,
kv_indices,
kv_indptr,
scale_src_stride,
KV_INDPTR_IDX: tl.constexpr,
NUM_TILES: tl.constexpr,
BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
num_programs = tl.num_programs(0)
active = tl.load(kv_indptr + KV_INDPTR_IDX)
block_start = pid * BLOCK
tiles = tl.arange(0, NUM_TILES)
while block_start < active:
offsets = block_start + tl.arange(0, BLOCK)
mask = offsets < active
rows = tl.load(kv_indices + offsets, mask=mask, other=0)
values = tl.load(
scale_src + rows[:, None] * scale_src_stride + tiles[None, :],
mask=mask[:, None],
)
tl.store(
scale_dst + rows[:, None] * NUM_TILES + tiles[None, :],
values,
mask=mask[:, None],
)
block_start += num_programs * BLOCK
def quantize_k_cache(cache_k, dv: int | None = None):
return _quantize_k_cache_fast_wrapped(cache_k, dv=dv)
@@ -169,89 +169,6 @@ def act_quant(
return y, s
@tilelang.jit(out_idx=[4], pass_configs=pass_configs)
def fp8_index_kernel(h: int, d: int, clear_accum=True):
b = T.symbolic("b")
m = T.symbolic("m")
n = T.symbolic("n")
blk_n1 = 512
blk_n2 = 128
@T.prim_func
def fp8_index_kernel_(
q: T.Tensor[(b, m, h, d), FP8],
q_s: T.Tensor[(b, m, h), FP32],
k: T.Tensor[(b, n, d), FP8],
k_s: T.Tensor[(b, n), FP32],
o: T.Tensor[(b, m, n), FP32],
) -> None:
with T.Kernel(b, m, T.ceildiv(n, blk_n1)) as (i_b, i_m, i1_n):
q_smem = T.alloc_shared((h, d), FP8)
T.copy(q[i_b, i_m, 0, 0], q_smem)
q_s_frag = T.alloc_fragment(h, FP32)
T.copy(q_s[i_b, i_m, 0], q_s_frag)
for i2_n in T.Pipelined(blk_n1 // blk_n2, num_stages=2):
k_smem = T.alloc_shared((blk_n2, d), FP8)
T.copy(k[i_b, i1_n * blk_n1 + i2_n * blk_n2, 0], k_smem)
k_s_frag = T.alloc_fragment(blk_n2, FP32)
T.copy(k_s[i_b, i1_n * blk_n1 + i2_n * blk_n2], k_s_frag)
logits = T.alloc_fragment((blk_n2, h), FP32)
if not clear_accum:
T.fill(logits, 0)
T.gemm(
k_smem,
q_smem,
logits,
transpose_A=False,
transpose_B=True,
clear_accum=clear_accum,
)
for i_h, i3_n in T.Parallel(h, blk_n2):
logits[i3_n, i_h] = T.max(logits[i3_n, i_h], 0) * q_s_frag[i_h]
logits_sum = T.alloc_fragment(blk_n2, FP32)
T.reduce_sum(logits, logits_sum, dim=1)
for i3_n in T.Parallel(blk_n2):
logits_sum[i3_n] *= k_s_frag[i3_n]
T.copy(logits_sum, o[i_b, i_m, i1_n * blk_n1 + i2_n * blk_n2])
return fp8_index_kernel_
def fp8_index(
q: torch.Tensor,
q_s: torch.Tensor,
k: torch.Tensor,
k_s: torch.Tensor,
) -> torch.Tensor:
"""
Perform index score using FP8 precision.
Args:
q (torch.Tensor): The Q tensor, must be contiguous.
q_s (torch.Tensor): The scaling factor for Q (float), must be contiguous.
k (torch.Tensor): The K tensor, must be contiguous.
k_s (torch.Tensor): The scaling factor for K (e8m0 here), must be contiguous.
fp8 q @ fp8 k -> fp32 logits
relu(fp32 logits) * q_s (weights) -> fp32 logits
fp32 logits -> fp32 logits_sum
fp32 logits_sum * k_s (e8m0) -> fp32 index_score
"""
if _is_hip:
return fp8_index_kernel(q.shape[2], q.shape[3], False)(q, q_s, k, k_s)
else:
return fp8_index_kernel(q.shape[2], q.shape[3])(q, q_s, k, k_s)
@tilelang.jit(
out_idx=[-1],
pass_configs={
@@ -371,56 +371,3 @@ def transform_index_page_table_prefill_fast(
num_warps=4,
)
return result
def transform_index_page_table_decode_ref(
page_table: torch.Tensor,
topk_indices: torch.Tensor,
result: Optional[torch.Tensor] = None,
page_size: int = 1,
) -> torch.Tensor:
assert page_size == 1
assert page_table.shape[0] == topk_indices.shape[0]
if result is None:
result = torch.empty_like(topk_indices, dtype=torch.int32)
assert result.shape == topk_indices.shape
torch.gather(
page_table.to(result.dtype),
dim=1,
index=topk_indices.clamp(min=0),
out=result,
)
result[topk_indices < 0] = -1
return result
def transform_index_page_table_prefill_ref(
page_table: torch.Tensor,
topk_indices: torch.Tensor,
extend_lens_cpu: List[int],
page_size: int = 1,
output_num_tokens: Optional[int] = None,
page_table_is_expanded: bool = False,
) -> torch.Tensor:
assert page_size == 1
real_num_tokens = sum(extend_lens_cpu)
result = _allocate_prefill_result(topk_indices, real_num_tokens, output_num_tokens)
if page_table_is_expanded:
if real_num_tokens > 0:
transform_index_page_table_decode_ref(
page_table[:real_num_tokens],
topk_indices[:real_num_tokens],
result=result[:real_num_tokens],
)
return result
offset = 0
for i, l in enumerate(extend_lens_cpu):
transform_index_page_table_decode_ref(
page_table[i].unsqueeze(0).expand(l, -1),
topk_indices[offset : offset + l],
result=result[offset : offset + l],
)
offset += l
return result
@@ -1,21 +1,18 @@
"""Triton sparse MLA decode kernel with FP8 and BF16 KV cache support.
"""Triton sparse MLA decode with FP8 and BF16 KV cache support.
Adapted from aiter's unified_attention_sparse_mla kernel for DSA shapes:
Decode-tuned launcher for the fused / split-K kernels of ``triton_sparse_mla``
on DSA shapes:
q: [bs, H, DIM] fp8/bf16 (DIM=576 = D_V+D_TAIL)
kv: [num_pages, 1, DIM] fp8/bf16
indices: [bs, 1, topk] int32
output: [1, bs, H, D_V] bf16
Two variants:
1. Base: single-pass per-token kernel (adapted from aiter)
2. Split-K: adaptive split-K with fused fast path (adapted from DSv4)
"""
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsa.triton_sparse_mla import (
_FP8_MAX,
_LOG2E,
_PREFERRED_BLOCK_K,
_cu_count,
_gfx950_sparse_mla_kv_splits,
@@ -28,14 +25,11 @@ from sglang.kernels.ops.attention.dsa.triton_sparse_mla import (
_reduce_d_chunk,
_row_strides,
_sparse_mla_block_k,
_sparse_mla_fused_kernel,
_sparse_mla_reduce_kernel,
_sparse_mla_split_k_kernel,
_validate_input_dtypes,
)
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
_IS_FNUZ = is_fp8_fnuz()
_FP8_MAX = 240.0 if _IS_FNUZ else 448.0
_G = tl.constexpr(128)
_SplitKWorkspace = list[tuple[torch.Tensor, torch.Tensor]]
_splitk_bufs: dict[tuple[torch.device, int], _SplitKWorkspace] = {}
@@ -82,420 +76,6 @@ def _get_splitk_bufs(
return lse, acc
# ---------------------------------------------------------------------------
# Split-K kernel (adapted from DSv4 paged_decode.py)
# ---------------------------------------------------------------------------
LOG2E = 1.4426950408889634
@triton.jit
def _sparse_mla_decode_fused_kernel(
q_nope_ptr, # [N, H, D_V]
q_rope_ptr, # [N, H, D_TAIL]
kv_ptr, # [num_pages, 1, KV_DIM]
idx_ptr, # [N, topk]
out_ptr, # [N, H, D_V]
qk_scale,
fp8_max,
topk: tl.constexpr,
H: tl.constexpr,
KV_DIM: tl.constexpr,
D_V: tl.constexpr,
D_TAIL: tl.constexpr,
NUM_GROUPS: tl.constexpr,
STRIDE_QN_T: tl.constexpr,
STRIDE_QN_H: tl.constexpr,
STRIDE_QR_T: tl.constexpr,
STRIDE_QR_H: tl.constexpr,
USE_FP8_DOT: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_K: tl.constexpr,
USE_I64_PAGE: tl.constexpr = True,
):
t = tl.program_id(0)
pid_h = tl.program_id(1)
h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
h_mask = h_offs < H
dt = tl.arange(0, D_TAIL)
g = tl.arange(0, _G)
input_type = kv_ptr.dtype.element_ty if USE_FP8_DOT else tl.bfloat16
if USE_FP8_DOT:
p_dot_scale = 1.0 / fp8_max
else:
p_dot_scale = 1.0
qn_base = q_nope_ptr + t * STRIDE_QN_T
qn_row = qn_base + h_offs[:, None] * STRIDE_QN_H
q0 = tl.load(
qn_row + g[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 2:
q1 = tl.load(
qn_row + (_G + g)[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 3:
q2 = tl.load(
qn_row + (2 * _G + g)[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 4:
q3 = tl.load(
qn_row + (3 * _G + g)[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
q_tail = tl.load(
q_rope_ptr + t * STRIDE_QR_T + h_offs[:, None] * STRIDE_QR_H + dt[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
neg_large = -3.4028234663852886e38
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
acc0 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
if NUM_GROUPS >= 2:
acc1 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
if NUM_GROUPS >= 3:
acc2 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
if NUM_GROUPS >= 4:
acc3 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
k_offs = tl.arange(0, BLOCK_K)
num_tiles = tl.cdiv(topk, BLOCK_K)
for j in tl.range(0, num_tiles, num_stages=3):
k_start = j * BLOCK_K
k_pos = k_start + k_offs
valid = k_pos < topk
slot = tl.load(idx_ptr + t * topk + k_pos, mask=valid, other=0)
valid = valid & (slot >= 0)
page = tl.where(valid, slot, 0)
if USE_I64_PAGE:
page = page.to(tl.int64)
kv_base = kv_ptr + page[:, None] * KV_DIM
kv0 = tl.load(
kv_base + g[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 2:
kv1 = tl.load(
kv_base + (_G + g)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 3:
kv2 = tl.load(
kv_base + (2 * _G + g)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 4:
kv3 = tl.load(
kv_base + (3 * _G + g)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
kv_tail = tl.load(
kv_base + (D_V + dt)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
scores = tl.dot(q0, tl.trans(kv0))
if NUM_GROUPS >= 2:
scores += tl.dot(q1, tl.trans(kv1))
if NUM_GROUPS >= 3:
scores += tl.dot(q2, tl.trans(kv2))
if NUM_GROUPS >= 4:
scores += tl.dot(q3, tl.trans(kv3))
scores += tl.dot(q_tail, tl.trans(kv_tail))
scores = scores * qk_scale
scores = tl.where(valid[None, :], scores, neg_large)
m_block = tl.max(scores, axis=1)
m_new = tl.maximum(m_i, m_block)
alpha = tl.exp2(m_i - m_new)
p = tl.exp2(scores - m_new[:, None])
l_new = l_i * alpha + tl.sum(p, axis=1)
if USE_FP8_DOT:
p_dot = (p * fp8_max).to(input_type)
else:
p_dot = p.to(input_type)
acc0 = acc0 * alpha[:, None] + tl.dot(p_dot, kv0).to(tl.float32) * p_dot_scale
if NUM_GROUPS >= 2:
acc1 = (
acc1 * alpha[:, None] + tl.dot(p_dot, kv1).to(tl.float32) * p_dot_scale
)
if NUM_GROUPS >= 3:
acc2 = (
acc2 * alpha[:, None] + tl.dot(p_dot, kv2).to(tl.float32) * p_dot_scale
)
if NUM_GROUPS >= 4:
acc3 = (
acc3 * alpha[:, None] + tl.dot(p_dot, kv3).to(tl.float32) * p_dot_scale
)
m_i = m_new
l_i = l_new
denom = tl.maximum(l_i, 1.0e-30)
inv_denom = 1.0 / denom
acc0 = tl.where(l_i[:, None] > 0.0, acc0 * inv_denom[:, None], 0.0)
if NUM_GROUPS >= 2:
acc1 = tl.where(l_i[:, None] > 0.0, acc1 * inv_denom[:, None], 0.0)
if NUM_GROUPS >= 3:
acc2 = tl.where(l_i[:, None] > 0.0, acc2 * inv_denom[:, None], 0.0)
if NUM_GROUPS >= 4:
acc3 = tl.where(l_i[:, None] > 0.0, acc3 * inv_denom[:, None], 0.0)
o_base = out_ptr + t * H * D_V
tl.store(
o_base + h_offs[:, None] * D_V + g[None, :],
acc0.to(tl.bfloat16),
mask=h_mask[:, None],
)
if NUM_GROUPS >= 2:
tl.store(
o_base + h_offs[:, None] * D_V + (_G + g)[None, :],
acc1.to(tl.bfloat16),
mask=h_mask[:, None],
)
if NUM_GROUPS >= 3:
tl.store(
o_base + h_offs[:, None] * D_V + (2 * _G + g)[None, :],
acc2.to(tl.bfloat16),
mask=h_mask[:, None],
)
if NUM_GROUPS >= 4:
tl.store(
o_base + h_offs[:, None] * D_V + (3 * _G + g)[None, :],
acc3.to(tl.bfloat16),
mask=h_mask[:, None],
)
@triton.jit
def _sparse_mla_decode_split_kernel(
q_nope_ptr, # [N, H, D_V]
q_rope_ptr, # [N, H, D_TAIL]
kv_ptr, # [num_pages, 1, KV_DIM]
idx_ptr, # [N, topk]
lse_partial_ptr, # [N, KV_SPLITS, H_padded] fp32
acc_partial_ptr, # [N, KV_SPLITS, H_padded, D_V] bf16
qk_scale,
fp8_max,
topk: tl.constexpr,
H: tl.constexpr,
KV_DIM: tl.constexpr,
D_V: tl.constexpr,
D_TAIL: tl.constexpr,
NUM_GROUPS: tl.constexpr,
STRIDE_QN_T: tl.constexpr,
STRIDE_QN_H: tl.constexpr,
STRIDE_QR_T: tl.constexpr,
STRIDE_QR_H: tl.constexpr,
USE_FP8_DOT: tl.constexpr,
KV_SPLITS: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_K: tl.constexpr,
):
t = tl.program_id(0)
pid_h = tl.program_id(1)
pid_k = tl.program_id(2)
h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
h_mask = h_offs < H
dt = tl.arange(0, D_TAIL)
g = tl.arange(0, _G)
input_type = kv_ptr.dtype.element_ty if USE_FP8_DOT else tl.bfloat16
if USE_FP8_DOT:
p_dot_scale = 1.0 / fp8_max
else:
p_dot_scale = 1.0
qn_base = q_nope_ptr + t * STRIDE_QN_T
qn_row = qn_base + h_offs[:, None] * STRIDE_QN_H
q0 = tl.load(
qn_row + g[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 2:
q1 = tl.load(
qn_row + (_G + g)[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 3:
q2 = tl.load(
qn_row + (2 * _G + g)[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 4:
q3 = tl.load(
qn_row + (3 * _G + g)[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
q_tail = tl.load(
q_rope_ptr + t * STRIDE_QR_T + h_offs[:, None] * STRIDE_QR_H + dt[None, :],
mask=h_mask[:, None],
other=0.0,
).to(input_type)
tiles_per_segment = tl.cdiv(topk, KV_SPLITS * BLOCK_K)
if pid_k * tiles_per_segment * BLOCK_K >= topk:
return
num_tiles = tl.cdiv(topk, BLOCK_K)
tile_start = pid_k * tiles_per_segment
tile_end = tl.minimum((pid_k + 1) * tiles_per_segment, num_tiles)
neg_large = -3.4028234663852886e38
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
acc0 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
if NUM_GROUPS >= 2:
acc1 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
if NUM_GROUPS >= 3:
acc2 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
if NUM_GROUPS >= 4:
acc3 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
k_offs = tl.arange(0, BLOCK_K)
for j in tl.range(tile_start, tile_end, num_stages=3):
k_start = j * BLOCK_K
k_pos = k_start + k_offs
valid = k_pos < topk
slot = tl.load(idx_ptr + t * topk + k_pos, mask=valid, other=0)
valid = valid & (slot >= 0)
page = tl.where(valid, slot, 0).to(tl.int64)
kv_base = kv_ptr + page[:, None] * KV_DIM
kv0 = tl.load(
kv_base + g[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 2:
kv1 = tl.load(
kv_base + (_G + g)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 3:
kv2 = tl.load(
kv_base + (2 * _G + g)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
if NUM_GROUPS >= 4:
kv3 = tl.load(
kv_base + (3 * _G + g)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
kv_tail = tl.load(
kv_base + (D_V + dt)[None, :],
mask=valid[:, None],
other=0.0,
).to(input_type)
scores = tl.dot(q0, tl.trans(kv0))
if NUM_GROUPS >= 2:
scores += tl.dot(q1, tl.trans(kv1))
if NUM_GROUPS >= 3:
scores += tl.dot(q2, tl.trans(kv2))
if NUM_GROUPS >= 4:
scores += tl.dot(q3, tl.trans(kv3))
scores += tl.dot(q_tail, tl.trans(kv_tail))
scores = scores * qk_scale
scores = tl.where(valid[None, :], scores, neg_large)
m_block = tl.max(scores, axis=1)
m_new = tl.maximum(m_i, m_block)
alpha = tl.exp2(m_i - m_new)
p = tl.exp2(scores - m_new[:, None])
l_new = l_i * alpha + tl.sum(p, axis=1)
if USE_FP8_DOT:
p_dot = (p * fp8_max).to(input_type)
else:
p_dot = p.to(input_type)
acc0 = acc0 * alpha[:, None] + tl.dot(p_dot, kv0).to(tl.float32) * p_dot_scale
if NUM_GROUPS >= 2:
acc1 = (
acc1 * alpha[:, None] + tl.dot(p_dot, kv1).to(tl.float32) * p_dot_scale
)
if NUM_GROUPS >= 3:
acc2 = (
acc2 * alpha[:, None] + tl.dot(p_dot, kv2).to(tl.float32) * p_dot_scale
)
if NUM_GROUPS >= 4:
acc3 = (
acc3 * alpha[:, None] + tl.dot(p_dot, kv3).to(tl.float32) * p_dot_scale
)
m_i = m_new
l_i = l_new
neg_large = -1073741824.0
denom = tl.maximum(l_i, 1.0e-30)
inv_denom = 1.0 / denom
has_data = l_i > 0.0
acc0 = tl.where(has_data[:, None], acc0 * inv_denom[:, None], 0.0)
if NUM_GROUPS >= 2:
acc1 = tl.where(has_data[:, None], acc1 * inv_denom[:, None], 0.0)
if NUM_GROUPS >= 3:
acc2 = tl.where(has_data[:, None], acc2 * inv_denom[:, None], 0.0)
if NUM_GROUPS >= 4:
acc3 = tl.where(has_data[:, None], acc3 * inv_denom[:, None], 0.0)
lse = tl.where(has_data, tl.log2(l_i) + m_i, neg_large)
H_padded = tl.cdiv(H, BLOCK_H) * BLOCK_H
lse_base = t * KV_SPLITS * H_padded + pid_k * H_padded
tl.store(lse_partial_ptr + lse_base + h_offs, lse, mask=h_mask)
ap_base = t * KV_SPLITS * H_padded * D_V + pid_k * H_padded * D_V
tl.store(
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + g[None, :],
acc0.to(tl.bfloat16),
mask=h_mask[:, None],
)
if NUM_GROUPS >= 2:
tl.store(
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (_G + g)[None, :],
acc1.to(tl.bfloat16),
mask=h_mask[:, None],
)
if NUM_GROUPS >= 3:
tl.store(
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (2 * _G + g)[None, :],
acc2.to(tl.bfloat16),
mask=h_mask[:, None],
)
if NUM_GROUPS >= 4:
tl.store(
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (3 * _G + g)[None, :],
acc3.to(tl.bfloat16),
mask=h_mask[:, None],
)
def triton_sparse_mla_decode_splitk(
q_nope: torch.Tensor,
q_rope: torch.Tensor,
@@ -581,7 +161,7 @@ def triton_sparse_mla_decode_splitk(
else:
kv_splits = min(kv_splits, max_kv_splits)
qk_scale = float(sm_scale) * LOG2E
qk_scale = float(sm_scale) * _LOG2E
if kv_splits == 1:
fused_num_warps = 4
@@ -589,11 +169,12 @@ def triton_sparse_mla_decode_splitk(
fused_num_warps = _gfx950_sparse_mla_num_warps(base_ctas, 1, num_cu)
out = torch.empty(bs, H, d_v, device=q_nope.device, dtype=torch.bfloat16)
with _no_async_copy():
_sparse_mla_decode_fused_kernel[(bs, n_head_blocks)](
_sparse_mla_fused_kernel[(bs, n_head_blocks)](
q_nope,
q_rope,
kv,
idx_flat,
idx_flat,
out,
qk_scale,
_FP8_MAX,
@@ -608,6 +189,7 @@ def triton_sparse_mla_decode_splitk(
STRIDE_QR_T=stride_qr_t,
STRIDE_QR_H=stride_qr_h,
USE_FP8_DOT=use_fp8_dot,
USE_TOPK_LENGTH=False,
BLOCK_H=BLOCK_H,
BLOCK_K=BLOCK_K,
USE_I64_PAGE=not (
@@ -637,7 +219,7 @@ def triton_sparse_mla_decode_splitk(
grid_split = (bs, n_head_blocks, kv_splits)
with _no_async_copy():
_sparse_mla_decode_split_kernel[grid_split](
_sparse_mla_split_k_kernel[grid_split](
q_nope,
q_rope,
kv,
@@ -13,9 +13,9 @@ from .compress import (
compress_forward,
compress_norm_rope_store,
)
from .compress_old import fused_norm_rope_inplace
from .elementwise import (
fused_k_norm_rope_flashmla,
fused_norm_rope_inplace,
fused_q_indexer_rope_first_quant,
fused_q_indexer_rope_hadamard_fp4_quant,
fused_q_indexer_rope_hadamard_quant,
@@ -150,7 +150,7 @@ _PREFILL_PLAN_BYTES = 24
# ----------------------------------------------------------------------------
# Plan dataclasses. The element at index 1 is the consumer for
# `compress_fused_norm_rope_inplace` (which reads ragged_id / seq_len from a
# `compress_norm_rope_store` (which reads ragged_id / seq_len from a
# 16-byte plan tensor --- both DecodePlan and CompressPlan satisfy that layout).
# ----------------------------------------------------------------------------
@@ -1,308 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Literal, NamedTuple, Optional, Union
import torch
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
from sglang.srt.environ import envs
from .utils import make_name
if TYPE_CHECKING:
from tvm_ffi.module import Module
@cache_once
def _jit_common_module() -> Module:
return load_jit(
make_name("common"),
cuda_files=["deepseek_v4/common.cuh"],
cuda_wrappers=[("plan_compress_prefill", "plan_compress_prefill")],
)
@cache_once
def _jit_compress_128_online_plan_module() -> Module:
"""Host-side plan generator for online compress 128 (no template args)."""
return load_jit(
make_name("compress_128_online_plan"),
cuda_files=["deepseek_v4/c128_online.cuh"],
cuda_wrappers=[
("plan_compress_online_prefill", "plan_compress_online_prefill"),
],
)
@cache_once
def _jit_compress_128_online_module(head_dim: int) -> Module:
"""Online compress 128 kernel: ring_size=1, per-index (max, sum, kv) state."""
args = make_cpp_args(head_dim, is_arch_support_pdl())
kernel_class = f"FlashCompress128OnlineKernel<{args}>"
return load_jit(
make_name("compress_128_online"),
*args,
cuda_files=["deepseek_v4/c128_online.cuh"],
cuda_wrappers=[
("decode", f"{kernel_class}::run_decode"),
("prefill", f"{kernel_class}::run_prefill"),
],
extra_cuda_cflags=["-use_fast_math"],
)
@cache_once
def norm_rope_module(
dtype: torch.dtype,
head_dim: int,
rope_dim: int,
) -> Module:
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
return load_jit(
make_name("fused_norm_rope"),
*args,
cuda_files=["deepseek_v4/fused_norm_rope.cuh"],
cuda_wrappers=[
("forward", f"FusedNormRopeKernel<{args}>::forward"),
],
)
@cache_once
def _jit_compress_module(
head_dim: int,
dtype_in: torch.dtype,
dtype_out: torch.dtype,
ratio: Literal[4, 128],
) -> Module:
args = make_cpp_args(head_dim, dtype_in, dtype_out, is_arch_support_pdl())
kernel_class = f"FlashCompress{ratio}Kernel<{args}>"
return load_jit(
make_name(f"compress_{ratio}"),
*args,
cuda_files=[f"deepseek_v4/c{ratio}.cuh"],
cuda_wrappers=[
("decode", f"{kernel_class}::run_decode"),
("prefill", f"{kernel_class}::run_prefill"),
],
extra_cuda_cflags=["-use_fast_math"],
)
class CompressorPrefillPlan(NamedTuple):
compress_ratio: int
compress_plan: torch.Tensor
write_plan: torch.Tensor
def copy_(self, other: CompressorPrefillPlan) -> None:
assert self.compress_ratio == other.compress_ratio
self.compress_plan.copy_(other.compress_plan)
self.write_plan.copy_(other.write_plan)
@staticmethod
def generate(
compress_ratio: Literal[4, 128],
num_q_tokens: int,
seq_lens: torch.Tensor,
extend_lens: torch.Tensor,
device: torch.device,
use_cuda_graph: bool = False,
) -> CompressorPrefillPlan:
from sglang.srt.environ import envs
# Online c128 keeps the same NamedTuple shape (compress_plan, write_plan)
# so call sites that splat `*plan[1:]` continue to work, but the C++
# plan struct semantics differ (last-token coords + window_len).
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
return CompressorPrefillPlan._generate_online(
num_q_tokens=num_q_tokens,
seq_lens=seq_lens,
extend_lens=extend_lens,
device=device,
use_cuda_graph=use_cuda_graph,
)
assert seq_lens.device == extend_lens.device
seq_lens = seq_lens.to(torch.int64)
extend_lens = extend_lens.to(torch.int64)
plan_tensor = torch.empty(
(2, num_q_tokens, 16),
dtype=torch.uint8,
device=seq_lens.device,
pin_memory=seq_lens.is_cpu,
)
module = _jit_common_module()
is_overlap = compress_ratio == 4
plan_lens = module.plan_compress_prefill(
extend_lens,
seq_lens,
plan_tensor[0],
plan_tensor[1],
compress_ratio,
is_overlap,
use_cuda_graph,
)
return CompressorPrefillPlan(
compress_ratio,
plan_tensor[0, : plan_lens[0]].to(device, non_blocking=True),
plan_tensor[1, : plan_lens[1]].to(device, non_blocking=True),
)
@staticmethod
def _generate_online(
num_q_tokens: int,
seq_lens: torch.Tensor,
extend_lens: torch.Tensor,
device: torch.device,
use_cuda_graph: bool,
) -> CompressorPrefillPlan:
# Online plan host-side path: only CPU/cuda-host implemented today.
# Move inputs to CPU pinned memory then bounce the result to device.
seq_lens_cpu = seq_lens.detach().to(torch.int64).cpu()
extend_lens_cpu = extend_lens.detach().to(torch.int64).cpu()
plan_tensor = torch.empty(
(2, num_q_tokens, 16),
dtype=torch.uint8,
device="cpu",
pin_memory=True,
)
module = _jit_compress_128_online_plan_module()
plan_lens = module.plan_compress_online_prefill(
extend_lens_cpu,
seq_lens_cpu,
plan_tensor[0],
plan_tensor[1],
use_cuda_graph,
)
return CompressorPrefillPlan(
128,
plan_tensor[0, : plan_lens[0]].to(device, non_blocking=True),
plan_tensor[1, : plan_lens[1]].to(device, non_blocking=True),
)
@property
def is_decode(self) -> bool:
return False
class CompressorDecodePlan(NamedTuple):
compress_ratio: int
seq_lens: torch.Tensor
def copy_(self, other: CompressorDecodePlan) -> None:
assert self.compress_ratio == other.compress_ratio
self.seq_lens.copy_(other.seq_lens)
@property
def is_decode(self) -> bool:
return True
def compress_plan(
compress_ratio: Literal[4, 128],
num_q_tokens: int,
seq_lens: torch.Tensor,
extend_lens: Optional[torch.Tensor],
device: torch.device,
) -> Union[CompressorDecodePlan, CompressorPrefillPlan]:
if extend_lens is not None:
return CompressorPrefillPlan.generate(
compress_ratio,
num_q_tokens,
seq_lens,
extend_lens,
device,
)
else:
assert num_q_tokens == len(seq_lens)
seq_lens = seq_lens.to(device, non_blocking=True)
return CompressorDecodePlan(compress_ratio, seq_lens)
def compress_forward(
kv_score_buffer: torch.Tensor,
kv_score_input: torch.Tensor,
ape: torch.Tensor,
indices: torch.Tensor,
plan: Union[CompressorDecodePlan, CompressorPrefillPlan, None] = None,
extra_data: Optional[torch.Tensor] = None,
*,
head_dim: int,
compress_ratio: Literal[4, 128],
out: Optional[torch.Tensor] = None,
seq_lens: Optional[torch.Tensor] = None,
extend_lens: Optional[torch.Tensor] = None,
) -> torch.Tensor:
assert head_dim % 128 == 0
num_q_tokens = kv_score_input.shape[0]
if out is None:
out = kv_score_input.new_empty((num_q_tokens, head_dim))
if plan is None:
assert seq_lens is not None
plan = compress_plan(
compress_ratio,
num_q_tokens,
seq_lens,
extend_lens,
kv_score_input.device,
)
assert plan.compress_ratio == compress_ratio, "Mismatched compress ratio in plan!"
# Online c128: separate JIT module, fp32 state, no compile-time dtypes.
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
online_module = _jit_compress_128_online_module(head_dim=head_dim)
F = online_module.decode if plan.is_decode else online_module.prefill
F(kv_score_buffer, kv_score_input, out, ape, indices, *plan[1:], extra_data)
return out
module = _jit_compress_module(
head_dim,
kv_score_input.dtype,
out.dtype,
compress_ratio,
)
F = module.decode if plan.is_decode else module.prefill
F(kv_score_buffer, kv_score_input, out, ape, indices, *plan[1:], extra_data)
return out
def compress_fused_norm_rope_inplace(
kv: torch.Tensor,
weight: torch.Tensor,
eps: float,
freq_cis: torch.Tensor,
plan: Union[CompressorDecodePlan, CompressorPrefillPlan],
) -> None:
freq_cis = torch.view_as_real(freq_cis).flatten(-2)
module = norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1])
module.forward(
kv,
weight,
plan[1],
freq_cis,
int(plan.is_decode),
eps,
plan.compress_ratio,
)
def fused_norm_rope_inplace(
kv: torch.Tensor,
weight: torch.Tensor,
eps: float,
freq_cis: torch.Tensor,
positions: torch.Tensor,
) -> None:
freq_cis = torch.view_as_real(freq_cis).flatten(-2)
module = norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1])
module.forward(
kv,
weight,
positions,
freq_cis,
2,
eps,
0,
)
@@ -509,86 +509,3 @@ def _gather_dequant_requant_fp8_paged_kernel(
output_ptr + out_row_base + DIM_NOPE + rope_offs,
rope_data.to(output_ptr.dtype.element_ty),
)
def dequantize_k_cache_paged_ref(
quant_k_cache: torch.Tensor,
page_table_1_flattened: torch.Tensor,
page_size: int,
) -> torch.Tensor:
"""Pure-torch reference for :func:`dequantize_k_cache_paged`.
Decodes the same v4 paged layout with vectorized torch indexing instead of
a Triton kernel. Used to validate the kernel (see the ``__main__`` block
below); not on any hot path.
"""
assert page_table_1_flattened.dtype in (torch.int32, torch.int64)
u8 = quant_k_cache.view(torch.uint8)
bytes_per_page = u8.shape[-1]
s_offset_bytes = page_size * NOPE_ROPE_BYTES
flat_u8 = u8.reshape(-1)
flat_fp8 = u8.view(fp8_dtype).reshape(-1)
flat_bf16 = u8.view(torch.bfloat16).reshape(-1)
loc = page_table_1_flattened.to(torch.int64)
page_idx = loc // page_size
in_page = loc % page_size
page_byte_base = page_idx * bytes_per_page
token_data_base = page_byte_base + in_page * NOPE_ROPE_BYTES
token_scale_base = (
page_byte_base + s_offset_bytes + in_page * PADDED_SCALE_PER_TOKEN
)
device = quant_k_cache.device
nope_byte = (
token_data_base[:, None] + torch.arange(DIM_NOPE, device=device)[None, :]
)
nope_fp8 = flat_fp8[nope_byte].to(torch.float32)
scale_byte = (
token_scale_base[:, None]
+ torch.arange(NUM_SCALE_TILES, device=device)[None, :]
)
scale_u8 = flat_u8[scale_byte].to(torch.int32)
scale_pow2 = torch.exp2((scale_u8 - 127).to(torch.float32))
scale_pow2 = torch.where(
scale_pow2 < (2.0**-126), torch.zeros_like(scale_pow2), scale_pow2
)
scale_full = scale_pow2.repeat_interleave(TILE_SIZE, dim=1)
nope = nope_fp8 * scale_full
rope_bf16_base = (token_data_base + DIM_NOPE) // 2
rope_idx = rope_bf16_base[:, None] + torch.arange(DIM_ROPE, device=device)[None, :]
rope = flat_bf16[rope_idx]
out = torch.empty(
(loc.shape[0], 1, DIM_NOPE + DIM_ROPE),
dtype=torch.bfloat16,
device=device,
)
out[:, 0, :DIM_NOPE] = nope.to(torch.bfloat16)
out[:, 0, DIM_NOPE:] = rope
return out
def gather_dequant_requant_fp8_paged_ref(
quant_k_cache: torch.Tensor,
page_table_1_flattened: torch.Tensor,
page_size: int,
extra_rows: int = 0,
) -> torch.Tensor:
"""Torch reference for :func:`gather_dequant_requant_fp8_paged`."""
active = dequantize_k_cache_paged_ref(
quant_k_cache,
page_table_1_flattened,
page_size,
).to(fp8_dtype)
if extra_rows == 0:
return active
out = torch.zeros(
(active.shape[0] + extra_rows, 1, DIM_NOPE + DIM_ROPE),
dtype=fp8_dtype,
device=active.device,
)
out[: active.shape[0]] = active
return out
@@ -32,6 +32,23 @@ def _jit_fused_rope_module():
)
@cache_once
def _jit_fused_norm_rope_module(
dtype: torch.dtype,
head_dim: int,
rope_dim: int,
):
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
return load_jit(
make_name("fused_norm_rope"),
*args,
cuda_files=["deepseek_v4/fused_norm_rope.cuh"],
cuda_wrappers=[
("forward", f"FusedNormRopeKernel<{args}>::forward"),
],
)
@cache_once
def _jit_main_q_norm_rope_module(
dtype: torch.dtype,
@@ -168,6 +185,26 @@ def fused_rope_inplace(
module.forward(q, k, freqs_real, positions, inverse)
def fused_norm_rope_inplace(
kv: torch.Tensor,
weight: torch.Tensor,
eps: float,
freq_cis: torch.Tensor,
positions: torch.Tensor,
) -> None:
freq_cis = torch.view_as_real(freq_cis).flatten(-2)
module = _jit_fused_norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1])
module.forward(
kv,
weight,
positions,
freq_cis,
2,
eps,
0,
)
def fused_q_norm_rope(
q_input: torch.Tensor,
q_output: torch.Tensor,
@@ -4,7 +4,6 @@ from __future__ import annotations
import functools
import itertools
import os
from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
import torch
@@ -22,6 +21,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import (
rope_tail_fake_quant_fp4_row,
)
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
from sglang.srt.environ import envs
if TYPE_CHECKING:
from sglang.kernels.ops.attention.dsv4.compress import (
@@ -62,9 +62,7 @@ C4_ROWGROUP_MIN_VERIFY_ROWS = 64
# (see `logits_rows_per_chunk`), so this caps the indexer's transient footprint
# independently of context length and chunked-prefill size; smaller budgets only
# buy more row chunks. 2 GiB covers 4096 rows over ~512K tokens of context.
_LOGITS_BUDGET_ELEMS = (
int(os.environ.get("SGLANG_DSV4_FP4_LOGITS_BUDGET_MB", "2048")) * 2**20 // 4
)
_LOGITS_BUDGET_ELEMS = envs.SGLANG_DSV4_FP4_LOGITS_BUDGET_MB.get() * 2**20 // 4
_LOGITS_POOL: dict = {}
File diff suppressed because it is too large Load Diff
@@ -36,10 +36,6 @@ class SetKAndS:
def execute(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack):
cls.triton(pool, buf, loc, nope_fp8_rope_bf16_pack)
@classmethod
def torch(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack):
_set_k_and_s_torch(buf, loc, nope_fp8_rope_bf16_pack, pool.page_size)
@classmethod
def triton(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack):
_set_k_and_s_triton(buf, loc, nope_fp8_rope_bf16_pack, pool.page_size)
@@ -183,77 +179,3 @@ def _set_k_and_s_triton_kernel(
tl.store(buf_fp8_ptr + out_k_nope_offsets, k_nope, mask=nope_mask)
tl.store(buf_bf16_ptr + out_k_rope_offsets, k_rope)
tl.store(buf_uint8_ptr + out_s_offsets, k_scale, mask=scale_mask)
def _set_k_and_s_torch(
buf: torch.Tensor,
loc: torch.Tensor,
nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack,
page_size: int,
):
num_pages, buf_numel_per_page = buf.shape
(num_tokens_to_write,) = loc.shape
k_nope, k_rope, scale_k_nope = (
nope_fp8_rope_bf16_pack.k_nope_fp8,
nope_fp8_rope_bf16_pack.k_rope_bf16,
nope_fp8_rope_bf16_pack.scale_k_nope_ue8m0,
)
num_tokens_to_write_nope, nope_dim = k_nope.shape
num_tokens_to_write_rope, rope_dim = k_rope.shape
num_tokens_to_write_scale, scale_dim = scale_k_nope.shape
assert (
num_tokens_to_write
== num_tokens_to_write_nope
== num_tokens_to_write_rope
== num_tokens_to_write_scale
), (
f"{num_tokens_to_write=} {num_tokens_to_write_nope=} {num_tokens_to_write_rope=} {num_tokens_to_write_scale=}"
)
assert buf.dtype == torch.uint8
assert loc.dtype in [
torch.int64,
torch.int32,
], f"{loc.dtype=}"
assert k_nope.dtype == fp8_dtype
assert k_rope.dtype == torch.bfloat16
assert scale_k_nope.dtype == torch.uint8
assert buf.is_contiguous()
assert loc.is_contiguous()
assert k_nope.is_contiguous()
assert k_rope.is_contiguous()
assert scale_k_nope.is_contiguous()
buf_fp8 = buf.view(fp8_dtype).flatten()
buf_bf16 = buf.view(torch.bfloat16).flatten()
buf_scale = buf.view(torch.uint8).flatten()
loc_page_index = loc // page_size
loc_token_offset_in_page = loc % page_size
s_offset_nbytes_in_page = page_size * (nope_dim + rope_dim * 2)
nope_offset = loc_page_index * buf_numel_per_page + loc_token_offset_in_page * (
nope_dim + rope_dim * 2
)
rope_offset = (
loc_page_index * buf_numel_per_page // 2
+ (loc_token_offset_in_page * (nope_dim + rope_dim * 2) + nope_dim) // 2
)
s_offset = (
loc_page_index * buf_numel_per_page
+ s_offset_nbytes_in_page
+ loc_token_offset_in_page * (scale_dim + 1)
)
for i in range(num_tokens_to_write):
buf_fp8[nope_offset[i] : nope_offset[i] + nope_dim] = k_nope[i]
buf_bf16[rope_offset[i] : rope_offset[i] + rope_dim] = k_rope[i]
buf_scale[s_offset[i] : s_offset[i] + scale_dim] = scale_k_nope[i]
@@ -1,120 +0,0 @@
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
@triton.jit
def _quant_k_cache_fused_kernel(
k_bf16_ptr,
k_nope_fp8_ptr,
k_rope_bf16_ptr,
scale_k_nope_uint8_ptr,
k_bf16_stride_0,
k_nope_fp8_stride_0,
k_rope_bf16_stride_0,
scale_stride_0,
DIM_NOPE: tl.constexpr,
DIM_ROPE: tl.constexpr,
TILE_SIZE: tl.constexpr,
NUM_TILES: tl.constexpr,
FP8_MIN: tl.constexpr,
FP8_MAX: tl.constexpr,
EPS: tl.constexpr,
):
token_id = tl.program_id(0).to(tl.int64)
tile_id = tl.program_id(1)
if tile_id == NUM_TILES:
rope_range = tl.arange(0, TILE_SIZE)
rope_mask = rope_range < DIM_ROPE
in_rope_offsets = token_id * k_bf16_stride_0 + DIM_NOPE + rope_range
rope_data = tl.load(k_bf16_ptr + in_rope_offsets, mask=rope_mask, other=0.0)
out_rope_offsets = token_id * k_rope_bf16_stride_0 + rope_range
tl.store(k_rope_bf16_ptr + out_rope_offsets, rope_data, mask=rope_mask)
else:
tile_range = tl.arange(0, TILE_SIZE)
in_tile_offsets = token_id * k_bf16_stride_0 + tile_id * TILE_SIZE + tile_range
x_bf16 = tl.load(k_bf16_ptr + in_tile_offsets)
x_fp32 = x_bf16.to(tl.float32)
abs_x = tl.abs(x_fp32)
max_abs = tl.max(abs_x)
max_abs_clamped = tl.maximum(max_abs, EPS)
scale = max_abs_clamped / FP8_MAX
log2_scale = tl.log2(scale)
ceil_log2 = tl.math.ceil(log2_scale)
scale_pow2_fp32 = tl.exp2(ceil_log2)
scale_inv = 1.0 / scale_pow2_fp32
x_scaled = x_fp32 * scale_inv
x_fp8 = tl.clamp(x_scaled, FP8_MIN, FP8_MAX).to(k_nope_fp8_ptr.dtype.element_ty)
out_fp8_offsets = (
token_id * k_nope_fp8_stride_0 + tile_id * TILE_SIZE + tile_range
)
tl.store(k_nope_fp8_ptr + out_fp8_offsets, x_fp8)
exponent = ceil_log2.to(tl.int32)
scale_uint8 = (exponent + 127).to(tl.uint8)
out_scale_offset = token_id * scale_stride_0 + tile_id
tl.store(scale_k_nope_uint8_ptr + out_scale_offset, scale_uint8)
def quant_to_nope_fp8_rope_bf16_pack_triton(
k_bf16: torch.Tensor,
) -> NopeFp8RopeBf16Pack:
assert k_bf16.dtype == torch.bfloat16
num_tokens, hidden_dim = k_bf16.shape
assert hidden_dim == 512
dim_nope = 448
dim_rope = 64
tile_size = 64
num_tiles = dim_nope // tile_size
k_bf16 = k_bf16.contiguous()
k_nope_fp8 = torch.empty(
(num_tokens, dim_nope), dtype=fp8_dtype, device=k_bf16.device
)
k_rope_bf16 = torch.empty(
(num_tokens, dim_rope), dtype=torch.bfloat16, device=k_bf16.device
)
scale_k_nope_ue8m0 = torch.empty(
(num_tokens, num_tiles), dtype=torch.uint8, device=k_bf16.device
)
fp8_dtype_info = torch.finfo(fp8_dtype)
grid = (num_tokens, num_tiles + 1)
_quant_k_cache_fused_kernel[grid](
k_bf16,
k_nope_fp8,
k_rope_bf16,
scale_k_nope_ue8m0,
k_bf16.stride(0),
k_nope_fp8.stride(0),
k_rope_bf16.stride(0),
scale_k_nope_ue8m0.stride(0),
DIM_NOPE=dim_nope,
DIM_ROPE=dim_rope,
TILE_SIZE=tile_size,
NUM_TILES=num_tiles,
FP8_MIN=fp8_dtype_info.min,
FP8_MAX=fp8_dtype_info.max,
EPS=1e-8,
)
return NopeFp8RopeBf16Pack(
k_nope_fp8=k_nope_fp8,
k_rope_bf16=k_rope_bf16,
scale_k_nope_ue8m0=scale_k_nope_ue8m0,
)
@@ -4,8 +4,6 @@ Indexer values use per-32 UE8M0 scales; compressed KV uses per-16 E4M3 scales.
Both paths round to the E2M1 grid with ties to even.
"""
from typing import Optional
import torch
FP8_MAX = 448.0
@@ -62,57 +60,3 @@ def fake_quant_compressed_kv(x: torch.Tensor) -> torch.Tensor:
scaled = (blocks / scale).clamp(-FP4_MAX, FP4_MAX)
deq = round_fp4(scaled) * scale
return deq.flatten(-2).to(x.dtype)
# ---------------------------------------------------------------------------
# Pure-torch reference of the paged V4.1 fp8 KV cache format read by the sparse
# decode kernel (528 B/token, "V41").
# ---------------------------------------------------------------------------
def ceil_pow2_scale(x: torch.Tensor) -> torch.Tensor:
"""``2 ** ceil(log2(max(x, 1e-4)))`` as fp32, computed on the IEEE bits so
that it is exact at (and just above) powers of two."""
x = x.float()
scale = ceil_pow2(torch.clamp_min(x, 1e-4))
# ceil_pow2 works on the bits of a finite value; a NaN or inf amax passes
# through (both become the ue8m0 NaN byte, but only the NaN one turns the
# whole tile's payload into NaN).
return torch.where(torch.isfinite(x), scale, x)
def quantize_k_cache_v41(
k: torch.Tensor, page_bytes: Optional[int] = None
) -> torch.Tensor:
"""``k`` ``[num_pages, page_size, 512]`` -> uint8 ``[num_pages, page_bytes]``
pages of the V41 layout: 512 e4m3 per token, then 16 ue8m0 scales per token
(one per 32 values), ``scale = 2 ** ceil(log2(max(amax / 448, 1e-4)))``."""
num_pages, page_size, d = k.shape
assert d == 512
x = k.float().view(num_pages, page_size, 16, 32)
scale = ceil_pow2_scale(x.abs().amax(dim=-1) / 448.0)
data = (x / scale.unsqueeze(-1)).to(torch.float8_e4m3fn).view(torch.uint8)
scale_u8 = scale.to(torch.float8_e8m0fnu).view(torch.uint8)
raw = page_size * 528
if page_bytes is None:
page_bytes = -(-raw // 512) * 512
assert page_bytes >= raw
out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device)
out[:, : page_size * 512] = data.reshape(num_pages, page_size * 512)
out[:, page_size * 512 : raw] = scale_u8.reshape(num_pages, page_size * 16)
return out
def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor:
"""Inverse of :func:`quantize_k_cache_v41`: ``[num_pages, page_size, 512]`` bf16."""
num_pages = pages.shape[0]
pages = pages.view(torch.uint8)
data = pages[:, : page_size * 512].reshape(num_pages, page_size, 512)
scale = pages[:, page_size * 512 : page_size * 528].reshape(
num_pages, page_size, 16
)
values = data.view(torch.float8_e4m3fn).to(torch.bfloat16)
scale_bf16 = scale.view(torch.float8_e8m0fnu).to(torch.bfloat16)
return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view(
num_pages, page_size, 512
)
@@ -52,13 +52,13 @@ the captured launch sequence.
from __future__ import annotations
import functools
import os
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.environ import envs
from sglang.srt.utils import is_hip
from sglang.srt.utils.common import is_gfx1250_supported
@@ -117,7 +117,7 @@ _FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
# End to end on DSv4 / MI355X / c128, n-weighted at matched batch size against
# the same config with the plain heuristic: decode step 121.76 -> 114.12 ms
# (-7.50 ms, -6.2 %). At c256 the gain is smaller (-5.37 ms of 156.19).
_HCA_KV_SPLITS = int(os.environ.get("SGLANG_MLA_HCA_KV_SPLITS", "4"))
_HCA_KV_SPLITS = envs.SGLANG_MLA_HCA_KV_SPLITS.get()
def _kv_splits_for_stream(compress_ratio: int) -> int | None:
@@ -24,7 +24,6 @@ import triton
import triton.language as tl
from sglang.kernels.ops.attention.decode_attention import _extract_kv_strides
from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd
from sglang.kernels.ops.attention.score_mod import unpack_aux_tensors
from sglang.srt.environ import envs
from sglang.srt.utils import is_cuda, is_gfx95_supported, is_gfx1250_supported, is_hip
@@ -1170,43 +1169,6 @@ def extend_attention_fwd(
)
def redundant_attention(
q_extend,
o_extend,
k_buffer,
v_buffer,
b_req_idx,
b_start_loc,
b_seq_len,
b_seq_len_prefix,
max_len_in_batch,
):
total_token_num = k_buffer.shape[0]
B, H_Q, D = b_req_idx.shape[0], q_extend.shape[-2], q_extend.shape[-1]
q_buffer = torch.empty(
(total_token_num, H_Q, D), dtype=q_extend.dtype, device=q_extend.device
)
pt = 0
for i in range(B):
cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i]
pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i]
q_buffer[pl:pr] = q_extend[pt : pt + cur_seq_len_extend]
pt += cur_seq_len_extend
o_buffer = torch.empty_like(q_buffer)
context_attention_fwd(
q_buffer, k_buffer, v_buffer, o_buffer, b_start_loc, b_seq_len, max_len_in_batch
)
pt = 0
for i in range(B):
cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i]
pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i]
o_extend[pt : pt + cur_seq_len_extend] = o_buffer[pl:pr]
pt += cur_seq_len_extend
@triton.jit
def _fwd_kernel_unified(
Q,
@@ -594,145 +594,6 @@ def gdn_replayssm_exact_fold_kernel(
tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h)
@triton.jit
def gdn_replayssm_circular_commit_kernel(
h0, # [num_layers, num_slots, HV, V, K]
rawv_cache, # [num_layers, num_slots, HV, L, V]
rawk_cache, # [num_layers, num_slots, H, L, K]
g_cache, # [num_layers, num_slots, HV, L]
beta_cache, # [num_layers, num_slots, HV, L]
ssm_state_indices, # [B]
write_pos, # [num_slots], already advanced by the accepted length
cache_base, # [num_slots]
is_flush_flags, # [num_slots]
accept_lens, # [B]
mamba_track_indices, # [B]
mamba_steps_to_track, # [B]
stride_state_slot: tl.constexpr,
stride_rawv_slot: tl.constexpr,
stride_rawk_slot: tl.constexpr,
stride_g_slot: tl.constexpr,
stride_beta_slot: tl.constexpr,
stride_state_layer: tl.constexpr,
stride_rawv_layer: tl.constexpr,
stride_rawk_layer: tl.constexpr,
stride_g_layer: tl.constexpr,
stride_beta_layer: tl.constexpr,
stride_indices: tl.constexpr,
stride_accept: tl.constexpr,
stride_track: tl.constexpr,
stride_steps: tl.constexpr,
H: tl.constexpr,
HV: tl.constexpr,
K: tl.constexpr,
V: tl.constexpr,
BK: tl.constexpr,
BV: tl.constexpr,
MAX_CACHE_LEN: tl.constexpr,
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
NULL_BLOCK_ID: tl.constexpr,
HAS_TRACK: tl.constexpr,
):
"""Materialize circular history only at capacity or radix-track boundaries."""
i_v = tl.program_id(0)
i_n = tl.program_id(1)
i_hvl = tl.program_id(2)
i_layer = (i_hvl // HV).to(tl.int64)
i_hv = i_hvl % HV
i_h = i_hv // (HV // H)
state_idx = tl.load(ssm_state_indices + i_n * stride_indices).to(tl.int64)
if state_idx <= NULL_BLOCK_ID:
return
n_history = tl.load(write_pos + state_idx).to(tl.int32)
fold_active = tl.load(is_flush_flags + state_idx) != 0
if HAS_TRACK:
track_idx = tl.load(mamba_track_indices + i_n * stride_track).to(tl.int64)
track_step = tl.load(mamba_steps_to_track + i_n * stride_steps).to(tl.int32)
has_track = (track_idx > NULL_BLOCK_ID) & (track_step >= 0)
n_accepted = tl.load(accept_lens + i_n * stride_accept).to(tl.int32)
previous_history = n_history - n_accepted
track_pos = previous_history + track_step + 1
else:
track_idx = NULL_BLOCK_ID
has_track = False
track_pos = 0
if (not fold_active) and (not has_track):
return
replay_len = tl.maximum(tl.where(fold_active, n_history, 0), track_pos)
if replay_len <= 0:
return
h0 = h0 + i_layer * stride_state_layer
rawv_cache = rawv_cache + i_layer * stride_rawv_layer
rawk_cache = rawk_cache + i_layer * stride_rawk_layer
g_cache = g_cache + i_layer * stride_g_layer
beta_cache = beta_cache + i_layer * stride_beta_layer
base = tl.load(cache_base + state_idx).to(tl.int32)
o_k = tl.arange(0, BK)
o_v = i_v * BV + tl.arange(0, BV)
mask_k = o_k < K
mask_v = o_v < V
mask_h = mask_k[:, None] & mask_v[None, :]
p_h0 = (
h0
+ state_idx * stride_state_slot
+ i_hv * V * K
+ o_v[None, :] * K
+ o_k[:, None]
)
b_h = tl.load(p_h0, mask=mask_h, other=0.0).to(tl.float32)
for t in range(0, replay_len):
phys = ((base + t) & (MAX_CACHE_LEN - 1)).to(tl.int64)
b_k = tl.load(
rawk_cache
+ state_idx * stride_rawk_slot
+ (i_h * MAX_CACHE_LEN + phys) * K
+ o_k,
mask=mask_k,
other=0.0,
).to(tl.float32)
b_v = tl.load(
rawv_cache
+ state_idx * stride_rawv_slot
+ (i_hv * MAX_CACHE_LEN + phys) * V
+ o_v,
mask=mask_v,
other=0.0,
).to(tl.float32)
b_g = tl.load(
g_cache + state_idx * stride_g_slot + i_hv * MAX_CACHE_LEN + phys
).to(tl.float32)
b_beta = tl.load(
beta_cache + state_idx * stride_beta_slot + i_hv * MAX_CACHE_LEN + phys
).to(tl.float32)
if USE_QK_L2NORM_IN_KERNEL:
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
b_h *= tl.exp(b_g)
b_v -= tl.sum(b_h * b_k[:, None], 0)
b_v *= b_beta
b_h += b_k[:, None] * b_v[None, :]
if HAS_TRACK:
if has_track and (t + 1 == track_pos):
tl.store(
h0
+ track_idx * stride_state_slot
+ i_hv * V * K
+ o_v[None, :] * K
+ o_k[:, None],
b_h.to(h0.dtype.element_ty),
mask=mask_h,
)
if fold_active:
tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h)
@triton.jit
def gdn_replayssm_compact_commit_kernel(
h0,
@@ -1011,39 +872,6 @@ def _advance_gdn_spec_cursors_kernel(
tl.store(is_flush_ptr + blk, next_is_flush, mask=valid)
@triton.jit
def _reset_gdn_replayssm_spec_cursors_kernel(
write_pos_ptr,
cache_base_ptr,
is_flush_ptr,
do_reset_ptr,
state_batch_indices_ptr,
n_rows,
stride_sbi: tl.constexpr,
stride_reset: tl.constexpr,
INIT_FLUSH: tl.constexpr,
BLOCK: tl.constexpr,
NULL_BLOCK_ID: tl.constexpr,
):
offs = tl.arange(0, BLOCK)
row_mask = offs < n_rows
blk = tl.load(
state_batch_indices_ptr + offs * stride_sbi, mask=row_mask, other=NULL_BLOCK_ID
).to(tl.int64)
do_reset = tl.load(do_reset_ptr + offs * stride_reset, mask=row_mask, other=0).to(
tl.int32
)
do = row_mask & (blk > NULL_BLOCK_ID) & (do_reset != 0)
tl.store(write_pos_ptr + blk, tl.zeros_like(blk).to(tl.int32), mask=do)
tl.store(cache_base_ptr + blk, tl.zeros_like(blk).to(tl.int32), mask=do)
tl.store(
is_flush_ptr + blk,
tl.full([BLOCK], INIT_FLUSH, dtype=tl.int8),
mask=do,
)
# ---------------------------------------------------------------------------
# Python wrappers.
# ---------------------------------------------------------------------------
@@ -1541,33 +1369,3 @@ def commit_gdn_replayssm_circular(
BLOCK=block,
NULL_BLOCK_ID=null_block_id,
)
def reset_gdn_replayssm_spec_cursors(
write_pos: torch.Tensor,
cache_base: torch.Tensor,
is_flush: torch.Tensor,
do_reset: torch.Tensor, # [n_rows] int/bool 1 for first-decode rows
state_batch_indices: torch.Tensor, # [n_rows] int
max_cache_len: int,
max_spec_len: int,
null_block_id: int = 0,
):
"""Reset the cursors of first-decode rows (prefill->decode handoff)."""
n_rows = state_batch_indices.shape[0]
BLOCK = triton.next_power_of_2(max(1, n_rows))
# Early-flush margin (2 * max_spec_len, strict '>'): mirror _advance_gdn_spec_cursors_kernel.
init_flush = 1 if 2 * max_spec_len > max_cache_len else 0
_reset_gdn_replayssm_spec_cursors_kernel[(1,)](
write_pos,
cache_base,
is_flush,
do_reset,
state_batch_indices,
n_rows,
stride_sbi=state_batch_indices.stride(0),
stride_reset=do_reset.stride(0),
INIT_FLUSH=init_flush,
BLOCK=BLOCK,
NULL_BLOCK_ID=null_block_id,
)
@@ -1,7 +1,5 @@
from __future__ import annotations
import inspect
import os
from functools import cache, lru_cache
from typing import Callable, Optional, Tuple, Union
@@ -11,14 +9,9 @@ import torch.nn.functional as F
from sglang.kernels.kernel_api_logging import debug_kernel_api
try:
if os.environ.get("SGLANG_INKLING_FA4_USE_PIP") == "1":
# A/B debug escape hatch: route through the pip flash-attn-4 package
# (dev's stack). rel_bias is vendored-only, so SHEARED must be 0.
from flash_attn.cute import flash_attn_varlen_func as _flash_attn_varlen_func
else:
from sglang.kernels.ops.attention.flash_attn.cute import (
flash_attn_varlen_func as _flash_attn_varlen_func,
)
from sglang.kernels.ops.attention.flash_attn.cute import (
flash_attn_varlen_func as _flash_attn_varlen_func,
)
except Exception as _e: # pragma: no cover
_flash_attn_varlen_func = None
_flash_attn_import_error = _e
@@ -26,14 +19,6 @@ else:
_flash_attn_import_error = None
def _takes_out(kernel) -> bool:
return kernel is not None and "out" in inspect.signature(kernel).parameters
# Released flash-attn-4 builds may lack the vendored kernel's out=.
_kernel_takes_out = _takes_out(_flash_attn_varlen_func)
def is_flash_attention_v4_available() -> bool:
return _flash_attn_varlen_func is not None
@@ -293,11 +278,7 @@ def flash_attn_varlen_func(
rel_bias_kwargs["rel_bias_prep_cache"] = rel_bias_prep_cache
# The kernel writes into out, except where it pads MLA heads: its output
# then has more heads than out, and the result is a new tensor.
out_kwargs = (
{"out": out}
if out is not None and mla_head_padding is None and _kernel_takes_out
else {}
)
out_kwargs = {"out": out} if out is not None and mla_head_padding is None else {}
result = _flash_attn_varlen_func(
q=q,
k=k,
@@ -3,13 +3,17 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Callable, Optional, Tuple, Union
import torch
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.kernels.ops.attention.fa4_sm120.dispatch import (
get_forward_arch,
resolve_runtime_policy,
try_cached_paged_decode,
)
from sglang.kernels.ops.attention.flash_attention_v4 import (
_flash_attn_import_error,
_flash_attn_varlen_func,
@@ -18,18 +22,6 @@ from sglang.kernels.ops.attention.flash_attention_v4 import (
_unpad_mla_result,
)
if os.environ.get("SGLANG_INKLING_FA4_USE_PIP") == "1":
# The pip escape hatch deliberately bypasses SGLang-owned SM12x kernels.
get_forward_arch = None
resolve_runtime_policy = None
try_cached_paged_decode = None
else:
from sglang.kernels.ops.attention.fa4_sm120.dispatch import (
get_forward_arch,
resolve_runtime_policy,
try_cached_paged_decode,
)
@dataclass(frozen=True)
class FlashAttentionV4SM120RuntimePolicy:
@@ -44,13 +36,6 @@ def get_flash_attention_v4_sm120_runtime_policy(
deterministic: bool,
) -> FlashAttentionV4SM120RuntimePolicy:
"""Resolve the SM12x FA4 launch policy exposed to SGLang."""
if resolve_runtime_policy is None:
num_splits = 1 if deterministic or device_capability < (9, 0) else 0
return FlashAttentionV4SM120RuntimePolicy(
num_splits=num_splits,
decode_num_splits=num_splits,
decode_uses_static_max_seqlen_k=False,
)
num_splits, decode_num_splits, decode_uses_static_max_seqlen_k = (
resolve_runtime_policy(
device_capability=device_capability,
@@ -251,7 +236,7 @@ def flash_attn_with_kvcache(
(k_cache.shape[0],), cache_seqlens, dtype=torch.int32, device=k_cache.device
)
forward_arch = get_forward_arch(q.device) if get_forward_arch is not None else None
forward_arch = get_forward_arch(q.device)
if (
forward_arch is not None
and not return_softmax_lse
@@ -5,13 +5,11 @@
import aiter as _aiter # noqa: F401
from .kimi_k3_kda_decode import (
flydsl_kimi_k3_kda_decode,
flydsl_kimi_k3_kda_decode_with_f_b,
is_flydsl_kimi_k3_kda_decode_supported,
)
__all__ = [
"flydsl_kimi_k3_kda_decode",
"flydsl_kimi_k3_kda_decode_with_f_b",
"is_flydsl_kimi_k3_kda_decode_supported",
]
@@ -1,450 +0,0 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
"""FlyDSL kernel for the fused Kimi-K3 KDA decode path on gfx950."""
import functools
import math
import flydsl.compiler as flyc
import flydsl.expr as fx
from aiter.ops.flydsl.kernels.tensor_shim import GTensor
from flydsl.expr import range_constexpr
from flydsl.expr.typing import T
_HEADS = 12
_DIM = 128
_LOG2E = math.log2(math.e)
_SCALE = _DIM**-0.5
_BLOCK_THREADS = 256
_NUM_WARPS = 4
_WARP_SIZE = 64
_WARP_THREADS_K = 8
_VALUES_PER_THREAD_K = 4
_WARP_TILE_K = _WARP_THREADS_K * _VALUES_PER_THREAD_K
_K_ITERS = _DIM // _WARP_TILE_K
_WARP_THREADS_V = _WARP_SIZE // _WARP_THREADS_K
_V_GROUP_TILE = _NUM_WARPS * _WARP_THREADS_V
_V_ITERS = _DIM // _V_GROUP_TILE
_WAVES_PER_EU = 3
@functools.cache
def create_kimi_k3_kda_decode_kernel(norm_eps: float, lower_bound: float):
"""Build the fixed gfx950 BF16 Kimi-K3 decode specialization."""
@fx.struct
class SharedStorage:
q: fx.Array[fx.BFloat16, _DIM, 16]
k: fx.Array[fx.BFloat16, _DIM, 16]
v: fx.Array[fx.BFloat16, _DIM, 16]
recurrent_out: fx.Array[fx.BFloat16, _DIM, 16]
norm_partial: fx.Array[fx.Float32, 2, 16]
@flyc.kernel(
name="kimi_k3_kda_decode_bf16_gfx950",
known_block_size=[_BLOCK_THREADS, 1, 1],
)
def kernel(
x_mem: fx.Tensor,
weight_mem: fx.Tensor,
conv_state_mem: fx.Tensor,
raw_g_mem: fx.Tensor,
raw_beta_mem: fx.Tensor,
A_log_mem: fx.Tensor,
dt_bias_mem: fx.Tensor,
state_mem: fx.Tensor,
state_indices_mem: fx.Tensor,
output_gate_mem: fx.Tensor,
norm_weight_mem: fx.Tensor,
out_mem: fx.Tensor,
batch_size: fx.Int32,
stride_x_token: fx.Int32,
stride_weight_channel: fx.Int32,
stride_weight_width: fx.Int32,
stride_conv_slot: fx.Int32,
stride_conv_channel: fx.Int32,
stride_conv_width: fx.Int32,
stride_g_token: fx.Int32,
stride_beta_token: fx.Int32,
stride_state_slot: fx.Int32,
stride_gate_token: fx.Int32,
stride_gate_head: fx.Int32,
stride_out_token: fx.Int32,
stride_out_head: fx.Int32,
):
del batch_size
x = GTensor(x_mem, dtype=T.bf16, shape=(-1,))
weight = GTensor(weight_mem, dtype=T.f32, shape=(-1,))
conv_state = GTensor(conv_state_mem, dtype=T.bf16, shape=(-1,))
raw_g = GTensor(raw_g_mem, dtype=T.bf16, shape=(-1,))
raw_beta = GTensor(raw_beta_mem, dtype=T.bf16, shape=(-1,))
A_log = GTensor(A_log_mem, dtype=T.f32, shape=(-1,))
dt_bias = GTensor(dt_bias_mem, dtype=T.f32, shape=(-1,))
state = GTensor(state_mem, dtype=T.f32, shape=(-1,))
state_indices = GTensor(state_indices_mem, dtype=T.i32, shape=(-1,))
output_gate = GTensor(output_gate_mem, dtype=T.bf16, shape=(-1,))
norm_weight = GTensor(norm_weight_mem, dtype=T.bf16, shape=(-1,))
out = GTensor(out_mem, dtype=T.bf16, shape=(-1,))
shared = fx.SharedAllocator().allocate(SharedStorage).peek()
q_lds = shared.q.ptr
k_lds = shared.k.ptr
v_lds = shared.v.ptr
out_lds = shared.recurrent_out.ptr
norm_lds = shared.norm_partial.ptr
tid = fx.thread_idx.x
block = fx.block_idx.x
batch = block // fx.Int32(_HEADS)
head = block % fx.Int32(_HEADS)
lane = tid % fx.Int32(_WARP_SIZE)
warp = tid // fx.Int32(_WARP_SIZE)
lane_k = lane % fx.Int32(_WARP_THREADS_K)
state_idx = fx.Int32(state_indices[batch])
valid = state_idx > fx.Int32(0)
# Branch bodies are closures: a bare `if` would make the AST rewriter
# carry the GTensors whose methods the body calls out as scf.if results.
def _valid_then():
# A workgroup exclusively owns all three convolution channels for
# its (batch, head), so every cache entry is shifted exactly once.
def _conv_then():
channel_local = tid
q_channel = head * fx.Int32(_DIM) + channel_local
k_channel = (
fx.Int32(_HEADS * _DIM) + head * fx.Int32(_DIM) + channel_local
)
v_channel = (
fx.Int32(2 * _HEADS * _DIM) + head * fx.Int32(_DIM) + channel_local
)
def convolve_channel(channel):
cs_base = (
state_idx * stride_conv_slot + channel * stride_conv_channel
)
c0 = fx.Float32(conv_state[cs_base])
c1 = fx.Float32(conv_state[cs_base + stride_conv_width])
c2 = fx.Float32(
conv_state[cs_base + fx.Int32(2) * stride_conv_width]
)
current = fx.BFloat16(x[batch * stride_x_token + channel])
current_f32 = fx.Float32(current)
w_base = channel * stride_weight_channel
acc = c0 * fx.Float32(weight[w_base])
acc = acc + c1 * fx.Float32(weight[w_base + stride_weight_width])
acc = acc + c2 * fx.Float32(
weight[w_base + fx.Int32(2) * stride_weight_width]
)
acc = acc + current_f32 * fx.Float32(
weight[w_base + fx.Int32(3) * stride_weight_width]
)
silu = acc / (
fx.Float32(1.0) + fx.math.exp2(-acc * fx.Float32(_LOG2E))
)
conv_state.store(
cs_base,
fx.BFloat16(c1),
)
conv_state.store(
cs_base + stride_conv_width,
fx.BFloat16(c2),
)
conv_state.store(
cs_base + fx.Int32(2) * stride_conv_width,
current,
)
return silu.to(fx.BFloat16)
q_conv = convolve_channel(q_channel)
k_conv = convolve_channel(k_channel)
v_conv = convolve_channel(v_channel)
fx.ptr_store(q_conv, q_lds + tid)
fx.ptr_store(k_conv, k_lds + tid)
fx.ptr_store(v_conv, v_lds + tid)
if tid < fx.Int32(_DIM):
_conv_then()
fx.gpu.barrier()
# Four waves split V into 32-row groups. Eight-lane subgroups
# reduce K; each lane issues one aligned f32x4 state transaction.
k_vec_start = lane_k * fx.Int32(_VALUES_PER_THREAD_K)
global_v_start = warp * fx.Int32(_WARP_THREADS_V) + lane // fx.Int32(
_WARP_THREADS_K
)
vec_f32 = T.vec(_VALUES_PER_THREAD_K, T.f32)
vec_bf16 = T.vec(_VALUES_PER_THREAD_K, T.bf16)
zero_vec = fx.full(
_VALUES_PER_THREAD_K,
0.0,
fx.Float32,
)
q_vecs = []
k_vecs = []
decay_vecs = []
sum_q_partial = fx.Float32(0.0)
sum_k_partial = fx.Float32(0.0)
a = fx.math.exp2(fx.Float32(A_log[head]) * fx.Float32(_LOG2E))
for ki in range_constexpr(_K_ITERS):
k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K)
q_bf16 = fx.ptr_load(
q_lds + k_base,
result_type=vec_bf16,
)
k_bf16 = fx.ptr_load(
k_lds + k_base,
result_type=vec_bf16,
)
q_f32 = q_bf16.extf(vec_f32)
k_f32 = k_bf16.extf(vec_f32)
q_vecs.append(q_f32)
k_vecs.append(k_f32)
sum_q_vec = q_f32 * q_f32
sum_k_vec = k_f32 * k_f32
sum_q_partial = sum_q_partial + fx.Vector(sum_q_vec).reduce("add")
sum_k_partial = sum_k_partial + fx.Vector(sum_k_vec).reduce("add")
gate_bf16 = raw_g.vec_load(
(batch * stride_g_token + head * fx.Int32(_DIM) + k_base,),
_VALUES_PER_THREAD_K,
)
gate_f32 = gate_bf16.extf(vec_f32)
dt = dt_bias.vec_load(
(head * fx.Int32(_DIM) + k_base,),
_VALUES_PER_THREAD_K,
)
sigmoid_arg = (gate_f32 + dt) * a
gate = fx.Float32(lower_bound) / (
fx.Float32(1.0) + fx.math.exp2(-sigmoid_arg * fx.Float32(_LOG2E))
)
decay_vecs.append(fx.math.exp2(gate * fx.Float32(_LOG2E)))
width = fx.Int32(_WARP_SIZE)
for offset in (1, 2, 4):
sum_q_partial = sum_q_partial + fx.gpu.shuffle_xor(
sum_q_partial, fx.Int32(offset), width
)
sum_k_partial = sum_k_partial + fx.gpu.shuffle_xor(
sum_k_partial, fx.Int32(offset), width
)
subgroup_leader = (lane // fx.Int32(_WARP_THREADS_K)) * fx.Int32(
_WARP_THREADS_K
)
norm_q = fx.gpu.shuffle_idx(sum_q_partial, subgroup_leader, width)
norm_k = fx.gpu.shuffle_idx(sum_k_partial, subgroup_leader, width)
inv_q = fx.math.rsqrt(fx.Float32(norm_q) + fx.Float32(1e-6))
inv_k = fx.math.rsqrt(fx.Float32(norm_k) + fx.Float32(1e-6))
for ki in range_constexpr(_K_ITERS):
q_vecs[ki] = q_vecs[ki] * fx.Float32(inv_q) * fx.Float32(_SCALE)
k_vecs[ki] = k_vecs[ki] * fx.Float32(inv_k)
dot_kq_vec = zero_vec
for ki in range_constexpr(_K_ITERS):
dot_kq_vec = fx.fma(k_vecs[ki], q_vecs[ki], dot_kq_vec)
dot_kq = fx.Vector(dot_kq_vec).reduce("add")
for offset in (1, 2, 4):
dot_kq = dot_kq + fx.gpu.shuffle_xor(dot_kq, fx.Int32(offset), width)
beta_value = fx.Float32(raw_beta[batch * stride_beta_token + head])
beta = fx.Float32(1.0) / (
fx.Float32(1.0) + fx.math.exp2(-beta_value * fx.Float32(_LOG2E))
)
state_head_base = state_idx * stride_state_slot + head * fx.Int32(
_DIM * _DIM
)
state_vecs = []
for vi in range_constexpr(_V_ITERS):
global_v = global_v_start + fx.Int32(vi * _V_GROUP_TILE)
for ki in range_constexpr(_K_ITERS):
k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K)
state_off = state_head_base + global_v * fx.Int32(_DIM) + k_base
state_vecs.append(
state.vec_load(
(state_off,),
_VALUES_PER_THREAD_K,
)
)
for vi in range_constexpr(_V_ITERS):
global_v = global_v_start + fx.Int32(vi * _V_GROUP_TILE)
sum_hk_vec = zero_vec
sum_hq_vec = zero_vec
for ki in range_constexpr(_K_ITERS):
state_pos = vi * _K_ITERS + ki
decayed = state_vecs[state_pos] * decay_vecs[ki]
state_vecs[state_pos] = decayed
sum_hk_vec = fx.fma(decayed, k_vecs[ki], sum_hk_vec)
sum_hq_vec = fx.fma(decayed, q_vecs[ki], sum_hq_vec)
sum_hk = fx.Vector(sum_hk_vec).reduce("add")
sum_hq = fx.Vector(sum_hq_vec).reduce("add")
for offset in (1, 2, 4):
sum_hk = sum_hk + fx.gpu.shuffle_xor(
sum_hk, fx.Int32(offset), width
)
sum_hq = sum_hq + fx.gpu.shuffle_xor(
sum_hq, fx.Int32(offset), width
)
conv_v = fx.Float32(fx.ptr_load(v_lds + global_v))
v_new = (conv_v - fx.Float32(sum_hk)) * beta
v_new = fx.gpu.shuffle_idx(v_new, subgroup_leader, width)
recurrent_value = fx.Float32(sum_hq) + fx.Float32(v_new) * fx.Float32(
dot_kq
)
v_new_vec = fx.full(_VALUES_PER_THREAD_K, v_new, fx.Float32)
for ki in range_constexpr(_K_ITERS):
state_pos = vi * _K_ITERS + ki
updated = fx.fma(k_vecs[ki], v_new_vec, state_vecs[state_pos])
k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K)
state_off = state_head_base + global_v * fx.Int32(_DIM) + k_base
state.vec_store(
(state_off,),
updated,
_VALUES_PER_THREAD_K,
)
if lane_k == fx.Int32(0):
fx.ptr_store(
fx.BFloat16(recurrent_value),
out_lds + global_v,
)
fx.gpu.barrier()
# Preserve the model's BF16 boundary before RMSNorm and gating.
def _norm_then():
recurrent_bf16 = fx.ptr_load(out_lds + tid)
recurrent_f32 = fx.Float32(recurrent_bf16)
square = recurrent_f32 * recurrent_f32
for offset in (32, 16, 8, 4, 2, 1):
square = square + fx.gpu.shuffle_xor(
square, fx.Int32(offset), width
)
if lane == fx.Int32(0):
fx.ptr_store(square, norm_lds + warp)
if tid < fx.Int32(_DIM):
_norm_then()
fx.gpu.barrier()
def _output_then():
norm_sum = fx.Float32(fx.ptr_load(norm_lds))
norm_sum = norm_sum + fx.Float32(fx.ptr_load(norm_lds + fx.Int32(1)))
inv_rms = fx.math.rsqrt(
norm_sum * fx.Float32(1.0 / _DIM) + fx.Float32(norm_eps)
)
recurrent_f32 = fx.Float32(fx.ptr_load(out_lds + tid))
norm_w = fx.Float32(norm_weight[tid])
gate_value = fx.Float32(
output_gate[
batch * stride_gate_token + head * stride_gate_head + tid
]
)
output_sigmoid = fx.Float32(1.0) / (
fx.Float32(1.0) + fx.math.exp2(-gate_value * fx.Float32(_LOG2E))
)
result = recurrent_f32 * inv_rms * norm_w * output_sigmoid
out.store(
batch * stride_out_token + head * stride_out_head + tid,
result.to(fx.BFloat16),
)
if tid < fx.Int32(_DIM):
_output_then()
def _zero_then():
out.store(
batch * stride_out_token + head * stride_out_head + tid,
fx.BFloat16(0.0),
)
if valid:
_valid_then()
elif tid < fx.Int32(_DIM):
_zero_then()
@flyc.jit
def launch(
x_mem: fx.Tensor,
weight_mem: fx.Tensor,
conv_state_mem: fx.Tensor,
raw_g_mem: fx.Tensor,
raw_beta_mem: fx.Tensor,
A_log_mem: fx.Tensor,
dt_bias_mem: fx.Tensor,
state_mem: fx.Tensor,
state_indices_mem: fx.Tensor,
output_gate_mem: fx.Tensor,
norm_weight_mem: fx.Tensor,
out_mem: fx.Tensor,
batch_size: fx.Int32,
stride_x_token: fx.Int32,
stride_weight_channel: fx.Int32,
stride_weight_width: fx.Int32,
stride_conv_slot: fx.Int32,
stride_conv_channel: fx.Int32,
stride_conv_width: fx.Int32,
stride_g_token: fx.Int32,
stride_beta_token: fx.Int32,
stride_state_slot: fx.Int32,
stride_gate_token: fx.Int32,
stride_gate_head: fx.Int32,
stride_out_token: fx.Int32,
stride_out_head: fx.Int32,
stream: fx.Stream = fx.Stream(None), # noqa: B008
):
kernel(
x_mem,
weight_mem,
conv_state_mem,
raw_g_mem,
raw_beta_mem,
A_log_mem,
dt_bias_mem,
state_mem,
state_indices_mem,
output_gate_mem,
norm_weight_mem,
out_mem,
batch_size,
stride_x_token,
stride_weight_channel,
stride_weight_width,
stride_conv_slot,
stride_conv_channel,
stride_conv_width,
stride_g_token,
stride_beta_token,
stride_state_slot,
stride_gate_token,
stride_gate_head,
stride_out_token,
stride_out_head,
).launch(
grid=(batch_size * fx.Int32(_HEADS), 1, 1),
block=(_BLOCK_THREADS, 1, 1),
stream=stream,
)
launch.compile_hints = {
"waves_per_eu": _WAVES_PER_EU,
"llvm_options": {
"amdgpu-expert-scheduling-mode": True,
},
}
return launch
__all__ = ["create_kimi_k3_kda_decode_kernel"]
@@ -11,9 +11,6 @@ from collections.abc import Iterable
import torch
from aiter.ops.flydsl.kernels.tensor_shim import _run_compiled
from .kernels.kimi_k3_kda_decode import (
create_kimi_k3_kda_decode_kernel,
)
from .kernels.kimi_k3_kda_decode_fb import (
create_kimi_k3_kda_decode_fb_kernel,
)
@@ -127,7 +124,7 @@ def _validate_kda_inputs(
norm_weight: torch.Tensor,
out: torch.Tensor | None,
) -> torch.Tensor:
"""Validate operands shared by both explicit KDA specializations."""
"""Validate the KDA decode operands."""
if not is_flydsl_kimi_k3_kda_decode_supported(device):
raise RuntimeError(f"`{api_name}` requires a gfx950 GPU.")
if batch <= 0:
@@ -256,109 +253,6 @@ def _validate_kda_inputs(
return out
def flydsl_kimi_k3_kda_decode(
x: torch.Tensor,
conv_weight: torch.Tensor,
conv_bias: torch.Tensor | None,
conv_state: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
lower_bound: float | None,
state: torch.Tensor,
state_indices: torch.Tensor,
output_gate: torch.Tensor,
norm_weight: torch.Tensor,
norm_eps: float,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run fused Kimi-K3 KDA decode on MI350-series GPUs.
This pure-decode specialization fuses the packed width-4 Q/K/V causal
convolution, the FP32 recurrent-state update, and the BF16
RMSNorm/sigmoid output gate. Slot zero is reserved: non-positive
``state_indices`` produce zero output without modifying either cache.
The layout is fixed to Kimi-K3 TP8: 12 local heads and 128-dimensional
key/value state. Call
:func:`is_flydsl_kimi_k3_kda_decode_supported` before dispatching from a
model implementation.
"""
if x.ndim != 2:
raise ValueError(f"`x` must have rank 2, got rank {x.ndim}.")
if not x.is_cuda:
raise ValueError("`x` must be a CUDA tensor.")
device = x.device
batch = x.shape[0]
out = _validate_kda_inputs(
api_name="flydsl_kimi_k3_kda_decode",
batch_source="x",
device=device,
batch=batch,
x=x,
conv_weight=conv_weight,
conv_bias=conv_bias,
conv_state=conv_state,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
state=state,
state_indices=state_indices,
output_gate=output_gate,
norm_weight=norm_weight,
out=out,
)
_check_same_device((("raw_g", raw_g),), device)
_check_tensor(
"raw_g",
raw_g,
shape=(1, batch, _HEADS, _DIM),
dtype=torch.bfloat16,
device=device,
inner_strides=(_DIM, 1),
)
executable = create_kimi_k3_kda_decode_kernel(
float(norm_eps),
float(lower_bound),
)
with torch.cuda.device(device):
stream = torch.cuda.current_stream(device)
_run_compiled(
executable,
x,
conv_weight,
conv_state,
raw_g,
raw_beta,
A_log,
dt_bias,
state,
state_indices,
output_gate,
norm_weight,
out,
batch,
x.stride(0),
conv_weight.stride(0),
conv_weight.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
raw_g.stride(1),
raw_beta.stride(1),
state.stride(0),
output_gate.stride(0),
output_gate.stride(1),
out.stride(1),
out.stride(2),
stream,
)
return out
def flydsl_kimi_k3_kda_decode_with_f_b(
f_a: torch.Tensor,
f_b_weight: torch.Tensor,
@@ -471,7 +365,6 @@ def flydsl_kimi_k3_kda_decode_with_f_b(
__all__ = [
"flydsl_kimi_k3_kda_decode",
"flydsl_kimi_k3_kda_decode_with_f_b",
"is_flydsl_kimi_k3_kda_decode_supported",
]
@@ -37,20 +37,12 @@ from sglang.kernels.ops.attention.fla.index import (
prepare_chunk_indices,
)
try:
from .Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host
from .fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM
from .fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM
from .fuse_k4_only_persistent import make_host_fn as _k4p_make_host
from .fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK
from .fuse_kernel123_persistent import make_host_function as _fused_make_host
except ImportError:
from Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host
from fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM
from fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM
from fuse_k4_only_persistent import make_host_fn as _k4p_make_host
from fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK
from fuse_kernel123_persistent import make_host_function as _fused_make_host
from .Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host
from .fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM
from .fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM
from .fuse_k4_only_persistent import make_host_fn as _k4p_make_host
from .fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK
from .fuse_kernel123_persistent import make_host_function as _fused_make_host
def _ct(t, etype):
@@ -219,62 +211,6 @@ _k4p_cache = {}
_k4p_tm_ws = {}
# Beta absorption (K1 fusion reverted; K123 stays at 375us baseline).
# Two-stage strategy on side stream:
# 1) v*beta runs PARALLEL with K123 (no data dep on K123 outputs)
# 2) k_scaled*beta runs AFTER K123 on side stream, PARALLEL with akk_inv
@torch.compile(fullgraph=True)
def _v_absorb(v, beta):
return v * beta.unsqueeze(-1).to(v.dtype)
@torch.compile(fullgraph=True)
def _ks_absorb(k_scaled, beta):
return k_scaled * beta.unsqueeze(-1).to(k_scaled.dtype)
# Side stream cache for v_absorb || K123 overlap.
_side_streams = {}
def _get_side_stream(dev):
idx = dev.index or 0
if idx not in _side_streams:
_side_streams[idx] = torch.cuda.Stream(device=dev)
return _side_streams[idx]
# ===== Per-section timing (K123 / K4) =====
# Set _TIMING_ENABLED=True to record CUDA-event timings for each call.
# Times accumulate in _TIMING_STATS; use reset_timings() / get_timings() to manage.
_TIMING_ENABLED = False
_TIMING_STATS = {"k123_us": 0.0, "k4_us": 0.0, "count": 0}
def enable_timing(enabled=True):
"""Enable/disable per-section CUDA event timing in chunk_kda_fwd.
Each call adds CUDA event records around K123 and K4 launches and a sync
at end of forward, so this slows down execution. Use only for benchmarking."""
global _TIMING_ENABLED
_TIMING_ENABLED = enabled
def reset_timings():
_TIMING_STATS["k123_us"] = 0.0
_TIMING_STATS["k4_us"] = 0.0
_TIMING_STATS["count"] = 0
def get_timings():
"""Returns (k123_avg_us, k4_avg_us, n_calls). Avgs are over all calls since last reset."""
n = max(_TIMING_STATS["count"], 1)
return (
_TIMING_STATS["k123_us"] / n,
_TIMING_STATS["k4_us"] / n,
_TIMING_STATS["count"],
)
# Buffer cache: avoid re-allocating ~67us of intermediate tensors per call.
# Also caches cute.Tensor wrappers (saves ~7us each call from from_dlpack).
_buf_cache = {}
@@ -406,11 +342,6 @@ def _get_buffers(dev, dtype_k, B, T, H, K_dim, V_dim, NT, N_seqs, BT):
s_ct = from_dlpack(S_out, assumed_align=16)
s_ct.element_type = fp32
# Cache PyTorch streams once per buf_cache entry — torch.cuda.current_stream
# is a torch C call that costs ~2us each invocation.
main_stream_cached = torch.cuda.current_stream(dev)
side_stream_cached = _get_side_stream(dev)
cute_wrappers = dict(
a_ct=a_ct,
aqc_ct=aqc_ct,
@@ -424,8 +355,6 @@ def _get_buffers(dev, dtype_k, B, T, H, K_dim, V_dim, NT, N_seqs, BT):
akk_in_view=akk_in_view,
akk_out_view=akk_out_view,
s_ct=s_ct,
main_stream=main_stream_cached,
side_stream=side_stream_cached,
# Filled lazily on first launch — saves cache_key tuple build +
# outer dict lookup on subsequent calls.
_k123_fns={},
@@ -936,18 +865,12 @@ def chunk_kda_fwd(
# only pays off for huge multi-seq varlen states (~192MB at N=32); for
# serving single-seq engagements the state is <1MB and the side stream
# was the last remaining cross-stream surface in this path.
main_stream = torch.cuda.current_stream(device)
needs_copy = S_in.data_ptr() != S_out.data_ptr()
if needs_copy:
S_out.copy_(S_in)
# Beta is fused entirely in akk_inv kernel epilogue (post-inv column-scale).
# No host v*beta and no K1 k_scaled*beta any more.
if _TIMING_ENABLED:
k123_s = torch.cuda.Event(enable_timing=True)
k123_e = torch.cuda.Event(enable_timing=True)
k123_s.record(stream=main_stream)
_launch_fused_k123_inv(
q,
k,
@@ -974,9 +897,6 @@ def chunk_kda_fwd(
varlen_pure_override=(True if needs_varlen_single_pad else varlen_pure),
)
if _TIMING_ENABLED:
k123_e.record(stream=main_stream)
# ===== K4: persistent kernel (eqlen + varlen via cu_seqlens) =====
if is_varlen:
# GPU-side cumsum + cache by id(cu_seqlens). No host sync.
@@ -987,11 +907,6 @@ def chunk_kda_fwd(
# State copy now runs on the caller's stream; K4 is stream-ordered after it.
if _TIMING_ENABLED:
k4_s = torch.cuda.Event(enable_timing=True)
k4_e = torch.cuda.Event(enable_timing=True)
k4_s.record(stream=main_stream)
_launch_k4_persistent(
cute_wrappers,
v,
@@ -1005,13 +920,6 @@ def chunk_kda_fwd(
use_fast_sync=(not is_varlen),
)
if _TIMING_ENABLED:
k4_e.record(stream=cute_wrappers["main_stream"])
torch.cuda.synchronize(device)
_TIMING_STATS["k123_us"] += k123_s.elapsed_time(k123_e) * 1000.0
_TIMING_STATS["k4_us"] += k4_s.elapsed_time(k4_e) * 1000.0
_TIMING_STATS["count"] += 1
o = O_flat
A_qk_out = A_qk
A_kk_out = A_kk
@@ -36,8 +36,6 @@ State management:
- single state_ready_mbar after both passes (TMEM no-overlap constraint)
"""
import os
import sys
import warnings
import cutlass.cutlass_dsl as _dsl_mod
@@ -62,54 +60,11 @@ from cutlass.cute.nvgpu import cpasync, tcgen05
from cutlass.cute.nvgpu.tcgen05 import Field, OperandMajorMode, OperandSource
from cutlass.cute.runtime import from_dlpack
from cutlass.cutlass_dsl import Int32, T
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
# GDN tile scheduler: import from the public FlashInfer release
# (`pip install flashinfer-python>=0.6.13`). Fall back to a local
# dynamic-kernel-generator checkout only if FlashInfer is unavailable.
try:
from flashinfer.gdn_kernels.blackwell.gated_delta_net_tile_scheduler import (
GDNTileScheduler,
GDNTileSchedulerParams,
)
except ImportError:
# Try multiple locations for GDNTileScheduler (workspace layout vs kda_optimized layout)
_GDN_PATHS = [
# workspace/flash-linear-attention/prefill -> ../../../dynamic-kernel-generator
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..",
"..",
"..",
"dynamic-kernel-generator",
"cutlass_ir",
"compiler",
"python",
"examples",
"blackwell",
"gated_delta_net",
),
# scripts/kda_optimized -> ../../dynamic-kernel-generator
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..",
"..",
"dynamic-kernel-generator",
"cutlass_ir",
"compiler",
"python",
"examples",
"blackwell",
"gated_delta_net",
),
]
for _p in _GDN_PATHS:
if os.path.isdir(_p):
sys.path.insert(0, _p)
break
from gated_delta_net_tile_scheduler import GDNTileSchedulerParams, GDNTileScheduler
from cutlass.utils import TensorMapManager, TensorMapUpdateMode
from flashinfer.gdn_kernels.blackwell.gated_delta_net_tile_scheduler import (
GDNTileScheduler,
GDNTileSchedulerParams,
)
def transform_partitioned_tensor_layout(tensor):
@@ -29,178 +29,6 @@ class SegLaMeta:
mask: Optional[torch.Tensor] = None # Currently not supported
# fused
@triton.jit
def seg_la_kernel(
Q,
K,
V,
S,
Out,
softmax_scale,
stride_q,
stride_k,
stride_v,
stride_s,
stride_o,
s_offsets,
q_offsets,
q_lengths,
s_scales,
decay_scales,
HEAD_DIM: tl.constexpr,
SPLIT_DIM: tl.constexpr,
BLOCK: tl.constexpr,
EVEN: tl.constexpr,
DECOUPLE: tl.constexpr,
):
bid = tl.program_id(0)
hid = tl.program_id(1)
sid = tl.program_id(2)
# s_scale is 0 (prefill) or 1 (decode)
s_scale = tl.load(s_scales + bid)
q_length = tl.load(q_lengths + bid)
q_offset = tl.load(q_offsets + bid)
s_offset = tl.load(s_offsets + bid)
decay_scale = -tl.load(decay_scales + hid)
offs_b = tl.arange(0, BLOCK)
offs_d = tl.arange(0, HEAD_DIM)
offs_s = tl.arange(0, SPLIT_DIM)
if s_offset == -1:
return
q_ptrs = (
Q
+ q_offset * stride_q
+ hid * HEAD_DIM
+ (offs_b[:, None] * stride_q + offs_d[None, :])
)
k_ptrs = (
K
+ q_offset * stride_k
+ hid * HEAD_DIM
+ (offs_b[:, None] * stride_k + offs_d[None, :])
)
v_ptrs = (
V
+ q_offset * stride_v
+ hid * HEAD_DIM
+ sid * SPLIT_DIM
+ (offs_b[:, None] * stride_v + offs_s[None, :])
)
out_ptrs = (
Out
+ q_offset * stride_o
+ hid * HEAD_DIM
+ sid * SPLIT_DIM
+ (offs_b[:, None] * stride_o + offs_s[None, :])
)
s_ptrs = (
S
+ s_offset * stride_s
+ hid * HEAD_DIM * HEAD_DIM
+ sid * SPLIT_DIM
+ (offs_d[:, None] * HEAD_DIM + offs_s[None, :])
)
state = tl.load(s_ptrs, mask=s_scale > 0).to(tl.float32)
if BLOCK > 1:
for n in range(0, q_length, BLOCK):
n = tl.multiple_of(n, BLOCK)
if EVEN:
q = tl.load(q_ptrs + n * stride_q).to(tl.float32)
k = tl.trans(tl.load(k_ptrs + n * stride_k)).to(tl.float32)
v = tl.load(v_ptrs + n * stride_k).to(tl.float32)
else:
q = tl.load(
q_ptrs + n * stride_q,
mask=(n + offs_b)[:, None] < q_length,
other=0.0,
).to(tl.float32)
k = tl.trans(
tl.load(
k_ptrs + n * stride_k,
mask=(n + offs_b)[:, None] < q_length,
other=0.0,
)
).to(tl.float32)
v = tl.load(
v_ptrs + n * stride_k,
mask=(n + offs_b)[:, None] < q_length,
other=0.0,
).to(tl.float32)
if DECOUPLE:
# only work with small scales
if EVEN:
b = BLOCK
else:
b = min(BLOCK, q_length - n)
b_offs = b - 1 - offs_b
edb = tl.exp(decay_scale * b_offs)
decays = tl.where(b_offs >= 0, edb, 0)
inv_decays = tl.where(b_offs >= 0, 1 / edb, 0)
q = q * inv_decays[:, None]
k = k * decays[None, :]
qk = tl.dot(q, k) * softmax_scale
qk = tl.where(offs_b[None, :] <= offs_b[:, None], qk, 0.0)
o = tl.dot(qk, v)
block_decay = tl.exp(decay_scale * b)
block_decay_plus = block_decay * softmax_scale
o = tl.dot(q, state) * block_decay_plus + o
state = state * block_decay + tl.dot(k, v)
else:
qk = tl.dot(q, k) * softmax_scale
decays = tl.exp(decay_scale * (offs_b[:, None] - offs_b[None, :]))
decays = tl.where(offs_b[None, :] <= offs_b[:, None], decays, 0.0)
qk *= decays
o = tl.dot(qk, v)
decay_arr = tl.exp(decay_scale * (offs_b[:, None] + 1)) * softmax_scale
o = tl.dot(q * decay_arr, state, acc=o)
if EVEN:
b = BLOCK
else:
b = min(BLOCK, q_length - n)
b_offs = b - 1 - offs_b
b_offs = tl.where(b_offs >= 0, b_offs, 10000)
decays = tl.exp(decay_scale * b_offs)
block_decay = tl.exp(decay_scale * b)
state = state * block_decay + tl.dot(k * decays[None, :], v)
if EVEN:
tl.store(out_ptrs + n * stride_o, o.to(Out.dtype.element_ty))
else:
tl.store(
out_ptrs + n * stride_o,
o.to(Out.dtype.element_ty),
mask=(n + offs_b)[:, None] < q_length,
)
tl.store(s_ptrs, state.to(S.dtype.element_ty))
else:
q = tl.trans(tl.load(q_ptrs)).to(tl.float32) * softmax_scale
k = tl.trans(tl.load(k_ptrs)).to(tl.float32)
v = tl.load(v_ptrs).to(tl.float32)
state = state * tl.exp(decay_scale) + k * v
o = tl.sum(q * state, axis=0, keep_dims=True)
tl.store(out_ptrs, o.to(Out.dtype.element_ty))
tl.store(s_ptrs, state.to(S.dtype.element_ty))
# used for prefilling
@triton.jit
def seg_la_p_kernel(
@@ -691,7 +519,6 @@ def seg_la_fwd(
track_lens=None,
track_state_indices=None,
softmax_scale=None,
decouple=False,
):
length, qo_heads, HEAD_DIM = q.shape
_, kv_heads, _ = k.shape
@@ -706,7 +533,6 @@ def seg_la_fwd(
if MAX_LENGTH > 1:
# prefill with partitioning q/k/v
# BLOCK should <= 64 with decouple
K_SPLIT_DIM = 32
V_SPLIT_DIM = 32 if bs <= 2 else 64
@@ -884,61 +710,4 @@ def seg_la_fwd(
else:
o = tmp[0]
# if fallback:
# # prefill/decode with partitioning v only
# o = torch.empty(q.shape, device=q.device, dtype=q.dtype)
# if MAX_LENGTH == 1:
# # decode
# BLOCK = 1
# EVEN = False
# SPLIT_DIM = 32
# num_warps = 8
# num_stages = 2
# num_dim_block = HEAD_DIM // SPLIT_DIM
# grid = (batch, kv_heads, num_dim_block)
# else:
# # prefill
# if decouple:
# BLOCK = 64
# SPLIT_DIM = 16
# else:
# BLOCK = HEAD_DIM
# SPLIT_DIM = 32
# # EVEN = all([x % BLOCK == 0 for x in meta.qls])
# EVEN = False
# num_warps = 8
# num_stages = 2
# # prop = torch.cuda.get_device_properties(q.device.index)
# # arch = prop.major * 10 + prop.minor
# # if arch not in (80, 90):
# # num_stages = 1
# num_dim_block = HEAD_DIM // SPLIT_DIM
# grid = (batch, kv_heads, num_dim_block)
# seg_la_kernel[grid](
# q,
# k,
# v,
# s,
# o,
# softmax_scale,
# q.stride(0),
# k.stride(0),
# v.stride(0),
# s.stride(0),
# o.stride(0),
# meta.s_offsets,
# meta.q_offsets,
# meta.q_lengths,
# meta.s_scales,
# decay_scales,
# HEAD_DIM=HEAD_DIM,
# SPLIT_DIM=SPLIT_DIM,
# BLOCK=BLOCK,
# EVEN=EVEN,
# DECOUPLE=decouple,
# num_warps=num_warps,
# num_stages=num_stages
# )
return o
@@ -4,10 +4,11 @@ import torch
import triton
import triton.language as tl
from sglang.srt.utils import get_bool_env_var, is_hip
from sglang.srt.environ import envs
from sglang.srt.utils import is_hip
_is_hip = is_hip()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip
# =============================================================================
# Fused kernel — reads INTERLEAVED input format
@@ -15,6 +15,7 @@ HIP-graph-capture safe (no host sync).
"""
import ctypes
import logging
import os
import re
import subprocess
@@ -24,6 +25,8 @@ import torch
from sglang.srt.distributed.device_communicators.cuda_wrapper import find_loaded_library
logger = logging.getLogger(__name__)
_DIR = os.path.dirname(os.path.abspath(__file__))
_LOG2E = 1.4426950408889634
@@ -205,10 +208,9 @@ def asm_kernel_available() -> bool:
def _disable(reason):
global _disabled_reason
_disabled_reason = reason
print(
logger.warning(
f"[vattn_asm] disabling the gfx950 asm attention kernel, falling back to "
f"the Triton path: {reason}",
flush=True,
f"the Triton path: {reason}"
)
raise AsmKernelUnavailable(reason)
@@ -264,10 +266,9 @@ def _get_kernel(hq):
except RuntimeError as e:
_disable(f"loading {co} failed: {e}")
_kernels[hq] = kern
print(
logger.info(
f"[vattn_asm] gfx950 asm verify-attention kernel active "
f"(in-tree loader, GQA ratio {hq}, built {co})",
flush=True,
f"(in-tree loader, GQA ratio {hq}, built {co})"
)
return kern
@@ -79,11 +79,13 @@ def can_use_fused_qk_complex_rope(
device: torch.device,
freqs_cis: torch.Tensor,
) -> bool:
"""Select the fused path before projecting the vision Q/K tensors."""
"""Select the fused path before projecting the vision Q/K tensors.
``freqs_cis`` sharing the activations' device is the kernel's contract,
which :func:`apply_fused_qk_complex_rope` asserts, not a dispatch choice.
"""
if not (
device.type == "cuda"
and freqs_cis.is_cuda
and device == freqs_cis.device
and dtype in (torch.bfloat16, torch.float16)
and freqs_cis.dtype == torch.complex64
):
@@ -115,7 +117,7 @@ def apply_fused_qk_complex_rope(
f"freqs={freqs_cis.shape}/{freqs_cis.dtype}/{freqs_cis.device}"
)
assert q.device == k.device and q.dtype == k.dtype
assert q.device == k.device == freqs_cis.device and q.dtype == k.dtype
assert q.shape == k.shape and q.ndim >= 3
assert q.shape[-1] % 2 == 0
assert freqs_cis.shape == q.shape[:-2] + (q.shape[-1] // 2,)
@@ -6,7 +6,12 @@ records metadata; importing this group does not load the implementations.
"""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
from sglang.kernels.spec import (
CUDA_ONLY,
CapabilityRequirement,
KernelBackend,
KernelSpec,
)
__all__ = []
@@ -16,7 +21,6 @@ register_kernel(
backend=KernelBackend.JIT,
target="sglang.kernels.ops.communication.all_reduce_fusion:moe_finalize_shared_gate_all_reduce",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
description="BF16 routed finalize, FP32 shared gate, and clustered all-reduce.",
)
)
@@ -39,7 +43,7 @@ for _mod, _fn in [
op=f"communication.{_fn}",
backend=KernelBackend.JIT,
target=f"sglang.kernels.ops.communication.{_mod}:{_fn}",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
del _mod, _fn
@@ -24,7 +24,6 @@ def _jit_inkling_all_reduce_module(dtype: torch.dtype, world_size: int) -> Modul
*args,
cuda_files=["inkling/inkling_all_reduce.cuh"],
cuda_wrappers=[
("two_shot_all_reduce", f"inkling_two_shot_all_reduce<{args}>"),
("two_shot_all_reduce_fused", f"inkling_two_shot_all_reduce_fused<{args}>"),
("multimem_one_shot_fused", f"inkling_multimem_one_shot_fused<{args}>"),
("multimem_full_oneshot", f"inkling_multimem_full_oneshot<{args}>"),
@@ -143,32 +142,6 @@ def compile_inkling_all_reduce(dtype: torch.dtype, world_size: int) -> None:
_jit_inkling_all_reduce_module(dtype, world_size)
def inkling_two_shot_all_reduce(
buffer: torch.Tensor,
peer_ptrs_dev: int,
rank: int,
world_size: int,
num_items: int,
) -> None:
"""Two-shot all-reduce in place over ``num_items`` elements of the symm buffer.
Args:
buffer: this rank's symm buffer (1D, contiguous, bf16), sliced to
``num_items``; used for device/dtype validation. The producer must
have already written this rank's shard into it.
peer_ptrs_dev: ``hdl.buffer_ptrs_dev`` -- device address of the array of
``world_size`` peer buffer base pointers.
rank: this rank within the TP group.
world_size: TP world size (compile-time template arg).
num_items: number of elements to reduce (multiple of 8 for bf16).
The caller is responsible for ``hdl.barrier()`` before (producers done) and
after (result visible) this call.
"""
module = _jit_inkling_all_reduce_module(buffer.dtype, world_size)
module.two_shot_all_reduce(buffer, peer_ptrs_dev, rank, num_items)
def inkling_two_shot_all_reduce_fused(
buffer: torch.Tensor,
data_ptrs_dev: int,
+2 -186
View File
@@ -1,30 +1,9 @@
"""Multi-process / multi-GPU launching utilities (torchrun-based).
Shared `multigpu_launch` helper that both `sglang.test.kernels.utils` and
`sglang.kernels.jit.benchmark.utils` build their domain-specific entry points on
top of (`multigpu_pytest_main`, `multigpu_bench_main`).
When a script that calls one of those wrappers is run with plain `python`, the
launcher relaunches the same file under `torchrun` once for each `N` in
`num_gpus`. When the inner workers run (identified by an env_key being set),
the same launcher calls `inner()` on every rank, silences stdout on non-zero
ranks, and exits with its return code.
"""
"""Shutdown helper for custom all-reduce communicators."""
from __future__ import annotations
import atexit
import logging
import os
import signal
import subprocess
import sys
from typing import Any, Callable, List, NoReturn, Optional, Sequence
import psutil
import torch
logger = logging.getLogger(__name__)
from typing import Any
def register_comm_cleanup(comm: Any) -> None:
@@ -52,166 +31,3 @@ def register_comm_cleanup(comm: Any) -> None:
pass
atexit.register(_safe_close)
def _kill_pgroup(pgid: int) -> None:
try:
os.killpg(pgid, signal.SIGKILL)
except ProcessLookupError:
pass
def _kill_descendants(pid: int) -> None:
"""Snapshot every descendant of `pid` *now* and SIGKILL them all.
Must be called BEFORE the direct child (torchrun) dies -- once it does,
its workers get reparented to init and we lose them via the process tree.
"""
try:
root = psutil.Process(pid)
except psutil.NoSuchProcess:
return
descendants = root.children(recursive=True)
for proc in descendants:
try:
proc.kill()
except psutil.Error:
# NoSuchProcess (already gone) or AccessDenied -- nothing to do.
pass
psutil.wait_procs(descendants, timeout=5)
def _extract_num_gpus_override(
argv: list[str],
) -> tuple[list[int] | None, list[str]]:
"""Pop `--num-gpu(s)` flags out of `argv` and return them separately.
Accepts `--num-gpu N`, `--num-gpu=N`, `--num-gpus ...`, and comma-separated
lists like `--num-gpu 2,4,8`. May be repeated.
"""
override: list[int] = []
remaining: list[str] = []
i = 0
while i < len(argv):
a = argv[i]
if a in ("--num-gpu", "--num-gpus"):
if i + 1 >= len(argv):
raise ValueError(f"missing value for {a} (expected e.g. `{a} 2,4`)")
override.extend(int(x) for x in argv[i + 1].split(","))
i += 2
elif a.startswith("--num-gpu=") or a.startswith("--num-gpus="):
_, val = a.split("=", 1)
override.extend(int(x) for x in val.split(","))
i += 1
else:
remaining.append(a)
i += 1
return (override if override else None), remaining
def multigpu_launch(
name: str,
file: str,
num_gpus: Sequence[int],
env_key: str,
inner: Callable[[], int],
kind: str,
pre_launch_fn: Optional[Callable[[List[int]], None]] = None,
timeout: Optional[int] = None,
) -> NoReturn | None:
"""Shared torchrun-based launcher.
See module docstring. `name` is the caller's `__name__`; `file` is its
`__file__`. `env_key` is a unique string per kind (test/benchmark) used to
detect the inside-torchrun state. `inner` returns an exit code.
`pre_launch_fn`, if given, runs once in the outer process *before* any
torchrun child is spawned. It receives the list of world sizes that will
actually be launched (already filtered against the host's GPU count and
any ``--num-gpu`` override). Use it for parallel JIT precompilation so the
on-disk kernel cache is warm by the time the torchrun children import
their kernels.
`timeout`, if given, bounds each per-world-size torchrun invocation (in
seconds). On expiry the child's whole process group is killed and the
launcher exits non-zero. `None` (the default) waits indefinitely.
"""
pid_key = env_key + "_PID"
if env_key in os.environ:
assert pid_key in os.environ
if name != "__main__":
return
rank = int(os.environ["LOCAL_RANK"])
if rank != 0:
sys.stdout = open(os.devnull, "w")
torch.cuda.set_device(rank)
return sys.exit(inner())
assert pid_key not in os.environ
if name != "__main__":
return logger.warning(
f"{file} can not directly run with `pytest`. "
"Use `python` to invoke it, which will internally relaunch it "
"under torchrun for each requested number of GPUs."
)
num_devices = torch.cuda.device_count()
override, forwarded_args = _extract_num_gpus_override(sys.argv[1:])
if override is not None:
logger.info(f"--num-gpu override: running only with {override}")
num_gpus = override
for N in num_gpus:
if N <= 1 or N > num_devices:
raise ValueError(
f"Invalid number of GPUs requested: {N} (available: {num_devices})"
)
os.environ[env_key] = "1"
os.environ[pid_key] = str(os.getpid())
os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("GLOO_SOCKET_IFNAME", "lo") # single-machine setup
# Unbuffered child stdout: when a worker is killed on timeout, pytest's
# block-buffered progress output is otherwise lost or flushed out of
# order into the CI log, making it impossible to tell which test hung.
os.environ.setdefault("PYTHONUNBUFFERED", "1")
signal.signal(signal.SIGINT, signal.default_int_handler)
runnable: List[int] = []
for N in sorted(num_gpus):
assert N > 1
if N > num_devices:
logger.warning(f"Skipping {kind} with {N} GPUs ({num_devices} available)")
continue
runnable.append(N)
if pre_launch_fn is not None and runnable:
logger.info(f"Running pre-launch hook for world sizes {runnable}")
pre_launch_fn(runnable)
for N in runnable:
logger.info(f"Running {kind} with {N} GPUs")
cmd = [
"torchrun",
"--nproc_per_node",
str(N),
"--local-addr",
"127.0.0.1",
file,
]
cmd += forwarded_args
proc = subprocess.Popen(cmd, start_new_session=True)
pgid = proc.pid
returncode = -1
timed_out = False
try:
returncode = proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
timed_out = True
finally:
_kill_descendants(os.getpid())
_kill_pgroup(pgid)
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
pass
if timed_out:
logger.error(f"{kind} (nproc={N}) timed out after {timeout} seconds")
sys.exit(1)
if returncode != 0:
logger.error(f"{kind} failed with {N} GPUs (exit {returncode})")
sys.exit(returncode)
logger.info(f"All {kind}s passed")
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
from sglang.kernels.ops.communication.all_reduce import Communicator
_PRIMITIVES: Final = ["all_reduce", "all_gather", "reduce_scatter"]
_PRIMITIVES: Final = ["all_gather"]
def get_multicast_ptr(tensor: torch.Tensor) -> int:
@@ -61,17 +61,6 @@ def _jit_push_module(dtype: torch.dtype, world_size: int) -> Module:
# `residual` on any of these is folded into the reduction; it may be shaped like
# this rank's shard or like the whole tensor, of which this rank's slice is taken.
def all_reduce_push(
comm: Communicator,
input: torch.Tensor,
output: torch.Tensor,
residual: torch.Tensor | None = None,
) -> None:
_jit_push_module(input.dtype, comm.world_size).all_reduce(
comm, input, output, residual
)
def all_gather_push(
comm: Communicator,
input: torch.Tensor,
@@ -83,39 +72,6 @@ def all_gather_push(
)
def reduce_scatter_push(
comm: Communicator,
input: torch.Tensor,
output: torch.Tensor,
residual: torch.Tensor | None = None,
) -> None:
_jit_push_module(input.dtype, comm.world_size).reduce_scatter(
comm, input, output, residual
)
def all_reduce_pull(
comm: Communicator,
input: torch.Tensor,
output: torch.Tensor,
residual: torch.Tensor | None = None,
*,
in_mc_ptr: int = 0,
out_mc_ptr: int = 0,
num_unroll=4,
num_blocks_hint: int = 0,
) -> None:
_jit_pull_module(input.dtype, num_unroll).all_reduce(
comm,
input,
output,
residual,
in_mc_ptr or get_multicast_ptr(input),
out_mc_ptr or get_multicast_ptr(output),
num_blocks_hint,
)
def all_gather_pull(
comm: Communicator,
input: torch.Tensor,
@@ -134,23 +90,3 @@ def all_gather_pull(
out_mc_ptr or get_multicast_ptr(output),
num_blocks_hint,
)
def reduce_scatter_pull(
comm: Communicator,
input: torch.Tensor,
output: torch.Tensor,
residual: torch.Tensor | None = None,
*,
in_mc_ptr: int = 0,
num_unroll=4,
num_blocks_hint: int = 0,
) -> None:
_jit_pull_module(input.dtype, num_unroll).reduce_scatter(
comm,
input,
output,
residual,
in_mc_ptr or get_multicast_ptr(input),
num_blocks_hint,
)
@@ -117,20 +117,6 @@ def get_dispatch(
)
def get_tuning(
kind: str,
world_size: int,
hidden_size: int,
num_tokens: int,
device: torch.device,
) -> Optional[Tuning]:
"""Compatibility helper for callers that only support staging push."""
dispatch = get_dispatch(kind, world_size, hidden_size, num_tokens, device)
if dispatch is None or dispatch.strategy != "push":
return None
return dispatch.tuning
def get_fusion_dispatch(
kind: str,
world_size: int,
+25 -173
View File
@@ -30,12 +30,7 @@ from __future__ import annotations
from typing import Any
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import (
CapabilityRequirement,
FormatSignature,
KernelBackend,
KernelSpec,
)
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
_CUDA = frozenset({CapabilityRequirement.CUDA})
_CUDA_SM90_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))})
@@ -49,654 +44,556 @@ _CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10,
# ``scale_residual_norm_scale_shift`` in Triton, CuTe-DSL and FlyDSL) are
# inventory: callers name the one they want via ``select_kernel``.
# ---------------------------------------------------------------------------
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
(
"diffusion.residual_gate_fp32",
KernelBackend.TRITON,
"modulate.residual_gate_fp32:residual_gate_fp32",
_CUDA,
"Residual gating with separate FP32 product and sum rounding.",
),
(
"diffusion.matrix_rope",
KernelBackend.TRITON,
"rope.matrix_rope:apply_matrix_rope",
_CUDA,
"Interleaved FP32 matrix RoPE with separate product and sum rounding.",
),
(
"diffusion.fp8_rowwise",
KernelBackend.TRITON,
"quantization.fp8_rowwise_triton:fp8_rowwise",
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
"FLUX.3 dynamic rowwise E4M3 quantization and zero row padding.",
),
(
"diffusion.gelu_tanh_cat",
KernelBackend.JIT,
"activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat",
_CUDA,
"BF16 tanh GELU plus channel concatenation.",
),
(
"diffusion.rmsnorm_cast_before_weight",
KernelBackend.TRITON,
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_bitexact",
_CUDA,
"MOSSImage RMSNorm with ATen-ordered reduction and BF16 rounding before gamma.",
),
(
"diffusion.rmsnorm_cast_before_weight_modulate",
KernelBackend.TRITON,
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_modulate_bitexact",
_CUDA,
"MOSSImage ATen-ordered RMSNorm with exact BF16 modulation.",
),
(
"diffusion.rmsnorm_cast_before_weight_rope",
KernelBackend.TRITON,
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_rope_bitexact",
_CUDA,
"MOSSImage ATen-ordered head RMSNorm with exact rotate-half RoPE.",
),
(
"diffusion.apply_group_norm_silu",
KernelBackend.TRITON,
"norm.group_norm_silu:apply_group_norm_silu",
_CUDA,
"Fused GroupNorm + SiLU.",
),
(
"diffusion.group_norm_silu_4d",
KernelBackend.TRITON,
"norm.group_norm_silu_twopass_triton:group_norm_silu_4d",
_CUDA,
"Channels-last two-pass GroupNorm(+SiLU), 4D.",
),
(
"diffusion.group_norm_silu_rows",
KernelBackend.TRITON,
"norm.group_norm_silu_twopass_triton:group_norm_silu_rows",
_CUDA,
"Channels-last two-pass GroupNorm(+SiLU) over (N, L, C) rows.",
),
(
"diffusion.wan_rmsnorm_silu",
KernelBackend.TRITON,
"norm.wan_rmsnorm_silu_triton:wan_rmsnorm_silu",
_CUDA,
"Wan VAE channels_last_3d RMSNorm + SiLU.",
),
(
"diffusion.wan_norm_silu_post",
KernelBackend.JIT,
"norm.wan_norm_silu_post:wan_norm_silu_post",
_CUDA,
"Wan VAE FP32 normalization post-ops with the native denominator.",
),
(
"diffusion.rmsnorm_scale_shift",
KernelBackend.TRITON,
"norm.rmsnorm_scale_shift_bitexact:fused_rmsnorm_scale_shift_bitexact",
_CUDA,
"Bit-exact RMSNorm + adaLN scale/shift.",
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_scale_residual_norm_scale_shift",
_CUDA_SM100_PLUS,
"KDA B200 native CUDA residual + LayerNorm + scale/shift (#27392).",
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.TRITON,
"norm.rmsnorm_scale_shift_bitexact:fused_scale_residual_rmsnorm_scale_shift_bitexact",
_CUDA,
"Bit-exact residual-gate add + RMSNorm + scale/shift.",
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.CUTE_DSL,
"norm.scale_residual_norm_cutedsl:fused_scale_residual_norm_scale_shift",
_CUDA,
"CuTe-DSL residual + norm + scale/shift.",
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.FLYDSL,
"norm.fused_residual_norm_flydsl:flydsl_fused_residual_norm_scale_shift",
_HIP,
"FlyDSL (ROCm gfx950) residual + norm + scale/shift.",
),
(
"diffusion.scale_residual_norm_scale_shift_nvfp4",
KernelBackend.JIT,
"sglang.kernels.kda_kernels.norm_scale_shift_jit:try_fused_scale_residual_norm_scale_shift_nvfp4",
_CUDA,
"Qwen residual LayerNorm/modulation + NVFP4 quantization.",
),
(
"diffusion.norm_scale_shift",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_norm_scale_shift",
_CUDA_SM100_PLUS,
"KDA B200 native CUDA LayerNorm + scale/shift (#27392).",
),
(
"diffusion.norm_scale_shift",
KernelBackend.CUTE_DSL,
"norm.scale_residual_norm_cutedsl:fused_norm_scale_shift",
_CUDA,
"CuTe-DSL norm + scale/shift.",
),
(
"diffusion.norm_scale_shift",
KernelBackend.FLYDSL,
"norm.fused_residual_norm_flydsl:flydsl_norm_scale_shift",
_HIP,
"FlyDSL (ROCm gfx950) norm + scale/shift.",
),
(
"diffusion.layernorm_modulate",
KernelBackend.TRITON,
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate",
_CUDA,
"Bit-exact LayerNorm + adaLN modulate.",
),
(
"diffusion.qk_head_layernorm",
KernelBackend.TRITON,
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_qk_head_layernorm",
_CUDA,
"Bit-exact per-head LayerNorm for q/k.",
),
(
"diffusion.qk_rmsnorm_native",
KernelBackend.TRITON,
"norm.zimage_qk_rmsnorm_triton:zimage_qk_rmsnorm_native",
_CUDA,
"Z-Image bf16-native per-head QK RMSNorm.",
),
(
"diffusion.rmsnorm_scale",
KernelBackend.TRITON,
"norm.native_bf16_rmsnorm_triton:rmsnorm_scale",
_CUDA,
"BF16-native RMSNorm * scale.",
),
(
"diffusion.rmsnorm_tanh_residual",
KernelBackend.TRITON,
"norm.native_bf16_rmsnorm_triton:rmsnorm_tanh_residual",
_CUDA,
"BF16-native x + tanh(gate) * RMSNorm(y).",
),
(
"diffusion.modulate_scale_shift",
KernelBackend.JIT,
"modulate.modulate_scale_shift_jit:modulate_scale_shift",
_CUDA,
"Bit-exact adaLN modulate x * (1 + scale) + shift.",
),
(
"diffusion.residual_gate_add",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.residual_gate_add_jit:residual_gate_add",
_CUDA,
"KDA native CUDA residual + gate * update (#29361).",
),
(
"diffusion.residual_gate_add2",
KernelBackend.JIT,
"modulate.residual_gate_add2_jit:residual_gate_add2",
_CUDA,
"Fused residual + gate * (update_a + update_b) for two-branch blocks.",
),
(
"diffusion.flux3_fp8_linear",
KernelBackend.FLASHINFER,
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_linear",
_CUDA_SM10X,
"FLUX.3 rowwise-FP8 PrimsTS GEMM.",
),
(
"diffusion.flux3_fp8_linear_swiglu",
KernelBackend.FLASHINFER,
"gemm.flux3_fp8_prims_ts:flux3_fused_mlp_up",
_CUDA_SM10X,
"FLUX.3 rowwise-FP8 PrimsTS GEMM with SwiGLU epilogue.",
),
(
"diffusion.flux3_fp8_qkv_qknorm_rope",
KernelBackend.FLASHINFER,
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_qkv_qknorm_rope",
_CUDA_SM10X,
"FLUX.3 rowwise-FP8 PrimsTS QKV GEMM with QK-norm/RoPE epilogue.",
),
(
"diffusion.flux3_nvfp4_linear",
KernelBackend.FLASHINFER,
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear",
_CUDA_SM10X,
"FLUX.3 NVFP4 PrimsTS GEMM.",
),
(
"diffusion.flux3_nvfp4_linear_swiglu",
KernelBackend.FLASHINFER,
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear_swiglu",
_CUDA_SM10X,
"FLUX.3 NVFP4 PrimsTS GEMM with SwiGLU epilogue.",
),
(
"diffusion.flux3_nvfp4_qkv_qknorm_rope",
KernelBackend.FLASHINFER,
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_qkv_qknorm_rope",
_CUDA_SM10X,
"FLUX.3 NVFP4 PrimsTS QKV GEMM with QK-norm/RoPE epilogue.",
),
(
"diffusion.timestep_embedding",
KernelBackend.JIT,
"modulate.timestep_embedding_jit:timestep_embedding",
_CUDA,
"Sinusoidal timestep embedding.",
),
(
"diffusion.temb_table_slices",
KernelBackend.TRITON,
"modulate.wan_temb_table_slices_triton:fused_temb_table_slices",
_CUDA,
"Contiguous adaLN slices for Wan2.2-TI2V.",
),
(
"diffusion.ltx2_ada_values",
KernelBackend.TRITON,
"modulate.ltx2_ada_values_triton:ltx2_ada_values9",
_CUDA,
"LTX-2 nine-way adaLN value split.",
),
(
"diffusion.fused_inplace_qknorm_rope",
KernelBackend.JIT,
"rope.qknorm_rope_jit:fused_inplace_qknorm_rope",
_CUDA,
"Fused in-place QK RMS-norm + RoPE.",
),
(
"diffusion.vsa_block_sparse_sm100",
KernelBackend.JIT,
"attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100",
frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}),
"FastVideo's warp-specialized tcgen05 block-sparse VSA forward (64-token tiles).",
),
(
"diffusion.h3_vae_rmsnorm",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_rmsnorm",
_CUDA,
"FP32-accumulation RMSNorm for the MiniMax-H3 video VAE decoder.",
),
(
"diffusion.h3_vae_scale_add_rmsnorm",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm",
_CUDA,
"Scaled residual add plus the following RMSNorm for the H3 VAE decoder.",
),
(
"diffusion.h3_vae_qk_rmsnorm_rope",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_qk_rmsnorm_rope",
_CUDA,
"Affine-free Q/K RMSNorm plus partial NeoX RoPE for the H3 VAE decoder.",
),
(
"diffusion.flux2_layernorm_modulate_fp8_quant",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate_fp8_quant_raw",
_CUDA,
"KDA-generated FLUX.2 LayerNorm + adaLN modulation + static FP8 quantization.",
),
(
"diffusion.flux2_qkv_epilogue",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit:try_fused_flux2_qkv_epilogue",
_CUDA,
"KDA-generated FLUX.2 QK RMS-norm + RoPE + joint QKV packing.",
),
(
"diffusion.flux2_token_cat_fp8",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton:try_flux2_token_cat_fp8",
_CUDA,
"KDA-generated FLUX.2 token concatenation + static FP8 quantization.",
),
(
"diffusion.qwen_qkv_epilogue",
KernelBackend.JIT,
"rope.qwen_qkv_epilogue_jit:try_fused_qwen_qkv_epilogue",
_CUDA_SM90_PLUS,
"Qwen-Image QK RMS-norm, RoPE, and joint QKV writes.",
),
(
"diffusion.ltx2_qknorm_split_rope",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit:ltx2_qknorm_split_rope_cuda",
_CUDA_SM100_PLUS,
"KDA native CUDA LTX-2 QK-norm + split RoPE (#29708).",
),
(
"diffusion.ltx25_decoder_rope",
KernelBackend.JIT,
"rope.ltx25_decoder_rope_jit:fused_ltx25_decoder_rope",
_CUDA,
"Paired LTX-2.5 decoder 3D RoPE.",
),
(
"diffusion.rope_rotate_half_fp32",
KernelBackend.TRITON,
"rope.rope_rotate_half_fp32:fused_rope_rotate_half_fp32",
_CUDA,
"Paired split-half RoPE with FP32 arithmetic and one output cast.",
),
(
"diffusion.rope_rotate_half",
KernelBackend.TRITON,
"rope.rope_rotate_half_bitexact:fused_rope_rotate_half_bitexact",
_CUDA,
"Bit-exact rotate-half RoPE.",
),
(
"diffusion.interleaved_rope_fp64",
KernelBackend.JIT,
"rope.interleaved_rope_fp64_jit:fused_interleaved_rope_fp64",
_CUDA,
"Paired interleaved RoPE with fp64 Diffusers semantics.",
),
(
"diffusion.helios_qk_rope",
KernelBackend.JIT,
"rope.helios_qk_rope_jit:fused_inplace_helios_qk_rope",
_CUDA,
"Paired in-place Helios transposed Q/K RoPE.",
),
(
"diffusion.complex_rope",
KernelBackend.TRITON,
"rope.complex_rope_triton:fused_complex_rope",
_CUDA,
"Paired RoPE preserving PyTorch complex64 multiplication rounding.",
),
(
"diffusion.hunyuan_qkv_rope_pack",
KernelBackend.TRITON,
"rope.hunyuan_qkv_pack_triton:hunyuan_qkv_rope_pack",
_CUDA,
"HunyuanVideo QKV pack + RoPE.",
),
(
"diffusion.joint_qkv_cat",
KernelBackend.TRITON,
"layout.joint_qkv_cat_triton:joint_qkv_cat",
_CUDA,
"Concatenate image/text QKV views into joint attention inputs.",
),
(
"diffusion.rmsnorm_preserve_reduction",
KernelBackend.TRITON,
"norm.rmsnorm_preserve_reduction:rmsnorm_preserve_reduction",
_CUDA,
"Cast-before-weight RMSNorm preserving the native FP32 mean reduction.",
),
(
"diffusion.silu_mul",
KernelBackend.TRITON,
"activation.silu_mul_bitexact:fused_silu_mul_bitexact",
_CUDA,
"Bit-exact silu(a) * b for split-projection SwiGLU.",
),
(
"diffusion.situ_softcap_mul",
KernelBackend.TRITON,
"activation.situ_softcap_mul_bitexact:fused_situ_softcap_mul_bitexact",
_CUDA,
"Bit-exact bounded SiTU for MOSSImage FFN projections.",
),
(
"diffusion.bias_silu",
KernelBackend.TRITON,
"activation.sana_conv_post_triton:fused_bias_silu",
_CUDA,
"Bit-exact conv bias + SiLU (Sana GLUMB).",
),
(
"diffusion.bias_glu",
KernelBackend.TRITON,
"activation.sana_conv_post_triton:fused_bias_glu",
_CUDA,
"Bit-exact conv bias + GLU (Sana GLUMB).",
),
(
"diffusion.linear_gelu_tanh",
KernelBackend.AOT,
"sites.fused_linear_gelu_site:fused_linear_gelu_tanh",
_CUDA,
"Linear + tanh-GELU via the cublasLt epilogue.",
),
(
"diffusion.sparse_linear_attn_fwd",
KernelBackend.TRITON,
"attention.sparse_linear_attn_triton:_attn_fwd",
_CUDA,
"Sparse linear attention forward.",
),
(
"diffusion.bigdn",
KernelBackend.TRITON,
"attention.sana_wm_gdn_triton:fused_bigdn_func",
_CUDA,
"Sana-WM bidirectional gated delta-net.",
),
(
"diffusion.fused_qknorm_rope_out_of_place",
KernelBackend.JIT,
"rope.qknorm_rope_jit:fused_qknorm_rope_out_of_place",
_CUDA,
"Out-of-place fused QK-norm + RoPE (raw q/k preserved).",
),
(
"diffusion.qknorm_complex_rope_cuda",
KernelBackend.JIT,
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda",
_CUDA,
"Bit-exact 128-wide RMSNorm + complex RoPE (Qwen-Image 2.1), CUDA.",
),
(
"diffusion.qknorm_complex_rope_pack_",
KernelBackend.JIT,
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_",
_CUDA,
"Q/K RMSNorm + complex RoPE with prefix K/V packing into [B, P+S, H, 128] buffers, CUDA.",
),
(
"diffusion.channel_rmsnorm_finish_silu",
KernelBackend.JIT,
"norm.channel_rmsnorm_finish_silu_jit:channel_rmsnorm_finish_silu",
_CUDA,
"Bit-exact channel-first RMSNorm finish + SiLU on aten's fp32 channel norm (Qwen-Image 2.1 VAE), CUDA.",
),
(
"diffusion.channel_rmsnorm_silu_nhwc",
KernelBackend.JIT,
"norm.channel_rmsnorm_silu_nhwc_jit:channel_rmsnorm_silu_nhwc",
_CUDA,
"Channels-last channel RMSNorm + SiLU with an in-warp reduction (Qwen-Image 2.1 VAE, quality-gated), CUDA.",
),
(
"diffusion.bias_residual_add",
KernelBackend.JIT,
"norm.bias_residual_add_jit:bias_residual_add",
_CUDA,
"Bit-exact conv bias + residual add in one pass for channels_last or NCHW bf16 tensors, CUDA.",
),
(
"diffusion.vdn_delta_factors",
KernelBackend.JIT,
"attention.vdn_delta_factors_jit:vdn_delta_factors",
_CUDA,
"VDN-H3 delta rule: fused (I + A)^-1 -> transition / injection (fp32, head_dim 128).",
),
(
"diffusion.vdn_temporal_conv_act",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_temporal_conv_act",
_CUDA,
"VDN-H3 linear branch: 5-tap temporal conv + SiLU + L2 norm.",
),
(
"diffusion.vdn_silu_l2norm",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_silu_l2norm",
_CUDA,
"VDN-H3 linear branch: SiLU + L2 norm over head_dim.",
),
(
"diffusion.vdn_frame_stats_prep",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_frame_stats_prep",
_CUDA,
"VDN-H3 linear branch: frame-statistics GEMM operands in one pass.",
),
(
"diffusion.vdn_gather_linear_state",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_gather_linear_state",
_CUDA,
"VDN-H3 linear branch: alpha-bridged boundary gather in one pass.",
),
(
"diffusion.vdn_linear_epilogue",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_linear_epilogue",
_CUDA,
"VDN-H3 linear branch: RMSNorm * gate readout epilogue.",
),
(
"diffusion.mxfp8_quantize_swizzled",
KernelBackend.TRITON,
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
_CUDA,
"bf16 -> MXFP8 (e4m3, block-32 E8M0 scales in the cuBLASLt swizzled layout).",
),
(
"diffusion.silu_mul_mxfp8",
KernelBackend.TRITON,
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
_CUDA,
"SwiGLU + MXFP8 quant for the online mxfp8 fc2 input.",
),
(
"diffusion.indexed_scale_shift_mxfp8_",
KernelBackend.TRITON,
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
_CUDA,
"Indexed adaLN modulation + MXFP8 quant for the online mxfp8 qkv/fc1 inputs.",
),
(
"diffusion.group_limited_topk",
KernelBackend.TRITON,
"routing.group_limited_topk_triton:group_limited_topk",
_CUDA,
"LingBot Video group-limited MoE top-k expert selection.",
),
(
"diffusion.usp_merge_heads",
KernelBackend.JIT,
"layout.usp_relayout_jit:usp_merge_heads",
_CUDA,
"USP all-to-all output head merge.",
),
(
"diffusion.pack_qkv_destination_major",
KernelBackend.TRITON,
"layout.ulysses_qkv_triton:pack_qkv_destination_major",
_CUDA,
"Ulysses destination-major QKV pack.",
),
(
"diffusion.varlen_pack_qkv",
KernelBackend.TRITON,
"layout.varlen_pack_pad_triton:fused_pack_qkv",
_CUDA,
"Varlen gather of Q/K/V at valid positions.",
),
(
"diffusion.varlen_pack_segmented_qkv",
KernelBackend.TRITON,
"layout.varlen_pack_pad_triton:fused_pack_segmented_qkv",
_CUDA,
"Varlen gather from a virtual prefix/main Q/K/V sequence.",
),
(
"diffusion.varlen_scatter_to_padded",
KernelBackend.TRITON,
"layout.varlen_pack_pad_triton:fused_scatter_to_padded",
_CUDA,
"Varlen scatter back to the dense layout.",
),
(
"diffusion.causal_conv3d_cat_pad",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit:fused_causal_conv3d_cat_pad_cuda",
_CUDA,
"KDA native CUDA causal Conv3d cat + pad (#29281).",
),
(
"diffusion.causal_conv3d_cat_pad",
KernelBackend.TRITON,
"layout.causal_conv3d_cat_pad_triton:fused_causal_conv3d_cat_pad",
_CUDA,
"Causal Conv3d cat + pad (Triton).",
),
(
"diffusion.cat_pad_channels_last_3d",
KernelBackend.TRITON,
"layout.wan_causal_cache_triton:cat_pad_channels_last_3d",
_CUDA,
"Wan causal VAE cat + pad in channels_last_3d.",
),
(
"diffusion.dup_up3d_add",
KernelBackend.TRITON,
"layout.wan_causal_cache_triton:dup_up3d_add",
_CUDA,
"Wan causal VAE main + DupUp3D(src).",
),
(
"diffusion.conv_bias_epilogue",
KernelBackend.TRITON,
"modulate.conv_bias_epilogue_triton:conv_bias_epilogue",
_CUDA,
"Bit-exact conv bias (+ residual) epilogue on channels_last outputs.",
),
(
"diffusion.nearest_upsample_nhwc",
KernelBackend.TRITON,
"layout.nearest_upsample_nhwc_triton:nearest_upsample_nhwc",
_CUDA,
"Wan-family VAE channels_last integer-factor nearest upsample.",
),
(
"diffusion.flux2_token_cat_nvfp4",
KernelBackend.JIT,
"layout.flux2_token_cat_nvfp4_jit:try_flux2_token_cat_nvfp4",
_CUDA,
"FLUX.2 single-block token concatenation + NVFP4 quantization.",
),
)
for _op, _backend, _target, _caps, _description in _SPECS:
for _op, _backend, _target, _caps in _SPECS:
register_kernel(
KernelSpec(
op=_op,
@@ -707,8 +604,6 @@ for _op, _backend, _target, _caps, _description in _SPECS:
else f"sglang.kernels.ops.diffusion.{_target}"
),
capabilities=_caps,
format_signature=FormatSignature(description=_description),
description=_description,
)
)
@@ -786,6 +681,12 @@ _EXPORTS: dict[str, str] = {
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
"zimage_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
# adaLN modulation, gating and timestep conditioning
"indexed_gate_bf16": "modulate.indexed_modulation_triton",
"indexed_gate_bf16_": "modulate.indexed_modulation_triton",
@@ -906,80 +807,31 @@ _EXPORTS: dict[str, str] = {
"BitExactFusionGate": "sites.bitexact_gate",
"flashinfer_rmsnorm_diagnostic_hint": "sites.bitexact_gate",
"tensors_equal": "sites.bitexact_gate",
"fused_gate_rmsnorm_active": "sites.fused_gate_rmsnorm_site",
"FLUX2_NVFP4_SWIGLU_QUANT": "sites.flux2_nvfp4_swiglu_quant_site",
"FUSED_GATE_RMSNORM": "sites.fused_gate_rmsnorm_site",
"fused_rmsnorm_scale": "sites.fused_gate_rmsnorm_site",
"fused_rmsnorm_tanh_residual": "sites.fused_gate_rmsnorm_site",
"mark_fused_gate_rmsnorm_site": "sites.fused_gate_rmsnorm_site",
"mount_fused_gate_rmsnorm": "sites.fused_gate_rmsnorm_site",
"unmount_fused_gate_rmsnorm": "sites.fused_gate_rmsnorm_site",
"FUSED_LINEAR_GELU": "sites.fused_linear_gelu_site",
"can_use_linear_gelu": "sites.fused_linear_gelu_site",
"fused_gelu_active": "sites.fused_linear_gelu_site",
"fused_linear_gelu_tanh": "sites.fused_linear_gelu_site",
"mark_fused_gelu_site": "sites.fused_linear_gelu_site",
"mount_fused_linear_gelu": "sites.fused_linear_gelu_site",
"unmount_fused_linear_gelu": "sites.fused_linear_gelu_site",
"flux2_nvfp4_swiglu_quant_active": "sites.flux2_nvfp4_swiglu_quant_site",
"mark_flux2_nvfp4_swiglu_quant_site": "sites.flux2_nvfp4_swiglu_quant_site",
"mount_flux2_nvfp4_swiglu_quant": "sites.flux2_nvfp4_swiglu_quant_site",
"unmount_flux2_nvfp4_swiglu_quant": "sites.flux2_nvfp4_swiglu_quant_site",
"mark_nvfp4_bias_gelu_site": "sites.nvfp4_bias_gelu_site",
"mount_nvfp4_bias_gelu": "sites.nvfp4_bias_gelu_site",
"nvfp4_bias_gelu_active": "sites.nvfp4_bias_gelu_site",
"unmount_nvfp4_bias_gelu": "sites.nvfp4_bias_gelu_site",
"mark_qwen_image_added_qkv_site": "sites.qwen_image_added_qkv_site",
"mount_qwen_image_added_qkv": "sites.qwen_image_added_qkv_site",
"qwen_image_added_qkv_active": "sites.qwen_image_added_qkv_site",
"unmount_qwen_image_added_qkv": "sites.qwen_image_added_qkv_site",
"FUSED_LN_MODULATE": "sites.fused_ln_modulate_site",
"can_use_ln_modulate": "sites.fused_ln_modulate_site",
"fused_ln_modulate": "sites.fused_ln_modulate_site",
"fused_ln_modulate_active": "sites.fused_ln_modulate_site",
"mark_fused_ln_modulate_site": "sites.fused_ln_modulate_site",
"mount_fused_ln_modulate": "sites.fused_ln_modulate_site",
"unmount_fused_ln_modulate": "sites.fused_ln_modulate_site",
"mark_hunyuan_qknorm_site": "sites.hunyuan_qknorm_site",
"mount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"HELIOS_GATED_RESIDUAL": "sites.gated_residual_site",
"LINGBOT_VIDEO_GATED_RESIDUAL": "sites.gated_residual_site",
"try_gated_residual": "sites.gated_residual_site",
"HUNYUAN_QKNORM": "sites.hunyuan_qknorm_site",
"try_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"unmount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"ltx2_rms_norm_modulate_active": "sites.ltx2_rmsnorm_modulate_site",
"mark_ltx2_rms_norm_modulate_site": "sites.ltx2_rmsnorm_modulate_site",
"mount_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"unmount_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"ltx2_qknorm_split_rope_active": "sites.ltx2_qknorm_split_rope_site",
"mark_ltx2_qknorm_split_rope_site": "sites.ltx2_qknorm_split_rope_site",
"mount_ltx2_qknorm_split_rope": "sites.ltx2_qknorm_split_rope_site",
"unmount_ltx2_qknorm_split_rope": "sites.ltx2_qknorm_split_rope_site",
"lingbot_video_rmsnorm_active": "sites.lingbot_video_rmsnorm_site",
"mark_lingbot_video_rmsnorm_site": "sites.lingbot_video_rmsnorm_site",
"mount_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
"LINGBOT_VIDEO_RMSNORM": "sites.lingbot_video_rmsnorm_site",
"try_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
"unmount_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
"helios_gated_residual_active": "sites.helios_gated_residual_site",
"mark_helios_gated_residual_site": "sites.helios_gated_residual_site",
"mount_helios_gated_residual": "sites.helios_gated_residual_site",
"try_helios_gated_residual": "sites.helios_gated_residual_site",
"unmount_helios_gated_residual": "sites.helios_gated_residual_site",
"lingbot_video_gated_residual_active": "sites.lingbot_video_gated_residual_site",
"mark_lingbot_video_gated_residual_site": "sites.lingbot_video_gated_residual_site",
"mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
"mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site",
"minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site",
"mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
"unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_qk_rmsnorm_rope": "norm.h3_vae_fused_norm",
"mark_sana_video_linear_attention_site": "sites.sana_video_linear_attention_site",
"mount_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
"sana_video_linear_attention_active": "sites.sana_video_linear_attention_site",
"LTX2_QKNORM_SPLIT_ROPE": "sites.ltx2_qknorm_split_rope_site",
"LTX2_RMS_NORM_MODULATE": "sites.ltx2_rmsnorm_modulate_site",
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"MINIMAX_H3_NORM_MODULATE": "sites.minimax_h3_norm_modulate_site",
"NVFP4_BIAS_GELU": "sites.nvfp4_bias_gelu_site",
"QWEN_IMAGE_ADDED_QKV": "sites.qwen_image_added_qkv_site",
"SANA_VIDEO_LINEAR_ATTENTION": "sites.sana_video_linear_attention_site",
"try_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
"unmount_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
"QualityGatedFusion": "sites.quality_gate",
# JIT C++/CUDA extensions (not kernels, not in the registry)
"interpolate": "ext.hunyuan3d_rasterizer",
@@ -166,7 +166,7 @@ _CHUNKWISE_TUNING: dict[tuple[str, str], _ChunkwiseCfg] = {
# nw=16 OOMs the 102 KB SRAM cap at TF32 (needs 131 KB); nw=8 fits and is
# within noise. The D-tile path (auto-enabled on spark, see
# `_pick_phase_b_d_splits`) is ~2.6× faster at TF32 / ~13% at IEEE — these
# baseline params only apply when PHASE_B_D_SPLITS=1 is forced.
# baseline params only apply where that picks d_splits=1.
B=_PhaseCfg(nw=8, use_acc=False, ns=1),
C=_PhaseCfg(nw=8, BS=16, ns=1), # binding constraint: M.fp32 64 KB + Q stage
),
@@ -1206,20 +1206,7 @@ def _pick_phase_b_d_splits(BLOCK_D: int, dot_precision: int = 0):
`d_splits>1` → `_phase_b_dtile_kernel` with nw/ns/acc overrides.
Per-arch optima from a 96-config sweep (2026-04-29, T=11 B=1 IEEE) and a
multi-arch bf16/TF32 sweep (2026-05-06, F=11 S=920); see inline cfg comments.
Env overrides: PHASE_B_D_SPLITS, PHASE_B_DTILE_NW/NS/ACC (1=True / 0=False).
"""
import os
env_d = os.environ.get("PHASE_B_D_SPLITS", None)
if env_d is not None:
d = int(env_d)
if d < 1 or BLOCK_D % d != 0:
return (1, None, None, None)
nw = int(os.environ.get("PHASE_B_DTILE_NW", "0")) or None
ns = int(os.environ.get("PHASE_B_DTILE_NS", "0")) or None
acc_env = os.environ.get("PHASE_B_DTILE_ACC", None)
acc = bool(int(acc_env)) if acc_env is not None else None
return (d, nw, ns, acc)
try:
import torch
@@ -1617,18 +1604,6 @@ def fused_bigdn_bidi_chunkwise(
return out
def _default_dot_prec() -> int:
try:
from sglang.kernels.ops.diffusion.attention.sana_wm_gdn_triton import (
_resolve_launch_config,
)
_, dot_prec, _, _ = _resolve_launch_config()
return dot_prec
except Exception:
return 0
def _cam_identity_tables(
*,
B: int,
@@ -1683,7 +1658,7 @@ def cam_scan_bidi_chunkwise(
assert decay.shape == (B, H, F)
if dot_precision is None:
dot_precision = _default_dot_prec()
dot_precision = 0
qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype)
qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2))
@@ -14,85 +14,16 @@
#
# SPDX-License-Identifier: Apache-2.0
"""Inference-side helpers for the bidirectional fused GDN path.
Precision knob: env var ``FUSED_GDN_PRECISION`` or ``PRECISION_OVERRIDE``:
0=IEEE fp32 dots, 1=TF32, 2=bf16 TC + fp32 state [default], 3=bf16 TC + bf16 state.
"""
"""Inference-side helpers for the bidirectional fused GDN path."""
# ruff: noqa: E501
from __future__ import annotations
import os
import torch
import triton
import triton.language as tl
# =====================================================================
# GPU-adaptive kernel config
# =====================================================================
def _get_kernel_config() -> dict:
"""Return optimal kernel parameters for the current GPU.
STATE_FP32 (fp32 state_prev) needs ~128KB SRAM (H100 228KB), vs ~96KB for
bf16 state_prev (fits GB10's 101KB).
"""
if not torch.cuda.is_available():
return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 4, "STATE_FP32": False}
smem = torch.cuda.get_device_properties(0).shared_memory_per_multiprocessor
state_fp32 = smem >= 150 * 1024 # H100 (228KB) yes, GB10 (101KB) no
return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 8, "STATE_FP32": state_fp32}
_KCFG = None
def _kcfg():
global _KCFG
if _KCFG is None:
_KCFG = _get_kernel_config()
return _KCFG
# precision=0 → IEEE fp32 dots + fp32 state (DOT_PRECISION=2, STATE_FP32=1)
# precision=1 → TF32 dots + fp32 state (DOT_PRECISION=1, STATE_FP32=1)
# precision=2 → bf16 dots + fp32 state (DOT_PRECISION=0, STATE_FP32=1) [default]
# precision=3 → bf16 dots + bf16 state (DOT_PRECISION=0, STATE_FP32=0)
def _precision_params(precision: int) -> tuple:
if precision == 0:
return 2, True
elif precision == 1:
return 1, True
elif precision == 3:
return 0, False
else: # default
return 0, True
_env_prec = os.environ.get("FUSED_GDN_PRECISION", None)
PRECISION_OVERRIDE: int | None = int(_env_prec) if _env_prec is not None else None
def _resolve_launch_config() -> tuple:
"""Returns (prec, dot_prec, state_fp32, num_warps).
Uses ``PRECISION_OVERRIDE`` when set, else ``_kcfg()`` (per-GPU SRAM).
num_warps clamped to 4 when dots run on fp32 operands (more registers).
"""
cfg = _kcfg()
prec = PRECISION_OVERRIDE if PRECISION_OVERRIDE is not None else 2
dot_prec, state_fp32 = _precision_params(prec)
if PRECISION_OVERRIDE is None:
state_fp32 = cfg["STATE_FP32"]
nw = cfg["num_warps"]
if dot_prec >= 1:
nw = min(nw, 4)
return prec, dot_prec, state_fp32, nw
def prepare_rope_tables(
rotary_emb, N: int, D: int, device
@@ -33,11 +33,16 @@ from typing import Any, Callable
import torch
from torch.utils._python_dispatch import TorchDispatchMode
from torch.utils._pytree import tree_leaves
from triton import knobs
from triton.knobs import HookChain
from triton.runtime import driver
from triton.runtime.jit import JITFunction
try:
from triton import knobs
from triton.knobs import HookChain
except ImportError:
# Vendor Triton builds may not expose the upstream launch-hook API.
knobs = None
logger = logging.getLogger(__name__)
_ENABLED = os.environ.get(
@@ -162,6 +167,9 @@ class _Plan:
def _launch_hooked() -> bool:
"""A profiler such as proton hooks Triton launches; those go through the runner."""
if knobs is None:
# Keep the regular runner when its hooks cannot be inspected safely.
return True
return any(
hook is not None and not (isinstance(hook, HookChain) and not hook.calls)
for hook in (knobs.runtime.launch_enter_hook, knobs.runtime.launch_exit_hook)
@@ -27,8 +27,6 @@ from typing import Callable, TypeVar
F = TypeVar("F", bound=Callable)
_CUDA_LIKE = frozenset({"cuda", "hip"})
def platform_key() -> str:
"""Return the live device family: ``cuda``/``hip``/``npu``/``mps``/``musa``/``xpu``/``cpu``.
@@ -72,11 +70,6 @@ def is_xpu() -> bool:
return current_platform.is_xpu()
def has_triton() -> bool:
"""True when the live device runs the Triton implementations."""
return platform_key() in _CUDA_LIKE
def lazy_fallback(kind: str, name: str) -> Callable:
"""Name a fallback without importing its module.
@@ -7,9 +7,6 @@ promotes the residual to FP32, so later layers carry an FP32 residual and an
FP16 attention/FFN update. ``h3_vae_scale_add_rmsnorm`` does the add and the
next RMSNorm in one launch: the sum stays in registers for the variance, and
the stored residual matches the eager FP32 add.
``h3_vae_qk_rmsnorm_rope`` is the affine-free head-64 path: RMSNorm, round to
the activation dtype, then partial NeoX RoPE on the first 48 channels.
"""
from __future__ import annotations
@@ -79,59 +76,6 @@ def _scale_add_rmsnorm_kernel(
tl.store(out_ptr + row * D + cols, acc * rstd * weight, mask=mask)
@triton.jit
def _qk_rmsnorm_rope_kernel(
q_ptr,
k_ptr,
qo_ptr,
ko_ptr,
cos_ptr,
sin_ptr,
stride_qb,
stride_qs,
stride_qh,
stride_kb,
stride_ks,
stride_kh,
seq_len,
n_heads,
eps,
):
row = tl.program_id(0).to(tl.int64)
heads = n_heads
token = row // heads
head = row % heads
seq = token % seq_len
batch = token // seq_len
cols = tl.arange(0, 64)
rope_mask = cols < 48
q_base = q_ptr + batch * stride_qb + seq * stride_qs + head * stride_qh
k_base = k_ptr + batch * stride_kb + seq * stride_ks + head * stride_kh
q = tl.load(q_base + cols).to(tl.float32)
k = tl.load(k_base + cols).to(tl.float32)
q = (q * tl.rsqrt(tl.sum(q * q, axis=0) / 64 + eps)).to(tl.float16)
k = (k * tl.rsqrt(tl.sum(k * k, axis=0) / 64 + eps)).to(tl.float16)
cos = tl.load(cos_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
sin = tl.load(sin_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
# NeoX rotate-half: [-x2, x1] on the first 48 channels.
src = tl.where(cols < 24, cols + 24, cols - 24)
sign = tl.where(cols < 24, -1.0, 1.0)
q_rot = (tl.gather(q.to(tl.float32), src, 0) * sign).to(tl.float16)
k_rot = (tl.gather(k.to(tl.float32), src, 0) * sign).to(tl.float16)
yq = (
(q.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
+ (q_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
).to(tl.float16)
yk = (
(k.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
+ (k_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
).to(tl.float16)
yq = tl.where(rope_mask, yq, q)
yk = tl.where(rope_mask, yk, k)
tl.store(qo_ptr + row * 64 + cols, yq)
tl.store(ko_ptr + row * 64 + cols, yk)
def _out_kind(dtype: torch.dtype) -> int:
if dtype == torch.float32:
return 0
@@ -239,61 +183,3 @@ def h3_vae_scale_add_rmsnorm(
if inplace:
return residual, normed.view(out_shape)
return residual_out, normed.view(out_shape)
def h3_vae_qk_rmsnorm_rope(
query: torch.Tensor,
key: torch.Tensor,
eps: float,
cos: torch.Tensor,
sin: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor] | None:
"""Affine-free RMSNorm plus partial NeoX RoPE for head 64 / rotary 48.
Returns None when the tensors are outside that contract so the caller can
keep the unfused norm and RoPE.
"""
if (
not query.is_cuda
or query.shape != key.shape
or query.dtype != key.dtype
or query.dtype != torch.float16
or query.dim() != 4
or query.shape[-1] != 64
or query.stride(-1) != 1
or key.stride(-1) != 1
or cos.shape != sin.shape
or cos.dim() != 2
or cos.shape[-1] != 48
or cos.shape[0] != query.shape[1]
or cos.device != query.device
or sin.device != query.device
or key.device != query.device
):
return None
batch, seq_len, n_heads, _ = query.shape
rows = batch * seq_len * n_heads
if rows == 0:
return None
query_out = torch.empty(rows, 64, device=query.device, dtype=query.dtype)
key_out = torch.empty(rows, 64, device=key.device, dtype=key.dtype)
_qk_rmsnorm_rope_kernel[(rows,)](
query,
key,
query_out,
key_out,
cos.contiguous(),
sin.contiguous(),
query.stride(0),
query.stride(1),
query.stride(2),
key.stride(0),
key.stride(1),
key.stride(2),
seq_len,
n_heads,
float(eps),
num_warps=1,
)
view = (batch, seq_len, n_heads, 64)
return query_out.view(view), key_out.view(view)
@@ -79,9 +79,9 @@ class BitExactFusionGate:
The first-sight check runs the eager reference chain plus a host sync, so
it must never happen inside ``torch.compile`` tracing or CUDA graph
capture. Once-for-all callers get both guards from
:meth:`can_attempt_once`; per-signature callers must keep their own
compile/capture checks next to the signature lookup (see FLUX / Sana).
capture. :meth:`run` applies both guards in either mode; hand-written
sites get them from :meth:`can_attempt_once` (once-for-all) or keep their
own compile/capture checks next to the signature lookup (FLUX / Sana).
"""
__slots__ = ("name", "disabled", "verified", "verified_sigs")
@@ -114,14 +114,49 @@ class BitExactFusionGate:
return False
if self.verified:
return True
# First-sight verify runs the eager reference chain and a host sync:
# attempt neither inside compile tracing nor CUDA graph capture (the
# sync would abort the capture; BCG then blocks the signature). Once
# verified, the fused kernel runs alone and is compile/capture-safe.
if torch.compiler.is_compiling():
return False
return not (
torch.cuda.is_available() and torch.cuda.is_current_stream_capturing()
return _can_verify_now()
def run(
self,
fused: Callable[[], T],
reference: Callable[[], T],
*,
sig: Any = None,
equal: EqualFn | None = None,
logger: logging.Logger | None = None,
diagnostic_hint: DiagnosticHintFn | None = None,
reraise: tuple[type[Exception], ...] = (),
) -> T:
"""Return ``fused()`` once it has matched ``reference()`` bit-for-bit.
The caller has already checked that the fast path applies. An
unverified ``sig`` is verified on this call unless it is inside compile
tracing or CUDA graph capture, where ``reference()`` runs instead. A
launch failure (other than ``reraise``) or a mismatch disables the
gate and returns ``reference()``.
"""
if self.disabled:
return reference()
sigs = self.verified_sigs
verified = self.verified if sigs is None else sig in sigs
if not verified and not _can_verify_now():
return reference()
try:
out = fused()
except reraise:
raise
except Exception as exc:
self.on_exception(exc, logger=logger)
return reference()
if verified:
return out
return self.accept_or_fallback(
out,
reference(),
sig=sig,
equal=equal,
logger=logger,
diagnostic_hint=diagnostic_hint,
)
def on_exception(
@@ -145,7 +180,6 @@ class BitExactFusionGate:
sig: Any = None,
equal: EqualFn | None = None,
logger: logging.Logger | None = None,
mismatch_msg: str | None = None,
diagnostic_hint: DiagnosticHintFn | None = None,
) -> T:
"""Return ``out`` when bit-exact; otherwise disable and return ``ref``."""
@@ -156,10 +190,6 @@ class BitExactFusionGate:
self.mark_verified(sig)
return out
if logger is not None:
message = mismatch_msg or (
f"{self.name} fast path is not bit-exact against this "
"platform's reference dispatch; falling back to eager"
)
details = (
"Correctness is preserved because the eager reference output "
"is used. A platform-specific reference kernel or reduction-order "
@@ -172,11 +202,24 @@ class BitExactFusionGate:
diagnostic_details = None
if diagnostic_details:
details = f"{details}. {diagnostic_details.rstrip('.')}"
logger.warning_once(f"{message.rstrip('.')}. {details}.")
logger.warning_once(
f"{self.name} fast path is not bit-exact against this platform's "
f"reference dispatch; falling back to eager. {details}."
)
self.disable()
return ref
def _can_verify_now() -> bool:
# First-sight verify runs the eager reference chain and a host sync:
# attempt neither inside compile tracing nor CUDA graph capture (the
# sync would abort the capture; BCG then blocks the signature). Once
# verified, the fused kernel runs alone and is compile/capture-safe.
if torch.compiler.is_compiling():
return False
return not (torch.cuda.is_available() and torch.cuda.is_current_stream_capturing())
def tensors_equal(a: Any, b: Any) -> bool:
"""``torch.equal`` for a tensor or a sequence of tensors."""
if isinstance(a, torch.Tensor):
@@ -8,29 +8,10 @@ Keep it disabled below the approximate tier and mount it only for
from __future__ import annotations
from torch import nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
_FUSION = QualityGatedFusion(
FLUX2_NVFP4_SWIGLU_QUANT = QualityGatedFusion(
name="FLUX.2 NVFP4 FC1+SwiGLU+quant",
marker_attr="_sgl_flux2_nvfp4_swiglu_quant_site",
enabled_attr="_sgl_flux2_nvfp4_swiglu_quant_enabled",
)
def mark_flux2_nvfp4_swiglu_quant_site(module: nn.Module) -> None:
"""Mark an eligible FLUX.2 feed-forward site, disabled by default."""
_FUSION.mark(module)
def flux2_nvfp4_swiglu_quant_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_flux2_nvfp4_swiglu_quant(root: nn.Module) -> bool:
return _FUSION.mount(root)
def unmount_flux2_nvfp4_swiglu_quant(root: nn.Module) -> None:
_FUSION.unmount(root)
@@ -12,9 +12,8 @@ native-bf16 reference RMSNorm. Ideogram's reference norm is ``F.rms_norm``
(fp32 internal statistics), so the fused path is numerically close (the norm
statistics round through bf16) but **not bit-exact**. Following the
fused-linear-GELU precedent, sites are therefore mounted only for
``quality="high"`` requests via :func:`mount_fused_gate_rmsnorm` /
:func:`unmount_fused_gate_rmsnorm` at batch boundaries; the default
``"exact"`` path keeps the unmodified reference chain bit-for-bit.
``quality="high"`` requests at batch boundaries; the default ``"exact"`` path
keeps the unmodified reference chain bit-for-bit.
Mounting is all-or-nothing per transformer: if any marked site fails the
static guards (non-bf16 norm weight, hidden size above the kernel limit, ...)
@@ -23,7 +22,6 @@ every site on that transformer stays on the reference path.
from __future__ import annotations
import logging
from importlib import import_module
import torch
@@ -31,17 +29,6 @@ import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
# Attributes of the site protocol (set by ``mark_fused_gate_rmsnorm_site``).
_SITE_NORM_ATTRS = "_sgl_fused_gate_rmsnorm_norm_attrs"
_SITE_ENABLED_ATTR = "_sgl_fused_gate_rmsnorm_enabled"
_FUSION = QualityGatedFusion(
name="fused gate RMSNorm",
marker_attr=_SITE_NORM_ATTRS,
enabled_attr=_SITE_ENABLED_ATTR,
)
# The Triton kernels mask a single block over the hidden dim.
_MAX_HIDDEN_SIZE = 8192
@@ -78,7 +65,7 @@ def _static_reject_reason(site: nn.Module) -> str | None:
import_module("triton")
except ImportError:
return "triton unavailable"
for attr in _FUSION.metadata(site, ()):
for attr in FUSED_GATE_RMSNORM.metadata(site, ()):
norm = getattr(site, attr, None)
weight = getattr(norm, "weight", None)
if weight is None or weight.dim() != 1:
@@ -90,33 +77,10 @@ def _static_reject_reason(site: nn.Module) -> str | None:
return None
def mark_fused_gate_rmsnorm_site(module: nn.Module, norm_attrs: tuple[str, ...]):
"""Declare ``module`` as a fused RMSNorm modulate/gate site.
``norm_attrs`` names the site's RMSNorm submodules (checked by the static
guards at mount time). The site starts unmounted
(``_sgl_fused_gate_rmsnorm_enabled = False``): the module's forward must
keep the reference path bit-exact until :func:`mount_fused_gate_rmsnorm`
enables it.
"""
_FUSION.mark(module, tuple(norm_attrs))
def fused_gate_rmsnorm_active(module: nn.Module) -> bool:
"""Whether the quality-gated fused path is mounted on ``module``."""
return _FUSION.is_enabled(module)
def mount_fused_gate_rmsnorm(root: nn.Module) -> bool:
"""Enable the fused kernels on every marked site under ``root``.
All-or-nothing: if any marked site fails the static guards, every site is
left (or reset) on the reference path and False is returned. Returns False
as well when ``root`` has no marked sites.
"""
return _FUSION.mount(root, reject_reason=_static_reject_reason, logger=logger)
def unmount_fused_gate_rmsnorm(root: nn.Module) -> None:
"""Reset every marked site under ``root`` to the bit-exact reference path."""
_FUSION.unmount(root)
# A site's marker metadata is the tuple of its RMSNorm submodule names.
FUSED_GATE_RMSNORM = QualityGatedFusion(
name="fused gate RMSNorm",
marker_attr="_sgl_fused_gate_rmsnorm_norm_attrs",
enabled_attr="_sgl_fused_gate_rmsnorm_enabled",
reject_reason=_static_reject_reason,
)
@@ -11,10 +11,9 @@ reference only at bf16/fp16 rounding-order level -- close, but not bit-exact.
Because it is not bit-exact, the fused path is **mounted only for
``quality="high"`` requests** (see ``SamplingParams.quality``): model code marks
its GELU up-projection sites with :func:`mark_fused_gelu_site` (default: off,
reference path, bit-exact), and the denoising stage calls
:func:`mount_fused_linear_gelu` / :func:`unmount_fused_linear_gelu` at batch
boundaries. Mounting is all-or-nothing per transformer: if any marked site
its GELU up-projection sites with ``FUSED_LINEAR_GELU.mark`` (default: off,
reference path, bit-exact), and the denoising stage mounts / unmounts
``FUSED_LINEAR_GELU`` at batch boundaries. Mounting is all-or-nothing per transformer: if any marked site
fails the static guards (quantized weights, missing bias, non-half dtype, ...)
no site on that transformer is fused.
@@ -25,7 +24,6 @@ single opaque op under ``torch.compile`` -- no graph break.
from __future__ import annotations
import logging
import weakref
from typing import Any
@@ -36,22 +34,11 @@ from sglang.kernels.jit.utils import get_jit_cuda_arch
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
from sglang.srt.utils.custom_op import register_custom_op
logger = logging.getLogger(__name__)
# ``torch._addmm_activation`` is the (private but stable) entry point to the
# cublasLt GEMM+bias+activation epilogue. Guard for builds where it is absent
# so the reference path is always available.
_HAS_ADDMM_ACTIVATION = hasattr(torch, "_addmm_activation")
# Attributes of the site protocol (set by ``mark_fused_gelu_site``).
_SITE_LINEAR_ATTR = "_sgl_fused_gelu_linear_attr"
_SITE_ENABLED_ATTR = "_sgl_fused_gelu_enabled"
_FUSION = QualityGatedFusion(
name="fused linear+GELU",
marker_attr=_SITE_LINEAR_ATTR,
enabled_attr=_SITE_ENABLED_ATTR,
)
def _fused_linear_gelu_tanh_fake(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor
@@ -157,39 +144,26 @@ def can_use_linear_gelu(linear: Any, x: torch.Tensor) -> bool:
return verdict
def mark_fused_gelu_site(module: nn.Module, linear_attr: str) -> None:
"""Declare ``module`` as a tanh-GELU up-projection fusion site.
``getattr(module, linear_attr)`` must be the up-projection linear whose
output feeds ``F.gelu(..., approximate="tanh")``. The site starts unmounted
(``_sgl_fused_gelu_enabled = False``): the module's forward must keep the
reference path bit-exact until :func:`mount_fused_linear_gelu` enables it.
"""
_FUSION.mark(module, linear_attr)
def fused_gelu_active(module: nn.Module) -> bool:
"""Whether the quality-gated fused path is mounted on ``module``."""
return _FUSION.is_enabled(module)
def _site_reject_reason(site: nn.Module) -> str | None:
linear = getattr(site, _FUSION.metadata(site), None)
linear = getattr(site, FUSED_LINEAR_GELU.metadata(site), None)
return "missing linear" if linear is None else _static_reject_reason(linear)
def mount_fused_linear_gelu(root: nn.Module) -> bool:
"""Enable the fused epilogue on every marked site under ``root``.
class _LinearGeluFusion(QualityGatedFusion):
def mount(self, root: nn.Module) -> bool:
_VERDICTS.clear()
return super().mount(root)
All-or-nothing: if any marked site fails the static guards, every site is
left (or reset) on the reference path and False is returned. Returns False
as well when ``root`` has no marked sites.
"""
_VERDICTS.clear()
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
def unmount(self, root: nn.Module) -> None:
_VERDICTS.clear()
super().unmount(root)
def unmount_fused_linear_gelu(root: nn.Module) -> None:
"""Reset every marked site under ``root`` to the bit-exact reference path."""
_VERDICTS.clear()
_FUSION.unmount(root)
# A site's marker metadata names its up-projection linear, whose output feeds
# ``F.gelu(..., approximate="tanh")``.
FUSED_LINEAR_GELU = _LinearGeluFusion(
name="fused linear+GELU",
marker_attr="_sgl_fused_gelu_linear_attr",
enabled_attr="_sgl_fused_gelu_enabled",
reject_reason=_site_reject_reason,
)
@@ -8,48 +8,27 @@ the result is not bit-exact vs the reference (half-precision rounding-order
differences only).
Because it is not bit-exact the fold is opt-in per batch: model code marks
its LN+modulate sites with :func:`mark_fused_ln_modulate_site` (default off,
reference path), and the denoising stage calls
:func:`mount_fused_ln_modulate` / :func:`unmount_fused_ln_modulate` at batch
boundaries for ``quality="high"`` requests.
its LN+modulate sites with ``FUSED_LN_MODULATE.mark`` (default off,
reference path), and the denoising stage mounts / unmounts
``FUSED_LN_MODULATE`` at batch boundaries for ``quality="high"`` requests.
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
from torch import nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
_SITE_MARKER_ATTR = "_sgl_fused_ln_modulate_site"
_SITE_ENABLED_ATTR = "_sgl_fused_ln_modulate_enabled"
_FUSION = QualityGatedFusion(
FUSED_LN_MODULATE = QualityGatedFusion(
name="fused LN+modulate",
marker_attr=_SITE_MARKER_ATTR,
enabled_attr=_SITE_ENABLED_ATTR,
marker_attr="_sgl_fused_ln_modulate_site",
enabled_attr="_sgl_fused_ln_modulate_enabled",
)
_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
def mark_fused_ln_modulate_site(module: nn.Module) -> None:
"""Mark ``module`` as an LN+modulate fusion site (mounted off)."""
_FUSION.mark(module)
def fused_ln_modulate_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_fused_ln_modulate(root: nn.Module) -> bool:
return _FUSION.mount(root)
def unmount_fused_ln_modulate(root: nn.Module) -> None:
_FUSION.unmount(root)
def can_use_ln_modulate(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> bool:
@@ -1,8 +1,8 @@
"""LingBot Video per-token gated-residual fusion, gated by request quality.
"""Per-token gated-residual fusion for Helios and LingBot Video, gated by quality.
Each LingBot block applies ``residual + (gate * update).to(residual.dtype)`` at
the attention and FFN updates, where ``update`` (post-norm) and the per-token
``gate`` (``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. The shared
Each block applies ``residual + (gate * update).to(residual.dtype)`` at its
attention and FFN updates, where ``update`` and the per-token ``gate``
(``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. The shared
``residual_gate_add`` kernel computes ``residual + update * gate`` in a single
pass but requires one dtype, so the gate and update are first cast to BF16.
That reordering of the FP32 multiply is numerically equivalent only at
@@ -13,51 +13,36 @@ half-precision rounding level (not bit-exact), so the fusion is opt-in:
from __future__ import annotations
import logging
import torch
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
HELIOS_GATED_RESIDUAL = QualityGatedFusion(
name="Helios per-token gated residual",
marker_attr="_sgl_helios_gated_residual_site",
enabled_attr="_sgl_helios_gated_residual_enabled",
)
LINGBOT_VIDEO_GATED_RESIDUAL = QualityGatedFusion(
name="LingBot Video per-token gated residual",
marker_attr="_sgl_lingbot_video_gated_residual_site",
enabled_attr="_sgl_lingbot_video_gated_residual_enabled",
)
def mark_lingbot_video_gated_residual_site(module: nn.Module) -> None:
"""Mark a LingBot block; it starts on the reference path."""
_FUSION.mark(module)
def lingbot_video_gated_residual_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_lingbot_video_gated_residual(root: nn.Module) -> bool:
return _FUSION.mount(root, logger=logger)
def unmount_lingbot_video_gated_residual(root: nn.Module) -> None:
_FUSION.unmount(root)
def try_lingbot_video_gated_residual(
def try_gated_residual(
fusion: QualityGatedFusion,
site: nn.Module,
residual: torch.Tensor,
update: torch.Tensor,
gate: torch.Tensor,
) -> torch.Tensor | None:
"""Return the fused ``residual + update * gate`` when the site is enabled.
"""Return the fused ``residual + update * gate`` when ``site`` is enabled.
Returns ``None`` (caller runs the reference path) when the site is off or
the tensors are not eligible for the per-token fast path.
"""
if not _FUSION.is_enabled(site):
if not fusion.is_enabled(site):
return None
from sglang.kernels.ops.diffusion import (
can_use_residual_gate_add_cuda,
@@ -1,72 +0,0 @@
"""Helios per-token gated-residual fusion, gated by request quality.
Each Helios block applies ``residual + (gate * update).to(residual.dtype)`` at
the self-attention and FFN updates, where ``update`` (post-attn / post-FFN) and
the per-token ``gate`` (``[B, S, 1]``) stay in FP32 while ``residual`` is BF16.
The shared ``residual_gate_add`` kernel computes ``residual + update * gate``
in a single pass but requires one dtype, so the gate and update are first cast
to BF16. That reordering of the FP32 multiply is numerically equivalent only at
half-precision rounding level (not bit-exact), so the fusion is opt-in:
``quality="lossless"`` and ``quality="high"`` mount it, while the default
``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit.
"""
from __future__ import annotations
import logging
import torch
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="Helios per-token gated residual",
marker_attr="_sgl_helios_gated_residual_site",
enabled_attr="_sgl_helios_gated_residual_enabled",
)
def mark_helios_gated_residual_site(module: nn.Module) -> None:
"""Mark a Helios block; it starts on the reference path."""
_FUSION.mark(module)
def helios_gated_residual_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_helios_gated_residual(root: nn.Module) -> bool:
return _FUSION.mount(root, logger=logger)
def unmount_helios_gated_residual(root: nn.Module) -> None:
_FUSION.unmount(root)
def try_helios_gated_residual(
site: nn.Module,
residual: torch.Tensor,
update: torch.Tensor,
gate: torch.Tensor,
) -> torch.Tensor | None:
"""Return the fused ``residual + update * gate`` when the site is enabled.
Returns ``None`` (caller runs the reference path) when the site is off or
the tensors are not eligible for the per-token fast path.
"""
if not _FUSION.is_enabled(site):
return None
from sglang.kernels.ops.diffusion import (
can_use_residual_gate_add_cuda,
residual_gate_add,
)
if residual.dtype != update.dtype or residual.dtype != gate.dtype:
update = update.to(residual.dtype)
gate = gate.to(residual.dtype)
if not can_use_residual_gate_add_cuda(residual, update, gate):
return None
return residual_gate_add(residual, update, gate)
@@ -2,7 +2,6 @@
from __future__ import annotations
import logging
from functools import cache
import torch
@@ -10,14 +9,6 @@ import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="HunyuanVideo strided QK RMSNorm",
marker_attr="_sgl_hunyuan_qknorm_site",
enabled_attr="_sgl_hunyuan_qknorm_enabled",
)
@cache
def _get_qk_rmsnorm_cute():
@@ -30,26 +21,18 @@ def _get_qk_rmsnorm_cute():
return qk_rmsnorm_cute
def mark_hunyuan_qknorm_site(module: nn.Module) -> None:
_FUSION.mark(module)
def _site_reject_reason(_site: nn.Module) -> str | None:
if _get_qk_rmsnorm_cute() is None:
return "FlashInfer CuTe QK RMSNorm unavailable"
return None
def mount_hunyuan_qknorm(root: nn.Module) -> bool:
return _FUSION.mount(
root,
reject_reason=_site_reject_reason,
logger=logger,
)
def unmount_hunyuan_qknorm(root: nn.Module) -> None:
_FUSION.unmount(root)
HUNYUAN_QKNORM = QualityGatedFusion(
name="HunyuanVideo strided QK RMSNorm",
marker_attr="_sgl_hunyuan_qknorm_site",
enabled_attr="_sgl_hunyuan_qknorm_enabled",
reject_reason=_site_reject_reason,
)
def try_hunyuan_qknorm(
@@ -62,7 +45,7 @@ def try_hunyuan_qknorm(
) -> tuple[torch.Tensor, torch.Tensor] | None:
"""Normalize strided Hunyuan Q/K views without materializing inputs."""
if not (
_FUSION.is_enabled(site)
HUNYUAN_QKNORM.is_enabled(site)
and not torch.compiler.is_compiling()
and q.is_cuda
and q.dtype == torch.bfloat16
@@ -13,7 +13,6 @@ remains unchanged.
from __future__ import annotations
import logging
from importlib import import_module
import torch
@@ -21,23 +20,6 @@ import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="LingBot Video fused RMSNorm",
marker_attr="_sgl_lingbot_video_rmsnorm_site",
enabled_attr="_sgl_lingbot_video_rmsnorm_enabled",
)
def mark_lingbot_video_rmsnorm_site(module: nn.Module) -> None:
"""Mark a LingBot RMSNorm module; it starts on the reference path."""
_FUSION.mark(module)
def lingbot_video_rmsnorm_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def _site_reject_reason(site: nn.Module) -> str | None:
try:
@@ -54,12 +36,12 @@ def _site_reject_reason(site: nn.Module) -> str | None:
return None
def mount_lingbot_video_rmsnorm(root: nn.Module) -> bool:
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
def unmount_lingbot_video_rmsnorm(root: nn.Module) -> None:
_FUSION.unmount(root)
LINGBOT_VIDEO_RMSNORM = QualityGatedFusion(
name="LingBot Video fused RMSNorm",
marker_attr="_sgl_lingbot_video_rmsnorm_site",
enabled_attr="_sgl_lingbot_video_rmsnorm_enabled",
reject_reason=_site_reject_reason,
)
def try_lingbot_video_rmsnorm(
@@ -70,7 +52,7 @@ def try_lingbot_video_rmsnorm(
) -> torch.Tensor | None:
"""Return the quality-gated RMSNorm result, or ``None`` to fall back."""
if not (
_FUSION.is_enabled(site)
LINGBOT_VIDEO_RMSNORM.is_enabled(site)
and hidden_states.is_cuda
and hidden_states.dtype in (torch.float16, torch.bfloat16)
and hidden_states.is_contiguous()
@@ -8,28 +8,10 @@ kernel fusions.
from __future__ import annotations
from torch import nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
_FUSION = QualityGatedFusion(
LTX2_QKNORM_SPLIT_ROPE = QualityGatedFusion(
name="LTX-2 Hopper QKNorm+split-RoPE",
marker_attr="_sgl_ltx2_qknorm_split_rope_site",
enabled_attr="_sgl_ltx2_qknorm_split_rope_enabled",
)
def mark_ltx2_qknorm_split_rope_site(module: nn.Module) -> None:
_FUSION.mark(module)
def ltx2_qknorm_split_rope_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_ltx2_qknorm_split_rope(root: nn.Module) -> bool:
return _FUSION.mount(root)
def unmount_ltx2_qknorm_split_rope(root: nn.Module) -> None:
_FUSION.unmount(root)
@@ -10,51 +10,30 @@ The fused kernel reproduces the RMSNorm math via ``rsqrt.approx`` (the
flashinfer CuTe form), which differs from aten's refined ``rsqrtf`` by at
most one bf16 ULP on a small fraction of elements. It is therefore *not*
bit-exact vs the eager reference, so the fold is opt-in per batch: model code
marks its adaLN sites with :func:`mark_ltx2_rms_norm_modulate_site` (default
off, reference path) and the denoising stage calls
:func:`mount_ltx2_rms_norm_modulate` / :func:`unmount_ltx2_rms_norm_modulate`
at batch boundaries for ``quality="high"`` requests.
marks its adaLN sites with ``LTX2_RMS_NORM_MODULATE.mark`` (default off,
reference path) and the denoising stage mounts / unmounts
``LTX2_RMS_NORM_MODULATE`` at batch boundaries for ``quality="high"`` requests.
"""
from __future__ import annotations
import torch
from torch import nn
from sglang.kernels.ops.diffusion.norm.rmsnorm_scale_shift_bitexact import (
fused_rmsnorm_scale_shift_bitexact,
)
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
_SITE_MARKER_ATTR = "_sgl_ltx2_rms_norm_modulate_site"
_SITE_ENABLED_ATTR = "_sgl_ltx2_rms_norm_modulate_enabled"
_FUSION = QualityGatedFusion(
LTX2_RMS_NORM_MODULATE = QualityGatedFusion(
name="LTX-2 RMSNorm+modulate",
marker_attr=_SITE_MARKER_ATTR,
enabled_attr=_SITE_ENABLED_ATTR,
marker_attr="_sgl_ltx2_rms_norm_modulate_site",
enabled_attr="_sgl_ltx2_rms_norm_modulate_enabled",
)
# ``RMSNormNoWeight`` applies no scale, so a ones weight reproduces it exactly.
_ONES_WEIGHT_CACHE: dict[tuple[torch.device, int], torch.Tensor] = {}
def mark_ltx2_rms_norm_modulate_site(module: nn.Module) -> None:
"""Mark ``module`` as an LTX-2 RMSNorm+modulate fusion site (mounted off)."""
_FUSION.mark(module)
def ltx2_rms_norm_modulate_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_ltx2_rms_norm_modulate(root: nn.Module) -> bool:
return _FUSION.mount(root)
def unmount_ltx2_rms_norm_modulate(root: nn.Module) -> None:
_FUSION.unmount(root)
def _ones_weight(x: torch.Tensor) -> torch.Tensor:
key = (x.device, int(x.shape[-1]))
w = _ONES_WEIGHT_CACHE.get(key)
@@ -9,33 +9,10 @@ mount it, while ``"exact"`` keeps the reference chain bit-for-bit.
from __future__ import annotations
import logging
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
MINIMAX_H3_NORM_MODULATE = QualityGatedFusion(
name="MiniMax-H3 fused RMSNorm + AdaLN",
marker_attr="_sgl_minimax_h3_norm_modulate_site",
enabled_attr="_sgl_minimax_h3_norm_modulate_enabled",
)
def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None:
"""Mark an H3 DiT block; it starts on the reference path."""
_FUSION.mark(module)
def minimax_h3_norm_modulate_active(module: nn.Module) -> bool:
return _FUSION.is_enabled(module)
def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool:
return _FUSION.mount(root, logger=logger)
def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None:
_FUSION.unmount(root)
@@ -8,30 +8,10 @@ only for the existing ``quality="high"`` contract.
from __future__ import annotations
import logging
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="Wan NVFP4 bias+GELU",
marker_attr="_sgl_nvfp4_bias_gelu_site",
enabled_attr="_sgl_nvfp4_bias_gelu_enabled",
)
def mark_nvfp4_bias_gelu_site(module: nn.Module) -> None:
"""Mark an MLP whose ``fc_in`` bias can be deferred for this fusion."""
_FUSION.mark(module)
def nvfp4_bias_gelu_active(module: nn.Module) -> bool:
"""Whether the quality-gated fusion is mounted on ``module``."""
return _FUSION.is_enabled(module)
def _site_reject_reason(site: nn.Module) -> str | None:
if not getattr(site, "fuse_bias_gelu_tanh", False):
@@ -51,17 +31,27 @@ def _site_reject_reason(site: nn.Module) -> str | None:
return None
def mount_nvfp4_bias_gelu(root: nn.Module) -> bool:
"""Enable every eligible marked site under ``root``."""
sites = list(_FUSION.iter_sites(root))
mounted = _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
for site in sites:
site.fc_in.skip_bias_add = mounted
return mounted
class _NVFP4BiasGeluFusion(QualityGatedFusion):
"""A mounted site defers its ``fc_in`` bias add to the fused GELU."""
__slots__ = ()
def mount(self, root: nn.Module) -> bool:
mounted = super().mount(root)
for site in self.iter_sites(root):
site.fc_in.skip_bias_add = mounted
return mounted
def unmount(self, root: nn.Module) -> None:
super().unmount(root)
for site in self.iter_sites(root):
site.fc_in.skip_bias_add = False
def unmount_nvfp4_bias_gelu(root: nn.Module) -> None:
"""Restore every marked site to its original linear+GELU path."""
_FUSION.unmount(root)
for site in _FUSION.iter_sites(root):
site.fc_in.skip_bias_add = False
# Marks an MLP whose ``fc_in`` bias can be deferred for this fusion.
NVFP4_BIAS_GELU = _NVFP4BiasGeluFusion(
name="Wan NVFP4 bias+GELU",
marker_attr="_sgl_nvfp4_bias_gelu_site",
enabled_attr="_sgl_nvfp4_bias_gelu_enabled",
reject_reason=_site_reject_reason,
)
@@ -8,6 +8,8 @@ from typing import Any
from torch import nn
logger = logging.getLogger(__name__)
RejectReason = Callable[[nn.Module], str | None]
@@ -16,15 +18,24 @@ class QualityGatedFusion:
The marker metadata describes the model attribute(s) owned by a site. The
enabled flag remains a plain module attribute so compiled model forwards
can read it without depending on this helper object.
can read it without depending on this helper object. ``reject_reason``
holds the static guards :meth:`mount` checks on every marked site.
"""
__slots__ = ("enabled_attr", "marker_attr", "name")
__slots__ = ("enabled_attr", "marker_attr", "name", "reject_reason")
def __init__(self, *, name: str, marker_attr: str, enabled_attr: str) -> None:
def __init__(
self,
*,
name: str,
marker_attr: str,
enabled_attr: str,
reject_reason: RejectReason | None = None,
) -> None:
self.name = name
self.marker_attr = marker_attr
self.enabled_attr = enabled_attr
self.reject_reason = reject_reason
def mark(self, module: nn.Module, metadata: Any = True) -> None:
setattr(module, self.marker_attr, metadata)
@@ -41,32 +52,25 @@ class QualityGatedFusion:
if hasattr(module, self.marker_attr):
yield module
def mount(
self,
root: nn.Module,
*,
reject_reason: RejectReason | None = None,
logger: logging.Logger | None = None,
) -> bool:
def mount(self, root: nn.Module) -> bool:
"""Enable every eligible site, or leave the whole family disabled."""
sites = list(self.iter_sites(root))
if not sites:
return False
if reject_reason is not None:
if self.reject_reason is not None:
for site in sites:
reason = reject_reason(site)
reason = self.reject_reason(site)
if reason is None:
continue
self._set_enabled(sites, False)
if logger is not None:
logger.info(
"%s: %s site failed static guards (%s); keeping the "
"whole model on the reference path",
self.name,
type(site).__name__,
reason,
)
logger.info(
"%s: %s site failed static guards (%s); keeping the "
"whole model on the reference path",
self.name,
type(site).__name__,
reason,
)
return False
self._set_enabled(sites, True)
@@ -8,30 +8,10 @@ slices independently. ``quality="high"`` mounts the single-GEMM path.
from __future__ import annotations
import logging
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="Qwen-Image fused added-QKV",
marker_attr="_sgl_qwen_image_added_qkv_site",
enabled_attr="_sgl_qwen_image_added_qkv_enabled",
)
def mark_qwen_image_added_qkv_site(module: nn.Module) -> None:
"""Mark an unquantized Qwen-Image attention site; it starts unmounted."""
_FUSION.mark(module)
def qwen_image_added_qkv_active(module: nn.Module) -> bool:
"""Whether the request-scoped packed added-QKV GEMM is mounted."""
return _FUSION.is_enabled(module)
def _site_reject_reason(site: nn.Module) -> str | None:
linear = getattr(site, "to_added_qkv", None)
@@ -44,9 +24,9 @@ def _site_reject_reason(site: nn.Module) -> str | None:
return None
def mount_qwen_image_added_qkv(root: nn.Module) -> bool:
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
def unmount_qwen_image_added_qkv(root: nn.Module) -> None:
_FUSION.unmount(root)
QWEN_IMAGE_ADDED_QKV = QualityGatedFusion(
name="Qwen-Image fused added-QKV",
marker_attr="_sgl_qwen_image_added_qkv_site",
enabled_attr="_sgl_qwen_image_added_qkv_enabled",
reject_reason=_site_reject_reason,
)
@@ -13,31 +13,11 @@ eligible; unsupported dtypes and layouts fall back to the reference path.
from __future__ import annotations
import logging
import torch
import torch.nn as nn
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
logger = logging.getLogger(__name__)
_FUSION = QualityGatedFusion(
name="SANA-Video BF16-input linear attention",
marker_attr="_sgl_sana_video_linear_attention_site",
enabled_attr="_sgl_sana_video_linear_attention_enabled",
)
def mark_sana_video_linear_attention_site(module: nn.Module) -> None:
"""Mark a SANA-Video linear-attention module; it starts unmounted."""
_FUSION.mark(module)
def sana_video_linear_attention_active(module: nn.Module) -> bool:
"""Whether the request-scoped BF16-input path is mounted on ``module``."""
return _FUSION.is_enabled(module)
def _site_reject_reason(_site: nn.Module) -> str | None:
if torch.version.cuda is None:
@@ -45,12 +25,12 @@ def _site_reject_reason(_site: nn.Module) -> str | None:
return None
def mount_sana_video_linear_attention(root: nn.Module) -> bool:
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
def unmount_sana_video_linear_attention(root: nn.Module) -> None:
_FUSION.unmount(root)
SANA_VIDEO_LINEAR_ATTENTION = QualityGatedFusion(
name="SANA-Video BF16-input linear attention",
marker_attr="_sgl_sana_video_linear_attention_site",
enabled_attr="_sgl_sana_video_linear_attention_enabled",
reject_reason=_site_reject_reason,
)
def try_sana_video_linear_attention(
@@ -62,7 +42,7 @@ def try_sana_video_linear_attention(
) -> torch.Tensor | None:
"""Return the quality-gated attention result, or ``None`` to fall back."""
if not (
_FUSION.is_enabled(site)
SANA_VIDEO_LINEAR_ATTENTION.is_enabled(site)
and query_rotate.is_cuda
and query_rotate.dtype == torch.bfloat16
and key_rotate.dtype == query_rotate.dtype
@@ -9,7 +9,12 @@ they logically belong to.
"""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
from sglang.kernels.spec import (
CUDA_ONLY,
CapabilityRequirement,
KernelBackend,
KernelSpec,
)
__all__ = []
@@ -20,7 +25,6 @@ for _fn in ("hc_combine_gate", "hc_combine_apply", "hc_combine_apply_norm"):
backend=KernelBackend.JIT,
target=f"sglang.kernels.ops.elementwise.hc_combine_decode:{_fn}",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
description="BF16 HC decode with four branches of width 2560 and PDL.",
)
)
del _fn
@@ -32,7 +36,7 @@ register_kernel(
op="elementwise.fused_qwen4_gate_value",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_value",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -40,7 +44,7 @@ register_kernel(
op="elementwise.fused_qwen4_gate_reduce",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_reduce",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)
register_kernel(
@@ -48,6 +52,6 @@ register_kernel(
op="elementwise.row_scale_bf16",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.elementwise.row_scale:row_scale_bf16",
capabilities=frozenset({CapabilityRequirement.CUDA}),
capabilities=CUDA_ONLY,
)
)

Some files were not shown because too many files have changed in this diff Show More