[Refactor] kernels/diffusion/tests: Prune dead code and fix tests that never ran (#44690)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
59fec011dd
commit
329072d99a
@@ -4,8 +4,8 @@ This package is the public in-tree import surface for callable kernels, per
|
||||
[RFC #29630](https://github.com/sgl-project/sglang/issues/29630).
|
||||
|
||||
```python
|
||||
from sglang.kernels.ops.layernorm import rmsnorm
|
||||
from sglang.kernels.ops.activation import silu_and_mul
|
||||
from sglang.kernels.ops.gemm import fp8_scaled_mm
|
||||
from sglang.kernels.ops.kvcache import reshape_and_cache_flash
|
||||
```
|
||||
|
||||
@@ -13,8 +13,8 @@ from sglang.kernels.ops.kvcache import reshape_and_cache_flash
|
||||
|
||||
```
|
||||
sglang/kernels/
|
||||
spec.py # KernelSpec, KernelBackend, FormatSignature,
|
||||
# CapabilityRequirement, PlatformInfo
|
||||
spec.py # KernelSpec, KernelBackend, CapabilityRequirement,
|
||||
# PlatformInfo
|
||||
registry.py # process-wide KernelRegistry + register_kernel()
|
||||
selector.py # heuristic select_kernel() and cached get_kernel()
|
||||
fused_op.py # BaseFusedOp: per-operator multi-backend contract
|
||||
@@ -94,7 +94,7 @@ explicitly, e.g.:
|
||||
|
||||
```python
|
||||
from sglang.kernels import select_kernel, KernelBackend
|
||||
jit_rmsnorm = select_kernel("layernorm.rmsnorm", backend=KernelBackend.JIT).load()
|
||||
jit_moe_align = select_kernel("moe.moe_align_block_size", backend=KernelBackend.JIT).load()
|
||||
```
|
||||
|
||||
## `BaseFusedOp` — the unified operator contract
|
||||
@@ -128,8 +128,11 @@ Dispatch priority, highest first: explicit `forward(..., backend=...)` →
|
||||
global forced backend (`SGLANG_FORCE_FUSED_OP_BACKEND`) → OOT platform
|
||||
override → declared optimized kernel backends by `priority` (filtered by
|
||||
`backend_eligible()`, a `CapabilityRequirement`-vs-`PlatformInfo` check
|
||||
extensible with per-call shape/dtype gates) → platform-specific forward →
|
||||
`forward_native`. The static part of the decision is resolved once and cached
|
||||
extensible with per-call applicability gates) → platform-specific forward →
|
||||
`forward_native`. A gate only answers whether the kernel covers these operands,
|
||||
so that falling through is a correct way to compute the same result; operands
|
||||
that violate the op's contract (mismatched shapes or devices) are a caller bug
|
||||
that the backend's launcher must raise, not a reason to fall back. The static part of the decision is resolved once and cached
|
||||
on the instance, so the hot path stays a single indirect call.
|
||||
|
||||
`BaseFusedOp` also owns the torch.compile mode protocol
|
||||
@@ -148,7 +151,7 @@ What this buys (see the
|
||||
|
||||
- **Unified correctness testing** — a generic harness enumerates
|
||||
`available_backends()` and asserts each one matches `forward_native`
|
||||
(`test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py`); new
|
||||
(`test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py`); new
|
||||
backends are picked up automatically.
|
||||
- **One-switch debugging** — `SGLANG_FORCE_FUSED_OP_BACKEND=torch` (or
|
||||
`set_fused_op_backend(KernelBackend.TORCH)`) flips *every* fused op to its
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
SGLang runtime code and tests should import callable kernels from
|
||||
``sglang.kernels.ops.<group>``, e.g.::
|
||||
|
||||
from sglang.kernels.ops.layernorm import rmsnorm
|
||||
from sglang.kernels.ops.activation import silu_and_mul
|
||||
from sglang.kernels.ops.gemm import fp8_scaled_mm
|
||||
from sglang.kernels.ops.kvcache import reshape_and_cache_flash
|
||||
|
||||
Implementations still live in ``sglang.kernels.jit`` (JIT CUDA), the
|
||||
@@ -39,7 +39,6 @@ from sglang.kernels.selector import get_kernel, select_kernel
|
||||
from sglang.kernels.spec import (
|
||||
CapabilityRequirement,
|
||||
DeviceType,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
PlatformInfo,
|
||||
@@ -57,7 +56,6 @@ __all__ = [
|
||||
"CapabilityRequirement",
|
||||
"DeviceType",
|
||||
"capabilities_satisfied",
|
||||
"FormatSignature",
|
||||
"FusedOpTraceRecord",
|
||||
"KernelBackend",
|
||||
"KernelRegistry",
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
"""Benchmark for DeepSeek-V4 fused norm + RoPE kernels."""
|
||||
|
||||
import itertools
|
||||
|
||||
import sgl_kernel
|
||||
import torch
|
||||
import triton
|
||||
import triton.testing
|
||||
|
||||
try:
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
except ImportError:
|
||||
IS_CI = False
|
||||
|
||||
batch_sizes = [1] if IS_CI else [1, 4, 16, 64, 256]
|
||||
num_heads_list = [8] if IS_CI else [8, 16, 64]
|
||||
head_dims = [192] if IS_CI else [128, 192]
|
||||
|
||||
configs = list(itertools.product(batch_sizes, num_heads_list, head_dims))
|
||||
|
||||
|
||||
def torch_rmsnorm_rope(
|
||||
q: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""Naive PyTorch reference: RMSNorm + RoPE."""
|
||||
rms = torch.sqrt(q.float().pow(2).mean(dim=-1, keepdim=True) + eps)
|
||||
q_normed = (q.float() / rms).to(q.dtype)
|
||||
return q_normed
|
||||
|
||||
|
||||
@triton.testing.perf_report(
|
||||
triton.testing.Benchmark(
|
||||
x_names=["batch_size", "num_heads", "head_dim"],
|
||||
x_vals=configs,
|
||||
line_arg="provider",
|
||||
line_vals=["sglang", "torch"],
|
||||
line_names=["SGL Kernel", "PyTorch"],
|
||||
styles=[("green", "-"), ("red", "--")],
|
||||
ylabel="µs (median)",
|
||||
plot_name="dsv4-q-norm-rope-performance",
|
||||
args={},
|
||||
)
|
||||
)
|
||||
def benchmark_q_norm_rope(batch_size, num_heads, head_dim, provider):
|
||||
torch.manual_seed(42)
|
||||
eps = 1e-6
|
||||
max_pos = 8192
|
||||
rope_dim = 64
|
||||
|
||||
q_input = torch.randn(
|
||||
batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
q_output = torch.empty_like(q_input)
|
||||
freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda")
|
||||
positions = torch.randint(
|
||||
0, max_pos, (batch_size,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
if provider == "sglang":
|
||||
fn = lambda: sgl_kernel.dsv4_fused_q_norm_rope(
|
||||
q_input, freqs_cis, positions, eps, q_output
|
||||
)
|
||||
else:
|
||||
fn = lambda: torch_rmsnorm_rope(q_input, freqs_cis, positions, eps)
|
||||
|
||||
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
|
||||
fn, quantiles=[0.5, 0.2, 0.8]
|
||||
)
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark_q_norm_rope.run(print_data=True)
|
||||
@@ -1,85 +0,0 @@
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import post_reorder_triton_kernel
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
# CI environment uses simplified parameters
|
||||
if IS_CI:
|
||||
batch_sizes = [64, 128] # Only test 2 values in CI
|
||||
else:
|
||||
batch_sizes = [64, 128, 256, 512, 640, 768, 1024, 2048, 4096]
|
||||
|
||||
configs = [(bs,) for bs in batch_sizes]
|
||||
|
||||
|
||||
@triton.testing.perf_report(
|
||||
triton.testing.Benchmark(
|
||||
x_names=["batch_size"],
|
||||
x_vals=[list(_) for _ in configs],
|
||||
line_arg="provider",
|
||||
line_vals=["triton"],
|
||||
line_names=["Triton Kernel"],
|
||||
styles=[("orange", "-")],
|
||||
ylabel="us",
|
||||
plot_name="ep-moe-post-reorder-performance",
|
||||
args={},
|
||||
)
|
||||
)
|
||||
def benchmark(batch_size, provider):
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device("cuda")
|
||||
hidden_size, topk, start_expert_id, end_expert_id, block_size = 4096, 8, 0, 255, 512
|
||||
|
||||
def alloc_tensors():
|
||||
down_output = torch.randn(
|
||||
batch_size * topk, hidden_size, dtype=dtype, device=device
|
||||
)
|
||||
output = torch.zeros(batch_size, hidden_size, dtype=dtype, device=device)
|
||||
src2dst = torch.randint(
|
||||
0, batch_size * topk, (batch_size, topk), dtype=torch.int32, device=device
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
start_expert_id,
|
||||
end_expert_id + 1,
|
||||
(batch_size, topk),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
topk_weights = torch.rand(batch_size, topk, dtype=dtype, device=device)
|
||||
return down_output, output, src2dst, topk_ids, topk_weights
|
||||
|
||||
quantiles = [0.5, 0.2, 0.8]
|
||||
|
||||
if provider == "triton":
|
||||
d_out, out, s2d, tk_ids, tk_weights = alloc_tensors()
|
||||
|
||||
def run_triton():
|
||||
post_reorder_triton_kernel[(batch_size,)](
|
||||
d_out.view(-1),
|
||||
out.view(-1),
|
||||
s2d.view(-1),
|
||||
tk_ids.view(-1),
|
||||
tk_weights.view(-1),
|
||||
start_expert_id,
|
||||
end_expert_id,
|
||||
topk,
|
||||
hidden_size,
|
||||
0,
|
||||
block_size,
|
||||
)
|
||||
|
||||
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
|
||||
run_triton, quantiles=quantiles
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown provider: {provider}")
|
||||
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run(print_data=True)
|
||||
@@ -5,14 +5,12 @@ import torch
|
||||
import triton
|
||||
from sgl_kernel.test_utils import create_per_token_group_quant_test_data
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
per_token_group_quant_8bit as triton_per_token_group_quant_8bit,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
sglang_per_token_group_quant_8bit,
|
||||
)
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.utils.bench_utils import bench_kineto
|
||||
from sglang.test.kernels.per_token_group_quant import (
|
||||
sglang_per_token_group_quant_8bit,
|
||||
triton_per_token_group_quant_8bit,
|
||||
)
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
@@ -178,16 +178,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.def("apply_shuffle_mul_sum(Tensor input, Tensor output, Tensor permutation, Tensor? factors) -> ()");
|
||||
m.impl("apply_shuffle_mul_sum", torch::kCUDA, &apply_shuffle_mul_sum);
|
||||
|
||||
// DeepSeek-V4 fused norm + rope
|
||||
m.def(
|
||||
"dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()");
|
||||
m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope);
|
||||
|
||||
m.def(
|
||||
"dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, "
|
||||
"Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()");
|
||||
m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla);
|
||||
|
||||
// DeepSeek-V4 fused indexer rope + hadamard + quant
|
||||
m.def(
|
||||
"dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, "
|
||||
"Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()");
|
||||
|
||||
@@ -52,15 +52,6 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) {
|
||||
"page_indices, int page_size, Tensor!? raw_indices, bool sort_output=False) -> ()");
|
||||
m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512);
|
||||
|
||||
m.def(
|
||||
"dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()");
|
||||
m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope);
|
||||
|
||||
m.def(
|
||||
"dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, "
|
||||
"Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()");
|
||||
m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla);
|
||||
|
||||
m.def(
|
||||
"dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, "
|
||||
"Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()");
|
||||
|
||||
@@ -13,9 +13,8 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// DeepSeek-V4 fused norm + RoPE kernels, ported from JIT kernel
|
||||
// python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh
|
||||
// to sgl-kernel AOT compilation with CUDA + HIP (ROCm) support.
|
||||
// DeepSeek-V4 indexer Q RoPE + Hadamard + FP8 quant kernel, ported from the JIT kernel in
|
||||
// python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh for the ROCm build.
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_bf16.h>
|
||||
@@ -59,14 +58,6 @@ using fp8x2_e4m3_t = uint16_t;
|
||||
|
||||
static constexpr uint32_t kWarpSize = 32;
|
||||
|
||||
template <uint32_t kNumThreads = kWarpSize>
|
||||
__device__ __forceinline__ float warp_reduce_sum(float val) {
|
||||
#pragma unroll
|
||||
for (uint32_t mask = kNumThreads / 2; mask > 0; mask >>= 1)
|
||||
val += SGLANG_SHFL_XOR_SYNC(FULL_MASK, val, mask);
|
||||
return val;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float warp_reduce_max(float val) {
|
||||
#pragma unroll
|
||||
for (uint32_t mask = kWarpSize / 2; mask > 0; mask >>= 1)
|
||||
@@ -96,31 +87,10 @@ __device__ __forceinline__ float bf16_to_float(bf16_t v) {
|
||||
return __bfloat162float(v);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ bf16_t float_to_bf16(float v) {
|
||||
#ifndef USE_ROCM
|
||||
return __float2bfloat16_rn(v);
|
||||
#else
|
||||
return __float2bfloat16(v);
|
||||
#endif
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// FP8 E4M3 helpers (portable CUDA + HIP)
|
||||
// ============================================================================
|
||||
|
||||
// UE8M0 scale: round a positive float to the nearest power-of-two
|
||||
// representable in UE8M0 (unsigned 8-bit exponent, no mantissa).
|
||||
__device__ __forceinline__ int32_t cast_to_ue8m0(float x) {
|
||||
uint32_t u = __float_as_uint(x);
|
||||
int32_t exp = static_cast<int32_t>((u >> 23) & 0xFFu);
|
||||
uint32_t mant = u & 0x7FFFFFu;
|
||||
return exp + (mant != 0);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float inv_scale_ue8m0(int32_t exp) {
|
||||
return __uint_as_float(static_cast<uint32_t>((127 + 127 - exp) << 23));
|
||||
}
|
||||
|
||||
static constexpr float kFP8Max = 448.0f;
|
||||
|
||||
#ifndef USE_ROCM
|
||||
@@ -188,236 +158,13 @@ __device__ __forceinline__ fp8x2_e4m3_t pack_fp8(float x, float y) {
|
||||
}
|
||||
#endif
|
||||
|
||||
// ============================================================================
|
||||
// Kernel 1: Fused Q Norm + RoPE
|
||||
// warp-per-(token, head), rmsnorm-self (no weight) + RoPE + write to q_out.
|
||||
// ============================================================================
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr uint32_t kFusedQBlockSize = 128;
|
||||
constexpr uint32_t kFusedQNumWarps = kFusedQBlockSize / kWarpSize;
|
||||
|
||||
constexpr uint32_t kFusedKBlockSize = 256;
|
||||
constexpr uint32_t kFusedKNumWarps = kFusedKBlockSize / kWarpSize;
|
||||
|
||||
struct FusedQNormRopeParams {
|
||||
const void* __restrict__ q_input;
|
||||
void* __restrict__ q_output;
|
||||
const float* __restrict__ freqs_cis;
|
||||
const int32_t* __restrict__ positions;
|
||||
int64_t q_input_stride_batch;
|
||||
int64_t q_output_stride_batch;
|
||||
uint32_t batch_size;
|
||||
uint32_t num_q_heads;
|
||||
float eps;
|
||||
};
|
||||
|
||||
// Compute the largest power-of-2 vec size that divides both kHeadDim and
|
||||
// fits in 16 bytes, while also dividing kRopeDim.
|
||||
template <int64_t kHeadDim, int64_t kRopeDim>
|
||||
struct QKernelTraits {
|
||||
static constexpr int64_t kMaxVecSize = 16 / sizeof(bf16_t); // 8
|
||||
// Use kRopeDim/kWarpSize (=2 for kRopeDim=64) as the vec size.
|
||||
// This guarantees kRopeDim % kVecSize == 0 and works for all head dims
|
||||
// that are multiples of kWarpSize*kVecSize.
|
||||
static constexpr int64_t kVecSize = kRopeDim / kWarpSize; // 2
|
||||
static constexpr int64_t kLocalSize = kHeadDim / (kWarpSize * kVecSize);
|
||||
static constexpr uint32_t kRopeSize = kRopeDim / kVecSize;
|
||||
static_assert(kHeadDim % (kWarpSize * kVecSize) == 0);
|
||||
static_assert(kRopeDim % kVecSize == 0);
|
||||
static_assert(kRopeDim == kWarpSize * 2, "1 (real, imag) pair per lane");
|
||||
};
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim>
|
||||
__global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_norm_rope_kernel(
|
||||
const __grid_constant__ FusedQNormRopeParams params) {
|
||||
using Traits = QKernelTraits<kHeadDim, kRopeDim>;
|
||||
constexpr int64_t kVecSize = Traits::kVecSize;
|
||||
constexpr int64_t kLocalSize = Traits::kLocalSize;
|
||||
constexpr uint32_t kRopeSize = Traits::kRopeSize;
|
||||
|
||||
using Storage = AlignedVec<bf16_t, kVecSize>;
|
||||
using Float2 = AlignedVec<float, 2>;
|
||||
|
||||
const auto warp_id = threadIdx.x / kWarpSize;
|
||||
const auto lane_id = threadIdx.x % kWarpSize;
|
||||
const auto work_id = blockIdx.x * kFusedQNumWarps + warp_id;
|
||||
|
||||
const uint32_t total_works = params.batch_size * params.num_q_heads;
|
||||
if (work_id >= total_works) return;
|
||||
|
||||
const uint32_t batch_id = work_id / params.num_q_heads;
|
||||
const uint32_t head_id = work_id % params.num_q_heads;
|
||||
const auto input_ptr =
|
||||
static_cast<const bf16_t*>(params.q_input) + batch_id * params.q_input_stride_batch + head_id * kHeadDim;
|
||||
const auto output_ptr =
|
||||
static_cast<bf16_t*>(params.q_output) + batch_id * params.q_output_stride_batch + head_id * kHeadDim;
|
||||
const auto position = params.positions[batch_id];
|
||||
|
||||
__shared__ Storage s_rope[kFusedQNumWarps][kRopeSize];
|
||||
|
||||
// Prefetch freq pair.
|
||||
Float2 freq;
|
||||
freq.load(params.freqs_cis + position * kRopeDim, lane_id);
|
||||
|
||||
// Part 1: rmsnorm-self (no weight).
|
||||
Storage input_vec[kLocalSize];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
input_vec[i].load(input_ptr, lane_id + i * kWarpSize);
|
||||
}
|
||||
|
||||
float sum_of_squares = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
float x = bf16_to_float(input_vec[i][j]);
|
||||
sum_of_squares += x * x;
|
||||
}
|
||||
}
|
||||
sum_of_squares = warp_reduce_sum(sum_of_squares);
|
||||
const float norm_factor = rsqrtf(sum_of_squares / static_cast<float>(kHeadDim) + params.eps);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
float x = bf16_to_float(input_vec[i][j]);
|
||||
input_vec[i][j] = float_to_bf16(x * norm_factor);
|
||||
}
|
||||
}
|
||||
|
||||
// Stash rope tail into shared memory; write nope tiles to gmem.
|
||||
const bool is_rope_lane = lane_id >= kWarpSize - kRopeSize;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
if (i == kLocalSize - 1 && is_rope_lane) {
|
||||
const auto rope_id = lane_id - (kWarpSize - kRopeSize);
|
||||
s_rope[warp_id][rope_id] = input_vec[i];
|
||||
} else {
|
||||
input_vec[i].store(output_ptr, lane_id + i * kWarpSize);
|
||||
}
|
||||
}
|
||||
__syncwarp();
|
||||
|
||||
// Part 2: RoPE on all 32 lanes -- one (real, imag) bf16x2 pair per lane.
|
||||
auto elem_ptr = reinterpret_cast<bf16x2_t*>(&s_rope[warp_id][0]);
|
||||
bf16x2_t elem = elem_ptr[lane_id];
|
||||
#ifndef USE_ROCM
|
||||
float2 elem_f = __bfloat1622float2(elem);
|
||||
float x_real = elem_f.x, x_imag = elem_f.y;
|
||||
#else
|
||||
float x_real = __bfloat162float(elem.x), x_imag = __bfloat162float(elem.y);
|
||||
#endif
|
||||
float freq_real = freq[0], freq_imag = freq[1];
|
||||
float rot_real = x_real * freq_real - x_imag * freq_imag;
|
||||
float rot_imag = x_real * freq_imag + x_imag * freq_real;
|
||||
bf16x2_t rotated = __float22bfloat162_rn(make_float2(rot_real, rot_imag));
|
||||
auto out_elem = reinterpret_cast<bf16x2_t*>(output_ptr + (kHeadDim - kRopeDim));
|
||||
out_elem[lane_id] = rotated;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Kernel 2: Fused K Norm + RoPE + FlashMLA Store
|
||||
// block-per-token, rmsnorm (with kv_weight) + RoPE + FP8 quantized store.
|
||||
// ============================================================================
|
||||
|
||||
struct FusedKNormRopeFlashMLAParams {
|
||||
const void* __restrict__ kv;
|
||||
const void* __restrict__ kv_weight;
|
||||
const float* __restrict__ freqs_cis;
|
||||
const int32_t* __restrict__ positions;
|
||||
const int32_t* __restrict__ out_loc;
|
||||
uint8_t* __restrict__ kvcache;
|
||||
int64_t kv_stride_batch;
|
||||
uint32_t batch_size;
|
||||
float eps;
|
||||
};
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, int32_t kPageBits>
|
||||
__global__ __launch_bounds__(kFusedKBlockSize, 8) void fused_k_norm_rope_flashmla_kernel(
|
||||
const __grid_constant__ FusedKNormRopeFlashMLAParams params) {
|
||||
constexpr int64_t kVecSize = 2;
|
||||
constexpr uint32_t kRopeWarp = kFusedKNumWarps - 1;
|
||||
constexpr int64_t kPageBytes = ((584ll << kPageBits) + 575) / 576 * 576;
|
||||
static_assert(kHeadDim == kFusedKBlockSize * kVecSize);
|
||||
static_assert(kRopeDim == kWarpSize * kVecSize);
|
||||
|
||||
using Storage = AlignedVec<bf16_t, kVecSize>;
|
||||
|
||||
const auto tx = threadIdx.x;
|
||||
const auto warp_id = tx / kWarpSize;
|
||||
const auto lane_id = tx % kWarpSize;
|
||||
const auto work_id = blockIdx.x;
|
||||
if (work_id >= params.batch_size) return;
|
||||
|
||||
const auto input_ptr = static_cast<const bf16_t*>(params.kv) + work_id * params.kv_stride_batch;
|
||||
const auto position = params.positions[work_id];
|
||||
const auto out_loc = params.out_loc[work_id];
|
||||
const auto freqs_cis = params.freqs_cis + position * kRopeDim;
|
||||
|
||||
AlignedVec<float, kVecSize> data, freq;
|
||||
|
||||
// Part 1: norm with block-wide reduction.
|
||||
{
|
||||
__shared__ float partial_sums[kFusedKNumWarps];
|
||||
|
||||
Storage input_vec, weight_vec;
|
||||
input_vec.load(input_ptr, tx);
|
||||
weight_vec.load(params.kv_weight, tx);
|
||||
if (warp_id == kRopeWarp) freq.load(freqs_cis, lane_id);
|
||||
|
||||
float sum_of_squares = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kVecSize; ++i) {
|
||||
float x = bf16_to_float(input_vec[i]);
|
||||
sum_of_squares += x * x;
|
||||
}
|
||||
const float warp_sum = warp_reduce_sum(sum_of_squares);
|
||||
if (lane_id == 0) partial_sums[warp_id] = warp_sum;
|
||||
__syncthreads();
|
||||
sum_of_squares = warp_reduce_sum<kFusedKNumWarps>(partial_sums[lane_id % kFusedKNumWarps]);
|
||||
const float norm_factor = rsqrtf(sum_of_squares / static_cast<float>(kHeadDim) + params.eps);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kVecSize; ++i) {
|
||||
float x = bf16_to_float(input_vec[i]);
|
||||
float w = bf16_to_float(weight_vec[i]);
|
||||
data[i] = x * norm_factor * w;
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t page = out_loc >> kPageBits;
|
||||
const int32_t offset = out_loc & ((1 << kPageBits) - 1);
|
||||
const auto page_ptr = params.kvcache + page * kPageBytes;
|
||||
const auto value_ptr = page_ptr + offset * 576;
|
||||
|
||||
// Part 2: rope on last warp (BF16 store), per-warp UE8M0 quant + store on others.
|
||||
if (warp_id == kRopeWarp) {
|
||||
float x_real = data[0], x_imag = data[1];
|
||||
float freq_real = freq[0], freq_imag = freq[1];
|
||||
float rot_real = x_real * freq_real - x_imag * freq_imag;
|
||||
float rot_imag = x_real * freq_imag + x_imag * freq_real;
|
||||
bf16x2_t result = __float22bfloat162_rn(make_float2(rot_real, rot_imag));
|
||||
auto rope_ptr = value_ptr + 448;
|
||||
reinterpret_cast<bf16x2_t*>(rope_ptr)[lane_id] = result;
|
||||
} else {
|
||||
float x = data[0], y = data[1];
|
||||
float abs_max = warp_reduce_max(fmaxf(fabsf(x), fabsf(y)));
|
||||
float scale_raw = fmaxf(1e-4f, abs_max) / kFP8Max;
|
||||
int32_t scale_ue8m0 = cast_to_ue8m0(scale_raw);
|
||||
float inv_scale = inv_scale_ue8m0(scale_ue8m0);
|
||||
fp8x2_e4m3_t result = pack_fp8(x * inv_scale, y * inv_scale);
|
||||
auto scale_ptr = page_ptr + (576ll << kPageBits) + offset * 8;
|
||||
reinterpret_cast<fp8x2_e4m3_t*>(value_ptr)[tx] = result;
|
||||
if (lane_id == 0) static_cast<uint8_t*>(scale_ptr)[warp_id] = static_cast<uint8_t>(scale_ue8m0);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Kernel 3: Fused Q Indexer RoPE + Hadamard + FP8 Quantization
|
||||
// Fused Q Indexer RoPE + Hadamard + FP8 Quantization
|
||||
// warp-per-(token, head), no norm, RoPE + Hadamard + fp8 act-quant.
|
||||
// ============================================================================
|
||||
|
||||
@@ -537,138 +284,6 @@ __global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_indexer_rope_had
|
||||
// Host-side launchers (PyTorch C++ extension API)
|
||||
// ============================================================================
|
||||
|
||||
void dsv4_fused_q_norm_rope(
|
||||
const at::Tensor& q_input,
|
||||
at::Tensor& q_output,
|
||||
const at::Tensor& freqs_cis,
|
||||
const at::Tensor& positions,
|
||||
double eps) {
|
||||
TORCH_CHECK(q_input.is_cuda(), "q_input must be a CUDA tensor");
|
||||
TORCH_CHECK(q_output.is_cuda(), "q_output must be a CUDA tensor");
|
||||
TORCH_CHECK(q_input.scalar_type() == at::ScalarType::BFloat16, "q_input must be bfloat16");
|
||||
TORCH_CHECK(q_output.scalar_type() == at::ScalarType::BFloat16, "q_output must be bfloat16");
|
||||
TORCH_CHECK(q_input.dim() == 3, "q_input must be 3D: (B, H, D)");
|
||||
TORCH_CHECK(q_output.dim() == 3, "q_output must be 3D: (B, H, D)");
|
||||
TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32");
|
||||
|
||||
const int64_t B = q_input.size(0);
|
||||
const int64_t H = q_input.size(1);
|
||||
const int64_t D = q_input.size(2);
|
||||
TORCH_CHECK(
|
||||
q_output.size(0) == B && q_output.size(1) == H && q_output.size(2) == D, "q_output shape must match q_input");
|
||||
TORCH_CHECK(q_input.stride(2) == 1 && q_output.stride(2) == 1, "last dim must be contiguous");
|
||||
TORCH_CHECK(q_input.stride(1) == D && q_output.stride(1) == D, "head dim must be contiguous");
|
||||
|
||||
if (B == 0) return;
|
||||
|
||||
const auto stream = at::cuda::getCurrentCUDAStream(q_input.get_device());
|
||||
const auto params = FusedQNormRopeParams{
|
||||
.q_input = q_input.data_ptr(),
|
||||
.q_output = q_output.data_ptr(),
|
||||
.freqs_cis = freqs_cis.data_ptr<float>(),
|
||||
.positions = positions.data_ptr<int32_t>(),
|
||||
.q_input_stride_batch = q_input.stride(0),
|
||||
.q_output_stride_batch = q_output.stride(0),
|
||||
.batch_size = static_cast<uint32_t>(B),
|
||||
.num_q_heads = static_cast<uint32_t>(H),
|
||||
.eps = static_cast<float>(eps),
|
||||
};
|
||||
const uint32_t total_works = static_cast<uint32_t>(B * H);
|
||||
const uint32_t num_blocks = CEILDIV(total_works, kFusedQNumWarps);
|
||||
|
||||
// Dispatch on head_dim. DeepSeek V4 uses D=192 with kRopeDim=64.
|
||||
constexpr int64_t kRopeDim = 64;
|
||||
switch (D) {
|
||||
case 128:
|
||||
fused_q_norm_rope_kernel<128, kRopeDim><<<num_blocks, kFusedQBlockSize, 0, stream>>>(params);
|
||||
break;
|
||||
case 192:
|
||||
fused_q_norm_rope_kernel<192, kRopeDim><<<num_blocks, kFusedQBlockSize, 0, stream>>>(params);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unsupported head_dim for dsv4_fused_q_norm_rope: ", D);
|
||||
}
|
||||
}
|
||||
|
||||
void dsv4_fused_k_norm_rope_flashmla(
|
||||
const at::Tensor& kv,
|
||||
const at::Tensor& kv_weight,
|
||||
const at::Tensor& freqs_cis,
|
||||
const at::Tensor& positions,
|
||||
const at::Tensor& out_loc,
|
||||
at::Tensor& kvcache,
|
||||
double eps,
|
||||
int64_t page_size) {
|
||||
TORCH_CHECK(kv.is_cuda(), "kv must be a CUDA tensor");
|
||||
TORCH_CHECK(kv.scalar_type() == at::ScalarType::BFloat16, "kv must be bfloat16");
|
||||
TORCH_CHECK(kv.dim() == 2, "kv must be 2D: (B, D)");
|
||||
TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32");
|
||||
TORCH_CHECK(out_loc.scalar_type() == at::ScalarType::Int, "out_loc must be int32");
|
||||
|
||||
const int64_t B = kv.size(0);
|
||||
const int64_t D = kv.size(1);
|
||||
TORCH_CHECK(D == 512, "kv head_dim must be 512 for FlashMLA");
|
||||
TORCH_CHECK(kv_weight.size(0) == D, "kv_weight size must match head_dim");
|
||||
|
||||
if (B == 0) return;
|
||||
|
||||
const auto stream = at::cuda::getCurrentCUDAStream(kv.get_device());
|
||||
const auto params = FusedKNormRopeFlashMLAParams{
|
||||
.kv = kv.data_ptr(),
|
||||
.kv_weight = kv_weight.data_ptr(),
|
||||
.freqs_cis = freqs_cis.data_ptr<float>(),
|
||||
.positions = positions.data_ptr<int32_t>(),
|
||||
.out_loc = out_loc.data_ptr<int32_t>(),
|
||||
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
|
||||
.kv_stride_batch = kv.stride(0),
|
||||
.batch_size = static_cast<uint32_t>(B),
|
||||
.eps = static_cast<float>(eps),
|
||||
};
|
||||
|
||||
constexpr int64_t kHeadDim = 512;
|
||||
constexpr int64_t kRopeDim = 64;
|
||||
|
||||
// Dispatch on page_size (must be power of 2).
|
||||
TORCH_CHECK(page_size > 0 && (page_size & (page_size - 1)) == 0, "page_size must be a power of 2");
|
||||
|
||||
#define LAUNCH_K_KERNEL(PAGE_BITS) \
|
||||
fused_k_norm_rope_flashmla_kernel<kHeadDim, kRopeDim, PAGE_BITS> \
|
||||
<<<static_cast<uint32_t>(B), kFusedKBlockSize, 0, stream>>>(params)
|
||||
|
||||
switch (page_size) {
|
||||
case 1:
|
||||
LAUNCH_K_KERNEL(0);
|
||||
break;
|
||||
case 2:
|
||||
LAUNCH_K_KERNEL(1);
|
||||
break;
|
||||
case 4:
|
||||
LAUNCH_K_KERNEL(2);
|
||||
break;
|
||||
case 8:
|
||||
LAUNCH_K_KERNEL(3);
|
||||
break;
|
||||
case 16:
|
||||
LAUNCH_K_KERNEL(4);
|
||||
break;
|
||||
case 32:
|
||||
LAUNCH_K_KERNEL(5);
|
||||
break;
|
||||
case 64:
|
||||
LAUNCH_K_KERNEL(6);
|
||||
break;
|
||||
case 128:
|
||||
LAUNCH_K_KERNEL(7);
|
||||
break;
|
||||
case 256:
|
||||
LAUNCH_K_KERNEL(8);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unsupported page_size: ", page_size);
|
||||
}
|
||||
#undef LAUNCH_K_KERNEL
|
||||
}
|
||||
|
||||
void dsv4_fused_q_indexer_rope_hadamard_quant(
|
||||
const at::Tensor& q_input,
|
||||
at::Tensor& q_fp8,
|
||||
|
||||
@@ -75,8 +75,13 @@ using Ws = CoopMbWorkspace<kHistBits, kTieCap>;
|
||||
// which is the 32768/b1 line above -- 18% on a case the kernel already wins by 2x.
|
||||
int row_split(int batch, int64_t row_len_hint) {
|
||||
// Escape hatch for benchmarking the two paths against each other; 0 forces the
|
||||
// one-block kernel. An empty value means unset, not zero.
|
||||
if (const char* e = std::getenv("SGL_DSA_TOPK_ROW_SPLIT"); e != nullptr && *e != '\0') {
|
||||
// one-block kernel. An empty value means unset, not zero. The legacy SGL_ name is
|
||||
// read here too: without sglang's environ import nothing rewrites it to SGLANG_.
|
||||
const char* e = std::getenv("SGLANG_DSA_TOPK_ROW_SPLIT");
|
||||
if (e == nullptr || *e == '\0') {
|
||||
e = std::getenv("SGL_DSA_TOPK_ROW_SPLIT");
|
||||
}
|
||||
if (e != nullptr && *e != '\0') {
|
||||
const int v = std::atoi(e);
|
||||
if (v >= 0) {
|
||||
return v;
|
||||
|
||||
@@ -175,25 +175,8 @@ void deepseek_v4_topk_transform_512(
|
||||
#endif
|
||||
|
||||
/*
|
||||
* From csrc/elementwise (DeepSeek-V4 norm + rope)
|
||||
* From csrc/elementwise (DeepSeek-V4 indexer rope + hadamard quant)
|
||||
*/
|
||||
void dsv4_fused_q_norm_rope(
|
||||
const at::Tensor& q_input,
|
||||
at::Tensor& q_output,
|
||||
const at::Tensor& freqs_cis,
|
||||
const at::Tensor& positions,
|
||||
double eps);
|
||||
|
||||
void dsv4_fused_k_norm_rope_flashmla(
|
||||
const at::Tensor& kv,
|
||||
const at::Tensor& kv_weight,
|
||||
const at::Tensor& freqs_cis,
|
||||
const at::Tensor& positions,
|
||||
const at::Tensor& out_loc,
|
||||
at::Tensor& kvcache,
|
||||
double eps,
|
||||
int64_t page_size);
|
||||
|
||||
void dsv4_fused_q_indexer_rope_hadamard_quant(
|
||||
const at::Tensor& q_input,
|
||||
at::Tensor& q_fp8,
|
||||
@@ -316,25 +299,8 @@ void apply_shuffle_mul_sum(
|
||||
const std::optional<torch::Tensor>& factors);
|
||||
|
||||
/*
|
||||
* From csrc/elementwise (DeepSeek-V4 norm + rope)
|
||||
* From csrc/elementwise (DeepSeek-V4 indexer rope + hadamard quant)
|
||||
*/
|
||||
void dsv4_fused_q_norm_rope(
|
||||
const at::Tensor& q_input,
|
||||
at::Tensor& q_output,
|
||||
const at::Tensor& freqs_cis,
|
||||
const at::Tensor& positions,
|
||||
double eps);
|
||||
|
||||
void dsv4_fused_k_norm_rope_flashmla(
|
||||
const at::Tensor& kv,
|
||||
const at::Tensor& kv_weight,
|
||||
const at::Tensor& freqs_cis,
|
||||
const at::Tensor& positions,
|
||||
const at::Tensor& out_loc,
|
||||
at::Tensor& kvcache,
|
||||
double eps,
|
||||
int64_t page_size);
|
||||
|
||||
void dsv4_fused_q_indexer_rope_hadamard_quant(
|
||||
const at::Tensor& q_input,
|
||||
at::Tensor& q_fp8,
|
||||
|
||||
@@ -32,9 +32,7 @@ else:
|
||||
concat_mla_absorb_q,
|
||||
concat_mla_k,
|
||||
copy_to_gpu_no_ce,
|
||||
dsv4_fused_k_norm_rope_flashmla,
|
||||
dsv4_fused_q_indexer_rope_hadamard_quant,
|
||||
dsv4_fused_q_norm_rope,
|
||||
fused_add_rmsnorm,
|
||||
gelu_and_mul,
|
||||
gelu_tanh_and_mul,
|
||||
@@ -155,9 +153,7 @@ else:
|
||||
"concat_mla_absorb_q",
|
||||
"concat_mla_k",
|
||||
"copy_to_gpu_no_ce",
|
||||
"dsv4_fused_k_norm_rope_flashmla",
|
||||
"dsv4_fused_q_indexer_rope_hadamard_quant",
|
||||
"dsv4_fused_q_norm_rope",
|
||||
"es_fp8_blockwise_scaled_grouped_mm",
|
||||
"es_sm100_mxfp8_blockscaled_grouped_mm",
|
||||
"es_sm100_mxfp8_blockscaled_grouped_quant",
|
||||
|
||||
@@ -324,59 +324,6 @@ if torch.version.hip is not None:
|
||||
return out
|
||||
|
||||
|
||||
def dsv4_fused_q_norm_rope(
|
||||
q_input: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
eps: float = 1e-6,
|
||||
q_output: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""DeepSeek-V4 fused Q RMSNorm (no weight) + RoPE.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
q_input : (B, num_q_heads, head_dim) bfloat16
|
||||
freqs_cis: (max_pos, rope_dim) float32, re/im interleaved
|
||||
positions: (B,) int32
|
||||
eps : RMSNorm epsilon
|
||||
q_output : optional pre-allocated output tensor
|
||||
"""
|
||||
if q_output is None:
|
||||
q_output = torch.empty_like(q_input)
|
||||
torch.ops.sgl_kernel.dsv4_fused_q_norm_rope.default(
|
||||
q_input, q_output, freqs_cis, positions, eps
|
||||
)
|
||||
return q_output
|
||||
|
||||
|
||||
def dsv4_fused_k_norm_rope_flashmla(
|
||||
kv: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out_loc: torch.Tensor,
|
||||
kvcache: torch.Tensor,
|
||||
eps: float = 1e-6,
|
||||
page_size: int = 1,
|
||||
) -> None:
|
||||
"""DeepSeek-V4 fused K RMSNorm + RoPE + FlashMLA FP8 store.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kv : (B, 512) bfloat16
|
||||
kv_weight: (512,) bfloat16
|
||||
freqs_cis: (max_pos, 64) float32
|
||||
positions: (B,) int32
|
||||
out_loc : (B,) int32 cache slot ids
|
||||
kvcache : (npages, page_bytes) uint8
|
||||
eps : RMSNorm epsilon
|
||||
page_size: page size (power of 2)
|
||||
"""
|
||||
torch.ops.sgl_kernel.dsv4_fused_k_norm_rope_flashmla.default(
|
||||
kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, page_size
|
||||
)
|
||||
|
||||
|
||||
def dsv4_fused_q_indexer_rope_hadamard_quant(
|
||||
q_input: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
|
||||
@@ -14,21 +14,9 @@
|
||||
# ==============================================================================
|
||||
|
||||
import functools
|
||||
from typing import Dict, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
_cache_buf: Dict[Tuple[str, torch.device], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_cache_buf(name: str, bytes: int, device: torch.device) -> torch.Tensor:
|
||||
key = (name, device)
|
||||
buf = _cache_buf.get(key)
|
||||
if buf is None:
|
||||
buf = torch.empty(bytes, dtype=torch.uint8, device=device)
|
||||
_cache_buf[key] = buf
|
||||
return buf
|
||||
|
||||
|
||||
def _to_tensor_scalar_tuple(x):
|
||||
if isinstance(x, torch.Tensor):
|
||||
|
||||
@@ -1,94 +1,10 @@
|
||||
"""Tests for DeepSeek-V4 fused norm + RoPE kernels."""
|
||||
"""Tests for the DeepSeek-V4 fused indexer Q RoPE + Hadamard + FP8 quant kernel."""
|
||||
|
||||
import pytest
|
||||
import sgl_kernel
|
||||
import torch
|
||||
|
||||
|
||||
def _ref_rmsnorm_self(x: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
"""Reference: RMSNorm without weight (identity weight)."""
|
||||
rms = torch.sqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + eps)
|
||||
return (x.float() / rms).to(x.dtype)
|
||||
|
||||
|
||||
def _ref_rope_interleaved(
|
||||
x: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, rope_dim: int
|
||||
) -> torch.Tensor:
|
||||
"""Reference: apply RoPE to the last `rope_dim` elements (interleaved re/im)."""
|
||||
out = x.clone()
|
||||
B = x.size(0)
|
||||
head_dim = x.size(-1)
|
||||
nope_dim = head_dim - rope_dim
|
||||
|
||||
for b in range(B):
|
||||
pos = positions[b].item()
|
||||
freq = freqs_cis[pos] # (rope_dim,) interleaved [re0, im0, re1, im1, ...]
|
||||
rope_part = out[b, ..., nope_dim:].float()
|
||||
# Reshape to pairs
|
||||
pairs = rope_part.reshape(*rope_part.shape[:-1], rope_dim // 2, 2)
|
||||
x_real = pairs[..., 0]
|
||||
x_imag = pairs[..., 1]
|
||||
freq_pairs = freq.reshape(rope_dim // 2, 2)
|
||||
f_real = freq_pairs[:, 0]
|
||||
f_imag = freq_pairs[:, 1]
|
||||
rot_real = x_real * f_real - x_imag * f_imag
|
||||
rot_imag = x_real * f_imag + x_imag * f_real
|
||||
result = torch.stack([rot_real, rot_imag], dim=-1).reshape(rope_part.shape)
|
||||
out[b, ..., nope_dim:] = result.to(x.dtype)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 16])
|
||||
@pytest.mark.parametrize("num_heads", [1, 8])
|
||||
@pytest.mark.parametrize("head_dim", [128, 192])
|
||||
def test_fused_q_norm_rope_correctness(batch_size, num_heads, head_dim):
|
||||
"""Test Q norm + rope against reference."""
|
||||
torch.manual_seed(42)
|
||||
rope_dim = 64
|
||||
max_pos = 512
|
||||
eps = 1e-6
|
||||
|
||||
q_input = torch.randn(
|
||||
batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda")
|
||||
positions = torch.randint(
|
||||
0, max_pos, (batch_size,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions, eps)
|
||||
|
||||
# Reference
|
||||
normed = _ref_rmsnorm_self(q_input, eps)
|
||||
expected = _ref_rope_interleaved(normed, freqs_cis, positions, rope_dim)
|
||||
|
||||
torch.testing.assert_close(q_output.float(), expected.float(), rtol=1e-2, atol=1e-2)
|
||||
|
||||
|
||||
def test_fused_q_norm_rope_zero_batch():
|
||||
"""Empty batch should not crash."""
|
||||
q_input = torch.empty(0, 8, 192, dtype=torch.bfloat16, device="cuda")
|
||||
freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda")
|
||||
positions = torch.empty(0, dtype=torch.int32, device="cuda")
|
||||
q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions)
|
||||
assert q_output.shape == q_input.shape
|
||||
|
||||
|
||||
def test_fused_q_norm_rope_preallocated_output():
|
||||
"""Test with pre-allocated output tensor."""
|
||||
torch.manual_seed(42)
|
||||
B, H, D = 4, 8, 192
|
||||
q_input = torch.randn(B, H, D, dtype=torch.bfloat16, device="cuda")
|
||||
freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda")
|
||||
positions = torch.randint(0, 512, (B,), dtype=torch.int32, device="cuda")
|
||||
q_output = torch.empty_like(q_input)
|
||||
|
||||
result = sgl_kernel.dsv4_fused_q_norm_rope(
|
||||
q_input, freqs_cis, positions, q_output=q_output
|
||||
)
|
||||
assert result is q_output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8])
|
||||
def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size):
|
||||
"""Basic launch coverage with finite output checks."""
|
||||
|
||||
@@ -8,21 +8,9 @@ from sglang.srt.server_args import (
|
||||
ServerArgs,
|
||||
set_global_server_args_for_scheduler,
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_hip,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
_is_cpu = is_cpu()
|
||||
_is_cpu_amx_available = cpu_has_amx_support()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
@@ -3,99 +3,9 @@ from typing import Optional
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
from sgl_kernel import merge_state_v2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def merge_state_kernel(
|
||||
output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_merged
|
||||
output_lse, # [NUM_TOKENS, NUM_HEADS] s_merged
|
||||
prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_a
|
||||
prefix_lse, # [NUM_TOKENS, NUM_HEADS] s_a
|
||||
suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_b
|
||||
suffix_lse, # [NUM_TOKENS, NUM_HEADS] s_b
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
PADDED_HEAD_SIZE: tl.constexpr,
|
||||
OUTPUT_LSE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
num_tokens = tl.num_programs(0)
|
||||
head_idx = tl.program_id(1)
|
||||
num_heads = tl.num_programs(1)
|
||||
|
||||
p_lse = tl.load(prefix_lse + token_idx * num_heads + head_idx)
|
||||
s_lse = tl.load(suffix_lse + token_idx * num_heads + head_idx)
|
||||
p_lse = float("-inf") if p_lse == float("inf") else p_lse
|
||||
s_lse = float("-inf") if s_lse == float("inf") else s_lse
|
||||
|
||||
max_lse = tl.maximum(p_lse, s_lse)
|
||||
p_lse = p_lse - max_lse
|
||||
s_lse = s_lse - max_lse
|
||||
out_se = tl.exp(p_lse) + tl.exp(s_lse)
|
||||
|
||||
if OUTPUT_LSE:
|
||||
out_lse = tl.log(out_se) + max_lse
|
||||
tl.store(output_lse + token_idx * num_heads + head_idx, out_lse)
|
||||
|
||||
head_arange = tl.arange(0, PADDED_HEAD_SIZE)
|
||||
head_mask = head_arange < HEAD_SIZE
|
||||
p_out = tl.load(
|
||||
prefix_output
|
||||
+ token_idx * num_heads * HEAD_SIZE
|
||||
+ head_idx * HEAD_SIZE
|
||||
+ head_arange,
|
||||
mask=head_mask,
|
||||
)
|
||||
s_out = tl.load(
|
||||
suffix_output
|
||||
+ token_idx * num_heads * HEAD_SIZE
|
||||
+ head_idx * HEAD_SIZE
|
||||
+ head_arange,
|
||||
mask=head_mask,
|
||||
)
|
||||
|
||||
p_scale = tl.exp(p_lse) / out_se
|
||||
s_scale = tl.exp(s_lse) / out_se
|
||||
out = p_out * p_scale + s_out * s_scale
|
||||
tl.store(
|
||||
output + token_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE + head_arange,
|
||||
out,
|
||||
mask=head_mask,
|
||||
)
|
||||
|
||||
|
||||
def merge_state_triton(
|
||||
prefix_output: torch.Tensor,
|
||||
prefix_lse: torch.Tensor,
|
||||
suffix_output: torch.Tensor,
|
||||
suffix_lse: torch.Tensor,
|
||||
output: Optional[torch.Tensor] = None,
|
||||
output_lse: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
num_tokens = output.shape[0]
|
||||
num_query_heads = output.shape[1]
|
||||
head_size = output.shape[2]
|
||||
padded_head_size = triton.next_power_of_2(head_size)
|
||||
# Avoid creating new tensors if they are already provided
|
||||
if output is None:
|
||||
output = torch.empty_like(prefix_output)
|
||||
if output_lse is None:
|
||||
output_lse = torch.empty_like(prefix_lse)
|
||||
|
||||
merge_state_kernel[(num_tokens, num_query_heads)](
|
||||
output,
|
||||
output_lse,
|
||||
prefix_output,
|
||||
prefix_lse,
|
||||
suffix_output,
|
||||
suffix_lse,
|
||||
head_size,
|
||||
padded_head_size,
|
||||
output_lse is not None,
|
||||
)
|
||||
return output, output_lse
|
||||
from sglang.kernels.ops.attention.merge_state import merge_state_triton
|
||||
|
||||
|
||||
# Naive PyTorch Implements of Merge Attn States
|
||||
|
||||
@@ -3,10 +3,10 @@ import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
from sgl_kernel import moe_align_block_size, moe_sum
|
||||
|
||||
from sglang.test.kernels.moe_align import moe_align_block_size_triton
|
||||
|
||||
|
||||
def is_hip() -> bool:
|
||||
return torch.version.hip is not None
|
||||
@@ -15,144 +15,14 @@ def is_hip() -> bool:
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
def ceil_div(a, b):
|
||||
return (a + b - 1) // b
|
||||
|
||||
|
||||
@triton.jit
|
||||
def moe_align_block_size_stage1(
|
||||
topk_ids_ptr,
|
||||
tokens_cnts_ptr,
|
||||
num_experts: tl.constexpr,
|
||||
numel: tl.constexpr,
|
||||
tokens_per_thread: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
start_idx = pid * tokens_per_thread
|
||||
off_c = (pid + 1) * num_experts
|
||||
|
||||
for i in range(tokens_per_thread):
|
||||
if start_idx + i < numel:
|
||||
idx = tl.load(topk_ids_ptr + start_idx + i)
|
||||
token_cnt = tl.load(tokens_cnts_ptr + off_c + idx)
|
||||
tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def moe_align_block_size_stage2(
|
||||
tokens_cnts_ptr,
|
||||
num_experts: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
last_cnt = 0
|
||||
for i in range(1, num_experts + 1):
|
||||
token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid)
|
||||
last_cnt = last_cnt + token_cnt
|
||||
tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def moe_align_block_size_stage3(
|
||||
total_tokens_post_pad_ptr,
|
||||
tokens_cnts_ptr,
|
||||
cumsum_ptr,
|
||||
num_experts: tl.constexpr,
|
||||
block_size: tl.constexpr,
|
||||
):
|
||||
last_cumsum = 0
|
||||
off_cnt = num_experts * num_experts
|
||||
for i in range(1, num_experts + 1):
|
||||
token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1)
|
||||
last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size
|
||||
tl.store(cumsum_ptr + i, last_cumsum)
|
||||
tl.store(total_tokens_post_pad_ptr, last_cumsum)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def moe_align_block_size_stage4(
|
||||
topk_ids_ptr,
|
||||
sorted_token_ids_ptr,
|
||||
expert_ids_ptr,
|
||||
tokens_cnts_ptr,
|
||||
cumsum_ptr,
|
||||
num_experts: tl.constexpr,
|
||||
block_size: tl.constexpr,
|
||||
numel: tl.constexpr,
|
||||
tokens_per_thread: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
start_idx = tl.load(cumsum_ptr + pid)
|
||||
end_idx = tl.load(cumsum_ptr + pid + 1)
|
||||
|
||||
for i in range(start_idx, end_idx, block_size):
|
||||
tl.store(expert_ids_ptr + i // block_size, pid)
|
||||
|
||||
start_idx = pid * tokens_per_thread
|
||||
off_t = pid * num_experts
|
||||
|
||||
for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)):
|
||||
expert_id = tl.load(topk_ids_ptr + i)
|
||||
token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id)
|
||||
rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id)
|
||||
tl.store(sorted_token_ids_ptr + rank_post_pad, i)
|
||||
tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1)
|
||||
|
||||
|
||||
def moe_align_block_size_triton(
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
block_size: int,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_pad: torch.Tensor,
|
||||
) -> None:
|
||||
numel = topk_ids.numel()
|
||||
grid = (num_experts,)
|
||||
tokens_cnts = torch.zeros(
|
||||
(num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device
|
||||
)
|
||||
cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device)
|
||||
tokens_per_thread = ceil_div(numel, num_experts)
|
||||
|
||||
moe_align_block_size_stage1[grid](
|
||||
topk_ids,
|
||||
tokens_cnts,
|
||||
num_experts,
|
||||
numel,
|
||||
tokens_per_thread,
|
||||
)
|
||||
moe_align_block_size_stage2[grid](
|
||||
tokens_cnts,
|
||||
num_experts,
|
||||
)
|
||||
moe_align_block_size_stage3[(1,)](
|
||||
num_tokens_post_pad,
|
||||
tokens_cnts,
|
||||
cumsum,
|
||||
num_experts,
|
||||
block_size,
|
||||
)
|
||||
moe_align_block_size_stage4[grid](
|
||||
topk_ids,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
tokens_cnts,
|
||||
cumsum,
|
||||
num_experts,
|
||||
block_size,
|
||||
numel,
|
||||
tokens_per_thread,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block_size,num_tokens,topk,num_experts,pad_sorted_token_ids",
|
||||
list(
|
||||
itertools.product(
|
||||
[32, 64, 128, 256], # block_size
|
||||
[1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096], # num_tokens
|
||||
[1, 2, 4, 8, 16, 32, 64], # topk
|
||||
[64, 160, 256, 257, 260, 264], # num_experts
|
||||
[32, 256], # block_size
|
||||
[1, 16, 4096], # num_tokens
|
||||
[1, 8, 64], # topk
|
||||
[64, 160, 257, 264], # num_experts
|
||||
[True, False], # pad_sorted_token_ids
|
||||
)
|
||||
),
|
||||
|
||||
@@ -8,58 +8,68 @@ from sgl_kernel.test_utils import (
|
||||
create_per_token_group_quant_test_data,
|
||||
)
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
per_token_group_quant_8bit as triton_per_token_group_quant_8bit,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
sglang_per_token_group_quant_8bit,
|
||||
)
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.test.kernels.per_token_group_quant import (
|
||||
sglang_per_token_group_quant_8bit,
|
||||
triton_per_token_group_quant_8bit,
|
||||
)
|
||||
|
||||
_is_hip = is_hip()
|
||||
fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn
|
||||
|
||||
configs = list(
|
||||
itertools.product(
|
||||
[1, 4, 16, 64, 127, 128, 512, 1024, 4096, 8192], # num_tokens
|
||||
[128, 256, 384, 512, 1024, 1536, 1664, 2048, 4096, 7168, 16384], # hidden_dim
|
||||
_ROW_MAJOR = dict(
|
||||
column_major_scales=False,
|
||||
scale_tma_aligned=False,
|
||||
scale_ue8m0=False,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
)
|
||||
_COLUMN_MAJOR_FLAGS = [
|
||||
dict(
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=False,
|
||||
scale_ue8m0=False,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
dict(
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=True,
|
||||
scale_ue8m0=False,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
dict(
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=True,
|
||||
scale_ue8m0=True,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _supported(group_size, dst_dtype, flags):
|
||||
# int8 has no column-major scale layout; UE8M0 scales are only built for group 128.
|
||||
if dst_dtype == torch.int8 and flags["column_major_scales"]:
|
||||
return False
|
||||
return not flags["scale_ue8m0"] or group_size == 128
|
||||
|
||||
|
||||
configs = [
|
||||
config
|
||||
for config in itertools.product(
|
||||
[1, 4, 127, 4096], # num_tokens
|
||||
[128, 384, 1664, 7168], # hidden_dim
|
||||
[16, 32, 64, 128], # group_size
|
||||
[None], # num_ranks
|
||||
[fp8_type_, torch.int8], # dtype
|
||||
[
|
||||
dict(
|
||||
column_major_scales=False,
|
||||
scale_tma_aligned=False,
|
||||
scale_ue8m0=False,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
dict(
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=False,
|
||||
scale_ue8m0=False,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
dict(
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=True,
|
||||
scale_ue8m0=False,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
dict(
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=True,
|
||||
scale_ue8m0=True,
|
||||
fuse_silu_and_mul=False,
|
||||
masked_layout_mode=None,
|
||||
),
|
||||
],
|
||||
[_ROW_MAJOR, *_COLUMN_MAJOR_FLAGS],
|
||||
)
|
||||
) + list(
|
||||
if _supported(config[2], config[4], config[5])
|
||||
] + list(
|
||||
itertools.product(
|
||||
[1, 4, 1 * 8, 4 * 8, 64 * 8, 256 * 8, 768 * 8],
|
||||
[1, 4 * 8, 768 * 8],
|
||||
# TODO support more
|
||||
[2048],
|
||||
[128],
|
||||
@@ -117,21 +127,11 @@ def test_per_token_group_quant_with_column_major(
|
||||
arch_major, _ = torch.cuda.get_device_capability(torch.cuda.current_device())
|
||||
if flags["scale_ue8m0"] and (arch_major <= 9):
|
||||
pytest.skip("Only Blackwell need ue8m0 fusion")
|
||||
return
|
||||
|
||||
if (flags["scale_ue8m0"] and (group_size != 128)) or (
|
||||
(dst_dtype == torch.int8) and flags["column_major_scales"]
|
||||
):
|
||||
pytest.skip()
|
||||
return
|
||||
|
||||
x, masked_m = create_per_token_group_quant_test_data(
|
||||
num_tokens=num_tokens, hidden_dim=hidden_dim, num_ranks=num_ranks, flags=flags
|
||||
)
|
||||
|
||||
# print("hack data!!!")
|
||||
# x = torch.full_like(x, fill_value=100)
|
||||
|
||||
execute_kwargs = dict(
|
||||
x=x,
|
||||
masked_m=masked_m,
|
||||
|
||||
@@ -539,10 +539,11 @@ def test_topk_variants_are_exact_on_variable_windows(
|
||||
kind: str, force_one_block: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Cover both dispatches and all three APIs on adversarial score distributions."""
|
||||
monkeypatch.delenv("SGL_DSA_TOPK_ROW_SPLIT", raising=False)
|
||||
if force_one_block:
|
||||
monkeypatch.setenv("SGL_DSA_TOPK_ROW_SPLIT", "0")
|
||||
monkeypatch.setenv("SGLANG_DSA_TOPK_ROW_SPLIT", "0")
|
||||
else:
|
||||
monkeypatch.delenv("SGL_DSA_TOPK_ROW_SPLIT", raising=False)
|
||||
monkeypatch.delenv("SGLANG_DSA_TOPK_ROW_SPLIT", raising=False)
|
||||
|
||||
bs, k = 4, 2048
|
||||
score = _make_scores(kind, bs, MAX_SEQ_LEN, seed=1200 + force_one_block)
|
||||
|
||||
@@ -35,7 +35,7 @@ Dispatch priority (highest first), resolved by :meth:`BaseFusedOp.forward`:
|
||||
whose method is overridden, that is *declared* in :attr:`capabilities`,
|
||||
and whose :class:`~sglang.kernels.spec.CapabilityRequirement` set matches
|
||||
the detected platform. Ops may extend :meth:`backend_eligible` with
|
||||
per-call shape/dtype gates; overriding it switches this step from a
|
||||
per-call applicability gates; overriding it switches this step from a
|
||||
statically cached choice to per-call selection.
|
||||
5. **Platform-specific forward** — ``forward_cuda`` on CUDA, ``forward_hip``
|
||||
(falling back to ``forward_cuda``) on ROCm, ``forward_musa`` on MUSA,
|
||||
@@ -86,12 +86,11 @@ from torch import nn
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.selector import _platform
|
||||
from sglang.kernels.spec import (
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
PlatformInfo,
|
||||
capabilities_satisfied,
|
||||
)
|
||||
|
||||
@@ -159,11 +158,6 @@ _PLATFORM_METHODS: Dict[str, Tuple[str, ...]] = {
|
||||
}
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _platform() -> PlatformInfo:
|
||||
return PlatformInfo.detect()
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _platform_key() -> str:
|
||||
"""In-tree platform dispatch key, or ``""`` for plain-native platforms.
|
||||
@@ -207,13 +201,6 @@ def _oot_dispatch_key() -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def clear_platform_caches() -> None:
|
||||
"""Drop the cached platform detection (used by tests that mock platforms)."""
|
||||
_platform.cache_clear()
|
||||
_platform_key.cache_clear()
|
||||
_oot_dispatch_key.cache_clear()
|
||||
|
||||
|
||||
# --- global backend override ------------------------------------------------
|
||||
|
||||
# Sentinel distinguishing "not resolved yet" from "resolved to None (no force)".
|
||||
@@ -350,7 +337,7 @@ class BaseFusedOp(nn.Module, ABC):
|
||||
Class attributes
|
||||
----------------
|
||||
op:
|
||||
Operator id, ``"<group>.<name>"`` (e.g. ``"layernorm.rmsnorm"``).
|
||||
Operator id, ``"<group>.<name>"`` (e.g. ``"activation.silu_and_mul"``).
|
||||
Required for :func:`register_fused_op`; layer-style subclasses that
|
||||
are not registered in the kernel registry may leave it empty.
|
||||
priority:
|
||||
@@ -368,10 +355,6 @@ class BaseFusedOp(nn.Module, ABC):
|
||||
the ``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.NPU`` shortcuts,
|
||||
e.g. ``{KernelBackend.AOT: {CapabilityRequirement.CUDA,
|
||||
CapabilityRequirement.HIP}}``.
|
||||
format_signature:
|
||||
Data-contract description shared by all backends of this op.
|
||||
descriptions:
|
||||
Optional per-backend one-liners for the registry inventory.
|
||||
"""
|
||||
|
||||
op: ClassVar[str] = ""
|
||||
@@ -379,8 +362,6 @@ class BaseFusedOp(nn.Module, ABC):
|
||||
capabilities: ClassVar[
|
||||
Mapping[KernelBackend, AbstractSet[CapabilityRequirement]]
|
||||
] = {}
|
||||
format_signature: ClassVar[FormatSignature] = FormatSignature()
|
||||
descriptions: ClassVar[Mapping[KernelBackend, str]] = {}
|
||||
|
||||
# OOT forward registry shared by all fused ops: platform dispatch key ->
|
||||
# {op class -> forward fn}. Populated by out-of-tree platform plugins.
|
||||
@@ -438,6 +419,9 @@ class BaseFusedOp(nn.Module, ABC):
|
||||
def forward_cute_dsl(self, *args, **kwargs):
|
||||
raise NotImplementedError(f"{self._op_label()}: no cute_dsl backend")
|
||||
|
||||
def forward_flydsl(self, *args, **kwargs):
|
||||
raise NotImplementedError(f"{self._op_label()}: no flydsl backend")
|
||||
|
||||
def forward_helion(self, *args, **kwargs):
|
||||
raise NotImplementedError(f"{self._op_label()}: no helion backend")
|
||||
|
||||
@@ -480,10 +464,12 @@ class BaseFusedOp(nn.Module, ABC):
|
||||
|
||||
The base implementation checks the backend's
|
||||
:class:`CapabilityRequirement` set (OR semantics) against the detected
|
||||
platform. Subclasses may extend it with per-call shape/dtype gates so
|
||||
auto-selection bounces to the next backend instead of raising;
|
||||
overriding this method switches backend auto-selection from a cached
|
||||
static choice to per-call resolution.
|
||||
platform. Subclasses may extend it with per-call gates for operands the
|
||||
kernel does not cover, where the next backend computes the same result;
|
||||
operands that violate the op's contract are a caller bug the backend's
|
||||
launcher must raise on, not a reason to bounce. Overriding this method
|
||||
switches backend auto-selection from a cached static choice to per-call
|
||||
resolution.
|
||||
"""
|
||||
return capabilities_satisfied(
|
||||
self.capabilities.get(backend, frozenset()), _platform()
|
||||
@@ -586,10 +572,6 @@ class BaseFusedOp(nn.Module, ABC):
|
||||
_record_trace(self, _dispatch_label(method), args, kwargs)
|
||||
return result
|
||||
|
||||
def dispatch_forward(self) -> Callable:
|
||||
"""The static dispatch target for this op on the current platform."""
|
||||
return self._resolve_forward_method()
|
||||
|
||||
# --- torch.compile mode ---
|
||||
|
||||
def _torch_compile_forward(self, num_tokens: int) -> Optional[Callable]:
|
||||
@@ -684,7 +666,7 @@ def register_fused_op(instance: BaseFusedOp, module: str, attr: str) -> BaseFuse
|
||||
``KernelSpec.load()`` can lazily resolve e.g.
|
||||
``"<module>:<attr>.forward_aot"`` to the bound backend method. Returns
|
||||
``instance`` so group packages can write
|
||||
``_RMSNORM = register_fused_op(_RMSNormOp(), __name__, "_RMSNORM")``.
|
||||
``_SILU_AND_MUL = register_fused_op(SiluAndMulOp(), __name__, "_SILU_AND_MUL")``.
|
||||
"""
|
||||
for backend in instance.available_backends():
|
||||
register_kernel(
|
||||
@@ -693,8 +675,6 @@ def register_fused_op(instance: BaseFusedOp, module: str, attr: str) -> BaseFuse
|
||||
backend=backend,
|
||||
target=f"{module}:{attr}.{BACKEND_METHODS[backend]}",
|
||||
capabilities=frozenset(instance.capabilities.get(backend, ())),
|
||||
format_signature=instance.format_signature,
|
||||
description=instance.descriptions.get(backend, ""),
|
||||
)
|
||||
)
|
||||
return instance
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Callable, List, Optional, Sequence, Tuple
|
||||
import torch
|
||||
import triton.testing
|
||||
|
||||
from sglang.kernels.ops.communication.mp import multigpu_launch
|
||||
from sglang.test.kernels.multigpu import multigpu_launch
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
|
||||
|
||||
@@ -1,522 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/tile.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
|
||||
#include <sgl_kernel/deepseek_v4/compress.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
#include <tvm/ffi/object.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using Plan128 = device::compress::PrefillPlan;
|
||||
using IndiceT = int32_t;
|
||||
|
||||
/// \brief Each thread will handle this many elements (split along head_dim)
|
||||
constexpr int32_t kTileElements = 2;
|
||||
/// \brief Each warp will handle this many elements (split along 128)
|
||||
constexpr int32_t kElementsPerWarp = 8;
|
||||
constexpr uint32_t kNumWarps = 128 / kElementsPerWarp;
|
||||
constexpr uint32_t kBlockSize = device::kWarpThreads * kNumWarps;
|
||||
|
||||
/// \brief Need to reduce register usage to increase occupancy
|
||||
#define C128_KERNEL __global__ __launch_bounds__(kBlockSize, 2)
|
||||
|
||||
struct Compress128DecodeParams {
|
||||
/**
|
||||
* \brief Shape: `[num_indices, 128, head_dim * 2]` \n
|
||||
* last dimension layout:
|
||||
* | kv current | score current |
|
||||
*/
|
||||
void* __restrict__ kv_score_buffer;
|
||||
/** \brief Shape: `[batch_size, head_dim * 2]` */
|
||||
const void* __restrict__ kv_score_input;
|
||||
/** \brief Shape: `[batch_size, head_dim]` */
|
||||
void* __restrict__ kv_compressed_output;
|
||||
/** \brief Shape: `[128, head_dim]` (called `ape`) */
|
||||
const void* __restrict__ score_bias;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ indices;
|
||||
/** \brief Shape: `[batch_size, ]` */
|
||||
const IndiceT* __restrict__ seq_lens;
|
||||
/** \NOTE: `batch_size` <= `num_indices` */
|
||||
uint32_t batch_size;
|
||||
};
|
||||
|
||||
struct Compress128PrefillParams {
|
||||
/**
|
||||
* \brief Shape: `[num_indices, 128, head_dim * 2]` \n
|
||||
* last dimension layout:
|
||||
* | kv current | score current |
|
||||
*/
|
||||
void* __restrict__ kv_score_buffer;
|
||||
/** \brief Shape: `[batch_size, head_dim * 2]` */
|
||||
const void* __restrict__ kv_score_input;
|
||||
/** \brief Shape: `[batch_size, head_dim]` */
|
||||
void* __restrict__ kv_compressed_output;
|
||||
/** \brief Shape: `[128, head_dim]` (called `ape`) */
|
||||
const void* __restrict__ score_bias;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ indices;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const int32_t* __restrict__ load_indices;
|
||||
/** \brief The following part is plan info. */
|
||||
const Plan128* __restrict__ compress_plan;
|
||||
const Plan128* __restrict__ write_plan;
|
||||
uint32_t num_compress;
|
||||
uint32_t num_write;
|
||||
};
|
||||
|
||||
struct Compress128SharedBuffer {
|
||||
using Storage = device::AlignedVector<float, kTileElements>;
|
||||
Storage data[kNumWarps][device::kWarpThreads + 1]; // padding to avoid bank conflict
|
||||
SGL_DEVICE Storage& operator()(uint32_t warp_id, uint32_t lane_id) {
|
||||
return data[warp_id][lane_id];
|
||||
}
|
||||
SGL_DEVICE float& operator()(uint32_t warp_id, uint32_t lane_id, uint32_t tile_id) {
|
||||
return data[warp_id][lane_id][tile_id];
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE void c128_write(
|
||||
T* kv_score_buf, //
|
||||
const T* kv_score_src,
|
||||
const int64_t head_dim,
|
||||
const int32_t write_pos,
|
||||
const uint32_t lane_id) {
|
||||
using namespace device;
|
||||
|
||||
using Storage = AlignedVector<T, kTileElements>;
|
||||
const auto element_size = head_dim * 2;
|
||||
const auto gmem = tile::Memory<Storage>{lane_id, kWarpThreads};
|
||||
kv_score_buf += write_pos * element_size;
|
||||
|
||||
/// NOTE: Layout | [0] = kv | [1] = score |
|
||||
Storage kv_score[2];
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 2; ++i) {
|
||||
kv_score[i] = gmem.load(kv_score_src + head_dim * i);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 2; ++i) {
|
||||
gmem.store(kv_score_buf + head_dim * i, kv_score[i]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename InFloat, typename OutFloat>
|
||||
SGL_DEVICE void c128_forward(
|
||||
const InFloat* kv_score_buf,
|
||||
const InFloat* kv_score_src,
|
||||
OutFloat* kv_out,
|
||||
const InFloat* score_bias,
|
||||
const int64_t head_dim,
|
||||
const int32_t window_len,
|
||||
const uint32_t warp_id,
|
||||
const uint32_t lane_id) {
|
||||
using namespace device;
|
||||
|
||||
const auto element_size = head_dim * 2;
|
||||
const auto score_offset = head_dim;
|
||||
|
||||
/// NOTE: part 1: load kv + score
|
||||
using StorageIn = AlignedVector<InFloat, kTileElements>;
|
||||
const auto gmem_in = tile::Memory<StorageIn>{lane_id, kWarpThreads};
|
||||
StorageIn kv[kElementsPerWarp];
|
||||
StorageIn score[kElementsPerWarp];
|
||||
StorageIn bias[kElementsPerWarp];
|
||||
const int32_t warp_offset = warp_id * kElementsPerWarp;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
const int32_t j = i + warp_offset;
|
||||
bias[i] = gmem_in.load(score_bias + j * head_dim);
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < kElementsPerWarp; ++i) {
|
||||
const int32_t j = i + warp_offset;
|
||||
const InFloat* src;
|
||||
__builtin_assume(j < 128);
|
||||
if (j < window_len) {
|
||||
src = kv_score_buf + j * element_size;
|
||||
} else {
|
||||
/// NOTE: k in [-127, 0]. We'll load from the ragged `kv_score_src`
|
||||
const int32_t k = j - 127;
|
||||
src = kv_score_src + k * element_size;
|
||||
}
|
||||
kv[i] = gmem_in.load(src);
|
||||
score[i] = gmem_in.load(src + score_offset);
|
||||
}
|
||||
|
||||
/// NOTE: part 2: safe online softmax + weighted sum
|
||||
using TmpStorage = typename Compress128SharedBuffer::Storage;
|
||||
__shared__ Compress128SharedBuffer s_local_val_max;
|
||||
__shared__ Compress128SharedBuffer s_local_exp_sum;
|
||||
__shared__ Compress128SharedBuffer s_local_product;
|
||||
|
||||
TmpStorage tmp_val_max;
|
||||
TmpStorage tmp_exp_sum;
|
||||
TmpStorage tmp_product;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < kTileElements; ++i) {
|
||||
float score_fp32[kElementsPerWarp];
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t j = 0; j < kElementsPerWarp; ++j) {
|
||||
score_fp32[j] = cast<float>(score[j][i]) + cast<float>(bias[j][i]);
|
||||
}
|
||||
|
||||
float max_value = score_fp32[0];
|
||||
float sum_exp_value = 0.0f;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t j = 1; j < kElementsPerWarp; ++j) {
|
||||
const auto fp32_score = score_fp32[j];
|
||||
max_value = fmaxf(max_value, fp32_score);
|
||||
}
|
||||
|
||||
float sum_product = 0.0f;
|
||||
#pragma unroll
|
||||
for (int32_t j = 0; j < 8; ++j) {
|
||||
const auto fp32_score = score_fp32[j];
|
||||
const auto exp_score = expf(fp32_score - max_value);
|
||||
sum_product += cast<float>(kv[j][i]) * exp_score;
|
||||
sum_exp_value += exp_score;
|
||||
}
|
||||
|
||||
tmp_val_max[i] = max_value;
|
||||
tmp_exp_sum[i] = sum_exp_value;
|
||||
tmp_product[i] = sum_product;
|
||||
}
|
||||
|
||||
// naturally aligned, so no bank conflict
|
||||
s_local_val_max(warp_id, lane_id) = tmp_val_max;
|
||||
s_local_exp_sum(warp_id, lane_id) = tmp_exp_sum;
|
||||
s_local_product(warp_id, lane_id) = tmp_product;
|
||||
|
||||
__syncthreads();
|
||||
|
||||
/// NOTE: part 3: online softmax
|
||||
/// NOTE: We have `kTileElements * kWarpThreads * kNumWarps` values to reduce
|
||||
/// each reduce will consume `kNumWarps` threads (use partial warp reduction)
|
||||
constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps;
|
||||
constexpr uint32_t kIteration = kReductionCount / kBlockSize;
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kIteration; ++i) {
|
||||
/// NOTE: Range `[0, kTileElements * kWarpThreads * kNumWarps)`
|
||||
const uint32_t j = i * kBlockSize + warp_id * kWarpThreads + lane_id;
|
||||
/// NOTE: Range `[0, kNumWarps)`
|
||||
const uint32_t local_warp_id = j % kNumWarps;
|
||||
/// NOTE: Range `[0, kTileElements * kWarpThreads)`
|
||||
const uint32_t local_elem_id = j / kNumWarps;
|
||||
/// NOTE: Range `[0, kTileElements)`
|
||||
const uint32_t local_tile_id = local_elem_id % kTileElements;
|
||||
/// NOTE: Range `[0, kWarpThreads)`
|
||||
const uint32_t local_lane_id = local_elem_id / kTileElements;
|
||||
/// NOTE: each warp will access the whole tile (all `kTileElements`)
|
||||
/// and for different lanes, the memory access only differ in `local_warp_id`
|
||||
/// so there's no bank conflict in shared memory access.
|
||||
static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs");
|
||||
const auto local_val_max = s_local_val_max(local_warp_id, local_lane_id, local_tile_id);
|
||||
const auto local_exp_sum = s_local_exp_sum(local_warp_id, local_lane_id, local_tile_id);
|
||||
const auto local_product = s_local_product(local_warp_id, local_lane_id, local_tile_id);
|
||||
const auto global_val_max = warp::reduce_max<kNumWarps>(local_val_max);
|
||||
const auto rescale = expf(local_val_max - global_val_max);
|
||||
const auto global_exp_sum = warp::reduce_sum<kNumWarps>(local_exp_sum * rescale);
|
||||
const auto final_scale = rescale / global_exp_sum;
|
||||
const auto global_product = warp::reduce_sum<kNumWarps>(local_product * final_scale);
|
||||
kv_out[local_elem_id] = cast<OutFloat>(global_product);
|
||||
}
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kUsePDL>
|
||||
C128_KERNEL void flash_c128_decode(const __grid_constant__ Compress128DecodeParams params) {
|
||||
using namespace device;
|
||||
|
||||
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64
|
||||
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
constexpr int64_t kElementSize = kHeadDim * 2;
|
||||
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
|
||||
|
||||
const auto& [
|
||||
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
|
||||
indices, seq_lens, batch_size // decode info
|
||||
] = params;
|
||||
const uint32_t warp_id = threadIdx.x / kWarpThreads;
|
||||
const uint32_t lane_id = threadIdx.x % kWarpThreads;
|
||||
|
||||
const uint32_t global_bid = blockIdx.x / kNumSplit; // batch id
|
||||
const uint32_t global_sid = blockIdx.x % kNumSplit; // split id
|
||||
if (global_bid >= batch_size) return;
|
||||
|
||||
const int32_t index = indices[global_bid];
|
||||
const int32_t seq_len = seq_lens[global_bid];
|
||||
const int64_t split_offset = global_sid * kTileDim;
|
||||
|
||||
// kv score
|
||||
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
|
||||
const auto kv_buf = kv_score_buffer + index * (kElementSize * 128) + split_offset;
|
||||
|
||||
// kv input
|
||||
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
|
||||
const auto kv_src = kv_score_input + global_bid * kElementSize + split_offset;
|
||||
|
||||
// kv output
|
||||
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
|
||||
const auto kv_out = kv_compressed_output + global_bid * kHeadDim + split_offset;
|
||||
|
||||
// score bias (ape)
|
||||
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
|
||||
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
/// NOTE: the write must be visible to the subsequent c128_forward,
|
||||
/// so only the last warp can write to HBM
|
||||
/// In addition, `position` = `seq_len - 1`. To avoid underflow, we use `seq_len + 127`
|
||||
if (warp_id == kNumWarps - 1) {
|
||||
c128_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 127) % 128, lane_id);
|
||||
}
|
||||
if (seq_len % 128 == 0) {
|
||||
c128_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, /*window_len=*/128, warp_id, lane_id);
|
||||
}
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
// compress kernel
|
||||
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kWrite, bool kUsePDL>
|
||||
C128_KERNEL void flash_c128_prefill(const __grid_constant__ Compress128PrefillParams params) {
|
||||
using namespace device;
|
||||
|
||||
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64
|
||||
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
constexpr int64_t kElementSize = kHeadDim * 2;
|
||||
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
|
||||
|
||||
const auto& [
|
||||
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
|
||||
indices, load_indices, compress_plan, write_plan, num_compress, num_write // prefill plan
|
||||
] = params;
|
||||
const uint32_t warp_id = threadIdx.x / kWarpThreads;
|
||||
const uint32_t lane_id = threadIdx.x % kWarpThreads;
|
||||
|
||||
uint32_t global_id;
|
||||
if constexpr (kWrite) {
|
||||
// for write kernel, we use global warp_id to dispatch work
|
||||
global_id = (blockIdx.x * blockDim.x + threadIdx.x) / kWarpThreads;
|
||||
} else {
|
||||
// for compress kernel, we use block id to dispatch work
|
||||
global_id = blockIdx.x; // block id
|
||||
}
|
||||
const uint32_t global_pid = global_id / kNumSplit; // plan id
|
||||
const uint32_t global_sid = global_id % kNumSplit; // split id
|
||||
|
||||
/// NOTE: compiler can optimize this if-else at compile time
|
||||
const auto num_plans = kWrite ? num_write : num_compress;
|
||||
const auto plan_ptr = kWrite ? write_plan : compress_plan;
|
||||
if (global_pid >= num_plans) return;
|
||||
|
||||
const auto& [ragged_id, global_bid, position, window_len] = plan_ptr[global_pid];
|
||||
const auto indices_ptr = kWrite ? indices : load_indices;
|
||||
|
||||
const int64_t split_offset = global_sid * kTileDim;
|
||||
|
||||
// kv input
|
||||
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
|
||||
const auto kv_src = kv_score_input + ragged_id * kElementSize + split_offset;
|
||||
|
||||
// kv output
|
||||
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
|
||||
const auto kv_out = kv_compressed_output + ragged_id * kHeadDim + split_offset;
|
||||
|
||||
// score bias (ape)
|
||||
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
|
||||
|
||||
if (ragged_id == 0xFFFFFFFF) [[unlikely]]
|
||||
return;
|
||||
|
||||
const int32_t index = indices_ptr[global_bid];
|
||||
// kv score
|
||||
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
|
||||
const auto kv_buf = kv_score_buffer + index * (kElementSize * 128) + split_offset;
|
||||
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
// only responsible for the compress part
|
||||
if constexpr (kWrite) {
|
||||
c128_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 128, lane_id);
|
||||
} else {
|
||||
c128_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, window_len, warp_id, lane_id);
|
||||
}
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kUsePDL>
|
||||
struct FlashCompress128Kernel {
|
||||
static constexpr auto decode_kernel = flash_c128_decode<kHeadDim, InFloat, OutFloat, kUsePDL>;
|
||||
template <bool kWrite>
|
||||
static constexpr auto prefill_kernel = flash_c128_prefill<kHeadDim, InFloat, OutFloat, kWrite, kUsePDL>;
|
||||
static constexpr auto prefill_c_kernel = prefill_kernel</*kWrite=*/false>;
|
||||
static constexpr auto prefill_w_kernel = prefill_kernel</*kWrite=*/true>;
|
||||
static constexpr int64_t kTileDim = kTileElements * device::kWarpThreads; // 64
|
||||
static constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
static constexpr uint32_t kWriteBlockSize = 128;
|
||||
static constexpr uint32_t kWarpsPerWriteBlock = kWriteBlockSize / device::kWarpThreads;
|
||||
|
||||
static void run_decode(
|
||||
const tvm::ffi::TensorView kv_score_buffer,
|
||||
const tvm::ffi::TensorView kv_score_input,
|
||||
const tvm::ffi::TensorView kv_compressed_output,
|
||||
const tvm::ffi::TensorView ape,
|
||||
const tvm::ffi::TensorView indices,
|
||||
const tvm::ffi::TensorView seq_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> /* UNUSED */) {
|
||||
using namespace host;
|
||||
|
||||
// this should not happen in practice
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({-1, 128, kHeadDim * 2}) // kv score
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device)
|
||||
.verify(kv_score_buffer);
|
||||
TensorMatcher({B, kHeadDim * 2}) // kv score input
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device)
|
||||
.verify(kv_score_input);
|
||||
TensorMatcher({B, kHeadDim}) // kv compressed output
|
||||
.with_dtype<OutFloat>()
|
||||
.with_device(device)
|
||||
.verify(kv_compressed_output);
|
||||
TensorMatcher({128, kHeadDim}) // ape
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device)
|
||||
.verify(ape);
|
||||
TensorMatcher({B}) // indices
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device)
|
||||
.verify(indices);
|
||||
TensorMatcher({B}) // seq lens
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device)
|
||||
.verify(seq_lens);
|
||||
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
const auto params = Compress128DecodeParams{
|
||||
.kv_score_buffer = kv_score_buffer.data_ptr(),
|
||||
.kv_score_input = kv_score_input.data_ptr(),
|
||||
.kv_compressed_output = kv_compressed_output.data_ptr(),
|
||||
.score_bias = ape.data_ptr(),
|
||||
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
|
||||
.seq_lens = static_cast<const IndiceT*>(seq_lens.data_ptr()),
|
||||
.batch_size = batch_size,
|
||||
};
|
||||
|
||||
const uint32_t num_blocks = batch_size * kNumSplit;
|
||||
LaunchKernel(num_blocks, kBlockSize, device.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(decode_kernel, params);
|
||||
}
|
||||
|
||||
static void run_prefill(
|
||||
const tvm::ffi::TensorView kv_score_buffer,
|
||||
const tvm::ffi::TensorView kv_score_input,
|
||||
const tvm::ffi::TensorView kv_compressed_output,
|
||||
const tvm::ffi::TensorView ape,
|
||||
const tvm::ffi::TensorView indices,
|
||||
const tvm::ffi::TensorView compress_plan,
|
||||
const tvm::ffi::TensorView write_plan,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto N = SymbolicSize{"num_q_tokens"};
|
||||
auto X = SymbolicSize{"compress_tokens"};
|
||||
auto Y = SymbolicSize{"write_tokens"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({-1, 128, kHeadDim * 2}) // kv score
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_buffer);
|
||||
TensorMatcher({N, kHeadDim * 2}) // kv score input
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_input);
|
||||
TensorMatcher({N, kHeadDim}) // kv compressed output
|
||||
.with_dtype<OutFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_compressed_output);
|
||||
TensorMatcher({128, kHeadDim}) // ape
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(ape);
|
||||
TensorMatcher({B}) // indices
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device_)
|
||||
.verify(indices);
|
||||
TensorMatcher({X, compress::kPrefillPlanDim}) // compress plan
|
||||
.with_dtype<compress::PrefillPlanTensorDtype>()
|
||||
.with_device(device_)
|
||||
.verify(compress_plan);
|
||||
TensorMatcher({Y, compress::kPrefillPlanDim}) // write plan
|
||||
.with_dtype<compress::PrefillPlanTensorDtype>()
|
||||
.with_device(device_)
|
||||
.verify(write_plan);
|
||||
|
||||
// might be needed for prefill write
|
||||
const auto load_indices = extra.value_or(indices);
|
||||
TensorMatcher({B}) // [read_positions]
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device_)
|
||||
.verify(load_indices);
|
||||
|
||||
const auto device = device_.unwrap();
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
const auto num_q_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
const auto num_c = static_cast<uint32_t>(X.unwrap());
|
||||
const auto num_w = static_cast<uint32_t>(Y.unwrap());
|
||||
const auto params = Compress128PrefillParams{
|
||||
.kv_score_buffer = kv_score_buffer.data_ptr(),
|
||||
.kv_score_input = kv_score_input.data_ptr(),
|
||||
.kv_compressed_output = kv_compressed_output.data_ptr(),
|
||||
.score_bias = ape.data_ptr(),
|
||||
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
|
||||
.load_indices = static_cast<const IndiceT*>(load_indices.data_ptr()),
|
||||
.compress_plan = static_cast<const Plan128*>(compress_plan.data_ptr()),
|
||||
.write_plan = static_cast<const Plan128*>(write_plan.data_ptr()),
|
||||
.num_compress = num_c,
|
||||
.num_write = num_w,
|
||||
};
|
||||
RuntimeCheck(num_q_tokens >= batch_size, "num_q_tokens must be >= batch_size");
|
||||
RuntimeCheck(num_q_tokens >= std::max(num_c, num_w), "invalid prefill plan");
|
||||
|
||||
constexpr auto kBlockSize_C = kBlockSize;
|
||||
constexpr auto kBlockSize_W = kWriteBlockSize;
|
||||
if (const auto num_c_blocks = num_c * kNumSplit) {
|
||||
LaunchKernel(num_c_blocks, kBlockSize_C, device) //
|
||||
.enable_pdl(kUsePDL)(prefill_c_kernel, params);
|
||||
}
|
||||
if (const auto num_w_blocks = div_ceil(num_w * kNumSplit, kWarpsPerWriteBlock)) {
|
||||
LaunchKernel(num_w_blocks, kBlockSize_W, device) //
|
||||
.enable_pdl(kUsePDL)(prefill_w_kernel, params);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
@@ -1,722 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/tile.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
|
||||
#include <sgl_kernel/deepseek_v4/compress.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
#include <tvm/ffi/container/tuple.h>
|
||||
#include <tvm/ffi/object.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cfloat>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
namespace device::compress {
|
||||
|
||||
/// \brief Plan entry for online compress 128 prefill.
|
||||
/// Each entry describes a contiguous segment of tokens that lies inside a
|
||||
/// single 128-chunk. Multiple segments can map to the same batch id when the
|
||||
/// extend tokens span chunk boundaries.
|
||||
///
|
||||
/// **Layout compatibility:** the field order/types match `PrefillPlan` so that
|
||||
/// downstream kernels (e.g. `fused_norm_rope` in `CompressExtend` mode) can
|
||||
/// consume the compress_plan tensor as-if it were a `PrefillPlan` tensor --
|
||||
/// they only read `ragged_id` and `position`, both of which carry identical
|
||||
/// semantics here (the LAST token of the segment in q-ragged and global
|
||||
/// coordinates respectively).
|
||||
///
|
||||
/// Note that `window_len` here means "number of real tokens in this segment"
|
||||
/// (1..128), which differs from `PrefillPlan::window_len`. Downstream kernels
|
||||
/// that share the tensor MUST NOT read it under that name.
|
||||
struct alignas(16) OnlinePrefillPlan {
|
||||
/// \brief Ragged-q position of the LAST token in this segment.
|
||||
/// Equal to `segment_start_ragged + window_len - 1`.
|
||||
uint32_t ragged_id;
|
||||
/// \brief Index into the `indices` / `load_indices` arrays.
|
||||
uint32_t batch_id;
|
||||
/// \brief Global position of the LAST token in this segment.
|
||||
/// For compress plans, `position % 128 == 127` (chunk-closing); for write
|
||||
/// plans, `position % 128 < 127`.
|
||||
uint32_t position;
|
||||
/// \brief Number of real tokens in this segment (1..128).
|
||||
/// The first segment token sits at `position - window_len + 1` (global) and
|
||||
/// at `ragged_id - window_len + 1` (ragged).
|
||||
uint32_t window_len;
|
||||
};
|
||||
|
||||
static_assert(alignof(OnlinePrefillPlan) == alignof(PrefillPlan));
|
||||
static_assert(sizeof(OnlinePrefillPlan) == sizeof(PrefillPlan));
|
||||
|
||||
} // namespace device::compress
|
||||
|
||||
namespace host::compress {
|
||||
|
||||
using device::compress::OnlinePrefillPlan;
|
||||
using OnlinePrefillPlanTensorDtype = uint8_t;
|
||||
inline constexpr int64_t kOnlinePrefillPlanDim = 16;
|
||||
|
||||
static_assert(alignof(OnlinePrefillPlan) == sizeof(OnlinePrefillPlan));
|
||||
static_assert(sizeof(OnlinePrefillPlan) == kOnlinePrefillPlanDim * sizeof(OnlinePrefillPlanTensorDtype));
|
||||
|
||||
} // namespace host::compress
|
||||
|
||||
using OnlinePlan = device::compress::OnlinePrefillPlan;
|
||||
using IndiceT = int32_t;
|
||||
|
||||
/// \brief Need to reduce register usage to increase occupancy
|
||||
struct Compress128OnlineDecodeParams {
|
||||
/** \brief Shape: `[num_indices, 1, head_dim * 3 (max, sum, kv) ]` \n */
|
||||
void* __restrict__ kv_score_buffer;
|
||||
/** \brief Shape: `[batch_size, head_dim * 2]` */
|
||||
const void* __restrict__ kv_score_input;
|
||||
/** \brief Shape: `[batch_size, head_dim]` */
|
||||
void* __restrict__ kv_compressed_output;
|
||||
/** \brief Shape: `[128, head_dim]` (called `ape`) */
|
||||
const void* __restrict__ score_bias;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ indices;
|
||||
/** \brief Shape: `[batch_size, ]` */
|
||||
const IndiceT* __restrict__ seq_lens;
|
||||
/** \NOTE: `batch_size` <= `num_indices` */
|
||||
uint32_t batch_size;
|
||||
};
|
||||
|
||||
/// \brief Need to reduce register usage to increase occupancy
|
||||
struct Compress128OnlinePrefillParams {
|
||||
/** \brief Shape: `[num_indices, 1, head_dim * 3 (max, sum, kv) ]` \n */
|
||||
void* __restrict__ kv_score_buffer;
|
||||
/** \brief Shape: `[num_q_tokens, head_dim * 2]` */
|
||||
const void* __restrict__ kv_score_input;
|
||||
/** \brief Shape: `[num_q_tokens, head_dim]` */
|
||||
void* __restrict__ kv_compressed_output;
|
||||
/** \brief Shape: `[128, head_dim]` (called `ape`) */
|
||||
const void* __restrict__ score_bias;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ indices;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ load_indices;
|
||||
/// \brief Plan for segments that close a chunk (write to `kv_compressed_output`).
|
||||
/// Shape: `[num_compress, 16]` (uint8).
|
||||
const OnlinePlan* __restrict__ compress_plan;
|
||||
/// \brief Plan for the trailing partial segment of each batch (write back to
|
||||
/// `kv_score_buffer`). Shape: `[num_write, 16]` (uint8).
|
||||
const OnlinePlan* __restrict__ write_plan;
|
||||
uint32_t num_compress;
|
||||
uint32_t num_write;
|
||||
};
|
||||
|
||||
// 4 elements per thread, kHeadDim / 4 threads per block
|
||||
template <int64_t kHeadDim, bool kUsePDL>
|
||||
__global__ void flash_c128_online_decode(const __grid_constant__ Compress128OnlineDecodeParams params) {
|
||||
using namespace device;
|
||||
constexpr uint32_t kVecSize = 4;
|
||||
constexpr uint32_t kBlockSize = kHeadDim / kVecSize;
|
||||
using Vec = AlignedVector<float, kVecSize>;
|
||||
const auto gmem = tile::Memory<Vec>::cta(kBlockSize);
|
||||
const auto batch_id = blockIdx.x;
|
||||
const auto index = params.indices[batch_id];
|
||||
const auto seq_len = params.seq_lens[batch_id];
|
||||
|
||||
const auto kv_score_buffer = static_cast<float*>(params.kv_score_buffer);
|
||||
const auto kv_buf = kv_score_buffer + index * (kHeadDim * 3);
|
||||
const auto kv_score_input = static_cast<const float*>(params.kv_score_input);
|
||||
const auto kv_src = kv_score_input + batch_id * (kHeadDim * 2);
|
||||
|
||||
/// NOTE: kv_score_buffer layout is [max, sum, kv] (slot 0 / 1 / 2). Reads,
|
||||
/// writes, and the prefill kernel must all agree on this order.
|
||||
const auto max_score_vec = gmem.load(kv_buf, 0);
|
||||
const auto sum_score_vec = gmem.load(kv_buf, 1);
|
||||
const auto old_kv_vec = gmem.load(kv_buf, 2);
|
||||
|
||||
/// NOTE: kv_score_input layout is | kv | score | (head_dim each), matching
|
||||
/// the offline c128 kernel and the online prefill kernel.
|
||||
const auto new_kv_vec = gmem.load(kv_src, 0);
|
||||
const auto new_score_raw_vec = gmem.load(kv_src, 1);
|
||||
|
||||
/// NOTE: the new token sits at global position `seq_len - 1`, so its
|
||||
/// position inside the 128-chunk is `(seq_len - 1) % 128`. The previous
|
||||
/// `seq_len % 128` was off by one (`bias[127]` vs `bias[0]`, etc.).
|
||||
const auto pos_in_chunk = (seq_len - 1) % 128;
|
||||
const auto bias_vec = gmem.load(params.score_bias, pos_in_chunk);
|
||||
|
||||
Vec out_kv_vec;
|
||||
Vec out_max_vec;
|
||||
Vec out_sum_vec;
|
||||
if (pos_in_chunk != 0) {
|
||||
// Mid-chunk: combine prior partial state with the new token via online softmax.
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 4; ++i) {
|
||||
const auto old_max = max_score_vec[i];
|
||||
const auto old_kv = old_kv_vec[i];
|
||||
const auto new_score = new_score_raw_vec[i] + bias_vec[i];
|
||||
const auto new_kv = new_kv_vec[i];
|
||||
const auto new_max = fmax(old_max, new_score);
|
||||
const auto old_sum = sum_score_vec[i] * expf(old_max - new_max);
|
||||
const auto new_exp = expf(new_score - new_max);
|
||||
const auto new_sum = old_sum + new_exp;
|
||||
out_kv_vec[i] = (old_kv * old_sum + new_kv * new_exp) / new_sum;
|
||||
out_max_vec[i] = new_max;
|
||||
out_sum_vec[i] = new_sum;
|
||||
}
|
||||
} else {
|
||||
// First token of a new 128-chunk: initialize state with this token alone.
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 4; ++i) {
|
||||
out_kv_vec[i] = new_kv_vec[i];
|
||||
out_max_vec[i] = new_score_raw_vec[i] + bias_vec[i];
|
||||
out_sum_vec[i] = 1.0f; // exp(score - max) with max == score
|
||||
}
|
||||
}
|
||||
|
||||
if (pos_in_chunk == 127) {
|
||||
// Chunk just closed: emit the compressed kv. No need to update the buffer
|
||||
// -- the next chunk's first token will overwrite it.
|
||||
const auto kv_out = static_cast<float*>(params.kv_compressed_output) + batch_id * kHeadDim;
|
||||
gmem.store(kv_out, out_kv_vec);
|
||||
} else {
|
||||
// Otherwise persist the running [max, sum, kv] state for the next step.
|
||||
gmem.store(kv_buf, out_max_vec, 0);
|
||||
gmem.store(kv_buf, out_sum_vec, 1);
|
||||
gmem.store(kv_buf, out_kv_vec, 2);
|
||||
}
|
||||
}
|
||||
|
||||
constexpr int32_t kTileElements = 2; // split (along head-dim)
|
||||
/// \brief Each warp will handle this many elements (split along softmax-128)
|
||||
constexpr int32_t kElementsPerWarp = 8;
|
||||
constexpr uint32_t kNumWarps = 128 / kElementsPerWarp;
|
||||
constexpr uint32_t kPrefillBlockSize = device::kWarpThreads * kNumWarps;
|
||||
using PrefillStorage = device::AlignedVector<float, kTileElements>;
|
||||
|
||||
struct Compress128SharedBuffer {
|
||||
using Storage = device::AlignedVector<float, 4>;
|
||||
Storage data[kNumWarps][device::kWarpThreads + 1]; // padding to avoid bank conflict
|
||||
SGL_DEVICE Storage& operator()(uint32_t warp_id, uint32_t lane_id) {
|
||||
return data[warp_id][lane_id];
|
||||
}
|
||||
SGL_DEVICE float& operator()(uint32_t warp_id, uint32_t lane_id, uint32_t tile_id) {
|
||||
return data[warp_id][lane_id][tile_id];
|
||||
}
|
||||
};
|
||||
|
||||
template <bool kNeedData>
|
||||
SGL_DEVICE void c128_prefill_forward(
|
||||
const PrefillStorage (&kv)[kElementsPerWarp],
|
||||
const PrefillStorage (&score)[kElementsPerWarp],
|
||||
float* kv_out,
|
||||
float* max_out,
|
||||
float* sum_out,
|
||||
const uint32_t warp_id,
|
||||
const uint32_t lane_id) {
|
||||
using namespace device;
|
||||
|
||||
/// NOTE: part 2: safe online softmax + weighted sum
|
||||
using TmpStorage = typename Compress128SharedBuffer::Storage;
|
||||
__shared__ Compress128SharedBuffer s_local_val_max;
|
||||
__shared__ Compress128SharedBuffer s_local_exp_sum;
|
||||
__shared__ Compress128SharedBuffer s_local_product;
|
||||
|
||||
TmpStorage tmp_val_max;
|
||||
TmpStorage tmp_exp_sum;
|
||||
TmpStorage tmp_product;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < kTileElements; ++i) {
|
||||
float score_fp32[kElementsPerWarp];
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t j = 0; j < kElementsPerWarp; ++j) {
|
||||
score_fp32[j] = score[j][i];
|
||||
}
|
||||
|
||||
float max_value = score_fp32[0];
|
||||
float sum_exp_value = 0.0f;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t j = 1; j < kElementsPerWarp; ++j) {
|
||||
const auto fp32_score = score_fp32[j];
|
||||
max_value = fmaxf(max_value, fp32_score);
|
||||
}
|
||||
|
||||
float sum_product = 0.0f;
|
||||
#pragma unroll
|
||||
for (int32_t j = 0; j < 8; ++j) {
|
||||
const auto fp32_score = score_fp32[j];
|
||||
const auto exp_score = expf(fp32_score - max_value);
|
||||
sum_product += cast<float>(kv[j][i]) * exp_score;
|
||||
sum_exp_value += exp_score;
|
||||
}
|
||||
|
||||
tmp_val_max[i] = max_value;
|
||||
tmp_exp_sum[i] = sum_exp_value;
|
||||
tmp_product[i] = sum_product;
|
||||
}
|
||||
|
||||
// naturally aligned, so no bank conflict
|
||||
s_local_val_max(warp_id, lane_id) = tmp_val_max;
|
||||
s_local_exp_sum(warp_id, lane_id) = tmp_exp_sum;
|
||||
s_local_product(warp_id, lane_id) = tmp_product;
|
||||
|
||||
__syncthreads();
|
||||
|
||||
/// NOTE: part 3: online softmax
|
||||
/// NOTE: We have `kTileElements * kWarpThreads * kNumWarps` values to reduce
|
||||
/// each reduce will consume `kNumWarps` threads (use partial warp reduction)
|
||||
constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps;
|
||||
constexpr uint32_t kIteration = kReductionCount / kPrefillBlockSize;
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kIteration; ++i) {
|
||||
/// NOTE: Range `[0, kTileElements * kWarpThreads * kNumWarps)`
|
||||
const uint32_t j = i * kPrefillBlockSize + warp_id * kWarpThreads + lane_id;
|
||||
/// NOTE: Range `[0, kNumWarps)`
|
||||
const uint32_t local_warp_id = j % kNumWarps;
|
||||
/// NOTE: Range `[0, kTileElements * kWarpThreads)`
|
||||
const uint32_t local_elem_id = j / kNumWarps;
|
||||
/// NOTE: Range `[0, kTileElements)`
|
||||
const uint32_t local_tile_id = local_elem_id % kTileElements;
|
||||
/// NOTE: Range `[0, kWarpThreads)`
|
||||
const uint32_t local_lane_id = local_elem_id / kTileElements;
|
||||
/// NOTE: each warp will access the whole tile (all `kTileElements`)
|
||||
/// and for different lanes, the memory access only differ in `local_warp_id`
|
||||
/// so there's no bank conflict in shared memory access.
|
||||
static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs");
|
||||
const auto local_val_max = s_local_val_max(local_warp_id, local_lane_id, local_tile_id);
|
||||
const auto local_exp_sum = s_local_exp_sum(local_warp_id, local_lane_id, local_tile_id);
|
||||
const auto local_product = s_local_product(local_warp_id, local_lane_id, local_tile_id);
|
||||
const auto global_val_max = warp::reduce_max<kNumWarps>(local_val_max);
|
||||
const auto rescale = expf(local_val_max - global_val_max);
|
||||
const auto global_exp_sum = warp::reduce_sum<kNumWarps>(local_exp_sum * rescale);
|
||||
const auto final_scale = rescale / global_exp_sum;
|
||||
const auto global_product = warp::reduce_sum<kNumWarps>(local_product * final_scale);
|
||||
kv_out[local_elem_id] = global_product;
|
||||
if constexpr (kNeedData) {
|
||||
max_out[local_elem_id] = global_val_max;
|
||||
sum_out[local_elem_id] = global_exp_sum;
|
||||
}
|
||||
}
|
||||
if constexpr (kNeedData) __syncthreads();
|
||||
}
|
||||
|
||||
/// \brief Sentinel score for padded positions in a 128-segment.
|
||||
/// Must be finite so that `score - max` never produces NaN even when an
|
||||
/// entire warp has only padded positions.
|
||||
constexpr float kPadScore = -FLT_MAX;
|
||||
|
||||
/// \brief Online compress 128 prefill. Two passes share this body:
|
||||
/// - `kWrite=false` (compress pass): handles segments that close a chunk.
|
||||
/// May load prior partial state from the buffer, but never writes to it,
|
||||
/// so concurrent blocks can read the same slot without racing.
|
||||
/// - `kWrite=true` (write pass): handles the trailing partial segment of each
|
||||
/// batch. Each batch contributes at most one such plan, so concurrent blocks
|
||||
/// touch disjoint buffer slots.
|
||||
///
|
||||
/// The two passes MUST run as separate kernel launches (in stream order) so
|
||||
/// that all reads in pass 1 finish before any writes in pass 2 start.
|
||||
template <int64_t kHeadDim, bool kWrite, bool kUsePDL>
|
||||
__global__ __launch_bounds__(kPrefillBlockSize, 2) //
|
||||
void flash_c128_online_prefill(const __grid_constant__ Compress128OnlinePrefillParams params) {
|
||||
using namespace device;
|
||||
|
||||
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64
|
||||
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
|
||||
|
||||
/// NOTE: the compiler folds the if-else at compile time.
|
||||
const auto num_plans = kWrite ? params.num_write : params.num_compress;
|
||||
const auto plan_ptr = kWrite ? params.write_plan : params.compress_plan;
|
||||
const uint32_t global_id = blockIdx.x;
|
||||
const uint32_t global_pid = global_id / kNumSplit; // plan id
|
||||
const uint32_t global_sid = global_id % kNumSplit; // split id
|
||||
if (global_pid >= num_plans) return;
|
||||
const auto [ragged_id, batch_id, position, window_len] = plan_ptr[global_pid];
|
||||
if (ragged_id == 0xFFFFFFFFu) [[unlikely]]
|
||||
return;
|
||||
|
||||
const uint32_t warp_id = threadIdx.x / kWarpThreads;
|
||||
const uint32_t lane_id = threadIdx.x % kWarpThreads;
|
||||
const int32_t split_offset = global_sid * kTileDim; // int32 is enough
|
||||
|
||||
const auto kv_score_buffer = static_cast<float*>(params.kv_score_buffer);
|
||||
const auto kv_score_input = static_cast<const float*>(params.kv_score_input);
|
||||
const auto kv_compressed_output = static_cast<float*>(params.kv_compressed_output);
|
||||
const auto score_bias_base = static_cast<const float*>(params.score_bias);
|
||||
|
||||
constexpr int64_t kElementSize = kHeadDim * 2; // | kv | score |
|
||||
const uint32_t chunk_offset = (position % 128u) + 1u - window_len;
|
||||
const uint32_t window_end = chunk_offset + window_len; // exclusive, in [1, 128]
|
||||
const int32_t segment_start = ragged_id - (position % 128u); // can be negative, but safe
|
||||
const int32_t load_index = chunk_offset != 0 ? params.load_indices[batch_id] : -1;
|
||||
const int32_t store_index = kWrite ? params.indices[batch_id] : -1;
|
||||
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
// 2 * 8 = 16 register per elem. in theory we should consume 48 register here
|
||||
PrefillStorage kv[kElementsPerWarp];
|
||||
PrefillStorage score[kElementsPerWarp];
|
||||
PrefillStorage bias[kElementsPerWarp];
|
||||
const auto warp_offset = warp_id * kElementsPerWarp;
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kElementsPerWarp; ++i) {
|
||||
const uint32_t j = i + warp_offset;
|
||||
if (j >= chunk_offset && j < window_end) {
|
||||
const auto kv_src_ptr = kv_score_input + (segment_start + j) * kElementSize + split_offset;
|
||||
const auto score_src_ptr = kv_src_ptr + kHeadDim;
|
||||
const auto bias_src_ptr = score_bias_base + j * kHeadDim + split_offset;
|
||||
kv[i].load(kv_src_ptr, lane_id);
|
||||
score[i].load(score_src_ptr, lane_id);
|
||||
bias[i].load(bias_src_ptr, lane_id);
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kElementsPerWarp; ++i) {
|
||||
const uint32_t j = i + warp_offset;
|
||||
const bool is_valid = (j >= chunk_offset && j < window_end);
|
||||
#pragma unroll
|
||||
for (uint32_t ii = 0; ii < kTileElements; ++ii) {
|
||||
score[i][ii] = is_valid ? score[i][ii] + bias[i][ii] : kPadScore;
|
||||
/// NOTE: must zero out kv on padded slots -- `c128_prefill_forward`
|
||||
/// computes `kv * exp_score` where `exp_score = expf(-FLT_MAX - max) ??? 0`,
|
||||
/// and IEEE-754 makes `NaN * 0 = NaN` / `+-inf * 0 = NaN`. An
|
||||
/// uninitialized register can hold a NaN/inf bit pattern, so without
|
||||
/// this reset a single padded warp can poison the whole softmax.
|
||||
kv[i][ii] = is_valid ? kv[i][ii] : 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
__shared__ alignas(16) float seg_kv[kTileDim];
|
||||
__shared__ alignas(16) float seg_max[kTileDim];
|
||||
__shared__ alignas(16) float seg_sum[kTileDim];
|
||||
|
||||
c128_prefill_forward<true>(kv, score, seg_kv, seg_max, seg_sum, warp_id, lane_id);
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
|
||||
if (warp_id == 0) {
|
||||
PrefillStorage out_kv_vec, out_max_vec, out_sum_vec;
|
||||
out_kv_vec.load(seg_kv, lane_id);
|
||||
out_max_vec.load(seg_max, lane_id);
|
||||
out_sum_vec.load(seg_sum, lane_id);
|
||||
if (chunk_offset != 0) {
|
||||
/// NOTE: load (max, sum, kv) of the in-progress chunk for this index.
|
||||
/// `load_indices` may differ from `indices` when the prior partial state
|
||||
/// lives on a different slot than the slot we ultimately write to.
|
||||
const auto buf_load = kv_score_buffer + load_index * (kHeadDim * 3) + split_offset;
|
||||
PrefillStorage buf_max_vec, buf_sum_vec, buf_kv_vec;
|
||||
buf_max_vec.load(buf_load + 0 * kHeadDim, lane_id);
|
||||
buf_sum_vec.load(buf_load + 1 * kHeadDim, lane_id);
|
||||
buf_kv_vec.load(buf_load + 2 * kHeadDim, lane_id);
|
||||
#pragma unroll
|
||||
for (uint32_t ii = 0; ii < kTileElements; ++ii) {
|
||||
const float m1 = buf_max_vec[ii];
|
||||
const float s1 = buf_sum_vec[ii];
|
||||
const float k1 = buf_kv_vec[ii];
|
||||
const float m2 = out_max_vec[ii];
|
||||
const float s2 = out_sum_vec[ii];
|
||||
const float k2 = out_kv_vec[ii];
|
||||
const float new_max = fmaxf(m1, m2);
|
||||
const float new_s1 = s1 * expf(m1 - new_max);
|
||||
const float new_s2 = s2 * expf(m2 - new_max);
|
||||
const float new_sum = new_s1 + new_s2;
|
||||
const float new_kv = (k1 * new_s1 + k2 * new_s2) / new_sum;
|
||||
out_max_vec[ii] = new_max;
|
||||
out_sum_vec[ii] = new_sum;
|
||||
out_kv_vec[ii] = new_kv;
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (kWrite) {
|
||||
const auto buf_store = kv_score_buffer + store_index * (kHeadDim * 3) + split_offset;
|
||||
reinterpret_cast<PrefillStorage*>(buf_store + 0 * kHeadDim)[lane_id] = out_max_vec;
|
||||
reinterpret_cast<PrefillStorage*>(buf_store + 1 * kHeadDim)[lane_id] = out_sum_vec;
|
||||
reinterpret_cast<PrefillStorage*>(buf_store + 2 * kHeadDim)[lane_id] = out_kv_vec;
|
||||
} else {
|
||||
const auto out_ptr = kv_compressed_output + ragged_id * kHeadDim + split_offset;
|
||||
reinterpret_cast<PrefillStorage*>(out_ptr)[lane_id] = out_kv_vec;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, bool kUsePDL>
|
||||
struct FlashCompress128OnlineKernel {
|
||||
static constexpr auto decode_kernel = flash_c128_online_decode<kHeadDim, kUsePDL>;
|
||||
template <bool kWrite>
|
||||
static constexpr auto prefill_kernel = flash_c128_online_prefill<kHeadDim, kWrite, kUsePDL>;
|
||||
static constexpr auto prefill_c_kernel = prefill_kernel</*kWrite=*/false>;
|
||||
static constexpr auto prefill_w_kernel = prefill_kernel</*kWrite=*/true>;
|
||||
static constexpr int64_t kTileDim = kTileElements * device::kWarpThreads; // 64
|
||||
static constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
static constexpr uint32_t kDecodeBlockSize = kHeadDim / 4;
|
||||
|
||||
static void run_decode(
|
||||
const tvm::ffi::TensorView kv_score_buffer,
|
||||
const tvm::ffi::TensorView kv_score_input,
|
||||
const tvm::ffi::TensorView kv_compressed_output,
|
||||
const tvm::ffi::TensorView ape,
|
||||
const tvm::ffi::TensorView indices,
|
||||
const tvm::ffi::TensorView seq_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> /* UNUSED */) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({-1, 1, kHeadDim * 3}) // kv score buffer (max, sum, kv)
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(kv_score_buffer);
|
||||
TensorMatcher({B, kHeadDim * 2}) // kv score input
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(kv_score_input);
|
||||
TensorMatcher({B, kHeadDim}) // kv compressed output
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(kv_compressed_output);
|
||||
TensorMatcher({128, kHeadDim}) // ape
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(ape);
|
||||
TensorMatcher({B}).with_dtype<IndiceT>().with_device(device).verify(indices);
|
||||
TensorMatcher({B}).with_dtype<IndiceT>().with_device(device).verify(seq_lens);
|
||||
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
const auto params = Compress128OnlineDecodeParams{
|
||||
.kv_score_buffer = kv_score_buffer.data_ptr(),
|
||||
.kv_score_input = kv_score_input.data_ptr(),
|
||||
.kv_compressed_output = kv_compressed_output.data_ptr(),
|
||||
.score_bias = ape.data_ptr(),
|
||||
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
|
||||
.seq_lens = static_cast<const IndiceT*>(seq_lens.data_ptr()),
|
||||
.batch_size = batch_size,
|
||||
};
|
||||
LaunchKernel(batch_size, kDecodeBlockSize, device.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(decode_kernel, params);
|
||||
}
|
||||
|
||||
static void run_prefill(
|
||||
const tvm::ffi::TensorView kv_score_buffer,
|
||||
const tvm::ffi::TensorView kv_score_input,
|
||||
const tvm::ffi::TensorView kv_compressed_output,
|
||||
const tvm::ffi::TensorView ape,
|
||||
const tvm::ffi::TensorView indices,
|
||||
const tvm::ffi::TensorView compress_plan,
|
||||
const tvm::ffi::TensorView write_plan,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
|
||||
using namespace host;
|
||||
using host::compress::kOnlinePrefillPlanDim;
|
||||
using host::compress::OnlinePrefillPlanTensorDtype;
|
||||
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto N = SymbolicSize{"num_q_tokens"};
|
||||
auto X = SymbolicSize{"compress_tokens"};
|
||||
auto Y = SymbolicSize{"write_tokens"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({-1, 1, kHeadDim * 3}) // kv score buffer (max, sum, kv) ??? 2D
|
||||
.with_dtype<float>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_buffer);
|
||||
TensorMatcher({N, kHeadDim * 2}) // kv score input
|
||||
.with_dtype<float>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_input);
|
||||
TensorMatcher({N, kHeadDim}) // kv compressed output
|
||||
.with_dtype<float>()
|
||||
.with_device(device_)
|
||||
.verify(kv_compressed_output);
|
||||
TensorMatcher({128, kHeadDim}) // ape
|
||||
.with_dtype<float>()
|
||||
.with_device(device_)
|
||||
.verify(ape);
|
||||
TensorMatcher({B}) // indices
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device_)
|
||||
.verify(indices);
|
||||
TensorMatcher({X, kOnlinePrefillPlanDim}) // compress plan
|
||||
.with_dtype<OnlinePrefillPlanTensorDtype>()
|
||||
.with_device(device_)
|
||||
.verify(compress_plan);
|
||||
TensorMatcher({Y, kOnlinePrefillPlanDim}) // write plan
|
||||
.with_dtype<OnlinePrefillPlanTensorDtype>()
|
||||
.with_device(device_)
|
||||
.verify(write_plan);
|
||||
|
||||
/// NOTE: `extra` is `load_indices`. When the previous partial state lives
|
||||
/// on a slot different from the destination slot (e.g. paged buffers), the
|
||||
/// caller must supply this; otherwise it defaults to `indices`.
|
||||
const auto load_indices = extra.value_or(indices);
|
||||
TensorMatcher({B}).with_dtype<IndiceT>().with_device(device_).verify(load_indices);
|
||||
|
||||
const auto device = device_.unwrap();
|
||||
const auto num_c = static_cast<uint32_t>(X.unwrap());
|
||||
const auto num_w = static_cast<uint32_t>(Y.unwrap());
|
||||
const auto params = Compress128OnlinePrefillParams{
|
||||
.kv_score_buffer = kv_score_buffer.data_ptr(),
|
||||
.kv_score_input = kv_score_input.data_ptr(),
|
||||
.kv_compressed_output = kv_compressed_output.data_ptr(),
|
||||
.score_bias = ape.data_ptr(),
|
||||
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
|
||||
.load_indices = static_cast<const IndiceT*>(load_indices.data_ptr()),
|
||||
.compress_plan = static_cast<const OnlinePlan*>(compress_plan.data_ptr()),
|
||||
.write_plan = static_cast<const OnlinePlan*>(write_plan.data_ptr()),
|
||||
.num_compress = num_c,
|
||||
.num_write = num_w,
|
||||
};
|
||||
|
||||
/// NOTE: pass 1 reads the buffer (for the first segment of each batch
|
||||
/// that started mid-chunk) and writes only to `kv_compressed_output`.
|
||||
/// Pass 2 then writes the trailing partial state of each batch back to
|
||||
/// the buffer. Stream serialization between the two launches enforces
|
||||
/// read-before-write on shared buffer slots.
|
||||
if (const auto num_c_blocks = num_c * kNumSplit) {
|
||||
LaunchKernel(num_c_blocks, kPrefillBlockSize, device) //
|
||||
.enable_pdl(kUsePDL)(prefill_c_kernel, params);
|
||||
}
|
||||
if (const auto num_w_blocks = num_w * kNumSplit) {
|
||||
LaunchKernel(num_w_blocks, kPrefillBlockSize, device) //
|
||||
.enable_pdl(kUsePDL)(prefill_w_kernel, params);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
namespace host::compress {
|
||||
|
||||
using OnlinePlanResult = tvm::ffi::Tuple<uint32_t, uint32_t>;
|
||||
|
||||
struct OnlinePrefillCompressParams {
|
||||
OnlinePrefillPlan* __restrict__ compress_plan;
|
||||
OnlinePrefillPlan* __restrict__ write_plan;
|
||||
const int64_t* __restrict__ seq_lens;
|
||||
const int64_t* __restrict__ extend_lens;
|
||||
uint32_t batch_size;
|
||||
uint32_t num_tokens;
|
||||
};
|
||||
|
||||
/// \brief Build the compress + write plans for online compress 128 prefill.
|
||||
///
|
||||
/// Each batch's `[prefix_len, prefix_len + extend_len)` range is split at
|
||||
/// 128-aligned boundaries. Every resulting segment falls into one of:
|
||||
/// - **compress**: closes a 128-chunk (`chunk_offset + window_len == 128`).
|
||||
/// These plans only read the buffer (when starting mid-chunk) and write the
|
||||
/// compressed kv to `kv_compressed_output`.
|
||||
/// - **write**: trailing partial of the batch (`chunk_offset + window_len < 128`).
|
||||
/// May read the buffer and always writes the new partial state back to it.
|
||||
/// Each batch produces at most one such plan.
|
||||
///
|
||||
/// The two plans MUST be dispatched as separate kernel launches in stream
|
||||
/// order so that pass-1 reads of a buffer slot complete before any pass-2
|
||||
/// write of the same slot.
|
||||
inline OnlinePlanResult plan_online_prefill_host(const OnlinePrefillCompressParams& params, const bool use_cuda_graph) {
|
||||
const auto& [compress_plan, write_plan, seq_lens, extend_lens, batch_size, num_tokens] = params;
|
||||
|
||||
uint32_t counter = 0;
|
||||
uint32_t compress_count = 0;
|
||||
uint32_t write_count = 0;
|
||||
for (const auto i : irange(batch_size)) {
|
||||
const uint32_t seq_len = static_cast<uint32_t>(seq_lens[i]);
|
||||
const uint32_t extend_len = static_cast<uint32_t>(extend_lens[i]);
|
||||
RuntimeCheck(0 < extend_len && extend_len <= seq_len);
|
||||
const uint32_t prefix_len = seq_len - extend_len;
|
||||
const uint32_t end_pos = prefix_len + extend_len;
|
||||
/// NOTE: split the extend range into per-128-chunk segments. Each segment
|
||||
/// stays inside one chunk, so the kernel can decide load/store from
|
||||
/// `chunk_offset` and `window_len` alone.
|
||||
uint32_t pos = prefix_len;
|
||||
while (pos < end_pos) {
|
||||
const uint32_t chunk_start = (pos / 128u) * 128u;
|
||||
const uint32_t seg_end = std::min(end_pos, chunk_start + 128u); // exclusive
|
||||
const uint32_t seg_len = seg_end - pos;
|
||||
const uint32_t chunk_off = pos - chunk_start;
|
||||
/// NOTE: store last-token coordinates so that downstream consumers
|
||||
/// (e.g. `fused_norm_rope`) can read `ragged_id` and `position` with the
|
||||
/// same semantics as `PrefillPlan`. The segment start is recoverable as
|
||||
/// `ragged_id - window_len + 1` and `position - window_len + 1`.
|
||||
const uint32_t last_pos = seg_end - 1;
|
||||
const uint32_t last_ragged = counter + (last_pos - prefix_len);
|
||||
const auto plan = OnlinePrefillPlan{
|
||||
.ragged_id = last_ragged,
|
||||
.batch_id = i,
|
||||
.position = last_pos,
|
||||
.window_len = seg_len,
|
||||
};
|
||||
if (chunk_off + seg_len == 128u) {
|
||||
// full chunk, must be complete, maybe read the buffer, no write
|
||||
RuntimeCheck(compress_count < num_tokens);
|
||||
compress_plan[compress_count++] = plan;
|
||||
} else {
|
||||
// last chunk, must be incomplete, maybe read the buffer, must write
|
||||
RuntimeCheck(write_count < num_tokens);
|
||||
write_plan[write_count++] = plan;
|
||||
}
|
||||
pos = seg_end;
|
||||
}
|
||||
counter += extend_len;
|
||||
}
|
||||
RuntimeCheck(counter == num_tokens, "input size ", counter, " != num_q_tokens ", num_tokens);
|
||||
if (!use_cuda_graph) return OnlinePlanResult{compress_count, write_count};
|
||||
/// NOTE: pad both plans with sentinel entries so cuda-graph runs always see
|
||||
/// the same number of blocks. The kernel skips plans whose `ragged_id` is -1.
|
||||
constexpr auto kInvalid = static_cast<uint32_t>(-1);
|
||||
constexpr auto kInvalidPlan = OnlinePrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid};
|
||||
for (const auto i : irange(compress_count, num_tokens)) {
|
||||
compress_plan[i] = kInvalidPlan;
|
||||
}
|
||||
for (const auto i : irange(write_count, num_tokens)) {
|
||||
write_plan[i] = kInvalidPlan;
|
||||
}
|
||||
return OnlinePlanResult{num_tokens, num_tokens};
|
||||
}
|
||||
|
||||
inline OnlinePlanResult plan_online_prefill(
|
||||
const tvm::ffi::TensorView extend_lens,
|
||||
const tvm::ffi::TensorView seq_lens,
|
||||
const tvm::ffi::TensorView compress_plan,
|
||||
const tvm::ffi::TensorView write_plan,
|
||||
const bool use_cuda_graph) {
|
||||
auto N = SymbolicSize{"batch_size"};
|
||||
auto M = SymbolicSize{"num_tokens"};
|
||||
auto device = SymbolicDevice{};
|
||||
/// NOTE: only host (CPU/cuda-host) planning is implemented for now. The
|
||||
device.set_options<kDLCPU, kDLCUDAHost>();
|
||||
TensorMatcher({N}) //
|
||||
.with_dtype<int64_t>()
|
||||
.with_device(device)
|
||||
.verify(extend_lens)
|
||||
.verify(seq_lens);
|
||||
TensorMatcher({M, kOnlinePrefillPlanDim}) //
|
||||
.with_dtype<OnlinePrefillPlanTensorDtype>()
|
||||
.with_device(device)
|
||||
.verify(compress_plan)
|
||||
.verify(write_plan);
|
||||
const auto params = OnlinePrefillCompressParams{
|
||||
.compress_plan = static_cast<OnlinePrefillPlan*>(compress_plan.data_ptr()),
|
||||
.write_plan = static_cast<OnlinePrefillPlan*>(write_plan.data_ptr()),
|
||||
.seq_lens = static_cast<const int64_t*>(seq_lens.data_ptr()),
|
||||
.extend_lens = static_cast<const int64_t*>(extend_lens.data_ptr()),
|
||||
.batch_size = static_cast<uint32_t>(N.unwrap()),
|
||||
.num_tokens = static_cast<uint32_t>(M.unwrap()),
|
||||
};
|
||||
return plan_online_prefill_host(params, use_cuda_graph);
|
||||
}
|
||||
|
||||
} // namespace host::compress
|
||||
|
||||
[[maybe_unused]]
|
||||
constexpr auto& plan_compress_online_prefill = host::compress::plan_online_prefill;
|
||||
|
||||
} // namespace sglang
|
||||
@@ -233,8 +233,8 @@ constexpr float kPadScore = -FLT_MAX;
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Cross-warp reduction. Same recipe as c128_online.cuh: each block-thread
|
||||
// pair reduces a (tile_id, lane_id) slot using a kNumWarps-wide warp shuffle.
|
||||
// Cross-warp reduction: each block-thread pair reduces a (tile_id, lane_id)
|
||||
// slot using a kNumWarps-wide warp shuffle.
|
||||
constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps;
|
||||
constexpr uint32_t kIteration = kReductionCount / kPrefillBlockSize;
|
||||
static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs");
|
||||
|
||||
@@ -1,549 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/tile.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
|
||||
#include <sgl_kernel/deepseek_v4/compress.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
#include <tvm/ffi/object.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using Plan4 = device::compress::PrefillPlan;
|
||||
using IndiceT = int32_t;
|
||||
|
||||
/// \brief Each thread will handle this many elements (split along head_dim)
|
||||
constexpr int kTileElements = 4;
|
||||
|
||||
/// \brief Need to improve register usage to reduce latency
|
||||
#define C4_KERNEL __global__ __launch_bounds__(128, 4)
|
||||
|
||||
enum class PageMode {
|
||||
RingBuffer = 8,
|
||||
Page4Align = 4,
|
||||
};
|
||||
|
||||
struct alignas(16) C4IndexBundle {
|
||||
int32_t load_first_page;
|
||||
int32_t load_second_page;
|
||||
int32_t write_first_page;
|
||||
int32_t last_position;
|
||||
};
|
||||
|
||||
struct Compress4DecodeParams {
|
||||
/**
|
||||
* \brief Shape: `[num_indices, 8, head_dim * 4]` \n
|
||||
* last dimension layout:
|
||||
* | kv overlap | kv | score overlap | score |
|
||||
*/
|
||||
void* __restrict__ kv_score_buffer;
|
||||
/** \brief Shape: `[batch_size, head_dim * 4]` */
|
||||
const void* __restrict__ kv_score_input;
|
||||
/** \brief Shape: `[batch_size, head_dim]` */
|
||||
void* __restrict__ kv_compressed_output;
|
||||
/** \brief Shape: `[8, head_dim]` (called `ape`) */
|
||||
const void* __restrict__ score_bias;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ indices;
|
||||
/** \brief Shape: `[batch_size, ]` */
|
||||
const IndiceT* __restrict__ seq_lens;
|
||||
/** \brief Shape: `[batch_size, 1]` */
|
||||
const int32_t* __restrict__ extra;
|
||||
/** \NOTE: `batch_size` <= `num_indices` */
|
||||
uint32_t batch_size;
|
||||
};
|
||||
|
||||
struct Compress4PrefillParams {
|
||||
/**
|
||||
* \brief Shape: `[num_indices, 8, head_dim * 4]` \n
|
||||
* last dimension layout:
|
||||
* | kv overlap | kv | score overlap | score |
|
||||
*/
|
||||
void* __restrict__ kv_score_buffer;
|
||||
/** \brief Shape: `[num_q_tokens, head_dim * 4]` */
|
||||
const void* __restrict__ kv_score_input;
|
||||
/** \brief Shape: `[num_q_tokens, head_dim]` */
|
||||
void* __restrict__ kv_compressed_output;
|
||||
/** \brief Shape: `[8, head_dim]` (called `ape`) */
|
||||
const void* __restrict__ score_bias;
|
||||
/** \brief Shape: `[batch_size, ]`*/
|
||||
const IndiceT* __restrict__ indices;
|
||||
/** \brief Shape: `[batch_size, 4]` */
|
||||
const C4IndexBundle* __restrict__ extra;
|
||||
/** \brief The following part is plan info. */
|
||||
|
||||
const Plan4* __restrict__ compress_plan;
|
||||
const Plan4* __restrict__ write_plan;
|
||||
uint32_t num_compress;
|
||||
uint32_t num_write;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE void c4_write(
|
||||
T* kv_score_buf, //
|
||||
const T* kv_score_src,
|
||||
const int64_t head_dim,
|
||||
const int32_t write_pos) {
|
||||
using namespace device;
|
||||
|
||||
using Storage = AlignedVector<T, kTileElements>;
|
||||
const auto element_size = head_dim * 4;
|
||||
const auto gmem = tile::Memory<Storage>::warp();
|
||||
kv_score_buf += write_pos * element_size;
|
||||
|
||||
/// NOTE: Layout | [0] = kv overlap | [1] = kv | [2] = score overlap | [3] = score |
|
||||
Storage kv_score[4];
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 4; ++i) {
|
||||
kv_score[i] = gmem.load(kv_score_src + head_dim * i);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 4; ++i) {
|
||||
gmem.store(kv_score_buf + head_dim * i, kv_score[i]);
|
||||
}
|
||||
}
|
||||
|
||||
template <bool kPaged, typename InFloat, typename OutFloat>
|
||||
SGL_DEVICE void c4_forward(
|
||||
const InFloat* kv_score_buf,
|
||||
const InFloat* kv_score_src,
|
||||
OutFloat* kv_out,
|
||||
const InFloat* score_bias,
|
||||
const int64_t head_dim,
|
||||
const int32_t seq_len,
|
||||
const int32_t window_len,
|
||||
[[maybe_unused]] const InFloat* kv_score_overlap_buf = nullptr) {
|
||||
using namespace device;
|
||||
|
||||
const auto element_size = head_dim * 4;
|
||||
const auto score_offset = head_dim * 2;
|
||||
const auto overlap_stride = head_dim;
|
||||
|
||||
/// NOTE: part 1: load kv + score
|
||||
using StorageIn = AlignedVector<InFloat, kTileElements>;
|
||||
const auto gmem_in = tile::Memory<StorageIn>::warp();
|
||||
StorageIn kv[8];
|
||||
StorageIn score[8];
|
||||
StorageIn bias[8];
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
bias[i] = gmem_in.load(score_bias + i * head_dim);
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
const bool is_overlap = i < 4;
|
||||
const InFloat* src;
|
||||
if (i < window_len) {
|
||||
/// NOTE: `seq_len` must be a multiple of 4 here
|
||||
if constexpr (kPaged) {
|
||||
const auto kv_score_ptr = is_overlap ? kv_score_overlap_buf : kv_score_buf;
|
||||
const int32_t k = i % 4;
|
||||
src = kv_score_ptr + k * element_size;
|
||||
} else {
|
||||
const int32_t k = (seq_len + i) % 8;
|
||||
src = kv_score_buf + k * element_size;
|
||||
}
|
||||
} else {
|
||||
/// NOTE: k in [-7, 0]. We'll load from the ragged `kv_score_src`
|
||||
const int32_t k = i - 7;
|
||||
src = kv_score_src + k * element_size;
|
||||
}
|
||||
src += (is_overlap ? 0 : overlap_stride);
|
||||
kv[i] = gmem_in.load(src);
|
||||
score[i] = gmem_in.load(src + score_offset);
|
||||
}
|
||||
|
||||
if (seq_len == 4) {
|
||||
[[unlikely]];
|
||||
constexpr float kFloatNegInf = -1e9f;
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < 4; ++i) {
|
||||
kv[i].fill(cast<InFloat>(0.0f));
|
||||
score[i].fill(cast<InFloat>(kFloatNegInf));
|
||||
}
|
||||
}
|
||||
|
||||
/// NOTE: part 2: safe online softmax + weighted sum
|
||||
using StorageOut = AlignedVector<OutFloat, kTileElements>;
|
||||
const auto gmem_out = tile::Memory<StorageOut>::warp();
|
||||
StorageOut result;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t i = 0; i < kTileElements; ++i) {
|
||||
float score_fp32[8];
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t j = 0; j < 8; ++j) {
|
||||
score_fp32[j] = cast<float>(score[j][i]) + cast<float>(bias[j][i]);
|
||||
}
|
||||
|
||||
float max_value = score_fp32[0];
|
||||
float sum_exp_value = 0.0f;
|
||||
|
||||
#pragma unroll
|
||||
for (int32_t j = 1; j < 8; ++j) {
|
||||
const auto fp32_score = score_fp32[j];
|
||||
max_value = fmaxf(max_value, fp32_score);
|
||||
}
|
||||
|
||||
float sum_product = 0.0f;
|
||||
#pragma unroll
|
||||
for (int32_t j = 0; j < 8; ++j) {
|
||||
const auto fp32_score = score_fp32[j];
|
||||
const auto exp_score = expf(fp32_score - max_value);
|
||||
sum_product += cast<float>(kv[j][i]) * exp_score;
|
||||
sum_exp_value += exp_score;
|
||||
}
|
||||
|
||||
result[i] = cast<OutFloat>(sum_product / sum_exp_value);
|
||||
}
|
||||
|
||||
gmem_out.store(kv_out, result);
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, typename InFloat, typename OutFloat, PageMode kMode, bool kUsePDL>
|
||||
C4_KERNEL void flash_c4_decode(const __grid_constant__ Compress4DecodeParams params) {
|
||||
using namespace device;
|
||||
|
||||
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 128
|
||||
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
constexpr int64_t kElementSize = kHeadDim * 4; // `* 4` due to overlap transform + score
|
||||
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
|
||||
|
||||
const auto& [
|
||||
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
|
||||
indices, seq_lens, extra, batch_size // decode info
|
||||
] = params;
|
||||
const uint32_t global_tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const uint32_t global_wid = global_tid / kWarpThreads; // warp id
|
||||
const uint32_t global_bid = global_wid / kNumSplit; // batch id
|
||||
const uint32_t global_sid = global_wid % kNumSplit; // split id
|
||||
|
||||
if (global_bid >= batch_size) return;
|
||||
|
||||
const int32_t index = indices[global_bid];
|
||||
const int32_t seq_len = seq_lens[global_bid];
|
||||
const int64_t split_offset = global_sid * kTileDim;
|
||||
|
||||
// kv score
|
||||
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
|
||||
|
||||
// kv input
|
||||
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
|
||||
const auto kv_src = kv_score_input + global_bid * kElementSize + split_offset;
|
||||
|
||||
// kv output
|
||||
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
|
||||
const auto kv_out = kv_compressed_output + global_bid * kHeadDim + split_offset;
|
||||
|
||||
// score bias (ape)
|
||||
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
|
||||
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
/// NOTE: `position` = `seq_len - 1`. To avoid underflow, we use `seq_len + page_size - 1`
|
||||
if constexpr (kMode == PageMode::Page4Align) {
|
||||
const auto index_prev = extra[global_bid];
|
||||
const auto kv_buf = kv_score_buffer + index * (kElementSize * 4) + split_offset;
|
||||
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 3) % 4);
|
||||
if (seq_len % 4 == 0) {
|
||||
const auto kv_overlap = kv_buf + (index_prev - index) * (kElementSize * 4);
|
||||
c4_forward<true>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, 8, kv_overlap);
|
||||
}
|
||||
} else {
|
||||
static_assert(kMode == PageMode::RingBuffer, "Unsupported PageMode");
|
||||
const auto kv_buf = kv_score_buffer + index * (kElementSize * 8) + split_offset;
|
||||
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 7) % 8);
|
||||
if (seq_len % 4 == 0) {
|
||||
c4_forward<false>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, /*window_size=*/8);
|
||||
}
|
||||
}
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, typename InFloat, typename OutFloat, PageMode kMode, bool kWrite, bool kUsePDL>
|
||||
C4_KERNEL void flash_c4_prefill(const __grid_constant__ Compress4PrefillParams params) {
|
||||
using namespace device;
|
||||
|
||||
constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 128
|
||||
constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
constexpr int64_t kElementSize = kHeadDim * 4; // `* 4` due to overlap transform + score
|
||||
static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim");
|
||||
|
||||
const auto& [
|
||||
_kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score
|
||||
indices, extra, compress_plan, write_plan, num_compress, num_write // prefill plan
|
||||
] = params;
|
||||
|
||||
const uint32_t global_tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const uint32_t global_wid = global_tid / kWarpThreads; // warp id
|
||||
const uint32_t global_pid = global_wid / kNumSplit; // plan id
|
||||
const uint32_t global_sid = global_wid % kNumSplit; // split id
|
||||
|
||||
/// NOTE: compiler can optimize this if-else at compile time
|
||||
const auto num_plans = kWrite ? num_write : num_compress;
|
||||
const auto plan_ptr = kWrite ? write_plan : compress_plan;
|
||||
if (global_pid >= num_plans) return;
|
||||
|
||||
const auto& [ragged_id, global_bid, position, window_len] = plan_ptr[global_pid];
|
||||
const int64_t split_offset = global_sid * kTileDim;
|
||||
|
||||
// kv score
|
||||
const auto kv_score_buffer = static_cast<InFloat*>(_kv_score_buffer);
|
||||
|
||||
// kv input
|
||||
const auto kv_score_input = static_cast<const InFloat*>(_kv_score_input);
|
||||
const auto kv_src = kv_score_input + ragged_id * kElementSize + split_offset;
|
||||
|
||||
// kv output
|
||||
const auto kv_compressed_output = static_cast<OutFloat*>(_kv_compressed_output);
|
||||
const auto kv_out = kv_compressed_output + ragged_id * kHeadDim + split_offset;
|
||||
|
||||
if (ragged_id == 0xFFFFFFFF) [[unlikely]]
|
||||
return;
|
||||
|
||||
// score bias (ape)
|
||||
const auto score_bias = static_cast<const InFloat*>(_score_bias) + split_offset;
|
||||
const auto seq_len = position + 1;
|
||||
const int32_t index = indices[global_bid];
|
||||
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
if constexpr (kMode == PageMode::Page4Align) {
|
||||
const auto write_second_page = index;
|
||||
const auto [load_first_page, load_second_page, write_first_page, last_pos] = extra[global_bid];
|
||||
if constexpr (kWrite) {
|
||||
int32_t index;
|
||||
if (position < static_cast<uint32_t>(last_pos)) {
|
||||
index = write_first_page;
|
||||
} else {
|
||||
index = write_second_page;
|
||||
}
|
||||
const auto kv_buf = kv_score_buffer + index * (kElementSize * 4) + split_offset;
|
||||
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 4);
|
||||
} else {
|
||||
int32_t index_overlap, index_normal;
|
||||
if (window_len <= 4) {
|
||||
index_overlap = load_second_page;
|
||||
index_normal = load_second_page; // not used
|
||||
} else {
|
||||
index_overlap = load_first_page;
|
||||
index_normal = load_second_page;
|
||||
}
|
||||
const auto kv_buf = kv_score_buffer + index_normal * (kElementSize * 4) + split_offset;
|
||||
const auto kv_overlap = kv_score_buffer + index_overlap * (kElementSize * 4) + split_offset;
|
||||
c4_forward<true>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, window_len, kv_overlap);
|
||||
}
|
||||
} else {
|
||||
static_assert(kMode == PageMode::RingBuffer, "Unsupported PageMode");
|
||||
const auto kv_buf = kv_score_buffer + index * (kElementSize * 8) + split_offset;
|
||||
if constexpr (kWrite) {
|
||||
c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 8);
|
||||
} else {
|
||||
c4_forward<false>(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, window_len);
|
||||
}
|
||||
}
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, typename InFloat, typename OutFloat, bool kUsePDL>
|
||||
struct FlashCompress4Kernel {
|
||||
template <PageMode kMode>
|
||||
static constexpr auto decode_kernel = flash_c4_decode<kHeadDim, InFloat, OutFloat, kMode, kUsePDL>;
|
||||
template <PageMode kMode, bool kWrite>
|
||||
static constexpr auto prefill_kernel = flash_c4_prefill<kHeadDim, InFloat, OutFloat, kMode, kWrite, kUsePDL>;
|
||||
template <PageMode kMode>
|
||||
static constexpr auto prefill_c_kernel = prefill_kernel<kMode, /*kWrite=*/false>;
|
||||
template <PageMode kMode>
|
||||
static constexpr auto prefill_w_kernel = prefill_kernel<kMode, /*kWrite=*/true>;
|
||||
static constexpr uint32_t kBlockSize = 128;
|
||||
static constexpr uint32_t kTileDim = kTileElements * device::kWarpThreads;
|
||||
static constexpr uint32_t kNumSplit = kHeadDim / kTileDim;
|
||||
static constexpr uint32_t kWarpsPerBlock = kBlockSize / device::kWarpThreads;
|
||||
|
||||
using Self = FlashCompress4Kernel;
|
||||
|
||||
static void run_decode(
|
||||
const tvm::ffi::TensorView kv_score_buffer,
|
||||
const tvm::ffi::TensorView kv_score_input,
|
||||
const tvm::ffi::TensorView kv_compressed_output,
|
||||
const tvm::ffi::TensorView ape,
|
||||
const tvm::ffi::TensorView indices,
|
||||
const tvm::ffi::TensorView seq_lens,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
|
||||
using namespace host;
|
||||
|
||||
// this should not happen in practice
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
const auto extra_ptr = _get_extra_pointer(B, device_, extra);
|
||||
const auto page_size = extra_ptr != nullptr ? 4 : 8;
|
||||
|
||||
TensorMatcher({-1, page_size, kHeadDim * 4}) // kv score
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_buffer);
|
||||
TensorMatcher({B, kHeadDim * 4}) // kv score input
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_input);
|
||||
TensorMatcher({B, kHeadDim}) // kv compressed output
|
||||
.with_dtype<OutFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_compressed_output);
|
||||
TensorMatcher({8, kHeadDim}) // ape
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(ape);
|
||||
TensorMatcher({B}) // indices
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device_)
|
||||
.verify(indices);
|
||||
TensorMatcher({B}) // seq lens
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device_)
|
||||
.verify(seq_lens);
|
||||
|
||||
const auto device = device_.unwrap();
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
const auto params = Compress4DecodeParams{
|
||||
.kv_score_buffer = kv_score_buffer.data_ptr(),
|
||||
.kv_score_input = kv_score_input.data_ptr(),
|
||||
.kv_compressed_output = kv_compressed_output.data_ptr(),
|
||||
.score_bias = ape.data_ptr(),
|
||||
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
|
||||
.seq_lens = static_cast<const IndiceT*>(seq_lens.data_ptr()),
|
||||
.extra = static_cast<const int32_t*>(extra_ptr),
|
||||
.batch_size = batch_size,
|
||||
};
|
||||
const auto kernel = extra_ptr != nullptr ? decode_kernel<PageMode::Page4Align> //
|
||||
: decode_kernel<PageMode::RingBuffer>;
|
||||
const uint32_t num_blocks = div_ceil(batch_size * kNumSplit, kWarpsPerBlock);
|
||||
LaunchKernel(num_blocks, kBlockSize, device) //
|
||||
.enable_pdl(kUsePDL)(kernel, params);
|
||||
}
|
||||
|
||||
static void run_prefill(
|
||||
const tvm::ffi::TensorView kv_score_buffer,
|
||||
const tvm::ffi::TensorView kv_score_input,
|
||||
const tvm::ffi::TensorView kv_compressed_output,
|
||||
const tvm::ffi::TensorView ape,
|
||||
const tvm::ffi::TensorView indices,
|
||||
const tvm::ffi::TensorView compress_plan,
|
||||
const tvm::ffi::TensorView write_plan,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> extra) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto N = SymbolicSize{"num_q_tokens"};
|
||||
auto X = SymbolicSize{"compress_tokens"};
|
||||
auto Y = SymbolicSize{"write_tokens"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
const auto extra_ptr = _get_extra_pointer(B, device_, extra, /*is_prefill=*/true);
|
||||
const auto page_size = extra_ptr != nullptr ? 4 : 8;
|
||||
|
||||
TensorMatcher({-1, page_size, kHeadDim * 4}) // kv score
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_buffer);
|
||||
TensorMatcher({N, kHeadDim * 4}) // kv score input
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_score_input);
|
||||
TensorMatcher({N, kHeadDim}) // kv compressed output
|
||||
.with_dtype<OutFloat>()
|
||||
.with_device(device_)
|
||||
.verify(kv_compressed_output);
|
||||
TensorMatcher({8, kHeadDim}) // ape
|
||||
.with_dtype<InFloat>()
|
||||
.with_device(device_)
|
||||
.verify(ape);
|
||||
TensorMatcher({B}) // indices
|
||||
.with_dtype<IndiceT>()
|
||||
.with_device(device_)
|
||||
.verify(indices);
|
||||
TensorMatcher({X, compress::kPrefillPlanDim}) // compress plan
|
||||
.with_dtype<compress::PrefillPlanTensorDtype>()
|
||||
.with_device(device_)
|
||||
.verify(compress_plan);
|
||||
TensorMatcher({Y, compress::kPrefillPlanDim}) // write plan
|
||||
.with_dtype<compress::PrefillPlanTensorDtype>()
|
||||
.with_device(device_)
|
||||
.verify(write_plan);
|
||||
|
||||
const auto device = device_.unwrap();
|
||||
const auto batch_size = static_cast<uint32_t>(B.unwrap());
|
||||
const auto num_q_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
const auto num_c = static_cast<uint32_t>(X.unwrap());
|
||||
const auto num_w = static_cast<uint32_t>(Y.unwrap());
|
||||
const auto params = Compress4PrefillParams{
|
||||
.kv_score_buffer = kv_score_buffer.data_ptr(),
|
||||
.kv_score_input = kv_score_input.data_ptr(),
|
||||
.kv_compressed_output = kv_compressed_output.data_ptr(),
|
||||
.score_bias = ape.data_ptr(),
|
||||
.indices = static_cast<const IndiceT*>(indices.data_ptr()),
|
||||
.extra = static_cast<const C4IndexBundle*>(extra_ptr),
|
||||
.compress_plan = static_cast<const Plan4*>(compress_plan.data_ptr()),
|
||||
.write_plan = static_cast<const Plan4*>(write_plan.data_ptr()),
|
||||
.num_compress = num_c,
|
||||
.num_write = num_w,
|
||||
};
|
||||
RuntimeCheck(num_q_tokens >= batch_size, "num_q_tokens must be >= batch_size");
|
||||
RuntimeCheck(num_q_tokens >= std::max(num_c, num_w), "invalid prefill plan");
|
||||
if (const auto num_c_blocks = div_ceil(num_c * kNumSplit, kWarpsPerBlock)) {
|
||||
const auto c_kernel = extra_ptr != nullptr ? prefill_c_kernel<PageMode::Page4Align> //
|
||||
: prefill_c_kernel<PageMode::RingBuffer>;
|
||||
LaunchKernel(num_c_blocks, kBlockSize, device) //
|
||||
.enable_pdl(kUsePDL)(c_kernel, params);
|
||||
}
|
||||
if (const auto num_w_blocks = div_ceil(num_w * kNumSplit, kWarpsPerBlock)) {
|
||||
const auto w_kernel = extra_ptr != nullptr ? prefill_w_kernel<PageMode::Page4Align> //
|
||||
: prefill_w_kernel<PageMode::RingBuffer>;
|
||||
LaunchKernel(num_w_blocks, kBlockSize, device) //
|
||||
.enable_pdl(kUsePDL)(w_kernel, params);
|
||||
}
|
||||
}
|
||||
|
||||
// some auxiliary functions
|
||||
private:
|
||||
static const void* _get_extra_pointer(
|
||||
host::SymbolicSize& B, // batch_size
|
||||
host::SymbolicDevice& device,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView>& extra,
|
||||
bool is_prefill = false) {
|
||||
// only have value when using page-aligned mode
|
||||
if (!extra.has_value()) return nullptr;
|
||||
const auto& extra_tensor = extra.value();
|
||||
/// NOTE: the metadata layout is different for prefill and decode:
|
||||
/// for prefill, last 4 are:
|
||||
/// load overlap | load normal | write overlap | last written page
|
||||
/// for decode, last 1 is the write (also load) overlap
|
||||
host::TensorMatcher({B, is_prefill ? 4 : 1}) // extra tensor
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(extra_tensor);
|
||||
const auto data_ptr = extra_tensor.data_ptr();
|
||||
host::RuntimeCheck(data_ptr != nullptr, "extra tensor data ptr is null");
|
||||
if (is_prefill) {
|
||||
static_assert(alignof(C4IndexBundle) == 16);
|
||||
host::RuntimeCheck(std::bit_cast<uintptr_t>(data_ptr) % 16 == 0, "extra tensor is not properly aligned");
|
||||
}
|
||||
return data_ptr;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
@@ -1,208 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/deepseek_v4/compress.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
namespace host::compress {
|
||||
|
||||
using PlanResult = tvm::ffi::Tuple<uint32_t, uint32_t>;
|
||||
|
||||
struct CompressParams {
|
||||
PrefillPlan* __restrict__ compress_plan;
|
||||
PrefillPlan* __restrict__ write_plan;
|
||||
const int64_t* __restrict__ seq_lens;
|
||||
const int64_t* __restrict__ extend_lens;
|
||||
uint32_t batch_size;
|
||||
uint32_t num_tokens;
|
||||
uint32_t compress_ratio;
|
||||
bool is_overlap;
|
||||
};
|
||||
|
||||
inline constexpr uint32_t kBlockSize = 1024;
|
||||
|
||||
#define PLAN_KERNEL __global__ __launch_bounds__(kBlockSize, 1) inline
|
||||
|
||||
PLAN_KERNEL void plan_prefill_cuda(const __grid_constant__ CompressParams params) {
|
||||
const auto &[
|
||||
compress_plan, write_plan, seq_lens, extend_lens, // pointers
|
||||
batch_size, num_tokens, compress_ratio, is_overlap // values
|
||||
] = params;
|
||||
|
||||
__shared__ uint32_t compress_counter;
|
||||
__shared__ uint32_t write_counter;
|
||||
|
||||
uint32_t batch_id = 0;
|
||||
uint32_t counter = 0;
|
||||
uint32_t extend_len = extend_lens[0];
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
if (tid == 0) {
|
||||
compress_counter = 0;
|
||||
write_counter = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (uint32_t i = tid; i < num_tokens; i += blockDim.x) {
|
||||
const uint32_t ragged_id = i;
|
||||
uint32_t j = ragged_id - counter;
|
||||
while (j >= extend_len) {
|
||||
j -= extend_len;
|
||||
batch_id += 1;
|
||||
if (batch_id >= batch_size) [[unlikely]]
|
||||
break;
|
||||
counter += extend_len;
|
||||
extend_len = extend_lens[batch_id];
|
||||
}
|
||||
if (batch_id >= batch_size) [[unlikely]]
|
||||
break;
|
||||
const uint32_t seq_len = seq_lens[batch_id];
|
||||
const uint32_t extend_len = extend_lens[batch_id];
|
||||
const uint32_t prefix_len = seq_len - extend_len;
|
||||
const uint32_t ratio = compress_ratio * (1 + is_overlap);
|
||||
const uint32_t window_len = j + 1 < ratio ? ratio - (j + 1) : 0;
|
||||
const uint32_t position = prefix_len + j;
|
||||
const auto plan = PrefillPlan{
|
||||
.ragged_id = ragged_id,
|
||||
.batch_id = batch_id,
|
||||
.position = position,
|
||||
.window_len = window_len,
|
||||
};
|
||||
const uint32_t start_write_pos = [seq_len, compress_ratio, is_overlap] {
|
||||
const uint32_t pos = seq_len / compress_ratio * compress_ratio;
|
||||
if (!is_overlap) return pos;
|
||||
return pos >= compress_ratio ? pos - compress_ratio : 0;
|
||||
}();
|
||||
if ((position + 1) % compress_ratio == 0) {
|
||||
const auto write_pos = atomicAdd(&compress_counter, 1);
|
||||
compress_plan[write_pos] = plan;
|
||||
}
|
||||
if (position >= start_write_pos) {
|
||||
const auto write_pos = atomicAdd(&write_counter, 1);
|
||||
write_plan[write_pos] = plan;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
constexpr auto kInvalid = static_cast<uint32_t>(-1);
|
||||
const auto kInvalidPlan = PrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid};
|
||||
const auto compress_count = compress_counter;
|
||||
const auto write_count = write_counter;
|
||||
for (uint32_t i = compress_count + tid; i < num_tokens; i += blockDim.x) {
|
||||
compress_plan[i] = kInvalidPlan;
|
||||
}
|
||||
for (uint32_t i = write_count + tid; i < num_tokens; i += blockDim.x) {
|
||||
write_plan[i] = kInvalidPlan;
|
||||
}
|
||||
}
|
||||
|
||||
inline PlanResult plan_prefill_host(const CompressParams& params, const bool use_cuda_graph) {
|
||||
const auto &[
|
||||
compress_ptr, write_ptr, seq_lens_ptr, extend_lens_ptr, // pointers
|
||||
batch_size, num_tokens, compress_ratio, is_overlap // values
|
||||
] = params;
|
||||
|
||||
uint32_t counter = 0;
|
||||
uint32_t compress_counter = 0;
|
||||
uint32_t write_counter = 0;
|
||||
const auto ratio = compress_ratio * (1 + is_overlap);
|
||||
for (const auto i : irange(batch_size)) {
|
||||
const uint32_t seq_len = seq_lens_ptr[i];
|
||||
const uint32_t extend_len = extend_lens_ptr[i];
|
||||
const uint32_t prefix_len = seq_len - extend_len;
|
||||
RuntimeCheck(0 < extend_len && extend_len <= seq_len);
|
||||
/// NOTE: `start_write_pos` must be a multiple of `compress_ratio`
|
||||
const uint32_t start_write_pos = [seq_len, compress_ratio, is_overlap] {
|
||||
const uint32_t pos = seq_len / compress_ratio * compress_ratio;
|
||||
if (!is_overlap) return pos;
|
||||
/// NOTE: to avoid unsigned integer underflow, don't use `pos - compress_ratio`
|
||||
return pos >= compress_ratio ? pos - compress_ratio : 0;
|
||||
}();
|
||||
/// NOTE: `position` is within [prefix_len, seq_len)
|
||||
for (const auto j : irange(extend_len)) {
|
||||
const uint32_t position = prefix_len + j;
|
||||
const auto plan = PrefillPlan{
|
||||
.ragged_id = counter + j,
|
||||
.batch_id = i,
|
||||
.position = position,
|
||||
.window_len = ratio - std::min(j + 1, ratio),
|
||||
};
|
||||
RuntimeCheck(plan.is_valid(compress_ratio, is_overlap), "Internal error!");
|
||||
if ((position + 1) % compress_ratio == 0) {
|
||||
compress_ptr[compress_counter++] = plan;
|
||||
}
|
||||
if (position >= start_write_pos) {
|
||||
write_ptr[write_counter++] = plan;
|
||||
}
|
||||
}
|
||||
counter += extend_len;
|
||||
}
|
||||
RuntimeCheck(counter == num_tokens, "input size ", counter, " != num_q_tokens ", num_tokens);
|
||||
if (!use_cuda_graph) return PlanResult{compress_counter, write_counter};
|
||||
constexpr auto kInvalid = static_cast<uint32_t>(-1);
|
||||
constexpr auto kInvalidPlan = PrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid};
|
||||
for (const auto i : irange(compress_counter, num_tokens)) {
|
||||
compress_ptr[i] = kInvalidPlan;
|
||||
}
|
||||
for (const auto i : irange(write_counter, num_tokens)) {
|
||||
write_ptr[i] = kInvalidPlan;
|
||||
}
|
||||
return PlanResult{num_tokens, num_tokens};
|
||||
}
|
||||
|
||||
inline PlanResult plan_prefill(
|
||||
const tvm::ffi::TensorView extend_lens,
|
||||
const tvm::ffi::TensorView seq_lens,
|
||||
const tvm::ffi::TensorView compress_plan,
|
||||
const tvm::ffi::TensorView write_plan,
|
||||
const uint32_t compress_ratio,
|
||||
const bool is_overlap, // for overlap transform, we have to keep 1 more extra window
|
||||
const bool use_cuda_graph) {
|
||||
auto N = SymbolicSize{"batch_size"};
|
||||
auto M = SymbolicSize{"num_tokens"};
|
||||
auto device = SymbolicDevice{};
|
||||
const bool is_cuda = [&] {
|
||||
if (extend_lens.device().device_type == kDLCUDA) {
|
||||
device.set_options<kDLCUDA>();
|
||||
return true;
|
||||
} else {
|
||||
device.set_options<kDLCPU, kDLCUDAHost>();
|
||||
return false;
|
||||
}
|
||||
}();
|
||||
TensorMatcher({N}) // extend_lens and seq_lens
|
||||
.with_dtype<int64_t>()
|
||||
.with_device(device)
|
||||
.verify(extend_lens)
|
||||
.verify(seq_lens);
|
||||
TensorMatcher({M, kPrefillPlanDim}) // compress_plan and write_plan
|
||||
.with_dtype<PrefillPlanTensorDtype>()
|
||||
.with_device(device)
|
||||
.verify(compress_plan)
|
||||
.verify(write_plan);
|
||||
|
||||
const auto params = CompressParams{
|
||||
.compress_plan = static_cast<PrefillPlan*>(compress_plan.data_ptr()),
|
||||
.write_plan = static_cast<PrefillPlan*>(write_plan.data_ptr()),
|
||||
.seq_lens = static_cast<const int64_t*>(seq_lens.data_ptr()),
|
||||
.extend_lens = static_cast<const int64_t*>(extend_lens.data_ptr()),
|
||||
.batch_size = static_cast<uint32_t>(N.unwrap()),
|
||||
.num_tokens = static_cast<uint32_t>(M.unwrap()),
|
||||
.compress_ratio = compress_ratio,
|
||||
.is_overlap = is_overlap,
|
||||
};
|
||||
|
||||
if (!is_cuda) return plan_prefill_host(params, use_cuda_graph);
|
||||
/// NOTE: cuda kernel plan is naturally compatible with cuda graph
|
||||
LaunchKernel(1, kBlockSize, device.unwrap())(plan_prefill_cuda, params);
|
||||
return PlanResult{params.num_tokens, params.num_tokens};
|
||||
}
|
||||
|
||||
} // namespace host::compress
|
||||
|
||||
[[maybe_unused]]
|
||||
constexpr auto& plan_compress_prefill = host::compress::plan_prefill;
|
||||
|
||||
} // namespace sglang
|
||||
@@ -43,7 +43,7 @@ struct MegaMoEPreDispatchParams {
|
||||
uint32_t top_k;
|
||||
};
|
||||
|
||||
// kGroupSize must match sglang_per_token_group_quant_fp8_ue8m0(group_size=).
|
||||
// kGroupSize must match the consuming MoE's activation scale group (quant_group_size).
|
||||
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
|
||||
// parameter rather than a runtime check because this kernel is issue-bound: the
|
||||
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
|
||||
|
||||
@@ -583,34 +583,12 @@ struct NVLinkComm {
|
||||
// specialized for each world size
|
||||
template <uint32_t kWorldSize>
|
||||
static void
|
||||
all_reduce_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional<TensorView> residual) {
|
||||
return run_push<Primitive::AR, kWorldSize>(comm->get_push_obj(), in, out, residual);
|
||||
}
|
||||
template <uint32_t kWorldSize>
|
||||
static void
|
||||
all_gather_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional<TensorView> residual) {
|
||||
return run_push<Primitive::AG, kWorldSize>(comm->get_push_obj(), in, out, residual);
|
||||
}
|
||||
template <uint32_t kWorldSize>
|
||||
static void
|
||||
reduce_scatter_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional<TensorView> residual) {
|
||||
return run_push<Primitive::RS, kWorldSize>(comm->get_push_obj(), in, out, residual);
|
||||
}
|
||||
|
||||
// only compile once for each world size
|
||||
template <uint32_t kPullUnroll>
|
||||
static void all_reduce_pull(
|
||||
CommunicatorRef comm, // only pull is needed
|
||||
TensorView in,
|
||||
TensorView out,
|
||||
tvm::ffi::Optional<TensorView> residual,
|
||||
int64_t in_mc_ptr,
|
||||
int64_t out_mc_ptr,
|
||||
uint32_t num_blocks_hint) {
|
||||
return run_pull<Primitive::AR, kPullUnroll>(
|
||||
comm->get_pull_obj(), in, out, residual, in_mc_ptr, out_mc_ptr, num_blocks_hint);
|
||||
}
|
||||
template <uint32_t kPullUnroll>
|
||||
static void all_gather_pull(
|
||||
CommunicatorRef comm, // only pull is needed
|
||||
TensorView in,
|
||||
@@ -621,16 +599,6 @@ struct NVLinkComm {
|
||||
return run_pull<Primitive::AG, kPullUnroll>(
|
||||
comm->get_pull_obj(), in, out, residual, 0, out_mc_ptr, num_blocks_hint);
|
||||
}
|
||||
template <uint32_t kPullUnroll>
|
||||
static void reduce_scatter_pull(
|
||||
CommunicatorRef comm, // only pull is needed
|
||||
TensorView in,
|
||||
TensorView out,
|
||||
tvm::ffi::Optional<TensorView> residual,
|
||||
int64_t in_mc_ptr,
|
||||
uint32_t num_blocks_hint) {
|
||||
return run_pull<Primitive::RS, kPullUnroll>(comm->get_pull_obj(), in, out, residual, in_mc_ptr, 0, num_blocks_hint);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
// Adler-32 checksum CUDA kernels for tensor verification.
|
||||
//
|
||||
// Two modes:
|
||||
// 1. Whole-tensor: checksum all bytes of a contiguous tensor
|
||||
// 2. Strided: checksum selected items across multiple tensors,
|
||||
// given precomputed byte pointers and per-item lengths
|
||||
// Checksums selected items across multiple tensors, given precomputed byte
|
||||
// pointers and per-item lengths.
|
||||
//
|
||||
// Adler-32:
|
||||
// A = 1 + sum(bytes) (mod 65521)
|
||||
@@ -164,75 +162,7 @@ __device__ Adler32State adler32_block_reduce(Adler32State state, Adler32State* s
|
||||
return state;
|
||||
}
|
||||
|
||||
// --- Kernel 1: Whole tensor ---
|
||||
|
||||
__global__ void adler32_whole_kernel(
|
||||
const uint8_t* __restrict__ data,
|
||||
uint32_t* __restrict__ out_a,
|
||||
uint32_t* __restrict__ out_b,
|
||||
uint64_t* __restrict__ out_len,
|
||||
uint64_t num_bytes,
|
||||
uint64_t chunk_size) {
|
||||
extern __shared__ Adler32State smem[];
|
||||
|
||||
uint64_t global_tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
|
||||
uint64_t start = global_tid * chunk_size;
|
||||
|
||||
Adler32State state;
|
||||
if (start < num_bytes) {
|
||||
uint64_t end = start + chunk_size;
|
||||
if (end > num_bytes) end = num_bytes;
|
||||
state = adler32_compute(data + start, end - start);
|
||||
} else {
|
||||
state = adler32_identity();
|
||||
}
|
||||
|
||||
uint32_t threads_in_block = blockDim.x;
|
||||
uint64_t block_start = (uint64_t)blockIdx.x * blockDim.x;
|
||||
uint64_t total_threads = (num_bytes + chunk_size - 1) / chunk_size;
|
||||
if (block_start + threads_in_block > total_threads) {
|
||||
threads_in_block = (uint32_t)(total_threads - block_start);
|
||||
}
|
||||
|
||||
Adler32State result = adler32_block_reduce(state, smem, threads_in_block);
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
out_a[blockIdx.x] = result.a;
|
||||
out_b[blockIdx.x] = result.b;
|
||||
out_len[blockIdx.x] = result.len;
|
||||
}
|
||||
}
|
||||
|
||||
struct Adler32WholeKernel {
|
||||
static void run(tvm::ffi::TensorView data, tvm::ffi::TensorView states, int64_t num_bytes, int64_t chunk_size) {
|
||||
using namespace host;
|
||||
|
||||
const uint8_t* data_ptr = static_cast<const uint8_t*>(data.data_ptr());
|
||||
|
||||
int64_t total_threads = (num_bytes + chunk_size - 1) / chunk_size;
|
||||
int64_t num_blocks = (total_threads + kBlockSize - 1) / kBlockSize;
|
||||
|
||||
uint32_t* out_a = static_cast<uint32_t*>(states.data_ptr());
|
||||
uint32_t* out_b = out_a + num_blocks;
|
||||
uint64_t* out_len = reinterpret_cast<uint64_t*>(out_b + num_blocks);
|
||||
|
||||
DLDevice device;
|
||||
device.device_type = kDLCUDA;
|
||||
device.device_id = data.device().device_id;
|
||||
|
||||
size_t smem_bytes = (kBlockSize / 32) * sizeof(Adler32State);
|
||||
LaunchKernel(num_blocks, kBlockSize, device, smem_bytes)(
|
||||
adler32_whole_kernel,
|
||||
data_ptr,
|
||||
out_a,
|
||||
out_b,
|
||||
out_len,
|
||||
static_cast<uint64_t>(num_bytes),
|
||||
static_cast<uint64_t>(chunk_size));
|
||||
}
|
||||
};
|
||||
|
||||
// --- Kernel 2: Strided items ---
|
||||
// --- Kernel 1: Strided items ---
|
||||
|
||||
__global__ void adler32_strided_kernel(
|
||||
const int64_t* __restrict__ ptrs,
|
||||
@@ -293,7 +223,7 @@ struct Adler32StridedKernel {
|
||||
}
|
||||
};
|
||||
|
||||
// --- Kernel 3: Final reduction ---
|
||||
// --- Kernel 2: Final reduction ---
|
||||
|
||||
__global__ void adler32_reduce_kernel(
|
||||
const uint32_t* __restrict__ in_a,
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/tile.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
|
||||
#include <cooperative_groups/reduce.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cooperative_groups.h>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
template <typename T, int VEC_SIZE_IN_BYTE>
|
||||
struct VecTypeTrait;
|
||||
|
||||
template <>
|
||||
struct VecTypeTrait<bf16_t, 16> {
|
||||
using packed_t = packed_t<bf16_t>;
|
||||
using vec_t = device::AlignedVector<packed_t, 4>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct VecTypeTrait<fp16_t, 16> {
|
||||
using packed_t = packed_t<fp16_t>;
|
||||
using vec_t = device::AlignedVector<packed_t, 4>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct VecTypeTrait<bf16_t, 32> {
|
||||
using packed_t = packed_t<bf16_t>;
|
||||
using vec_t = device::AlignedVector<packed_t, 8>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct VecTypeTrait<fp16_t, 32> {
|
||||
using packed_t = packed_t<fp16_t>;
|
||||
using vec_t = device::AlignedVector<packed_t, 8>;
|
||||
};
|
||||
|
||||
template <typename packed_t>
|
||||
SGL_DEVICE packed_t rms(const packed_t& val, const packed_t& weight, float rsqrt_square_sum) {
|
||||
float2 valf = device::cast<fp32x2_t, packed_t>(val);
|
||||
float2 weightf = device::cast<fp32x2_t, packed_t>(weight);
|
||||
return device::cast<packed_t, fp32x2_t>(
|
||||
make_float2(valf.x * weightf.x * rsqrt_square_sum, valf.y * weightf.y * rsqrt_square_sum));
|
||||
}
|
||||
|
||||
template <typename T, int VEC_SIZE_IN_BYTE>
|
||||
__global__ void qknorm_across_heads_reg_kernel(
|
||||
T* __restrict__ q,
|
||||
T* __restrict__ k,
|
||||
const T* __restrict__ q_weight,
|
||||
const T* __restrict__ k_weight,
|
||||
int vec_hidden_size,
|
||||
float eps) {
|
||||
constexpr int inner_loop = VEC_SIZE_IN_BYTE == 16 ? 4 : 8;
|
||||
|
||||
__shared__ float shared_memory[32];
|
||||
|
||||
using vec_t = typename VecTypeTrait<T, VEC_SIZE_IN_BYTE>::vec_t;
|
||||
using packed_t = typename VecTypeTrait<T, VEC_SIZE_IN_BYTE>::packed_t;
|
||||
vec_t v_data;
|
||||
vec_t v_weight;
|
||||
const int warp_id = threadIdx.x >> 5;
|
||||
const int lane_id = threadIdx.x & 31;
|
||||
const int warp_count = (blockDim.x + 31) >> 5;
|
||||
const float inv_hidden_size = 1.0f / static_cast<float>(vec_hidden_size * (VEC_SIZE_IN_BYTE / sizeof(T)));
|
||||
const bool is_q = blockIdx.y == 0;
|
||||
|
||||
const auto token_id = blockIdx.x;
|
||||
float2 acc_square = make_float2(0.0f, 0.0f);
|
||||
vec_t* data = reinterpret_cast<vec_t*>(is_q ? q : k) + token_id * vec_hidden_size;
|
||||
const vec_t* weight = reinterpret_cast<const vec_t*>(is_q ? q_weight : k_weight);
|
||||
|
||||
if (threadIdx.x < vec_hidden_size) {
|
||||
v_data = data[threadIdx.x];
|
||||
v_weight = weight[threadIdx.x];
|
||||
for (int i = 0; i < inner_loop; i++) {
|
||||
float2 val = device::cast<fp32x2_t, packed_t>(v_data[i]);
|
||||
acc_square.x += val.x * val.x;
|
||||
acc_square.y += val.y * val.y;
|
||||
}
|
||||
}
|
||||
|
||||
auto cg_warp = cooperative_groups::tiled_partition<32>(cooperative_groups::this_thread_block());
|
||||
float* buffer = shared_memory;
|
||||
float warp_sum = cooperative_groups::reduce(cg_warp, acc_square.x + acc_square.y, cooperative_groups::plus<float>());
|
||||
if (lane_id == 0) {
|
||||
buffer[warp_id] = warp_sum;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
if (threadIdx.x < 32) {
|
||||
float cta_sum = cooperative_groups::reduce(
|
||||
cg_warp, (threadIdx.x < warp_count) ? buffer[threadIdx.x] : 0.0f, cooperative_groups::plus<float>());
|
||||
if (threadIdx.x == 0) {
|
||||
buffer[0] = rsqrtf(eps + cta_sum * inv_hidden_size);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (threadIdx.x < vec_hidden_size) {
|
||||
float rsqrt_val = buffer[0];
|
||||
for (int i = 0; i < inner_loop; i++) {
|
||||
v_data[i] = rms(v_data[i], v_weight[i], rsqrt_val);
|
||||
}
|
||||
data[threadIdx.x] = v_data;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename DType>
|
||||
struct QKNormAcrossHeadsKernel {
|
||||
static void
|
||||
run(const tvm::ffi::TensorView q,
|
||||
const tvm::ffi::TensorView k,
|
||||
const tvm::ffi::TensorView q_weight,
|
||||
const tvm::ffi::TensorView k_weight,
|
||||
float eps) {
|
||||
using namespace host;
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
auto D = SymbolicSize{"hidden_size"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({N, D}) // q
|
||||
.with_strides({D, 1})
|
||||
.with_dtype<DType>()
|
||||
.with_device(device)
|
||||
.verify(q);
|
||||
TensorMatcher({N, D}) // k
|
||||
.with_strides({D, 1})
|
||||
.with_dtype<DType>()
|
||||
.with_device(device)
|
||||
.verify(k);
|
||||
TensorMatcher({D}) // q_weight
|
||||
.with_dtype<DType>()
|
||||
.with_device(device)
|
||||
.verify(q_weight);
|
||||
TensorMatcher({D}) // k_weight
|
||||
.with_dtype<DType>()
|
||||
.with_device(device)
|
||||
.verify(k_weight);
|
||||
|
||||
int hidden_size = static_cast<int>(D.unwrap());
|
||||
if (hidden_size <= (device::kMaxVecBytes == 32 ? 12288 : 8192)) {
|
||||
int elements_in_vec = device::kMaxVecBytes / sizeof(DType);
|
||||
int vec_hidden_size = hidden_size / elements_in_vec;
|
||||
uint threads = (vec_hidden_size + 31) / 32 * 32;
|
||||
|
||||
// Runtime check
|
||||
host::RuntimeCheck(
|
||||
hidden_size % elements_in_vec == 0,
|
||||
"hidden_size",
|
||||
hidden_size,
|
||||
" can not align to elements_in_vec ",
|
||||
elements_in_vec);
|
||||
|
||||
auto kernel = qknorm_across_heads_reg_kernel<DType, device::kMaxVecBytes>;
|
||||
|
||||
LaunchKernel(dim3(static_cast<uint>(N.unwrap()), 2), threads, device.unwrap())
|
||||
.enable_pdl(false)(
|
||||
kernel,
|
||||
reinterpret_cast<DType*>(q.data_ptr()),
|
||||
reinterpret_cast<DType*>(k.data_ptr()),
|
||||
reinterpret_cast<DType*>(q_weight.data_ptr()),
|
||||
reinterpret_cast<DType*>(k_weight.data_ptr()),
|
||||
vec_hidden_size,
|
||||
eps);
|
||||
} else {
|
||||
host::RuntimeCheck(false, "Large hidden_sizes are not supported for now.");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
@@ -12,16 +12,13 @@
|
||||
// slice S_r (in any buffer), so there is no write-write conflict, and each
|
||||
// per-element load completes before its store (data dependency).
|
||||
//
|
||||
// Two variants:
|
||||
// * ..._kernel (v1): no in-kernel sync; the caller fences with the symm-mem
|
||||
// handle's barrier() on each side (3 launches total).
|
||||
// * ..._fused_kernel (v2): an in-kernel per-block system barrier (entry:
|
||||
// producers done + visible; exit: broadcasts done + visible), so the whole
|
||||
// all-reduce is a single launch. The barrier uses a DEDICATED symmetric
|
||||
// flags buffer (independent of torch's signal pad, so no interference with
|
||||
// multimem) and a device-resident monotonic epoch counter per block, which
|
||||
// keeps advancing across launches -- including CUDA-graph replays -- so
|
||||
// flags never go stale (spin is `flag < epoch`, epoch strictly increasing).
|
||||
// ..._fused_kernel (v2) uses an in-kernel per-block system barrier (entry:
|
||||
// producers done + visible; exit: broadcasts done + visible), so the whole
|
||||
// all-reduce is a single launch. The barrier uses a DEDICATED symmetric
|
||||
// flags buffer (independent of torch's signal pad, so no interference with
|
||||
// multimem) and a device-resident monotonic epoch counter per block, which
|
||||
// keeps advancing across launches -- including CUDA-graph replays -- so
|
||||
// flags never go stale (spin is `flag < epoch`, epoch strictly increasing).
|
||||
//
|
||||
// Fusion seam: the reduced `result` Storage below is where an epilogue (RMSNorm
|
||||
// / short-conv / bias) plugs in -- applied in registers before the broadcast
|
||||
@@ -162,15 +159,6 @@ __device__ __forceinline__ void two_shot_reduce_local(DType* (&input)[kNumGPU],
|
||||
}
|
||||
}
|
||||
|
||||
// v1: no in-kernel barrier (caller fences via hdl.barrier()).
|
||||
template <typename DType, uint32_t kNumGPU>
|
||||
__global__ __launch_bounds__(1024, 1) void inkling_two_shot_all_reduce_kernel(
|
||||
void* const* __restrict__ peer_ptrs, const uint32_t rank, const uint32_t num_items) {
|
||||
DType* input[kNumGPU];
|
||||
const uint32_t local_vecs = slice_setup<DType, kNumGPU>(input, peer_ptrs, rank, num_items);
|
||||
two_shot_reduce_local<DType, kNumGPU>(input, local_vecs);
|
||||
}
|
||||
|
||||
// v2: single-launch, fused entry + exit system barrier. `shared` (optional):
|
||||
// this rank's LOCAL shared-expert partials, folded into its own buffer before
|
||||
// the entry barrier (which then must publish -- see fold_shared_local).
|
||||
@@ -408,7 +396,7 @@ __global__ __launch_bounds__(1024, 1) void inkling_multimem_full_oneshot_kernel(
|
||||
// NO exit barrier: result is local & complete; input reuse is caller-managed.
|
||||
}
|
||||
|
||||
// Blocks needed to cover this rank's two-shot slice (v1/v2/v3 partition).
|
||||
// Blocks needed to cover this rank's two-shot slice (v2/v3 partition).
|
||||
template <typename DType, uint32_t kNumGPU>
|
||||
uint32_t work_num_blocks(uint32_t n, uint32_t block_size) {
|
||||
using Trait = InklingAllReduceTrait<DType, kNumGPU>;
|
||||
@@ -483,22 +471,6 @@ void validate(tvm::ffi::TensorView buf, int64_t peer_ptrs_dev, int64_t rank, int
|
||||
RuntimeCheck(rank >= 0 && rank < kNumGPU, "rank out of range");
|
||||
}
|
||||
|
||||
template <typename DType, uint32_t kNumGPU>
|
||||
void inkling_two_shot_all_reduce(
|
||||
tvm::ffi::TensorView local_buffer, int64_t peer_ptrs_dev, int64_t rank, int64_t num_items) {
|
||||
using namespace host;
|
||||
uint32_t n;
|
||||
validate<DType, kNumGPU>(local_buffer, peer_ptrs_dev, rank, num_items, n);
|
||||
const auto device = local_buffer.device();
|
||||
const uint32_t num_blocks = work_num_blocks<DType, kNumGPU>(n, 1024u); // no in-kernel barrier -> uncapped
|
||||
const auto stream = LaunchKernel::resolve_device(device);
|
||||
LaunchKernel(num_blocks, 1024u, stream)(
|
||||
inkling_two_shot_all_reduce_kernel<DType, kNumGPU>,
|
||||
reinterpret_cast<void* const*>(peer_ptrs_dev),
|
||||
static_cast<uint32_t>(rank),
|
||||
n);
|
||||
}
|
||||
|
||||
template <typename DType, uint32_t kNumGPU>
|
||||
void inkling_two_shot_all_reduce_fused(
|
||||
tvm::ffi::TensorView local_buffer,
|
||||
|
||||
@@ -19,81 +19,12 @@ namespace sglang {
|
||||
using deepseek_v4::fp8::cast_to_ue8m0;
|
||||
using deepseek_v4::fp8::pack_fp8;
|
||||
|
||||
// Per-token group quant to FP8-e4m3 with a fused UE8M0 scale. Each group of
|
||||
// kGroupSize columns gets one UE8M0 exponent byte written contiguously in
|
||||
// row-major order into ``x_sf`` (int32 [num_tokens, num_groups/4], 4 group
|
||||
// bytes per int32). This is the deep_gemm "transform_sf" pack done inline in
|
||||
// the quant, reusing the dsv4 ``cast_to_ue8m0``/``pack_fp8`` primitives -- it
|
||||
// is byte-identical to ``per_token_group_quant_fp8(scale_ue8m0=True)`` followed
|
||||
// by ``transform_sf_into_required_layout`` (both round via ceil(log2(absmax/
|
||||
// FP8_MAX))), but emits no separate transpose/pack kernel.
|
||||
struct PerTokenQuantUe8m0Params {
|
||||
const bf16_t* __restrict__ x; // [num_tokens, hidden]
|
||||
fp8_e4m3_t* __restrict__ x_q; // [num_tokens, hidden]
|
||||
int32_t* __restrict__ x_sf; // [num_tokens, num_groups/4]; written as bytes
|
||||
uint32_t num_tokens;
|
||||
uint32_t hidden;
|
||||
uint32_t num_groups; // hidden / kGroupSize
|
||||
};
|
||||
|
||||
template <uint32_t kGroupSize, bool kUsePDL>
|
||||
__global__ __launch_bounds__(1024, 2) void //
|
||||
per_token_quant_ue8m0_kernel(const PerTokenQuantUe8m0Params __grid_constant__ params) {
|
||||
using namespace device;
|
||||
constexpr uint32_t kVecElems = 8; // 8 bf16 = 16B load per thread
|
||||
static_assert(kGroupSize % kVecElems == 0, "group_size must be a multiple of 8");
|
||||
constexpr uint32_t kThreadsPerGroup = kGroupSize / kVecElems;
|
||||
using InputVec = AlignedVector<bf16x2_t, kVecElems / 2>;
|
||||
using OutputVec = AlignedVector<fp8x2_e4m3_t, kVecElems / 2>;
|
||||
|
||||
const uint32_t token_id = blockIdx.x;
|
||||
const uint32_t tid = threadIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
|
||||
const auto token_in = params.x + static_cast<uint64_t>(token_id) * params.hidden;
|
||||
const auto token_out = params.x_q + static_cast<uint64_t>(token_id) * params.hidden;
|
||||
|
||||
InputVec in_vec;
|
||||
in_vec.load(token_in, tid);
|
||||
float local_max = 0.0f;
|
||||
float vals[kVecElems];
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecElems / 2; ++i) {
|
||||
const auto [v0, v1] = cast<fp32x2_t>(in_vec[i]);
|
||||
vals[2 * i + 0] = v0;
|
||||
vals[2 * i + 1] = v1;
|
||||
local_max = fmaxf(local_max, fmaxf(fabsf(v0), fabsf(v1)));
|
||||
}
|
||||
// Absmax across the kThreadsPerGroup threads that cover one group.
|
||||
local_max = warp::reduce_max<kThreadsPerGroup>(local_max);
|
||||
const float absmax = fmaxf(local_max, 1e-10f);
|
||||
const float raw_scale = absmax / math::FP8_E4M3_MAX;
|
||||
const uint32_t ue8m0_exp = cast_to_ue8m0(raw_scale);
|
||||
const float inv_scale = __uint_as_float((127u + 127u - ue8m0_exp) << 23);
|
||||
|
||||
OutputVec out_vec;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecElems / 2; ++i) {
|
||||
out_vec[i] = pack_fp8(vals[2 * i + 0] * inv_scale, vals[2 * i + 1] * inv_scale);
|
||||
}
|
||||
out_vec.store(token_out, tid);
|
||||
|
||||
const uint32_t group_id = tid / kThreadsPerGroup;
|
||||
const uint32_t within_group_id = tid % kThreadsPerGroup;
|
||||
if (within_group_id == 0 && group_id < params.num_groups) {
|
||||
const uint32_t byte_off = token_id * params.num_groups + group_id;
|
||||
reinterpret_cast<uint8_t*>(params.x_sf)[byte_off] = static_cast<uint8_t>(ue8m0_exp);
|
||||
}
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
// Fused quant + scatter: like per_token_quant_ue8m0_kernel, but instead of
|
||||
// writing the fp8/scale for the single source token, it scatters them straight
|
||||
// into the permuted grouped-GEMM input -- replicating each token to its ``topk``
|
||||
// destination rows -- so the separate fill_gateup_input_triton_kernel launch (and
|
||||
// the intermediate x_q/x_sf buffers) are eliminated. The fp8 value + UE8M0 scale
|
||||
// are computed exactly once per token (identical to the non-fused kernel); only
|
||||
// the stores differ.
|
||||
// Fused per-token group quant to FP8-e4m3 with a UE8M0 scale + scatter: each
|
||||
// group of kGroupSize columns gets one UE8M0 exponent (ceil(log2(absmax /
|
||||
// FP8_MAX)), via the dsv4 ``cast_to_ue8m0``/``pack_fp8`` primitives), and the
|
||||
// fp8 row + scale bytes are written straight into the permuted grouped-GEMM
|
||||
// input -- replicating each token to its ``topk`` destination rows -- so no
|
||||
// separate fill_gateup_input_triton_kernel launch or x_q/x_sf buffers are needed.
|
||||
struct PerTokenQuantUe8m0ScatterParams {
|
||||
const bf16_t* __restrict__ x; // [num_tokens, hidden]
|
||||
fp8_e4m3_t* __restrict__ gateup_input; // [E, m_max, hidden]
|
||||
@@ -242,38 +173,4 @@ void per_token_quant_ue8m0_scatter(
|
||||
.enable_pdl(kUsePDL)(kernel, params);
|
||||
}
|
||||
|
||||
template <int64_t kGroupSize, bool kUsePDL>
|
||||
void per_token_quant_ue8m0(tvm::ffi::TensorView x, tvm::ffi::TensorView x_q, tvm::ffi::TensorView x_sf) {
|
||||
using namespace host;
|
||||
auto device = SymbolicDevice{};
|
||||
auto M = SymbolicSize{"num_tokens"};
|
||||
auto H = SymbolicSize{"hidden"};
|
||||
auto G4 = SymbolicSize{"num_groups_div_4"};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({M, H}).with_dtype<bf16_t>().with_device(device).verify(x);
|
||||
TensorMatcher({M, H}).with_dtype<fp8_e4m3_t>().with_device(device).verify(x_q);
|
||||
TensorMatcher({M, G4}).with_dtype<int32_t>().with_device(device).verify(x_sf);
|
||||
|
||||
const uint32_t num_tokens = static_cast<uint32_t>(M.unwrap());
|
||||
const uint32_t hidden = static_cast<uint32_t>(H.unwrap());
|
||||
RuntimeCheck(hidden % kGroupSize == 0, "hidden ", hidden, " not divisible by group_size ", kGroupSize);
|
||||
const uint32_t num_groups = hidden / static_cast<uint32_t>(kGroupSize);
|
||||
RuntimeCheck(static_cast<uint32_t>(G4.unwrap()) * 4 == num_groups);
|
||||
const uint32_t threads = hidden / 8; // kVecElems
|
||||
RuntimeCheck(threads <= 1024, "hidden/8 must be <= 1024, got ", threads);
|
||||
|
||||
const auto params = PerTokenQuantUe8m0Params{
|
||||
.x = static_cast<const bf16_t*>(x.data_ptr()),
|
||||
.x_q = static_cast<fp8_e4m3_t*>(x_q.data_ptr()),
|
||||
.x_sf = static_cast<int32_t*>(x_sf.data_ptr()),
|
||||
.num_tokens = num_tokens,
|
||||
.hidden = hidden,
|
||||
.num_groups = num_groups,
|
||||
};
|
||||
if (num_tokens == 0) return;
|
||||
constexpr auto kernel = per_token_quant_ue8m0_kernel<kGroupSize, kUsePDL>;
|
||||
LaunchKernel(num_tokens, threads, device.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(kernel, params);
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -18,7 +18,6 @@ static constexpr int kInklingRoutedExperts = 256;
|
||||
static constexpr int kInklingSharedExperts = 2;
|
||||
static constexpr int kInklingTotalExperts = kInklingRoutedExperts + kInklingSharedExperts;
|
||||
static constexpr int kInklingTopK = 6;
|
||||
static constexpr int kInklingTopPow2 = 8;
|
||||
static constexpr int kInklingWarpSize = 32;
|
||||
static constexpr int kInklingValuesPerLane = kInklingRoutedExperts / kInklingWarpSize;
|
||||
|
||||
@@ -26,10 +25,6 @@ __device__ __forceinline__ float inkling_sigmoid(float x) {
|
||||
return 1.0f / (1.0f + __expf(-x));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ bool inkling_score_better(float score, int idx, float best_score, int best_idx) {
|
||||
return score > best_score || (score == best_score && idx < best_idx);
|
||||
}
|
||||
|
||||
// Order-preserving uint32 key of an fp32 (same total order as the triton
|
||||
// kernel's fpval_to_key): flip sign bit for positives, all bits for negatives.
|
||||
__device__ __forceinline__ uint32_t inkling_fp_key(float f) {
|
||||
@@ -44,158 +39,6 @@ __device__ __forceinline__ int32_t inkling_pack_routed(int32_t id, float w) {
|
||||
return static_cast<int32_t>((static_cast<uint32_t>(id) << 16) | wbits);
|
||||
}
|
||||
|
||||
template <int WarpsPerBlock, bool ReturnPacked>
|
||||
__launch_bounds__(kInklingWarpSize* WarpsPerBlock) __global__ void inkling_gate_topk_renorm_kernel(
|
||||
const float* __restrict__ logits,
|
||||
const float* __restrict__ bias,
|
||||
const float* __restrict__ global_scale,
|
||||
float* __restrict__ routed_w,
|
||||
float* __restrict__ shared_w,
|
||||
int64_t* __restrict__ indices,
|
||||
int32_t* __restrict__ packed,
|
||||
int64_t M,
|
||||
int64_t logits_stride_m,
|
||||
float route_scale) {
|
||||
const int lane = threadIdx.x;
|
||||
const int warp_in_block = threadIdx.y;
|
||||
const int64_t row = static_cast<int64_t>(blockIdx.x) * WarpsPerBlock + warp_in_block;
|
||||
if (row >= M) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t row_base = row * logits_stride_m;
|
||||
float local_scores[kInklingValuesPerLane];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingValuesPerLane; ++i) {
|
||||
const int expert = lane + i * kInklingWarpSize;
|
||||
const float raw = logits[row_base + expert];
|
||||
local_scores[i] = inkling_sigmoid(raw) + bias[expert];
|
||||
}
|
||||
|
||||
int selected_idx[kInklingTopK];
|
||||
#pragma unroll
|
||||
for (int k = 0; k < kInklingTopK; ++k) {
|
||||
float best_score = -FLT_MAX;
|
||||
int best_idx = INT_MAX;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingValuesPerLane; ++i) {
|
||||
const int expert = lane + i * kInklingWarpSize;
|
||||
const float score = local_scores[i];
|
||||
if (inkling_score_better(score, expert, best_score, best_idx)) {
|
||||
best_score = score;
|
||||
best_idx = expert;
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = kInklingWarpSize / 2; offset > 0; offset >>= 1) {
|
||||
const float other_score = __shfl_xor_sync(0xffffffff, best_score, offset);
|
||||
const int other_idx = __shfl_xor_sync(0xffffffff, best_idx, offset);
|
||||
if (inkling_score_better(other_score, other_idx, best_score, best_idx)) {
|
||||
best_score = other_score;
|
||||
best_idx = other_idx;
|
||||
}
|
||||
}
|
||||
|
||||
selected_idx[k] = best_idx;
|
||||
if (best_idx % kInklingWarpSize == lane) {
|
||||
local_scores[best_idx / kInklingWarpSize] = -FLT_MAX;
|
||||
}
|
||||
__syncwarp();
|
||||
}
|
||||
|
||||
if (lane != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
float active[kInklingTopPow2];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingTopK; ++i) {
|
||||
active[i] = inkling_sigmoid(logits[row_base + selected_idx[i]]);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingSharedExperts; ++i) {
|
||||
active[kInklingTopK + i] = inkling_sigmoid(logits[row_base + kInklingRoutedExperts + i]);
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingTopPow2; ++i) {
|
||||
sum += active[i];
|
||||
}
|
||||
const float scale = route_scale * global_scale[0] / sum;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingTopK; ++i) {
|
||||
const float w = active[i] * scale;
|
||||
if constexpr (ReturnPacked) {
|
||||
packed[row * kInklingTopK + i] = inkling_pack_routed(selected_idx[i], w);
|
||||
} else {
|
||||
routed_w[row * kInklingTopK + i] = w;
|
||||
indices[row * kInklingTopK + i] = static_cast<int64_t>(selected_idx[i]);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kInklingSharedExperts; ++i) {
|
||||
shared_w[row * kInklingSharedExperts + i] = active[kInklingTopK + i] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
template <int WarpsPerBlock, bool ReturnPacked>
|
||||
void launch_inkling_gate_topk_renorm(
|
||||
const float* logits,
|
||||
const float* bias,
|
||||
const float* global_scale,
|
||||
float* routed_w,
|
||||
float* shared_w,
|
||||
int64_t* indices,
|
||||
int32_t* packed,
|
||||
int64_t tokens,
|
||||
int64_t logits_stride_m,
|
||||
float route_scale,
|
||||
DLDevice device) {
|
||||
using namespace host;
|
||||
const dim3 block(kInklingWarpSize, WarpsPerBlock);
|
||||
const dim3 grid(static_cast<unsigned int>(div_ceil(tokens, static_cast<int64_t>(WarpsPerBlock))));
|
||||
LaunchKernel(grid, block, device)(
|
||||
inkling_gate_topk_renorm_kernel<WarpsPerBlock, ReturnPacked>,
|
||||
logits,
|
||||
bias,
|
||||
global_scale,
|
||||
routed_w,
|
||||
shared_w,
|
||||
indices,
|
||||
packed,
|
||||
tokens,
|
||||
logits_stride_m,
|
||||
route_scale);
|
||||
}
|
||||
|
||||
template <bool ReturnPacked>
|
||||
void dispatch_inkling_gate_topk_renorm(
|
||||
const float* logits,
|
||||
const float* bias,
|
||||
const float* global_scale,
|
||||
float* routed_w,
|
||||
float* shared_w,
|
||||
int64_t* indices,
|
||||
int32_t* packed,
|
||||
int64_t tokens,
|
||||
int64_t logits_stride_m,
|
||||
float route_scale,
|
||||
DLDevice device) {
|
||||
if (tokens <= 64) {
|
||||
launch_inkling_gate_topk_renorm<1, ReturnPacked>(
|
||||
logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device);
|
||||
} else if (tokens <= 1024) {
|
||||
launch_inkling_gate_topk_renorm<4, ReturnPacked>(
|
||||
logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device);
|
||||
} else {
|
||||
launch_inkling_gate_topk_renorm<8, ReturnPacked>(
|
||||
logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device);
|
||||
}
|
||||
}
|
||||
|
||||
// Warp-per-row gate kernel with optional expert-per-block GEMV fusion.
|
||||
|
||||
// Inkling gate GEMM shape: x [M, 6144] bf16 x W [264 (row-padded), 6144] bf16.
|
||||
@@ -736,122 +579,6 @@ void check_gate_row_alignment(const void* ptr, int64_t stride_m) {
|
||||
"logits rows must be device::kMaxVecBytes-aligned (production pitch is 264 floats)");
|
||||
}
|
||||
|
||||
void inkling_gate_topk_renorm(
|
||||
tvm::ffi::TensorView logits,
|
||||
tvm::ffi::TensorView bias,
|
||||
tvm::ffi::TensorView global_scale,
|
||||
tvm::ffi::TensorView routed_w,
|
||||
tvm::ffi::TensorView shared_w,
|
||||
tvm::ffi::TensorView indices,
|
||||
double route_scale) {
|
||||
using namespace host;
|
||||
|
||||
SymbolicSize M{"tokens"};
|
||||
SymbolicDevice device_;
|
||||
device_.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({M, kInklingTotalExperts})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<fp32_t>()
|
||||
.with_device<kDLCUDA>(device_)
|
||||
.verify(logits);
|
||||
TensorMatcher({kInklingRoutedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(bias);
|
||||
TensorMatcher({1}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(global_scale);
|
||||
TensorMatcher({M, kInklingTopK}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(routed_w);
|
||||
TensorMatcher({M, kInklingSharedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(shared_w);
|
||||
TensorMatcher({M, kInklingTopK}).with_dtype<int64_t>().with_device<kDLCUDA>(device_).verify(indices);
|
||||
|
||||
RuntimeCheck(logits.stride(1) == 1, "logits must be contiguous along the expert dimension");
|
||||
RuntimeCheck(bias.stride(0) == 1, "bias must be contiguous");
|
||||
RuntimeCheck(routed_w.stride(1) == 1, "routed_w must be contiguous along top-k dimension");
|
||||
RuntimeCheck(shared_w.stride(1) == 1, "shared_w must be contiguous along shared dimension");
|
||||
RuntimeCheck(indices.stride(1) == 1, "indices must be contiguous along top-k dimension");
|
||||
|
||||
const int64_t tokens = M.unwrap();
|
||||
if (tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto* logits_ptr = static_cast<const float*>(logits.data_ptr());
|
||||
const auto* bias_ptr = static_cast<const float*>(bias.data_ptr());
|
||||
const auto* global_scale_ptr = static_cast<const float*>(global_scale.data_ptr());
|
||||
auto* routed_w_ptr = static_cast<float*>(routed_w.data_ptr());
|
||||
auto* shared_w_ptr = static_cast<float*>(shared_w.data_ptr());
|
||||
auto* indices_ptr = static_cast<int64_t*>(indices.data_ptr());
|
||||
const float route_scale_f = static_cast<float>(route_scale);
|
||||
const DLDevice device = device_.unwrap();
|
||||
|
||||
dispatch_inkling_gate_topk_renorm<false>(
|
||||
logits_ptr,
|
||||
bias_ptr,
|
||||
global_scale_ptr,
|
||||
routed_w_ptr,
|
||||
shared_w_ptr,
|
||||
indices_ptr,
|
||||
nullptr,
|
||||
tokens,
|
||||
logits.stride(0),
|
||||
route_scale_f,
|
||||
device);
|
||||
}
|
||||
|
||||
// Packed variant: emits packed[M, kInklingTopK] int32 ((id<<16)|bf16 weight) instead of
|
||||
// the routed_w + indices pair; shared_w still written.
|
||||
void inkling_gate_topk_renorm_packed(
|
||||
tvm::ffi::TensorView logits,
|
||||
tvm::ffi::TensorView bias,
|
||||
tvm::ffi::TensorView global_scale,
|
||||
tvm::ffi::TensorView packed,
|
||||
tvm::ffi::TensorView shared_w,
|
||||
double route_scale) {
|
||||
using namespace host;
|
||||
|
||||
SymbolicSize M{"tokens"};
|
||||
SymbolicDevice device_;
|
||||
device_.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({M, kInklingTotalExperts})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<fp32_t>()
|
||||
.with_device<kDLCUDA>(device_)
|
||||
.verify(logits);
|
||||
TensorMatcher({kInklingRoutedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(bias);
|
||||
TensorMatcher({1}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(global_scale);
|
||||
TensorMatcher({M, kInklingTopK}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(packed);
|
||||
TensorMatcher({M, kInklingSharedExperts}).with_dtype<fp32_t>().with_device<kDLCUDA>(device_).verify(shared_w);
|
||||
|
||||
RuntimeCheck(logits.stride(1) == 1, "logits must be contiguous along the expert dimension");
|
||||
RuntimeCheck(bias.stride(0) == 1, "bias must be contiguous");
|
||||
RuntimeCheck(packed.stride(1) == 1, "packed must be contiguous along top-k dimension");
|
||||
RuntimeCheck(shared_w.stride(1) == 1, "shared_w must be contiguous along shared dimension");
|
||||
|
||||
const int64_t tokens = M.unwrap();
|
||||
if (tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto* logits_ptr = static_cast<const float*>(logits.data_ptr());
|
||||
const auto* bias_ptr = static_cast<const float*>(bias.data_ptr());
|
||||
const auto* global_scale_ptr = static_cast<const float*>(global_scale.data_ptr());
|
||||
auto* packed_ptr = static_cast<int32_t*>(packed.data_ptr());
|
||||
auto* shared_w_ptr = static_cast<float*>(shared_w.data_ptr());
|
||||
const float route_scale_f = static_cast<float>(route_scale);
|
||||
const DLDevice device = device_.unwrap();
|
||||
|
||||
dispatch_inkling_gate_topk_renorm<true>(
|
||||
logits_ptr,
|
||||
bias_ptr,
|
||||
global_scale_ptr,
|
||||
nullptr,
|
||||
shared_w_ptr,
|
||||
nullptr,
|
||||
packed_ptr,
|
||||
tokens,
|
||||
logits.stride(0),
|
||||
route_scale_f,
|
||||
device);
|
||||
}
|
||||
|
||||
// Uses int32 indices for MoeRunner, optional PDL, and tunable warps (0 = auto).
|
||||
// Logit rows must be device::kMaxVecBytes-aligned (the standard pitch is 264).
|
||||
void inkling_gate_topk_renorm_v2(
|
||||
|
||||
@@ -1,376 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cfloat>
|
||||
#include <cstdint>
|
||||
|
||||
// gfx1250 needs a 64-bit __shfl_*_sync mask (static_assert sizeof == 8); it's
|
||||
// masked to wave32 internally, so 64-bit is fine everywhere. __gfx1250__ is
|
||||
// device-pass only, so widen in the host pass too or the launch stub won't build.
|
||||
#if defined(__gfx1250__) || (defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_DEVICE_COMPILE__))
|
||||
#define SGL_WARP_SYNC_MASK 0xFFFFFFFFFFFFFFFFULL
|
||||
#else
|
||||
#define SGL_WARP_SYNC_MASK 0xFFFFFFFF
|
||||
#endif
|
||||
|
||||
namespace sglang {
|
||||
|
||||
constexpr uint32_t kWarpSize = 32;
|
||||
constexpr uint32_t kWarpsPerCTA = 6;
|
||||
constexpr uint32_t kSmallTokenThreshold = 512;
|
||||
constexpr uint32_t kMaxExperts = 512;
|
||||
constexpr uint32_t kMaxTopK = 16;
|
||||
|
||||
enum class ScoringFunc : uint32_t {
|
||||
kSigmoid = 0,
|
||||
kSqrtSoftplus = 1,
|
||||
};
|
||||
|
||||
struct MoEFusedGateParams {
|
||||
const float* __restrict__ input;
|
||||
const float* __restrict__ bias;
|
||||
float* __restrict__ output;
|
||||
int32_t* __restrict__ indices;
|
||||
uint32_t num_rows;
|
||||
uint32_t num_experts;
|
||||
uint32_t topk;
|
||||
uint32_t num_fused_shared_experts;
|
||||
bool renormalize;
|
||||
float routed_scaling_factor;
|
||||
bool apply_routed_scaling_factor_on_output;
|
||||
};
|
||||
|
||||
template <ScoringFunc kScoringFunc>
|
||||
__device__ __forceinline__ float compute_score(float x) {
|
||||
if constexpr (kScoringFunc == ScoringFunc::kSigmoid) {
|
||||
// sigmoid(x) = 1 / (1 + exp(-x))
|
||||
return 1.0f / (1.0f + expf(-x));
|
||||
} else {
|
||||
// sqrt(softplus(x)); sign folded out because expf overflows above 88.7.
|
||||
const float softplus = fmaxf(x, 0.0f) + log1pf(expf(-fabsf(x)));
|
||||
return sqrtf(softplus);
|
||||
}
|
||||
}
|
||||
|
||||
template <uint32_t kWarpsPerToken, ScoringFunc kScoringFunc>
|
||||
__global__ void moe_fused_gate_kernel_small_token(const MoEFusedGateParams __grid_constant__ params) {
|
||||
const auto& [input, bias, output, indices, num_rows, num_experts, topk, num_fused_shared_experts, renormalize, routed_scaling_factor, apply_routed_scaling_factor_on_output] =
|
||||
params;
|
||||
|
||||
uint32_t row_idx = blockIdx.x;
|
||||
if (row_idx >= num_rows) return;
|
||||
|
||||
// number of routed experts to select (excluding fused shared experts)
|
||||
const uint32_t topk_routed = topk - num_fused_shared_experts;
|
||||
|
||||
uint32_t tid = threadIdx.x;
|
||||
uint32_t warp_id = tid / kWarpSize;
|
||||
uint32_t lane_id = tid % kWarpSize;
|
||||
// Actual warps launched (<= kWarpsPerToken). num_experts that need fewer than
|
||||
// kWarpsPerToken warps leave the upper warp_maxs/warp_experts slots unwritten,
|
||||
// so the cross-warp reduction below must only read the launched warps.
|
||||
const uint32_t num_warps = blockDim.x / kWarpSize;
|
||||
|
||||
extern __shared__ float shared_mem[];
|
||||
float* shared_scores = shared_mem;
|
||||
float* shared_original_scores = shared_mem + num_experts;
|
||||
|
||||
// For warp-level reduction
|
||||
__shared__ float warp_maxs[kWarpsPerToken];
|
||||
__shared__ int warp_experts[kWarpsPerToken];
|
||||
__shared__ int selected_experts[kMaxTopK];
|
||||
|
||||
for (uint32_t e = tid; e < num_experts; e += blockDim.x) {
|
||||
float input_val = input[row_idx * num_experts + e];
|
||||
float bias_val = bias[e];
|
||||
float score_val = compute_score<kScoringFunc>(input_val);
|
||||
float biased_val = score_val + bias_val;
|
||||
shared_scores[e] = biased_val;
|
||||
shared_original_scores[e] = score_val;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// only select topk_routed experts (excluding shared experts)
|
||||
for (uint32_t k = 0; k < topk_routed; k++) {
|
||||
float my_val = -FLT_MAX;
|
||||
int my_expert = -1;
|
||||
for (uint32_t e = tid; e < num_experts; e += blockDim.x) {
|
||||
if (shared_scores[e] > my_val) {
|
||||
my_val = shared_scores[e];
|
||||
my_expert = e;
|
||||
}
|
||||
}
|
||||
|
||||
float warp_max_val = my_val;
|
||||
int warp_max_expert = my_expert;
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, warp_max_val, offset);
|
||||
int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, warp_max_expert, offset);
|
||||
if (other_val > warp_max_val) {
|
||||
warp_max_val = other_val;
|
||||
warp_max_expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
if (lane_id == 0 && warp_id < kWarpsPerToken) {
|
||||
warp_maxs[warp_id] = warp_max_val;
|
||||
warp_experts[warp_id] = warp_max_expert;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
if (warp_id == 0) {
|
||||
float final_max = (lane_id < num_warps) ? warp_maxs[lane_id] : -FLT_MAX;
|
||||
int final_expert = (lane_id < num_warps) ? warp_experts[lane_id] : -1;
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, final_max, offset);
|
||||
int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, final_expert, offset);
|
||||
if (other_val > final_max) {
|
||||
final_max = other_val;
|
||||
final_expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
if (lane_id == 0) {
|
||||
selected_experts[k] = final_expert;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int selected = selected_experts[k];
|
||||
if (selected >= 0 && tid == 0) {
|
||||
shared_scores[selected] = -FLT_MAX;
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
static_assert(kMaxTopK <= device::kWarpThreads);
|
||||
if (tid >= device::kWarpThreads) return;
|
||||
|
||||
// only use the first warp to perform write to global operation
|
||||
float routed_weight = 0.0f;
|
||||
int32_t selected_expert = 0;
|
||||
if (tid < topk_routed) {
|
||||
int expert_id = selected_experts[tid];
|
||||
float score = shared_original_scores[expert_id];
|
||||
if (expert_id >= 0 && expert_id < static_cast<int>(num_experts)) {
|
||||
routed_weight = score;
|
||||
selected_expert = expert_id;
|
||||
}
|
||||
}
|
||||
const auto routed_sum = device::warp::reduce_sum<kMaxTopK>(routed_weight);
|
||||
if (tid < topk) {
|
||||
const bool is_shared = tid >= topk_routed;
|
||||
const auto output_offset = row_idx * topk + tid;
|
||||
const auto weight = is_shared ? (routed_sum / routed_scaling_factor) : routed_weight;
|
||||
const auto expert_id = is_shared ? (num_experts + tid - topk_routed) : selected_expert;
|
||||
const auto scale = apply_routed_scaling_factor_on_output ? routed_scaling_factor : 1.0f;
|
||||
const auto norm = renormalize && routed_sum > 0.0f ? routed_sum : 1.0f;
|
||||
output[output_offset] = weight / norm * scale;
|
||||
indices[output_offset] = expert_id;
|
||||
}
|
||||
}
|
||||
|
||||
template <ScoringFunc kScoringFunc>
|
||||
__global__ void moe_fused_gate_kernel(const MoEFusedGateParams __grid_constant__ params) {
|
||||
const auto& [input, bias, output, indices, num_rows, num_experts, topk, num_fused_shared_experts, renormalize, routed_scaling_factor, apply_routed_scaling_factor_on_output] =
|
||||
params;
|
||||
|
||||
uint32_t row_idx = blockIdx.x * kWarpsPerCTA + threadIdx.y;
|
||||
if (row_idx >= num_rows) return;
|
||||
|
||||
// number of routed experts to select (excluding fused shared experts)
|
||||
const uint32_t topk_routed = topk - num_fused_shared_experts;
|
||||
|
||||
uint32_t lane_id = threadIdx.x;
|
||||
uint32_t warp_id = threadIdx.y;
|
||||
|
||||
extern __shared__ float shared_mem[];
|
||||
float* shared_scores = shared_mem + warp_id * num_experts * 2;
|
||||
float* shared_original_scores = shared_scores + num_experts;
|
||||
__shared__ int selected_experts[kWarpsPerCTA][kMaxTopK];
|
||||
int* warp_selected_experts = selected_experts[warp_id];
|
||||
|
||||
for (uint32_t e = lane_id; e < num_experts; e += kWarpSize) {
|
||||
float input_val = input[row_idx * num_experts + e];
|
||||
float bias_val = bias[e];
|
||||
float score_val = compute_score<kScoringFunc>(input_val);
|
||||
float biased_val = score_val + bias_val;
|
||||
shared_scores[e] = biased_val;
|
||||
shared_original_scores[e] = score_val;
|
||||
}
|
||||
|
||||
__syncwarp();
|
||||
|
||||
// only select topk_routed experts
|
||||
for (uint32_t k = 0; k < topk_routed; k++) {
|
||||
float max_val = -FLT_MAX;
|
||||
int max_expert = -1;
|
||||
|
||||
for (uint32_t expert = lane_id; expert < num_experts; expert += kWarpSize) {
|
||||
if (shared_scores[expert] > max_val) {
|
||||
max_val = shared_scores[expert];
|
||||
max_expert = expert;
|
||||
}
|
||||
}
|
||||
|
||||
for (int offset = kWarpSize / 2; offset > 0; offset /= 2) {
|
||||
float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, max_val, offset);
|
||||
int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, max_expert, offset);
|
||||
|
||||
if (other_val > max_val || (other_val == max_val && other_expert < max_expert)) {
|
||||
max_val = other_val;
|
||||
max_expert = other_expert;
|
||||
}
|
||||
}
|
||||
|
||||
if (lane_id == 0) {
|
||||
warp_selected_experts[k] = max_expert;
|
||||
if (max_expert != -1) {
|
||||
shared_scores[max_expert] = -FLT_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
__syncwarp();
|
||||
}
|
||||
|
||||
static_assert(kMaxTopK <= device::kWarpThreads);
|
||||
|
||||
float routed_weight = 0.0f;
|
||||
int32_t selected_expert = 0;
|
||||
if (lane_id < topk_routed) {
|
||||
int expert_id = warp_selected_experts[lane_id];
|
||||
if (expert_id >= 0 && expert_id < static_cast<int>(num_experts)) {
|
||||
routed_weight = shared_original_scores[expert_id];
|
||||
selected_expert = expert_id;
|
||||
}
|
||||
}
|
||||
const auto routed_sum = device::warp::reduce_sum<kMaxTopK>(routed_weight);
|
||||
if (lane_id < topk) {
|
||||
const bool is_shared = lane_id >= topk_routed;
|
||||
const auto output_idx = row_idx * topk + lane_id;
|
||||
const auto weight = is_shared ? (routed_sum / routed_scaling_factor) : routed_weight;
|
||||
const auto expert_id = is_shared ? (num_experts + lane_id - topk_routed) : selected_expert;
|
||||
const auto scale = apply_routed_scaling_factor_on_output ? routed_scaling_factor : 1.0f;
|
||||
const auto norm = renormalize && routed_sum > 0.0f ? routed_sum : 1.0f;
|
||||
output[output_idx] = weight / norm * scale;
|
||||
indices[output_idx] = expert_id;
|
||||
}
|
||||
}
|
||||
|
||||
template <ScoringFunc kScoringFunc>
|
||||
void dispatch_small_token_kernel(
|
||||
uint32_t num_rows,
|
||||
uint32_t threads_per_block,
|
||||
uint32_t warps_per_token,
|
||||
DLDevice device,
|
||||
size_t smem_per_row,
|
||||
const MoEFusedGateParams& params) {
|
||||
using namespace host;
|
||||
if (warps_per_token <= 8) {
|
||||
LaunchKernel(num_rows, threads_per_block, device, smem_per_row)(
|
||||
moe_fused_gate_kernel_small_token<8, kScoringFunc>, params);
|
||||
} else if (warps_per_token <= 12) {
|
||||
LaunchKernel(num_rows, threads_per_block, device, smem_per_row)(
|
||||
moe_fused_gate_kernel_small_token<12, kScoringFunc>, params);
|
||||
} else {
|
||||
LaunchKernel(num_rows, threads_per_block, device, smem_per_row)(
|
||||
moe_fused_gate_kernel_small_token<16, kScoringFunc>, params);
|
||||
}
|
||||
}
|
||||
|
||||
struct MoEFusedGateKernel {
|
||||
static void
|
||||
run(const tvm::ffi::TensorView input,
|
||||
const tvm::ffi::TensorView bias,
|
||||
const tvm::ffi::TensorView output,
|
||||
const tvm::ffi::TensorView indices,
|
||||
uint32_t topk,
|
||||
uint32_t scoring_func, // 0 = sigmoid, 1 = sqrtsoftplus
|
||||
uint32_t num_fused_shared_experts,
|
||||
bool renormalize,
|
||||
float routed_scaling_factor,
|
||||
bool apply_routed_scaling_factor_on_output) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_rows"};
|
||||
auto E = SymbolicSize{"num_experts"};
|
||||
auto K = SymbolicSize{"topk"};
|
||||
auto device = SymbolicDevice{};
|
||||
K.set_value(topk);
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({N, E}).with_dtype<float>().with_device(device).verify(input);
|
||||
TensorMatcher({E}).with_dtype<float>().with_device(device).verify(bias);
|
||||
TensorMatcher({N, K}).with_dtype<float>().with_device(device).verify(output);
|
||||
TensorMatcher({N, K}).with_dtype<int32_t>().with_device(device).verify(indices);
|
||||
|
||||
const auto num_rows = static_cast<uint32_t>(N.unwrap());
|
||||
const auto num_experts = static_cast<uint32_t>(E.unwrap());
|
||||
|
||||
RuntimeCheck(num_experts <= kMaxExperts, "num_experts exceeds maximum supported value");
|
||||
RuntimeCheck(scoring_func <= 1, "scoring_func must be 0 (sigmoid) or 1 (sqrtsoftplus)");
|
||||
RuntimeCheck(topk > num_fused_shared_experts, "topk must be greater than num_fused_shared_experts");
|
||||
|
||||
const auto params = MoEFusedGateParams{
|
||||
.input = static_cast<const float*>(input.data_ptr()),
|
||||
.bias = static_cast<const float*>(bias.data_ptr()),
|
||||
.output = static_cast<float*>(output.data_ptr()),
|
||||
.indices = static_cast<int32_t*>(indices.data_ptr()),
|
||||
.num_rows = num_rows,
|
||||
.num_experts = num_experts,
|
||||
.topk = topk,
|
||||
.num_fused_shared_experts = num_fused_shared_experts,
|
||||
.renormalize = renormalize,
|
||||
.routed_scaling_factor = routed_scaling_factor,
|
||||
.apply_routed_scaling_factor_on_output = apply_routed_scaling_factor_on_output,
|
||||
};
|
||||
|
||||
const size_t smem_per_row = 2 * num_experts * sizeof(float);
|
||||
|
||||
bool use_small_token_kernel = num_rows <= kSmallTokenThreshold;
|
||||
|
||||
if (use_small_token_kernel) {
|
||||
// 1 token per block
|
||||
uint32_t warps_per_token = div_ceil(num_experts, kWarpSize);
|
||||
warps_per_token = std::min(warps_per_token, 16u);
|
||||
uint32_t threads_per_block = warps_per_token * kWarpSize;
|
||||
|
||||
if (scoring_func == 0) {
|
||||
dispatch_small_token_kernel<ScoringFunc::kSigmoid>(
|
||||
num_rows, threads_per_block, warps_per_token, device.unwrap(), smem_per_row, params);
|
||||
} else {
|
||||
dispatch_small_token_kernel<ScoringFunc::kSqrtSoftplus>(
|
||||
num_rows, threads_per_block, warps_per_token, device.unwrap(), smem_per_row, params);
|
||||
}
|
||||
} else {
|
||||
// multiple tokens per block
|
||||
uint32_t num_blocks = div_ceil(num_rows, kWarpsPerCTA);
|
||||
dim3 block_dim(kWarpSize, kWarpsPerCTA);
|
||||
size_t large_smem = smem_per_row * kWarpsPerCTA;
|
||||
|
||||
if (scoring_func == 0) {
|
||||
LaunchKernel(num_blocks, block_dim, device.unwrap(), large_smem)(
|
||||
moe_fused_gate_kernel<ScoringFunc::kSigmoid>, params);
|
||||
} else {
|
||||
LaunchKernel(num_blocks, block_dim, device.unwrap(), large_smem)(
|
||||
moe_fused_gate_kernel<ScoringFunc::kSqrtSoftplus>, params);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
@@ -29,7 +29,7 @@
|
||||
// The reader requires the rows of a page to be contiguous and the page stride
|
||||
// to be a multiple of kPageAlign (its TMA row stride), which is what
|
||||
// kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is
|
||||
// `sglang.kernels.ops.attention.dsv4.torch_quant`.
|
||||
// `sglang.test.kernels.deepseek_v4.k_cache_reference`.
|
||||
|
||||
namespace sglang {
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
"""Fused gated-activation kernels (``act(x[:h]) * x[h:]``).
|
||||
"""Activation kernels.
|
||||
|
||||
Each operator is a :class:`~sglang.kernels.fused_op.BaseFusedOp` with a
|
||||
pure-``torch`` reference (``forward_native``) plus AOT (``sgl_kernel``) and
|
||||
JIT CUDA backends behind one ``(input, out)`` signature. The JIT backend
|
||||
additionally accepts ``expert_ids`` / ``expert_step`` — call
|
||||
``forward_jit`` directly when those are needed.
|
||||
``silu_and_mul`` (``silu(x[..., :d]) * x[..., d:]``) is a
|
||||
:class:`~sglang.kernels.fused_op.BaseFusedOp` with a pure-``torch`` reference
|
||||
(``forward_native``) plus AOT (``sgl_kernel``), JIT CUDA and aiter backends
|
||||
behind one ``(input, out)`` signature. The JIT backend additionally accepts
|
||||
``expert_ids`` / ``expert_step`` — call ``forward_jit`` directly when those are
|
||||
needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,8 +15,8 @@ from typing import TYPE_CHECKING, Optional
|
||||
from sglang.kernels.fused_op import BaseFusedOp, register_fused_op
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import (
|
||||
CUDA_ONLY,
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
@@ -23,85 +24,11 @@ from sglang.kernels.spec import (
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
_ACT_DTYPES = ("float16", "bfloat16")
|
||||
_CUDA = frozenset({CapabilityRequirement.CUDA})
|
||||
_HIP = frozenset({CapabilityRequirement.HIP})
|
||||
# sgl_kernel's gated-activation ops build for CUDA *and* ROCm (production
|
||||
# imports them from sgl_kernel on both), so the AOT backend spans both devices
|
||||
# — the canonical OR-semantics case that a device-baked backend name couldn't.
|
||||
_CUDA_HIP = frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP})
|
||||
# JIT before AOT to match the production path (srt/layers/activation.py imports
|
||||
# from sglang.kernels.ops.activation.activation on CUDA); auto-selection must not invert it.
|
||||
_ACT_PRIORITY = (
|
||||
KernelBackend.JIT,
|
||||
KernelBackend.AOT,
|
||||
KernelBackend.TORCH,
|
||||
)
|
||||
|
||||
|
||||
class _GatedActivationOp(BaseFusedOp):
|
||||
"""Shared structure for ``act(x[..., :d]) * x[..., d:]`` operators."""
|
||||
|
||||
# Set by subclasses: sgl_kernel / jit_kernel attr name (same for both).
|
||||
kernel_attr: str
|
||||
|
||||
priority = _ACT_PRIORITY
|
||||
capabilities = {
|
||||
KernelBackend.AOT: _CUDA_HIP,
|
||||
KernelBackend.JIT: _CUDA,
|
||||
}
|
||||
format_signature = FormatSignature(
|
||||
supported_dtypes=_ACT_DTYPES,
|
||||
description="gated activation; returns tensor",
|
||||
)
|
||||
|
||||
def _act(self, gate: torch.Tensor) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_native(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
d = input.shape[-1] // 2
|
||||
result = self._act(input[..., :d]) * input[..., d:]
|
||||
if out is None:
|
||||
return result
|
||||
out.copy_(result)
|
||||
return out
|
||||
|
||||
def forward_aot(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
import sgl_kernel
|
||||
|
||||
return getattr(sgl_kernel, self.kernel_attr)(input, out)
|
||||
|
||||
def forward_jit(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
expert_ids: Optional[torch.Tensor] = None,
|
||||
expert_step: int = 1,
|
||||
) -> torch.Tensor:
|
||||
import sglang.kernels.ops.activation.activation as jit_activation
|
||||
|
||||
return getattr(jit_activation, self.kernel_attr)(
|
||||
input, out, expert_ids, expert_step
|
||||
)
|
||||
|
||||
|
||||
class SiluAndMulOp(_GatedActivationOp):
|
||||
"""``out = silu(input[..., :d]) * input[..., d:]`` with ``d = input.shape[-1] // 2``.
|
||||
|
||||
Adds an ``AITER`` backend on ``device=HIP``: on ROCm this op has a native
|
||||
``aiter`` kernel (``srt/layers/activation.py`` uses it in production). Note
|
||||
the sibling gelu ops below deliberately do *not* register AITER — ROCm/aiter
|
||||
coverage is a per-``(op, backend)`` subset, which the decoupled backend/device
|
||||
model expresses directly (a device-agnostic ``KernelBackend`` name plus a
|
||||
per-backend ``CapabilityRequirement``).
|
||||
"""
|
||||
class SiluAndMulOp(BaseFusedOp):
|
||||
"""``out = silu(input[..., :d]) * input[..., d:]`` with ``d = input.shape[-1] // 2``."""
|
||||
|
||||
op = "activation.silu_and_mul"
|
||||
kernel_attr = "silu_and_mul"
|
||||
# AOT spans CUDA+HIP; JIT is CUDA; AITER is an opt-in HIP path. By priority,
|
||||
# CUDA resolves to JIT and HIP resolves to AOT (matching production
|
||||
# defaults); AITER is registered and HIP-eligible but sits below AOT, so it
|
||||
@@ -113,21 +40,44 @@ class SiluAndMulOp(_GatedActivationOp):
|
||||
KernelBackend.TORCH,
|
||||
)
|
||||
capabilities = {
|
||||
KernelBackend.AOT: _CUDA_HIP,
|
||||
KernelBackend.JIT: _CUDA,
|
||||
KernelBackend.AITER: _HIP,
|
||||
}
|
||||
descriptions = {
|
||||
KernelBackend.AOT: "silu_and_mul (sgl_kernel wheel).",
|
||||
KernelBackend.JIT: "silu_and_mul (sglang.kernels.jit).",
|
||||
KernelBackend.AITER: "silu_and_mul (aiter, ROCm).",
|
||||
KernelBackend.TORCH: "silu_and_mul (pure-torch reference).",
|
||||
KernelBackend.AOT: frozenset(
|
||||
{CapabilityRequirement.CUDA, CapabilityRequirement.HIP}
|
||||
),
|
||||
KernelBackend.JIT: CUDA_ONLY,
|
||||
KernelBackend.AITER: frozenset({CapabilityRequirement.HIP}),
|
||||
}
|
||||
|
||||
def _act(self, gate: torch.Tensor) -> torch.Tensor:
|
||||
def forward_native(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
import torch.nn.functional as F
|
||||
|
||||
return F.silu(gate)
|
||||
d = input.shape[-1] // 2
|
||||
result = F.silu(input[..., :d]) * input[..., d:]
|
||||
if out is None:
|
||||
return result
|
||||
out.copy_(result)
|
||||
return out
|
||||
|
||||
def forward_aot(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
from sgl_kernel import silu_and_mul as aot_silu_and_mul
|
||||
|
||||
return aot_silu_and_mul(input, out)
|
||||
|
||||
def forward_jit(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
expert_ids: Optional[torch.Tensor] = None,
|
||||
expert_step: int = 1,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.activation.activation import (
|
||||
silu_and_mul as jit_silu_and_mul,
|
||||
)
|
||||
|
||||
return jit_silu_and_mul(input, out, expert_ids, expert_step)
|
||||
|
||||
def forward_aiter(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
@@ -146,132 +96,7 @@ class SiluAndMulOp(_GatedActivationOp):
|
||||
return out
|
||||
|
||||
|
||||
class GeluAndMulOp(_GatedActivationOp):
|
||||
"""``out = gelu(input[..., :d]) * input[..., d:]`` (erf-based GELU)."""
|
||||
|
||||
op = "activation.gelu_and_mul"
|
||||
kernel_attr = "gelu_and_mul"
|
||||
descriptions = {
|
||||
KernelBackend.AOT: "gelu_and_mul (sgl_kernel wheel).",
|
||||
KernelBackend.JIT: "gelu_and_mul (sglang.kernels.jit).",
|
||||
KernelBackend.TORCH: "gelu_and_mul (pure-torch reference).",
|
||||
}
|
||||
|
||||
def _act(self, gate: torch.Tensor) -> torch.Tensor:
|
||||
import torch.nn.functional as F
|
||||
|
||||
return F.gelu(gate, approximate="none")
|
||||
|
||||
|
||||
class GeluTanhAndMulOp(_GatedActivationOp):
|
||||
"""``out = gelu_tanh(input[..., :d]) * input[..., d:]`` (tanh-approximated GELU)."""
|
||||
|
||||
op = "activation.gelu_tanh_and_mul"
|
||||
kernel_attr = "gelu_tanh_and_mul"
|
||||
descriptions = {
|
||||
KernelBackend.AOT: "gelu_tanh_and_mul (sgl_kernel wheel).",
|
||||
KernelBackend.JIT: "gelu_tanh_and_mul (sglang.kernels.jit).",
|
||||
KernelBackend.TORCH: "gelu_tanh_and_mul (pure-torch reference).",
|
||||
}
|
||||
|
||||
def _act(self, gate: torch.Tensor) -> torch.Tensor:
|
||||
import torch.nn.functional as F
|
||||
|
||||
return F.gelu(gate, approximate="tanh")
|
||||
|
||||
|
||||
class ReLU2Op(BaseFusedOp):
|
||||
"""``out = relu(input) ** 2`` (single-input, not gated).
|
||||
|
||||
The real kernel is the CUDA JIT path (``sglang.kernels.ops.activation.activation.relu2``,
|
||||
used in production on CUDA); elsewhere the torch reference runs.
|
||||
"""
|
||||
|
||||
op = "activation.relu2"
|
||||
priority = (KernelBackend.JIT, KernelBackend.TORCH)
|
||||
capabilities = {KernelBackend.JIT: _CUDA}
|
||||
format_signature = FormatSignature(
|
||||
supported_dtypes=_ACT_DTYPES,
|
||||
description="relu(x) ** 2; returns tensor",
|
||||
)
|
||||
descriptions = {
|
||||
KernelBackend.JIT: "relu(x)**2 (sglang.kernels.jit).",
|
||||
KernelBackend.TORCH: "relu(x)**2 (pure-torch reference).",
|
||||
}
|
||||
|
||||
def forward_native(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
import torch.nn.functional as F
|
||||
|
||||
x = F.relu(input)
|
||||
result = x * x
|
||||
if out is None:
|
||||
return result
|
||||
out.copy_(result)
|
||||
return out
|
||||
|
||||
def forward_jit(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.activation.activation import relu2
|
||||
|
||||
result = relu2(input)
|
||||
if out is None:
|
||||
return result
|
||||
out.copy_(result)
|
||||
return out
|
||||
|
||||
|
||||
class QuickGELUOp(BaseFusedOp):
|
||||
"""``out = input * sigmoid(1.702 * input)`` (single-input, not gated).
|
||||
|
||||
Only ROCm has a native kernel (``sgl_kernel.gelu_quick``, AOT on HIP); CUDA
|
||||
uses the torch reference in production, so no CUDA backend is registered.
|
||||
"""
|
||||
|
||||
op = "activation.gelu_quick"
|
||||
priority = (KernelBackend.AOT, KernelBackend.TORCH)
|
||||
capabilities = {KernelBackend.AOT: _HIP}
|
||||
format_signature = FormatSignature(
|
||||
supported_dtypes=_ACT_DTYPES,
|
||||
description="x * sigmoid(1.702 * x); returns tensor",
|
||||
)
|
||||
descriptions = {
|
||||
KernelBackend.AOT: "quick GELU (sgl_kernel wheel, ROCm).",
|
||||
KernelBackend.TORCH: "quick GELU (pure-torch reference).",
|
||||
}
|
||||
|
||||
def forward_native(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
import torch
|
||||
|
||||
result = input * torch.sigmoid(1.702 * input)
|
||||
if out is None:
|
||||
return result
|
||||
out.copy_(result)
|
||||
return out
|
||||
|
||||
def forward_aot(
|
||||
self, input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
import torch
|
||||
from sgl_kernel import gelu_quick
|
||||
|
||||
if out is None:
|
||||
out = torch.empty(input.shape, dtype=input.dtype, device=input.device)
|
||||
gelu_quick(input, out)
|
||||
return out
|
||||
|
||||
|
||||
_SILU_AND_MUL = register_fused_op(SiluAndMulOp(), __name__, "_SILU_AND_MUL")
|
||||
_GELU_AND_MUL = register_fused_op(GeluAndMulOp(), __name__, "_GELU_AND_MUL")
|
||||
_GELU_TANH_AND_MUL = register_fused_op(
|
||||
GeluTanhAndMulOp(), __name__, "_GELU_TANH_AND_MUL"
|
||||
)
|
||||
_RELU2 = register_fused_op(ReLU2Op(), __name__, "_RELU2")
|
||||
_GELU_QUICK = register_fused_op(QuickGELUOp(), __name__, "_GELU_QUICK")
|
||||
|
||||
|
||||
def silu_and_mul(
|
||||
@@ -281,30 +106,6 @@ def silu_and_mul(
|
||||
return _SILU_AND_MUL(input, out)
|
||||
|
||||
|
||||
def gelu_and_mul(
|
||||
input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""``out = gelu(input[..., :d]) * input[..., d:]``."""
|
||||
return _GELU_AND_MUL(input, out)
|
||||
|
||||
|
||||
def gelu_tanh_and_mul(
|
||||
input: torch.Tensor, out: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""``out = gelu_tanh(input[..., :d]) * input[..., d:]``."""
|
||||
return _GELU_TANH_AND_MUL(input, out)
|
||||
|
||||
|
||||
def relu2(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
"""``out = relu(input) ** 2``."""
|
||||
return _RELU2(input, out)
|
||||
|
||||
|
||||
def gelu_quick(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
"""``out = input * sigmoid(1.702 * input)``."""
|
||||
return _GELU_QUICK(input, out)
|
||||
|
||||
|
||||
def situ_and_mul(
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor],
|
||||
@@ -319,29 +120,19 @@ def situ_and_mul(
|
||||
__all__ = [
|
||||
"situ_and_mul",
|
||||
"SiluAndMulOp",
|
||||
"GeluAndMulOp",
|
||||
"GeluTanhAndMulOp",
|
||||
"ReLU2Op",
|
||||
"QuickGELUOp",
|
||||
"silu_and_mul",
|
||||
"gelu_and_mul",
|
||||
"gelu_tanh_and_mul",
|
||||
"relu2",
|
||||
"gelu_quick",
|
||||
]
|
||||
|
||||
|
||||
# Triton kernel migrated into this group (from layers/triton_ops/softcap);
|
||||
# registered for inventory. Import it from its module.
|
||||
for _fn in ("softcap_out", "softcap_inplace_logits"):
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"activation.{_fn}",
|
||||
backend=KernelBackend.TRITON,
|
||||
target=f"sglang.kernels.ops.activation.softcap:{_fn}",
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="activation.softcap_inplace_logits",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.activation.softcap:softcap_inplace_logits",
|
||||
)
|
||||
del _fn
|
||||
)
|
||||
|
||||
|
||||
register_kernel(
|
||||
@@ -349,11 +140,7 @@ register_kernel(
|
||||
op="activation.softcap_to_float32_logits",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.activation.softcap:softcap_to_float32_logits",
|
||||
capabilities=_CUDA,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float16", "bfloat16", "float32"),
|
||||
description="Softcap logits into a caller-provided FP32 output.",
|
||||
),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -364,6 +151,6 @@ register_kernel(
|
||||
op="activation.situ_and_mul",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3,70 +3,6 @@ import triton
|
||||
import triton.language as tl
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
softcap_out_autotune = triton.autotune(
|
||||
configs=[
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=4),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=8),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=16),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=4),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=8),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=4),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=8),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=16),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=32),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=32),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=32),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32),
|
||||
triton.Config(kwargs={"BLOCK_SIZE": 32768}, num_warps=32),
|
||||
],
|
||||
key=["n_ele"],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def softcap_out_kernel(
|
||||
output_ptr,
|
||||
input_ptr,
|
||||
n_ele,
|
||||
softcap_const: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(axis=0)
|
||||
block_start = pid * BLOCK_SIZE
|
||||
offsets = block_start + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < n_ele
|
||||
x = tl.load(input_ptr + offsets, mask=mask)
|
||||
fx = x.to(tl.float32)
|
||||
fxs = fx / softcap_const
|
||||
exped = tl.exp(2 * fxs)
|
||||
top = exped - 1
|
||||
bottom = exped + 1
|
||||
output = top / bottom * softcap_const
|
||||
tl.store(output_ptr + offsets, output, mask=mask)
|
||||
|
||||
|
||||
softcap_out_kernel_autotuned = softcap_out_autotune(softcap_out_kernel)
|
||||
|
||||
|
||||
def softcap_out(x, softcap_const, autotune=False):
|
||||
output = torch.empty_like(x, dtype=torch.float32)
|
||||
n_elements = output.numel()
|
||||
if autotune:
|
||||
|
||||
def grid(meta):
|
||||
return (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)
|
||||
|
||||
softcap_out_kernel_autotuned[grid](output, x, n_elements, softcap_const)
|
||||
else:
|
||||
softcap_out_kernel[(triton.cdiv(n_elements, 128),)](
|
||||
output, x, n_elements, softcap_const, BLOCK_SIZE=128, num_warps=8
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def softcap_inplace_logits_kernel(
|
||||
|
||||
@@ -13,8 +13,8 @@ from typing import TYPE_CHECKING
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.selector import get_kernel
|
||||
from sglang.kernels.spec import (
|
||||
CUDA_ONLY,
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
@@ -66,15 +66,6 @@ register_kernel(
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(12, 1), max_sm=(12, 1))}
|
||||
),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("bfloat16",),
|
||||
description=(
|
||||
"Qwen3.8 packed QSA decode: D=256, 12:1 GQA, 1 <= q_rows <= 128"
|
||||
),
|
||||
),
|
||||
description=(
|
||||
"SM121 Qwen3.8 QSA decode optimized by Codex/Kimi K3 through KDA-1.5."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -265,12 +256,6 @@ for _mod, _fn, _backend, _device in [
|
||||
KernelBackend.CUTE_DSL,
|
||||
CapabilityRequirement.CUDA,
|
||||
),
|
||||
(
|
||||
"kda_flydsl.kimi_k3_kda_decode",
|
||||
"flydsl_kimi_k3_kda_decode",
|
||||
KernelBackend.FLYDSL,
|
||||
CapabilityRequirement.HIP,
|
||||
),
|
||||
(
|
||||
"kda_flydsl.kimi_k3_kda_decode",
|
||||
"flydsl_kimi_k3_kda_decode_with_f_b",
|
||||
@@ -308,7 +293,7 @@ register_kernel(
|
||||
op="attention.get_block_table",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.minicpm_sala.get_block_table:get_block_table",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -316,7 +301,7 @@ register_kernel(
|
||||
op="attention.fast_kpool_topk_transform_fused",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsa.kpool_topk_transform:fast_kpool_topk_transform_fused",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -324,7 +309,7 @@ register_kernel(
|
||||
op="attention.fast_topk",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.fast_topk:fast_topk",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -344,7 +329,7 @@ register_kernel(
|
||||
op="attention.fused_k_indexer_norm_rope",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -352,7 +337,7 @@ register_kernel(
|
||||
op="attention.fused_k_indexer_norm_rope_store",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope_store",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -370,7 +355,7 @@ register_kernel(
|
||||
op="attention.fp4_index_logits_decode",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.attention.dsv4.fp4_indexer:fp4_index_logits_decode",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -379,8 +364,7 @@ register_kernel(
|
||||
op="attention.pack_prefix_current",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.attention.dllm_kv_pack:pack_prefix_current",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
description="Pack CSR prefix and current K/V into caller-provided dense buffers.",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -391,10 +375,6 @@ register_kernel(
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(11, 9))}
|
||||
),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float16", "bfloat16"),
|
||||
description="SM100/SM110 D512 GQA with separate K/V and optional page tables.",
|
||||
),
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -402,12 +382,7 @@ register_kernel(
|
||||
op="attention.gemma_qkv_norm_rope",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.attention.gemma_qkv_norm_rope:gemma_qkv_norm_rope",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("float16", "bfloat16"),
|
||||
in_place=True,
|
||||
description="Strided QKV normalization and partial NeoX RoPE.",
|
||||
),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -417,7 +392,7 @@ for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"):
|
||||
op=f"attention.{_fn}",
|
||||
backend=KernelBackend.TRITON,
|
||||
target=f"sglang.kernels.ops.attention.dsv4.fp4_indexer:{_fn}",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
del _fn
|
||||
|
||||
@@ -6,11 +6,7 @@ import torch
|
||||
|
||||
from sglang.kernels.fused_op import BaseFusedOp, register_fused_op
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.kernels.spec import (
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
)
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
@@ -51,13 +47,6 @@ class ClampPositionOp(BaseFusedOp):
|
||||
{CapabilityRequirement.CUDA, CapabilityRequirement.HIP}
|
||||
)
|
||||
}
|
||||
format_signature = FormatSignature(
|
||||
supported_dtypes=("int32", "int64"),
|
||||
description="clamp(seq_lens - 1, min=0); returns int64 positions",
|
||||
)
|
||||
descriptions = {
|
||||
KernelBackend.JIT: "Fused clamp-position kernel (sglang.kernels.jit).",
|
||||
}
|
||||
|
||||
def forward_native(self, seq_lens: torch.Tensor) -> torch.Tensor:
|
||||
return torch.clamp(seq_lens - 1, min=0).to(torch.int64)
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/cute_utils/cvt.py
|
||||
from cutlass import Constexpr, Float32, Uint32, cute
|
||||
from cutlass._mlir import ir
|
||||
from cutlass._mlir.dialects import llvm, vector
|
||||
from cutlass import Float32, Uint32, cute
|
||||
from cutlass._mlir.dialects import llvm
|
||||
from cutlass.cutlass_dsl import T, dsl_user_op
|
||||
|
||||
|
||||
@@ -48,99 +47,3 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]:
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported type {type(data)}")
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
|
||||
# there is only fp8->fp16 conversion, hence we need to go
|
||||
# round trip through fp16.
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.i32()] * 2),
|
||||
[x.ir_value(loc=loc, ip=ip)],
|
||||
"{\n\t"
|
||||
".reg .b16 x0, x1;\n\t"
|
||||
".reg .b16 t00, t01, t10, t11;\n\t"
|
||||
"mov.b32 {x0, x1}, $2;\n\t"
|
||||
"cvt.rn.f16x2.e4m3x2 $0, x0;\n\t"
|
||||
"cvt.rn.f16x2.e4m3x2 $1, x1;\n\t"
|
||||
"mov.b32 {t00, t01}, $0;\n\t"
|
||||
"mov.b32 {t10, t11}, $1;\n\t"
|
||||
"cvt.rn.bf16.f16 t00, t00;\n\t"
|
||||
"cvt.rn.bf16.f16 t01, t01;\n\t"
|
||||
"cvt.rn.bf16.f16 t10, t10;\n\t"
|
||||
"cvt.rn.bf16.f16 t11, t11;\n\t"
|
||||
"mov.b32 $0, {t00, t01};\n\t"
|
||||
"mov.b32 $1, {t10, t11};\n\t"
|
||||
"}\n",
|
||||
"=r,=r,r",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
vec = vector.from_elements(
|
||||
ir.VectorType.get([2], T.i32(), loc=loc),
|
||||
[llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(vec, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x4_to_fp8x4(
|
||||
a0: Float32,
|
||||
a1: Float32,
|
||||
a2: Float32,
|
||||
a3: Float32,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> Uint32:
|
||||
# Pack four FP32 values into one b32 of four e4m3 bytes, byte order
|
||||
# {a0, a1, a2, a3} from low to high address.
|
||||
out = llvm.inline_asm(
|
||||
T.i32(),
|
||||
[
|
||||
a0.ir_value(loc=loc, ip=ip),
|
||||
a1.ir_value(loc=loc, ip=ip),
|
||||
a2.ir_value(loc=loc, ip=ip),
|
||||
a3.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"{\n\t"
|
||||
".reg .b16 t0, t1;\n\t"
|
||||
"cvt.rn.satfinite.e4m3x2.f32 t0, $2, $1;\n\t"
|
||||
"cvt.rn.satfinite.e4m3x2.f32 t1, $4, $3;\n\t"
|
||||
"mov.b32 $0, {t0, t1};\n\t"
|
||||
"}\n",
|
||||
"=r,f,f,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint32(out)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x8_to_fp4x8(
|
||||
vals: cute.Tensor,
|
||||
offset: Constexpr[int],
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> Uint32:
|
||||
# Pack eight scaled FP32 values into four E2M1x2 bytes, returned as one b32.
|
||||
assert vals.element_type is Float32
|
||||
out = llvm.inline_asm(
|
||||
T.i32(),
|
||||
[vals[offset + i].ir_value(loc=loc, ip=ip) for i in range(8)],
|
||||
"{\n\t"
|
||||
".reg .b8 x0, x1, x2, x3;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x0, $2, $1;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x1, $4, $3;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x2, $6, $5;\n\t"
|
||||
"cvt.rn.satfinite.e2m1x2.f32 x3, $8, $7;\n\t"
|
||||
"mov.b32 $0, {x0, x1, x2, x3};\n\t"
|
||||
"}\n",
|
||||
"=r,f,f,f,f,f,f,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint32(out)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -671,46 +671,3 @@ def dcp_lse_combine_triton(
|
||||
RETURN_LSE=return_lse,
|
||||
)
|
||||
return out, (out_lse if return_lse else None)
|
||||
|
||||
|
||||
def _lse_weighted_combine_cpu(
|
||||
partial_outputs: torch.Tensor,
|
||||
partial_lses: torch.Tensor,
|
||||
is_lse_base_on_e: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""CPU reference: combine N partial attention outputs using LSE weights.
|
||||
|
||||
Args:
|
||||
partial_outputs: [N, B, H_local, D]
|
||||
partial_lses: [N, B, H_local]
|
||||
is_lse_base_on_e: base-e (True) or base-2 (False)
|
||||
|
||||
Returns:
|
||||
[B, H_local, D] combined output
|
||||
"""
|
||||
N, B, H_local, D = partial_outputs.shape
|
||||
partial_outputs = partial_outputs.float()
|
||||
partial_lses = partial_lses.float()
|
||||
|
||||
# Sanitize
|
||||
partial_lses = torch.where(
|
||||
torch.isnan(partial_lses) | torch.isinf(partial_lses),
|
||||
torch.full_like(partial_lses, float("-inf")),
|
||||
partial_lses,
|
||||
)
|
||||
|
||||
# max LSE for numerical stability
|
||||
lse_max, _ = partial_lses.max(dim=0)
|
||||
lse_max = torch.where(lse_max == float("-inf"), torch.zeros_like(lse_max), lse_max)
|
||||
|
||||
centered = partial_lses - lse_max.unsqueeze(0)
|
||||
if is_lse_base_on_e:
|
||||
weights = torch.exp(centered)
|
||||
else:
|
||||
weights = torch.pow(2.0, centered)
|
||||
|
||||
weight_sum = weights.sum(dim=0, keepdim=True)
|
||||
weights = weights / weight_sum.clamp_min(1.0)
|
||||
|
||||
combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0)
|
||||
return combined
|
||||
|
||||
@@ -519,92 +519,6 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_softmax_pool_kernel(
|
||||
kv_score_ptr,
|
||||
out_ptr,
|
||||
stride_bs: tl.constexpr,
|
||||
stride_k: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
HEAD_BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
base = pid * stride_bs
|
||||
|
||||
offs = tl.arange(0, HEAD_BLOCK)
|
||||
mask = offs < HEAD_DIM
|
||||
|
||||
max_val = tl.full([HEAD_BLOCK], float("-inf"), dtype=tl.float32)
|
||||
for k in range(K):
|
||||
s = tl.load(
|
||||
kv_score_ptr + base + k * stride_k + HEAD_DIM + offs,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
max_val = tl.maximum(max_val, s)
|
||||
|
||||
sum_exp = tl.zeros([HEAD_BLOCK], dtype=tl.float32)
|
||||
weighted = tl.zeros([HEAD_BLOCK], dtype=tl.float32)
|
||||
for k in range(K):
|
||||
s = tl.load(
|
||||
kv_score_ptr + base + k * stride_k + HEAD_DIM + offs,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
kv_score_ptr + base + k * stride_k + offs,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
w = tl.exp(s - max_val)
|
||||
sum_exp += w
|
||||
weighted += v * w
|
||||
|
||||
result = weighted / sum_exp
|
||||
tl.store(
|
||||
out_ptr + pid * HEAD_DIM + offs, result.to(out_ptr.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
def fused_softmax_pool_triton(
|
||||
kv_score: torch.Tensor,
|
||||
head_dim: int,
|
||||
) -> torch.Tensor:
|
||||
"""Fused softmax-weighted-sum: out = (kv * softmax(score, dim=1)).sum(dim=1).
|
||||
|
||||
Replaces the generic cunn_SpatialSoftMaxForward + elementwise multiply + sum
|
||||
with a single Triton kernel.
|
||||
|
||||
Args:
|
||||
kv_score: [bs, K, 2 * head_dim] where first head_dim is kv, second is score.
|
||||
head_dim: dimension of each of kv and score.
|
||||
Returns:
|
||||
output: [bs, head_dim]
|
||||
"""
|
||||
assert kv_score.dim() == 3
|
||||
bs, K, last = kv_score.shape
|
||||
assert last == 2 * head_dim
|
||||
assert kv_score.is_contiguous()
|
||||
|
||||
out = torch.empty(bs, head_dim, dtype=kv_score.dtype, device=kv_score.device)
|
||||
if bs == 0:
|
||||
return out
|
||||
|
||||
HEAD_BLOCK = triton.next_power_of_2(head_dim)
|
||||
grid = (bs,)
|
||||
_fused_softmax_pool_kernel[grid](
|
||||
kv_score,
|
||||
out,
|
||||
stride_bs=kv_score.stride(0),
|
||||
stride_k=kv_score.stride(1),
|
||||
K=K,
|
||||
HEAD_DIM=head_dim,
|
||||
HEAD_BLOCK=HEAD_BLOCK,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def fused_norm_rope_inplace_triton(
|
||||
kv: torch.Tensor,
|
||||
weight: Optional[torch.Tensor],
|
||||
|
||||
@@ -9,12 +9,11 @@ from sglang.srt.layers.attention.dsa.utils import (
|
||||
INDEXER_K_CACHE_PRESHUFFLE_TILE,
|
||||
aiter_can_use_preshuffle_paged_mqa,
|
||||
)
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, is_xpu
|
||||
from sglang.srt.utils import is_hip, is_xpu
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_xpu = is_xpu()
|
||||
_is_fp8_fnuz = is_fp8_fnuz()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
# aiter cp_gather kernel with preshuffle=True is only valid when the indexer
|
||||
# uses the page_size=64 preshuffle layout (i.e. when the matching MQA gluon path
|
||||
# is also enabled).
|
||||
@@ -32,149 +31,6 @@ s: scale, 1 item per token, fp32
|
||||
"""
|
||||
|
||||
|
||||
class GetK:
|
||||
@classmethod
|
||||
def execute(cls, *args, **kwargs):
|
||||
return cls.triton(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def slow(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
page_size = pool.index_page_size
|
||||
num_pages = (seq_len + page_size - 1) // page_size
|
||||
seq_len_ = num_pages * page_size
|
||||
index_k_fp8 = torch.empty(
|
||||
(seq_len_, pool.index_head_dim),
|
||||
dtype=torch.uint8,
|
||||
device=pool.device,
|
||||
)
|
||||
for i in range(num_pages):
|
||||
page_index = page_indices[i]
|
||||
index_k_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
|
||||
: page_size * pool.index_head_dim
|
||||
].view(-1, pool.index_head_dim)
|
||||
|
||||
return index_k_fp8[:seq_len]
|
||||
|
||||
@classmethod
|
||||
def torch_fast(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
"""
|
||||
:param page_indices: (num_pages,), int32
|
||||
:return: (seq_len, index_head_dim), uint8
|
||||
"""
|
||||
|
||||
# can handle per 128B instead of per element
|
||||
|
||||
# page_indices: (num_pages,), element := a page index
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
|
||||
num_k_bytes_per_page = pool.index_page_size * pool.index_head_dim
|
||||
num_k_bytes_per_token = pool.index_head_dim
|
||||
|
||||
# buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8
|
||||
# flat_buf: (whatever,), uint8
|
||||
flat_buf = buf.flatten()
|
||||
|
||||
# flat_indices: (num_pages, num_k_bytes_per_page), int32, element := an index into flat_buf that we want to access
|
||||
flat_indices = (page_indices * buf_numel_per_page)[:, None] + torch.arange(
|
||||
num_k_bytes_per_page, dtype=torch.int32, device="cuda"
|
||||
)[None, :]
|
||||
flat_indices = flat_indices.flatten()[: seq_len * num_k_bytes_per_token]
|
||||
|
||||
out = flat_buf[flat_indices]
|
||||
return out.view(-1, 128)
|
||||
|
||||
@classmethod
|
||||
def triton(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
"""
|
||||
Triton implementation for gathering K data from paged buffer.
|
||||
:param page_indices: (num_pages,), int32/int64
|
||||
:return: (seq_len, index_head_dim), uint8
|
||||
"""
|
||||
return _get_k_triton(
|
||||
buf=buf,
|
||||
page_indices=page_indices,
|
||||
seq_len=seq_len,
|
||||
page_size=pool.index_page_size,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
|
||||
class GetS:
|
||||
@classmethod
|
||||
def execute(cls, *args, **kwargs):
|
||||
return cls.triton(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def slow(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
page_size = pool.index_page_size
|
||||
num_pages = (seq_len + page_size - 1) // page_size
|
||||
seq_len_ = num_pages * page_size
|
||||
assert pool.index_head_dim // pool.quant_block_size == 1
|
||||
index_k_scale_fp8 = torch.empty(
|
||||
(seq_len_, 4),
|
||||
dtype=torch.uint8,
|
||||
device=pool.device,
|
||||
)
|
||||
for i in range(num_pages):
|
||||
page_index = page_indices[i]
|
||||
index_k_scale_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][
|
||||
page_size * pool.index_head_dim :
|
||||
].view(-1, 4)
|
||||
return index_k_scale_fp8[:seq_len]
|
||||
|
||||
@classmethod
|
||||
def torch_fast(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
"""
|
||||
:param page_indices: (num_pages,), int32
|
||||
:return: (seq_len, index_head_dim // quant_block_size), uint8
|
||||
"""
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
|
||||
s_offset_in_page = pool.index_page_size * pool.index_head_dim
|
||||
num_s_bytes_per_page = buf.shape[1] - s_offset_in_page
|
||||
num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4
|
||||
|
||||
flat_buf = buf.flatten()
|
||||
flat_indices = (
|
||||
(page_indices * buf_numel_per_page)[:, None]
|
||||
+ torch.arange(num_s_bytes_per_page, dtype=torch.int32, device="cuda")[
|
||||
None, :
|
||||
]
|
||||
+ s_offset_in_page
|
||||
)
|
||||
flat_indices = flat_indices.flatten()[: seq_len * num_s_bytes_per_token]
|
||||
|
||||
out = flat_buf[flat_indices]
|
||||
return out.view(-1, 4)
|
||||
|
||||
@classmethod
|
||||
def triton(
|
||||
cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor
|
||||
):
|
||||
"""
|
||||
Triton implementation for gathering S (scale) data from paged buffer.
|
||||
:param page_indices: (num_pages,), int32/int64
|
||||
:return: (seq_len, 4), uint8
|
||||
"""
|
||||
return _get_s_triton(
|
||||
buf=buf,
|
||||
page_indices=page_indices,
|
||||
seq_len=seq_len,
|
||||
page_size=pool.index_page_size,
|
||||
index_head_dim=pool.index_head_dim,
|
||||
)
|
||||
|
||||
|
||||
class GetKAndS:
|
||||
@classmethod
|
||||
def execute(cls, *args, **kwargs):
|
||||
@@ -415,161 +271,6 @@ def _set_k_and_s_triton_kernel(
|
||||
tl.store(buf_fp32_ptr + out_s_offset, k_scale)
|
||||
|
||||
|
||||
def _get_k_triton(
|
||||
buf: torch.Tensor,
|
||||
page_indices: torch.Tensor,
|
||||
seq_len: int,
|
||||
page_size: int,
|
||||
index_head_dim: int,
|
||||
):
|
||||
"""
|
||||
Gather K (key) data from paged buffer using Triton.
|
||||
|
||||
:param buf: (num_pages, page_size * 128 + page_size * 4), uint8
|
||||
:param page_indices: (num_pages,), int32/int64
|
||||
:param seq_len: int, number of tokens to gather
|
||||
:param page_size: int, typically 64
|
||||
:param index_head_dim: int, typically 128
|
||||
:return: (seq_len, index_head_dim), uint8
|
||||
"""
|
||||
num_pages, buf_numel_per_page = buf.shape
|
||||
|
||||
# Allocate output
|
||||
out = torch.empty((seq_len, index_head_dim), dtype=torch.uint8, device=buf.device)
|
||||
|
||||
# Launch kernel with one thread per token
|
||||
grid = (seq_len,)
|
||||
_get_k_triton_kernel[grid](
|
||||
buf,
|
||||
page_indices,
|
||||
out,
|
||||
seq_len,
|
||||
page_size,
|
||||
buf_numel_per_page,
|
||||
index_head_dim,
|
||||
BLOCK_SIZE=128,
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _get_k_triton_kernel(
|
||||
buf_ptr,
|
||||
page_indices_ptr,
|
||||
out_ptr,
|
||||
seq_len: tl.constexpr,
|
||||
page_size: tl.constexpr,
|
||||
buf_numel_per_page: tl.constexpr,
|
||||
index_head_dim: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Each program handles one token (seq_len tokens total).
|
||||
Loads 128 bytes from the appropriate page.
|
||||
"""
|
||||
token_id = tl.program_id(0)
|
||||
|
||||
# Calculate which page and offset within page
|
||||
page_idx = token_id // page_size
|
||||
token_offset_in_page = token_id % page_size
|
||||
|
||||
# Load the page index from page_indices
|
||||
page_index = tl.load(page_indices_ptr + page_idx)
|
||||
|
||||
# Calculate source offset in buf
|
||||
# buf[page_index, token_offset_in_page * index_head_dim : ...]
|
||||
src_base_offset = (
|
||||
page_index * buf_numel_per_page + token_offset_in_page * index_head_dim
|
||||
)
|
||||
|
||||
# Load 128 bytes (index_head_dim elements)
|
||||
offsets = tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < index_head_dim
|
||||
data = tl.load(buf_ptr + src_base_offset + offsets, mask=mask)
|
||||
|
||||
# Store to output
|
||||
dst_offset = token_id * index_head_dim
|
||||
tl.store(out_ptr + dst_offset + offsets, data, mask=mask)
|
||||
|
||||
|
||||
def _get_s_triton(
|
||||
buf: torch.Tensor,
|
||||
page_indices: torch.Tensor,
|
||||
seq_len: int,
|
||||
page_size: int,
|
||||
index_head_dim: int,
|
||||
):
|
||||
"""
|
||||
Gather S (scale) data from paged buffer using Triton.
|
||||
|
||||
:param buf: (num_pages, page_size * 128 + page_size * 4), uint8
|
||||
:param page_indices: (num_pages,), int32/int64
|
||||
:param seq_len: int, number of tokens to gather
|
||||
:param page_size: int, typically 64
|
||||
:param index_head_dim: int, typically 128
|
||||
:return: (seq_len, 4), uint8 (representing fp32 scale)
|
||||
"""
|
||||
num_pages, buf_numel_per_page = buf.shape
|
||||
s_offset_in_page = page_size * index_head_dim # Scales start after K data
|
||||
|
||||
# Allocate output
|
||||
out = torch.empty((seq_len, 4), dtype=torch.uint8, device=buf.device)
|
||||
|
||||
# Launch kernel with one thread per token
|
||||
grid = (seq_len,)
|
||||
_get_s_triton_kernel[grid](
|
||||
buf,
|
||||
page_indices,
|
||||
out,
|
||||
seq_len,
|
||||
page_size,
|
||||
buf_numel_per_page,
|
||||
s_offset_in_page,
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _get_s_triton_kernel(
|
||||
buf_ptr,
|
||||
page_indices_ptr,
|
||||
out_ptr,
|
||||
seq_len: tl.constexpr,
|
||||
page_size: tl.constexpr,
|
||||
buf_numel_per_page: tl.constexpr,
|
||||
s_offset_in_page: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Each program handles one token (seq_len tokens total).
|
||||
Loads 4 bytes (fp32 scale) from the appropriate page.
|
||||
"""
|
||||
token_id = tl.program_id(0)
|
||||
|
||||
# Calculate which page and offset within page
|
||||
page_idx = token_id // page_size
|
||||
token_offset_in_page = token_id % page_size
|
||||
|
||||
# Load the page index from page_indices
|
||||
page_index = tl.load(page_indices_ptr + page_idx)
|
||||
|
||||
# Calculate source offset in buf
|
||||
# Scales are stored after K data: page_size * index_head_dim offset
|
||||
# buf[page_index, s_offset_in_page + token_offset_in_page * 4 : ...]
|
||||
src_base_offset = (
|
||||
page_index * buf_numel_per_page + s_offset_in_page + token_offset_in_page * 4
|
||||
)
|
||||
|
||||
# Load 4 bytes (fp32 scale)
|
||||
offsets = tl.arange(0, 4)
|
||||
data = tl.load(buf_ptr + src_base_offset + offsets)
|
||||
|
||||
# Store to output
|
||||
dst_offset = token_id * 4
|
||||
tl.store(out_ptr + dst_offset + offsets, data)
|
||||
|
||||
|
||||
def _get_k_and_s_triton(
|
||||
buf: torch.Tensor,
|
||||
page_indices: torch.Tensor,
|
||||
@@ -581,7 +282,6 @@ def _get_k_and_s_triton(
|
||||
):
|
||||
"""
|
||||
Fused gather of both K (key) and S (scale) data from paged buffer using Triton.
|
||||
This is more efficient than calling GetK and GetS separately.
|
||||
|
||||
:param buf: (num_pages, page_size * 128 + page_size * 4), uint8
|
||||
:param page_indices: (num_pages,), int32/int64
|
||||
|
||||
@@ -3,57 +3,6 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
def gather_dsa_kv_scales(
|
||||
scale_src,
|
||||
scale_dst,
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
kv_indptr_idx,
|
||||
):
|
||||
_gather_dsa_kv_scales[(32,)](
|
||||
scale_src,
|
||||
scale_dst,
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
scale_src.stride(0),
|
||||
KV_INDPTR_IDX=kv_indptr_idx,
|
||||
NUM_TILES=scale_src.shape[-1],
|
||||
BLOCK=256,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_dsa_kv_scales(
|
||||
scale_src,
|
||||
scale_dst,
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
scale_src_stride,
|
||||
KV_INDPTR_IDX: tl.constexpr,
|
||||
NUM_TILES: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
num_programs = tl.num_programs(0)
|
||||
active = tl.load(kv_indptr + KV_INDPTR_IDX)
|
||||
block_start = pid * BLOCK
|
||||
tiles = tl.arange(0, NUM_TILES)
|
||||
while block_start < active:
|
||||
offsets = block_start + tl.arange(0, BLOCK)
|
||||
mask = offsets < active
|
||||
rows = tl.load(kv_indices + offsets, mask=mask, other=0)
|
||||
values = tl.load(
|
||||
scale_src + rows[:, None] * scale_src_stride + tiles[None, :],
|
||||
mask=mask[:, None],
|
||||
)
|
||||
tl.store(
|
||||
scale_dst + rows[:, None] * NUM_TILES + tiles[None, :],
|
||||
values,
|
||||
mask=mask[:, None],
|
||||
)
|
||||
block_start += num_programs * BLOCK
|
||||
|
||||
|
||||
def quantize_k_cache(cache_k, dv: int | None = None):
|
||||
return _quantize_k_cache_fast_wrapped(cache_k, dv=dv)
|
||||
|
||||
|
||||
@@ -169,89 +169,6 @@ def act_quant(
|
||||
return y, s
|
||||
|
||||
|
||||
@tilelang.jit(out_idx=[4], pass_configs=pass_configs)
|
||||
def fp8_index_kernel(h: int, d: int, clear_accum=True):
|
||||
b = T.symbolic("b")
|
||||
m = T.symbolic("m")
|
||||
n = T.symbolic("n")
|
||||
|
||||
blk_n1 = 512
|
||||
blk_n2 = 128
|
||||
|
||||
@T.prim_func
|
||||
def fp8_index_kernel_(
|
||||
q: T.Tensor[(b, m, h, d), FP8],
|
||||
q_s: T.Tensor[(b, m, h), FP32],
|
||||
k: T.Tensor[(b, n, d), FP8],
|
||||
k_s: T.Tensor[(b, n), FP32],
|
||||
o: T.Tensor[(b, m, n), FP32],
|
||||
) -> None:
|
||||
with T.Kernel(b, m, T.ceildiv(n, blk_n1)) as (i_b, i_m, i1_n):
|
||||
q_smem = T.alloc_shared((h, d), FP8)
|
||||
T.copy(q[i_b, i_m, 0, 0], q_smem)
|
||||
|
||||
q_s_frag = T.alloc_fragment(h, FP32)
|
||||
T.copy(q_s[i_b, i_m, 0], q_s_frag)
|
||||
|
||||
for i2_n in T.Pipelined(blk_n1 // blk_n2, num_stages=2):
|
||||
k_smem = T.alloc_shared((blk_n2, d), FP8)
|
||||
T.copy(k[i_b, i1_n * blk_n1 + i2_n * blk_n2, 0], k_smem)
|
||||
|
||||
k_s_frag = T.alloc_fragment(blk_n2, FP32)
|
||||
T.copy(k_s[i_b, i1_n * blk_n1 + i2_n * blk_n2], k_s_frag)
|
||||
|
||||
logits = T.alloc_fragment((blk_n2, h), FP32)
|
||||
if not clear_accum:
|
||||
T.fill(logits, 0)
|
||||
T.gemm(
|
||||
k_smem,
|
||||
q_smem,
|
||||
logits,
|
||||
transpose_A=False,
|
||||
transpose_B=True,
|
||||
clear_accum=clear_accum,
|
||||
)
|
||||
|
||||
for i_h, i3_n in T.Parallel(h, blk_n2):
|
||||
logits[i3_n, i_h] = T.max(logits[i3_n, i_h], 0) * q_s_frag[i_h]
|
||||
|
||||
logits_sum = T.alloc_fragment(blk_n2, FP32)
|
||||
T.reduce_sum(logits, logits_sum, dim=1)
|
||||
|
||||
for i3_n in T.Parallel(blk_n2):
|
||||
logits_sum[i3_n] *= k_s_frag[i3_n]
|
||||
|
||||
T.copy(logits_sum, o[i_b, i_m, i1_n * blk_n1 + i2_n * blk_n2])
|
||||
|
||||
return fp8_index_kernel_
|
||||
|
||||
|
||||
def fp8_index(
|
||||
q: torch.Tensor,
|
||||
q_s: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
k_s: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Perform index score using FP8 precision.
|
||||
|
||||
Args:
|
||||
q (torch.Tensor): The Q tensor, must be contiguous.
|
||||
q_s (torch.Tensor): The scaling factor for Q (float), must be contiguous.
|
||||
k (torch.Tensor): The K tensor, must be contiguous.
|
||||
k_s (torch.Tensor): The scaling factor for K (e8m0 here), must be contiguous.
|
||||
|
||||
fp8 q @ fp8 k -> fp32 logits
|
||||
relu(fp32 logits) * q_s (weights) -> fp32 logits
|
||||
fp32 logits -> fp32 logits_sum
|
||||
fp32 logits_sum * k_s (e8m0) -> fp32 index_score
|
||||
"""
|
||||
if _is_hip:
|
||||
return fp8_index_kernel(q.shape[2], q.shape[3], False)(q, q_s, k, k_s)
|
||||
else:
|
||||
return fp8_index_kernel(q.shape[2], q.shape[3])(q, q_s, k, k_s)
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
out_idx=[-1],
|
||||
pass_configs={
|
||||
|
||||
@@ -371,56 +371,3 @@ def transform_index_page_table_prefill_fast(
|
||||
num_warps=4,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def transform_index_page_table_decode_ref(
|
||||
page_table: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
result: Optional[torch.Tensor] = None,
|
||||
page_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
assert page_size == 1
|
||||
assert page_table.shape[0] == topk_indices.shape[0]
|
||||
if result is None:
|
||||
result = torch.empty_like(topk_indices, dtype=torch.int32)
|
||||
assert result.shape == topk_indices.shape
|
||||
torch.gather(
|
||||
page_table.to(result.dtype),
|
||||
dim=1,
|
||||
index=topk_indices.clamp(min=0),
|
||||
out=result,
|
||||
)
|
||||
result[topk_indices < 0] = -1
|
||||
return result
|
||||
|
||||
|
||||
def transform_index_page_table_prefill_ref(
|
||||
page_table: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
extend_lens_cpu: List[int],
|
||||
page_size: int = 1,
|
||||
output_num_tokens: Optional[int] = None,
|
||||
page_table_is_expanded: bool = False,
|
||||
) -> torch.Tensor:
|
||||
assert page_size == 1
|
||||
real_num_tokens = sum(extend_lens_cpu)
|
||||
result = _allocate_prefill_result(topk_indices, real_num_tokens, output_num_tokens)
|
||||
|
||||
if page_table_is_expanded:
|
||||
if real_num_tokens > 0:
|
||||
transform_index_page_table_decode_ref(
|
||||
page_table[:real_num_tokens],
|
||||
topk_indices[:real_num_tokens],
|
||||
result=result[:real_num_tokens],
|
||||
)
|
||||
return result
|
||||
|
||||
offset = 0
|
||||
for i, l in enumerate(extend_lens_cpu):
|
||||
transform_index_page_table_decode_ref(
|
||||
page_table[i].unsqueeze(0).expand(l, -1),
|
||||
topk_indices[offset : offset + l],
|
||||
result=result[offset : offset + l],
|
||||
)
|
||||
offset += l
|
||||
return result
|
||||
|
||||
@@ -1,21 +1,18 @@
|
||||
"""Triton sparse MLA decode kernel with FP8 and BF16 KV cache support.
|
||||
"""Triton sparse MLA decode with FP8 and BF16 KV cache support.
|
||||
|
||||
Adapted from aiter's unified_attention_sparse_mla kernel for DSA shapes:
|
||||
Decode-tuned launcher for the fused / split-K kernels of ``triton_sparse_mla``
|
||||
on DSA shapes:
|
||||
q: [bs, H, DIM] fp8/bf16 (DIM=576 = D_V+D_TAIL)
|
||||
kv: [num_pages, 1, DIM] fp8/bf16
|
||||
indices: [bs, 1, topk] int32
|
||||
output: [1, bs, H, D_V] bf16
|
||||
|
||||
Two variants:
|
||||
1. Base: single-pass per-token kernel (adapted from aiter)
|
||||
2. Split-K: adaptive split-K with fused fast path (adapted from DSv4)
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsa.triton_sparse_mla import (
|
||||
_FP8_MAX,
|
||||
_LOG2E,
|
||||
_PREFERRED_BLOCK_K,
|
||||
_cu_count,
|
||||
_gfx950_sparse_mla_kv_splits,
|
||||
@@ -28,14 +25,11 @@ from sglang.kernels.ops.attention.dsa.triton_sparse_mla import (
|
||||
_reduce_d_chunk,
|
||||
_row_strides,
|
||||
_sparse_mla_block_k,
|
||||
_sparse_mla_fused_kernel,
|
||||
_sparse_mla_reduce_kernel,
|
||||
_sparse_mla_split_k_kernel,
|
||||
_validate_input_dtypes,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
|
||||
_IS_FNUZ = is_fp8_fnuz()
|
||||
_FP8_MAX = 240.0 if _IS_FNUZ else 448.0
|
||||
_G = tl.constexpr(128)
|
||||
|
||||
_SplitKWorkspace = list[tuple[torch.Tensor, torch.Tensor]]
|
||||
_splitk_bufs: dict[tuple[torch.device, int], _SplitKWorkspace] = {}
|
||||
@@ -82,420 +76,6 @@ def _get_splitk_bufs(
|
||||
return lse, acc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Split-K kernel (adapted from DSv4 paged_decode.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_mla_decode_fused_kernel(
|
||||
q_nope_ptr, # [N, H, D_V]
|
||||
q_rope_ptr, # [N, H, D_TAIL]
|
||||
kv_ptr, # [num_pages, 1, KV_DIM]
|
||||
idx_ptr, # [N, topk]
|
||||
out_ptr, # [N, H, D_V]
|
||||
qk_scale,
|
||||
fp8_max,
|
||||
topk: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
KV_DIM: tl.constexpr,
|
||||
D_V: tl.constexpr,
|
||||
D_TAIL: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
STRIDE_QN_T: tl.constexpr,
|
||||
STRIDE_QN_H: tl.constexpr,
|
||||
STRIDE_QR_T: tl.constexpr,
|
||||
STRIDE_QR_H: tl.constexpr,
|
||||
USE_FP8_DOT: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
USE_I64_PAGE: tl.constexpr = True,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
|
||||
h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
h_mask = h_offs < H
|
||||
dt = tl.arange(0, D_TAIL)
|
||||
g = tl.arange(0, _G)
|
||||
|
||||
input_type = kv_ptr.dtype.element_ty if USE_FP8_DOT else tl.bfloat16
|
||||
if USE_FP8_DOT:
|
||||
p_dot_scale = 1.0 / fp8_max
|
||||
else:
|
||||
p_dot_scale = 1.0
|
||||
|
||||
qn_base = q_nope_ptr + t * STRIDE_QN_T
|
||||
qn_row = qn_base + h_offs[:, None] * STRIDE_QN_H
|
||||
q0 = tl.load(
|
||||
qn_row + g[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 2:
|
||||
q1 = tl.load(
|
||||
qn_row + (_G + g)[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 3:
|
||||
q2 = tl.load(
|
||||
qn_row + (2 * _G + g)[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 4:
|
||||
q3 = tl.load(
|
||||
qn_row + (3 * _G + g)[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
q_tail = tl.load(
|
||||
q_rope_ptr + t * STRIDE_QR_T + h_offs[:, None] * STRIDE_QR_H + dt[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
|
||||
neg_large = -3.4028234663852886e38
|
||||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
acc0 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
if NUM_GROUPS >= 2:
|
||||
acc1 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
if NUM_GROUPS >= 3:
|
||||
acc2 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
if NUM_GROUPS >= 4:
|
||||
acc3 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
|
||||
k_offs = tl.arange(0, BLOCK_K)
|
||||
num_tiles = tl.cdiv(topk, BLOCK_K)
|
||||
|
||||
for j in tl.range(0, num_tiles, num_stages=3):
|
||||
k_start = j * BLOCK_K
|
||||
k_pos = k_start + k_offs
|
||||
valid = k_pos < topk
|
||||
|
||||
slot = tl.load(idx_ptr + t * topk + k_pos, mask=valid, other=0)
|
||||
valid = valid & (slot >= 0)
|
||||
page = tl.where(valid, slot, 0)
|
||||
if USE_I64_PAGE:
|
||||
page = page.to(tl.int64)
|
||||
|
||||
kv_base = kv_ptr + page[:, None] * KV_DIM
|
||||
kv0 = tl.load(
|
||||
kv_base + g[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 2:
|
||||
kv1 = tl.load(
|
||||
kv_base + (_G + g)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 3:
|
||||
kv2 = tl.load(
|
||||
kv_base + (2 * _G + g)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 4:
|
||||
kv3 = tl.load(
|
||||
kv_base + (3 * _G + g)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
kv_tail = tl.load(
|
||||
kv_base + (D_V + dt)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
|
||||
scores = tl.dot(q0, tl.trans(kv0))
|
||||
if NUM_GROUPS >= 2:
|
||||
scores += tl.dot(q1, tl.trans(kv1))
|
||||
if NUM_GROUPS >= 3:
|
||||
scores += tl.dot(q2, tl.trans(kv2))
|
||||
if NUM_GROUPS >= 4:
|
||||
scores += tl.dot(q3, tl.trans(kv3))
|
||||
scores += tl.dot(q_tail, tl.trans(kv_tail))
|
||||
scores = scores * qk_scale
|
||||
scores = tl.where(valid[None, :], scores, neg_large)
|
||||
|
||||
m_block = tl.max(scores, axis=1)
|
||||
m_new = tl.maximum(m_i, m_block)
|
||||
alpha = tl.exp2(m_i - m_new)
|
||||
p = tl.exp2(scores - m_new[:, None])
|
||||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||||
|
||||
if USE_FP8_DOT:
|
||||
p_dot = (p * fp8_max).to(input_type)
|
||||
else:
|
||||
p_dot = p.to(input_type)
|
||||
acc0 = acc0 * alpha[:, None] + tl.dot(p_dot, kv0).to(tl.float32) * p_dot_scale
|
||||
if NUM_GROUPS >= 2:
|
||||
acc1 = (
|
||||
acc1 * alpha[:, None] + tl.dot(p_dot, kv1).to(tl.float32) * p_dot_scale
|
||||
)
|
||||
if NUM_GROUPS >= 3:
|
||||
acc2 = (
|
||||
acc2 * alpha[:, None] + tl.dot(p_dot, kv2).to(tl.float32) * p_dot_scale
|
||||
)
|
||||
if NUM_GROUPS >= 4:
|
||||
acc3 = (
|
||||
acc3 * alpha[:, None] + tl.dot(p_dot, kv3).to(tl.float32) * p_dot_scale
|
||||
)
|
||||
m_i = m_new
|
||||
l_i = l_new
|
||||
|
||||
denom = tl.maximum(l_i, 1.0e-30)
|
||||
inv_denom = 1.0 / denom
|
||||
acc0 = tl.where(l_i[:, None] > 0.0, acc0 * inv_denom[:, None], 0.0)
|
||||
if NUM_GROUPS >= 2:
|
||||
acc1 = tl.where(l_i[:, None] > 0.0, acc1 * inv_denom[:, None], 0.0)
|
||||
if NUM_GROUPS >= 3:
|
||||
acc2 = tl.where(l_i[:, None] > 0.0, acc2 * inv_denom[:, None], 0.0)
|
||||
if NUM_GROUPS >= 4:
|
||||
acc3 = tl.where(l_i[:, None] > 0.0, acc3 * inv_denom[:, None], 0.0)
|
||||
|
||||
o_base = out_ptr + t * H * D_V
|
||||
tl.store(
|
||||
o_base + h_offs[:, None] * D_V + g[None, :],
|
||||
acc0.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
if NUM_GROUPS >= 2:
|
||||
tl.store(
|
||||
o_base + h_offs[:, None] * D_V + (_G + g)[None, :],
|
||||
acc1.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
if NUM_GROUPS >= 3:
|
||||
tl.store(
|
||||
o_base + h_offs[:, None] * D_V + (2 * _G + g)[None, :],
|
||||
acc2.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
if NUM_GROUPS >= 4:
|
||||
tl.store(
|
||||
o_base + h_offs[:, None] * D_V + (3 * _G + g)[None, :],
|
||||
acc3.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_mla_decode_split_kernel(
|
||||
q_nope_ptr, # [N, H, D_V]
|
||||
q_rope_ptr, # [N, H, D_TAIL]
|
||||
kv_ptr, # [num_pages, 1, KV_DIM]
|
||||
idx_ptr, # [N, topk]
|
||||
lse_partial_ptr, # [N, KV_SPLITS, H_padded] fp32
|
||||
acc_partial_ptr, # [N, KV_SPLITS, H_padded, D_V] bf16
|
||||
qk_scale,
|
||||
fp8_max,
|
||||
topk: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
KV_DIM: tl.constexpr,
|
||||
D_V: tl.constexpr,
|
||||
D_TAIL: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
STRIDE_QN_T: tl.constexpr,
|
||||
STRIDE_QN_H: tl.constexpr,
|
||||
STRIDE_QR_T: tl.constexpr,
|
||||
STRIDE_QR_H: tl.constexpr,
|
||||
USE_FP8_DOT: tl.constexpr,
|
||||
KV_SPLITS: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
pid_k = tl.program_id(2)
|
||||
|
||||
h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
h_mask = h_offs < H
|
||||
dt = tl.arange(0, D_TAIL)
|
||||
g = tl.arange(0, _G)
|
||||
|
||||
input_type = kv_ptr.dtype.element_ty if USE_FP8_DOT else tl.bfloat16
|
||||
if USE_FP8_DOT:
|
||||
p_dot_scale = 1.0 / fp8_max
|
||||
else:
|
||||
p_dot_scale = 1.0
|
||||
|
||||
qn_base = q_nope_ptr + t * STRIDE_QN_T
|
||||
qn_row = qn_base + h_offs[:, None] * STRIDE_QN_H
|
||||
q0 = tl.load(
|
||||
qn_row + g[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 2:
|
||||
q1 = tl.load(
|
||||
qn_row + (_G + g)[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 3:
|
||||
q2 = tl.load(
|
||||
qn_row + (2 * _G + g)[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 4:
|
||||
q3 = tl.load(
|
||||
qn_row + (3 * _G + g)[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
q_tail = tl.load(
|
||||
q_rope_ptr + t * STRIDE_QR_T + h_offs[:, None] * STRIDE_QR_H + dt[None, :],
|
||||
mask=h_mask[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
|
||||
tiles_per_segment = tl.cdiv(topk, KV_SPLITS * BLOCK_K)
|
||||
if pid_k * tiles_per_segment * BLOCK_K >= topk:
|
||||
return
|
||||
num_tiles = tl.cdiv(topk, BLOCK_K)
|
||||
tile_start = pid_k * tiles_per_segment
|
||||
tile_end = tl.minimum((pid_k + 1) * tiles_per_segment, num_tiles)
|
||||
|
||||
neg_large = -3.4028234663852886e38
|
||||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
acc0 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
if NUM_GROUPS >= 2:
|
||||
acc1 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
if NUM_GROUPS >= 3:
|
||||
acc2 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
if NUM_GROUPS >= 4:
|
||||
acc3 = tl.zeros((BLOCK_H, _G), dtype=tl.float32)
|
||||
|
||||
k_offs = tl.arange(0, BLOCK_K)
|
||||
for j in tl.range(tile_start, tile_end, num_stages=3):
|
||||
k_start = j * BLOCK_K
|
||||
k_pos = k_start + k_offs
|
||||
valid = k_pos < topk
|
||||
|
||||
slot = tl.load(idx_ptr + t * topk + k_pos, mask=valid, other=0)
|
||||
valid = valid & (slot >= 0)
|
||||
page = tl.where(valid, slot, 0).to(tl.int64)
|
||||
|
||||
kv_base = kv_ptr + page[:, None] * KV_DIM
|
||||
kv0 = tl.load(
|
||||
kv_base + g[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 2:
|
||||
kv1 = tl.load(
|
||||
kv_base + (_G + g)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 3:
|
||||
kv2 = tl.load(
|
||||
kv_base + (2 * _G + g)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
if NUM_GROUPS >= 4:
|
||||
kv3 = tl.load(
|
||||
kv_base + (3 * _G + g)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
kv_tail = tl.load(
|
||||
kv_base + (D_V + dt)[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
).to(input_type)
|
||||
|
||||
scores = tl.dot(q0, tl.trans(kv0))
|
||||
if NUM_GROUPS >= 2:
|
||||
scores += tl.dot(q1, tl.trans(kv1))
|
||||
if NUM_GROUPS >= 3:
|
||||
scores += tl.dot(q2, tl.trans(kv2))
|
||||
if NUM_GROUPS >= 4:
|
||||
scores += tl.dot(q3, tl.trans(kv3))
|
||||
scores += tl.dot(q_tail, tl.trans(kv_tail))
|
||||
scores = scores * qk_scale
|
||||
scores = tl.where(valid[None, :], scores, neg_large)
|
||||
|
||||
m_block = tl.max(scores, axis=1)
|
||||
m_new = tl.maximum(m_i, m_block)
|
||||
alpha = tl.exp2(m_i - m_new)
|
||||
p = tl.exp2(scores - m_new[:, None])
|
||||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||||
|
||||
if USE_FP8_DOT:
|
||||
p_dot = (p * fp8_max).to(input_type)
|
||||
else:
|
||||
p_dot = p.to(input_type)
|
||||
acc0 = acc0 * alpha[:, None] + tl.dot(p_dot, kv0).to(tl.float32) * p_dot_scale
|
||||
if NUM_GROUPS >= 2:
|
||||
acc1 = (
|
||||
acc1 * alpha[:, None] + tl.dot(p_dot, kv1).to(tl.float32) * p_dot_scale
|
||||
)
|
||||
if NUM_GROUPS >= 3:
|
||||
acc2 = (
|
||||
acc2 * alpha[:, None] + tl.dot(p_dot, kv2).to(tl.float32) * p_dot_scale
|
||||
)
|
||||
if NUM_GROUPS >= 4:
|
||||
acc3 = (
|
||||
acc3 * alpha[:, None] + tl.dot(p_dot, kv3).to(tl.float32) * p_dot_scale
|
||||
)
|
||||
m_i = m_new
|
||||
l_i = l_new
|
||||
|
||||
neg_large = -1073741824.0
|
||||
denom = tl.maximum(l_i, 1.0e-30)
|
||||
inv_denom = 1.0 / denom
|
||||
has_data = l_i > 0.0
|
||||
acc0 = tl.where(has_data[:, None], acc0 * inv_denom[:, None], 0.0)
|
||||
if NUM_GROUPS >= 2:
|
||||
acc1 = tl.where(has_data[:, None], acc1 * inv_denom[:, None], 0.0)
|
||||
if NUM_GROUPS >= 3:
|
||||
acc2 = tl.where(has_data[:, None], acc2 * inv_denom[:, None], 0.0)
|
||||
if NUM_GROUPS >= 4:
|
||||
acc3 = tl.where(has_data[:, None], acc3 * inv_denom[:, None], 0.0)
|
||||
|
||||
lse = tl.where(has_data, tl.log2(l_i) + m_i, neg_large)
|
||||
|
||||
H_padded = tl.cdiv(H, BLOCK_H) * BLOCK_H
|
||||
lse_base = t * KV_SPLITS * H_padded + pid_k * H_padded
|
||||
tl.store(lse_partial_ptr + lse_base + h_offs, lse, mask=h_mask)
|
||||
|
||||
ap_base = t * KV_SPLITS * H_padded * D_V + pid_k * H_padded * D_V
|
||||
tl.store(
|
||||
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + g[None, :],
|
||||
acc0.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
if NUM_GROUPS >= 2:
|
||||
tl.store(
|
||||
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (_G + g)[None, :],
|
||||
acc1.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
if NUM_GROUPS >= 3:
|
||||
tl.store(
|
||||
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (2 * _G + g)[None, :],
|
||||
acc2.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
if NUM_GROUPS >= 4:
|
||||
tl.store(
|
||||
acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (3 * _G + g)[None, :],
|
||||
acc3.to(tl.bfloat16),
|
||||
mask=h_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
def triton_sparse_mla_decode_splitk(
|
||||
q_nope: torch.Tensor,
|
||||
q_rope: torch.Tensor,
|
||||
@@ -581,7 +161,7 @@ def triton_sparse_mla_decode_splitk(
|
||||
else:
|
||||
kv_splits = min(kv_splits, max_kv_splits)
|
||||
|
||||
qk_scale = float(sm_scale) * LOG2E
|
||||
qk_scale = float(sm_scale) * _LOG2E
|
||||
|
||||
if kv_splits == 1:
|
||||
fused_num_warps = 4
|
||||
@@ -589,11 +169,12 @@ def triton_sparse_mla_decode_splitk(
|
||||
fused_num_warps = _gfx950_sparse_mla_num_warps(base_ctas, 1, num_cu)
|
||||
out = torch.empty(bs, H, d_v, device=q_nope.device, dtype=torch.bfloat16)
|
||||
with _no_async_copy():
|
||||
_sparse_mla_decode_fused_kernel[(bs, n_head_blocks)](
|
||||
_sparse_mla_fused_kernel[(bs, n_head_blocks)](
|
||||
q_nope,
|
||||
q_rope,
|
||||
kv,
|
||||
idx_flat,
|
||||
idx_flat,
|
||||
out,
|
||||
qk_scale,
|
||||
_FP8_MAX,
|
||||
@@ -608,6 +189,7 @@ def triton_sparse_mla_decode_splitk(
|
||||
STRIDE_QR_T=stride_qr_t,
|
||||
STRIDE_QR_H=stride_qr_h,
|
||||
USE_FP8_DOT=use_fp8_dot,
|
||||
USE_TOPK_LENGTH=False,
|
||||
BLOCK_H=BLOCK_H,
|
||||
BLOCK_K=BLOCK_K,
|
||||
USE_I64_PAGE=not (
|
||||
@@ -637,7 +219,7 @@ def triton_sparse_mla_decode_splitk(
|
||||
|
||||
grid_split = (bs, n_head_blocks, kv_splits)
|
||||
with _no_async_copy():
|
||||
_sparse_mla_decode_split_kernel[grid_split](
|
||||
_sparse_mla_split_k_kernel[grid_split](
|
||||
q_nope,
|
||||
q_rope,
|
||||
kv,
|
||||
|
||||
@@ -13,9 +13,9 @@ from .compress import (
|
||||
compress_forward,
|
||||
compress_norm_rope_store,
|
||||
)
|
||||
from .compress_old import fused_norm_rope_inplace
|
||||
from .elementwise import (
|
||||
fused_k_norm_rope_flashmla,
|
||||
fused_norm_rope_inplace,
|
||||
fused_q_indexer_rope_first_quant,
|
||||
fused_q_indexer_rope_hadamard_fp4_quant,
|
||||
fused_q_indexer_rope_hadamard_quant,
|
||||
|
||||
@@ -150,7 +150,7 @@ _PREFILL_PLAN_BYTES = 24
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Plan dataclasses. The element at index 1 is the consumer for
|
||||
# `compress_fused_norm_rope_inplace` (which reads ragged_id / seq_len from a
|
||||
# `compress_norm_rope_store` (which reads ragged_id / seq_len from a
|
||||
# 16-byte plan tensor --- both DecodePlan and CompressPlan satisfy that layout).
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -1,308 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Literal, NamedTuple, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
from .utils import make_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_common_module() -> Module:
|
||||
return load_jit(
|
||||
make_name("common"),
|
||||
cuda_files=["deepseek_v4/common.cuh"],
|
||||
cuda_wrappers=[("plan_compress_prefill", "plan_compress_prefill")],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_compress_128_online_plan_module() -> Module:
|
||||
"""Host-side plan generator for online compress 128 (no template args)."""
|
||||
return load_jit(
|
||||
make_name("compress_128_online_plan"),
|
||||
cuda_files=["deepseek_v4/c128_online.cuh"],
|
||||
cuda_wrappers=[
|
||||
("plan_compress_online_prefill", "plan_compress_online_prefill"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_compress_128_online_module(head_dim: int) -> Module:
|
||||
"""Online compress 128 kernel: ring_size=1, per-index (max, sum, kv) state."""
|
||||
args = make_cpp_args(head_dim, is_arch_support_pdl())
|
||||
kernel_class = f"FlashCompress128OnlineKernel<{args}>"
|
||||
return load_jit(
|
||||
make_name("compress_128_online"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/c128_online.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode", f"{kernel_class}::run_decode"),
|
||||
("prefill", f"{kernel_class}::run_prefill"),
|
||||
],
|
||||
extra_cuda_cflags=["-use_fast_math"],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def norm_rope_module(
|
||||
dtype: torch.dtype,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
) -> Module:
|
||||
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
make_name("fused_norm_rope"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/fused_norm_rope.cuh"],
|
||||
cuda_wrappers=[
|
||||
("forward", f"FusedNormRopeKernel<{args}>::forward"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_compress_module(
|
||||
head_dim: int,
|
||||
dtype_in: torch.dtype,
|
||||
dtype_out: torch.dtype,
|
||||
ratio: Literal[4, 128],
|
||||
) -> Module:
|
||||
args = make_cpp_args(head_dim, dtype_in, dtype_out, is_arch_support_pdl())
|
||||
kernel_class = f"FlashCompress{ratio}Kernel<{args}>"
|
||||
return load_jit(
|
||||
make_name(f"compress_{ratio}"),
|
||||
*args,
|
||||
cuda_files=[f"deepseek_v4/c{ratio}.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode", f"{kernel_class}::run_decode"),
|
||||
("prefill", f"{kernel_class}::run_prefill"),
|
||||
],
|
||||
extra_cuda_cflags=["-use_fast_math"],
|
||||
)
|
||||
|
||||
|
||||
class CompressorPrefillPlan(NamedTuple):
|
||||
compress_ratio: int
|
||||
compress_plan: torch.Tensor
|
||||
write_plan: torch.Tensor
|
||||
|
||||
def copy_(self, other: CompressorPrefillPlan) -> None:
|
||||
assert self.compress_ratio == other.compress_ratio
|
||||
self.compress_plan.copy_(other.compress_plan)
|
||||
self.write_plan.copy_(other.write_plan)
|
||||
|
||||
@staticmethod
|
||||
def generate(
|
||||
compress_ratio: Literal[4, 128],
|
||||
num_q_tokens: int,
|
||||
seq_lens: torch.Tensor,
|
||||
extend_lens: torch.Tensor,
|
||||
device: torch.device,
|
||||
use_cuda_graph: bool = False,
|
||||
) -> CompressorPrefillPlan:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
# Online c128 keeps the same NamedTuple shape (compress_plan, write_plan)
|
||||
# so call sites that splat `*plan[1:]` continue to work, but the C++
|
||||
# plan struct semantics differ (last-token coords + window_len).
|
||||
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
|
||||
return CompressorPrefillPlan._generate_online(
|
||||
num_q_tokens=num_q_tokens,
|
||||
seq_lens=seq_lens,
|
||||
extend_lens=extend_lens,
|
||||
device=device,
|
||||
use_cuda_graph=use_cuda_graph,
|
||||
)
|
||||
assert seq_lens.device == extend_lens.device
|
||||
seq_lens = seq_lens.to(torch.int64)
|
||||
extend_lens = extend_lens.to(torch.int64)
|
||||
plan_tensor = torch.empty(
|
||||
(2, num_q_tokens, 16),
|
||||
dtype=torch.uint8,
|
||||
device=seq_lens.device,
|
||||
pin_memory=seq_lens.is_cpu,
|
||||
)
|
||||
module = _jit_common_module()
|
||||
is_overlap = compress_ratio == 4
|
||||
plan_lens = module.plan_compress_prefill(
|
||||
extend_lens,
|
||||
seq_lens,
|
||||
plan_tensor[0],
|
||||
plan_tensor[1],
|
||||
compress_ratio,
|
||||
is_overlap,
|
||||
use_cuda_graph,
|
||||
)
|
||||
return CompressorPrefillPlan(
|
||||
compress_ratio,
|
||||
plan_tensor[0, : plan_lens[0]].to(device, non_blocking=True),
|
||||
plan_tensor[1, : plan_lens[1]].to(device, non_blocking=True),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _generate_online(
|
||||
num_q_tokens: int,
|
||||
seq_lens: torch.Tensor,
|
||||
extend_lens: torch.Tensor,
|
||||
device: torch.device,
|
||||
use_cuda_graph: bool,
|
||||
) -> CompressorPrefillPlan:
|
||||
# Online plan host-side path: only CPU/cuda-host implemented today.
|
||||
# Move inputs to CPU pinned memory then bounce the result to device.
|
||||
seq_lens_cpu = seq_lens.detach().to(torch.int64).cpu()
|
||||
extend_lens_cpu = extend_lens.detach().to(torch.int64).cpu()
|
||||
plan_tensor = torch.empty(
|
||||
(2, num_q_tokens, 16),
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
module = _jit_compress_128_online_plan_module()
|
||||
plan_lens = module.plan_compress_online_prefill(
|
||||
extend_lens_cpu,
|
||||
seq_lens_cpu,
|
||||
plan_tensor[0],
|
||||
plan_tensor[1],
|
||||
use_cuda_graph,
|
||||
)
|
||||
return CompressorPrefillPlan(
|
||||
128,
|
||||
plan_tensor[0, : plan_lens[0]].to(device, non_blocking=True),
|
||||
plan_tensor[1, : plan_lens[1]].to(device, non_blocking=True),
|
||||
)
|
||||
|
||||
@property
|
||||
def is_decode(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class CompressorDecodePlan(NamedTuple):
|
||||
compress_ratio: int
|
||||
seq_lens: torch.Tensor
|
||||
|
||||
def copy_(self, other: CompressorDecodePlan) -> None:
|
||||
assert self.compress_ratio == other.compress_ratio
|
||||
self.seq_lens.copy_(other.seq_lens)
|
||||
|
||||
@property
|
||||
def is_decode(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def compress_plan(
|
||||
compress_ratio: Literal[4, 128],
|
||||
num_q_tokens: int,
|
||||
seq_lens: torch.Tensor,
|
||||
extend_lens: Optional[torch.Tensor],
|
||||
device: torch.device,
|
||||
) -> Union[CompressorDecodePlan, CompressorPrefillPlan]:
|
||||
if extend_lens is not None:
|
||||
return CompressorPrefillPlan.generate(
|
||||
compress_ratio,
|
||||
num_q_tokens,
|
||||
seq_lens,
|
||||
extend_lens,
|
||||
device,
|
||||
)
|
||||
else:
|
||||
assert num_q_tokens == len(seq_lens)
|
||||
seq_lens = seq_lens.to(device, non_blocking=True)
|
||||
return CompressorDecodePlan(compress_ratio, seq_lens)
|
||||
|
||||
|
||||
def compress_forward(
|
||||
kv_score_buffer: torch.Tensor,
|
||||
kv_score_input: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
plan: Union[CompressorDecodePlan, CompressorPrefillPlan, None] = None,
|
||||
extra_data: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
head_dim: int,
|
||||
compress_ratio: Literal[4, 128],
|
||||
out: Optional[torch.Tensor] = None,
|
||||
seq_lens: Optional[torch.Tensor] = None,
|
||||
extend_lens: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
assert head_dim % 128 == 0
|
||||
num_q_tokens = kv_score_input.shape[0]
|
||||
if out is None:
|
||||
out = kv_score_input.new_empty((num_q_tokens, head_dim))
|
||||
if plan is None:
|
||||
assert seq_lens is not None
|
||||
plan = compress_plan(
|
||||
compress_ratio,
|
||||
num_q_tokens,
|
||||
seq_lens,
|
||||
extend_lens,
|
||||
kv_score_input.device,
|
||||
)
|
||||
assert plan.compress_ratio == compress_ratio, "Mismatched compress ratio in plan!"
|
||||
# Online c128: separate JIT module, fp32 state, no compile-time dtypes.
|
||||
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
|
||||
online_module = _jit_compress_128_online_module(head_dim=head_dim)
|
||||
F = online_module.decode if plan.is_decode else online_module.prefill
|
||||
F(kv_score_buffer, kv_score_input, out, ape, indices, *plan[1:], extra_data)
|
||||
return out
|
||||
module = _jit_compress_module(
|
||||
head_dim,
|
||||
kv_score_input.dtype,
|
||||
out.dtype,
|
||||
compress_ratio,
|
||||
)
|
||||
F = module.decode if plan.is_decode else module.prefill
|
||||
F(kv_score_buffer, kv_score_input, out, ape, indices, *plan[1:], extra_data)
|
||||
return out
|
||||
|
||||
|
||||
def compress_fused_norm_rope_inplace(
|
||||
kv: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
freq_cis: torch.Tensor,
|
||||
plan: Union[CompressorDecodePlan, CompressorPrefillPlan],
|
||||
) -> None:
|
||||
freq_cis = torch.view_as_real(freq_cis).flatten(-2)
|
||||
module = norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1])
|
||||
module.forward(
|
||||
kv,
|
||||
weight,
|
||||
plan[1],
|
||||
freq_cis,
|
||||
int(plan.is_decode),
|
||||
eps,
|
||||
plan.compress_ratio,
|
||||
)
|
||||
|
||||
|
||||
def fused_norm_rope_inplace(
|
||||
kv: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
freq_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
freq_cis = torch.view_as_real(freq_cis).flatten(-2)
|
||||
module = norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1])
|
||||
module.forward(
|
||||
kv,
|
||||
weight,
|
||||
positions,
|
||||
freq_cis,
|
||||
2,
|
||||
eps,
|
||||
0,
|
||||
)
|
||||
@@ -509,86 +509,3 @@ def _gather_dequant_requant_fp8_paged_kernel(
|
||||
output_ptr + out_row_base + DIM_NOPE + rope_offs,
|
||||
rope_data.to(output_ptr.dtype.element_ty),
|
||||
)
|
||||
|
||||
|
||||
def dequantize_k_cache_paged_ref(
|
||||
quant_k_cache: torch.Tensor,
|
||||
page_table_1_flattened: torch.Tensor,
|
||||
page_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Pure-torch reference for :func:`dequantize_k_cache_paged`.
|
||||
|
||||
Decodes the same v4 paged layout with vectorized torch indexing instead of
|
||||
a Triton kernel. Used to validate the kernel (see the ``__main__`` block
|
||||
below); not on any hot path.
|
||||
"""
|
||||
assert page_table_1_flattened.dtype in (torch.int32, torch.int64)
|
||||
u8 = quant_k_cache.view(torch.uint8)
|
||||
bytes_per_page = u8.shape[-1]
|
||||
s_offset_bytes = page_size * NOPE_ROPE_BYTES
|
||||
|
||||
flat_u8 = u8.reshape(-1)
|
||||
flat_fp8 = u8.view(fp8_dtype).reshape(-1)
|
||||
flat_bf16 = u8.view(torch.bfloat16).reshape(-1)
|
||||
|
||||
loc = page_table_1_flattened.to(torch.int64)
|
||||
page_idx = loc // page_size
|
||||
in_page = loc % page_size
|
||||
page_byte_base = page_idx * bytes_per_page
|
||||
token_data_base = page_byte_base + in_page * NOPE_ROPE_BYTES
|
||||
token_scale_base = (
|
||||
page_byte_base + s_offset_bytes + in_page * PADDED_SCALE_PER_TOKEN
|
||||
)
|
||||
|
||||
device = quant_k_cache.device
|
||||
nope_byte = (
|
||||
token_data_base[:, None] + torch.arange(DIM_NOPE, device=device)[None, :]
|
||||
)
|
||||
nope_fp8 = flat_fp8[nope_byte].to(torch.float32)
|
||||
scale_byte = (
|
||||
token_scale_base[:, None]
|
||||
+ torch.arange(NUM_SCALE_TILES, device=device)[None, :]
|
||||
)
|
||||
scale_u8 = flat_u8[scale_byte].to(torch.int32)
|
||||
scale_pow2 = torch.exp2((scale_u8 - 127).to(torch.float32))
|
||||
scale_pow2 = torch.where(
|
||||
scale_pow2 < (2.0**-126), torch.zeros_like(scale_pow2), scale_pow2
|
||||
)
|
||||
scale_full = scale_pow2.repeat_interleave(TILE_SIZE, dim=1)
|
||||
nope = nope_fp8 * scale_full
|
||||
|
||||
rope_bf16_base = (token_data_base + DIM_NOPE) // 2
|
||||
rope_idx = rope_bf16_base[:, None] + torch.arange(DIM_ROPE, device=device)[None, :]
|
||||
rope = flat_bf16[rope_idx]
|
||||
|
||||
out = torch.empty(
|
||||
(loc.shape[0], 1, DIM_NOPE + DIM_ROPE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
out[:, 0, :DIM_NOPE] = nope.to(torch.bfloat16)
|
||||
out[:, 0, DIM_NOPE:] = rope
|
||||
return out
|
||||
|
||||
|
||||
def gather_dequant_requant_fp8_paged_ref(
|
||||
quant_k_cache: torch.Tensor,
|
||||
page_table_1_flattened: torch.Tensor,
|
||||
page_size: int,
|
||||
extra_rows: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Torch reference for :func:`gather_dequant_requant_fp8_paged`."""
|
||||
active = dequantize_k_cache_paged_ref(
|
||||
quant_k_cache,
|
||||
page_table_1_flattened,
|
||||
page_size,
|
||||
).to(fp8_dtype)
|
||||
if extra_rows == 0:
|
||||
return active
|
||||
out = torch.zeros(
|
||||
(active.shape[0] + extra_rows, 1, DIM_NOPE + DIM_ROPE),
|
||||
dtype=fp8_dtype,
|
||||
device=active.device,
|
||||
)
|
||||
out[: active.shape[0]] = active
|
||||
return out
|
||||
|
||||
@@ -32,6 +32,23 @@ def _jit_fused_rope_module():
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_fused_norm_rope_module(
|
||||
dtype: torch.dtype,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
):
|
||||
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
make_name("fused_norm_rope"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/fused_norm_rope.cuh"],
|
||||
cuda_wrappers=[
|
||||
("forward", f"FusedNormRopeKernel<{args}>::forward"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_main_q_norm_rope_module(
|
||||
dtype: torch.dtype,
|
||||
@@ -168,6 +185,26 @@ def fused_rope_inplace(
|
||||
module.forward(q, k, freqs_real, positions, inverse)
|
||||
|
||||
|
||||
def fused_norm_rope_inplace(
|
||||
kv: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
freq_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
freq_cis = torch.view_as_real(freq_cis).flatten(-2)
|
||||
module = _jit_fused_norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1])
|
||||
module.forward(
|
||||
kv,
|
||||
weight,
|
||||
positions,
|
||||
freq_cis,
|
||||
2,
|
||||
eps,
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
def fused_q_norm_rope(
|
||||
q_input: torch.Tensor,
|
||||
q_output: torch.Tensor,
|
||||
|
||||
@@ -4,7 +4,6 @@ from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -22,6 +21,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import (
|
||||
rope_tail_fake_quant_fp4_row,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.attention.dsv4.compress import (
|
||||
@@ -62,9 +62,7 @@ C4_ROWGROUP_MIN_VERIFY_ROWS = 64
|
||||
# (see `logits_rows_per_chunk`), so this caps the indexer's transient footprint
|
||||
# independently of context length and chunked-prefill size; smaller budgets only
|
||||
# buy more row chunks. 2 GiB covers 4096 rows over ~512K tokens of context.
|
||||
_LOGITS_BUDGET_ELEMS = (
|
||||
int(os.environ.get("SGLANG_DSV4_FP4_LOGITS_BUDGET_MB", "2048")) * 2**20 // 4
|
||||
)
|
||||
_LOGITS_BUDGET_ELEMS = envs.SGLANG_DSV4_FP4_LOGITS_BUDGET_MB.get() * 2**20 // 4
|
||||
_LOGITS_POOL: dict = {}
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -36,10 +36,6 @@ class SetKAndS:
|
||||
def execute(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack):
|
||||
cls.triton(pool, buf, loc, nope_fp8_rope_bf16_pack)
|
||||
|
||||
@classmethod
|
||||
def torch(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack):
|
||||
_set_k_and_s_torch(buf, loc, nope_fp8_rope_bf16_pack, pool.page_size)
|
||||
|
||||
@classmethod
|
||||
def triton(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack):
|
||||
_set_k_and_s_triton(buf, loc, nope_fp8_rope_bf16_pack, pool.page_size)
|
||||
@@ -183,77 +179,3 @@ def _set_k_and_s_triton_kernel(
|
||||
tl.store(buf_fp8_ptr + out_k_nope_offsets, k_nope, mask=nope_mask)
|
||||
tl.store(buf_bf16_ptr + out_k_rope_offsets, k_rope)
|
||||
tl.store(buf_uint8_ptr + out_s_offsets, k_scale, mask=scale_mask)
|
||||
|
||||
|
||||
def _set_k_and_s_torch(
|
||||
buf: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack,
|
||||
page_size: int,
|
||||
):
|
||||
num_pages, buf_numel_per_page = buf.shape
|
||||
(num_tokens_to_write,) = loc.shape
|
||||
|
||||
k_nope, k_rope, scale_k_nope = (
|
||||
nope_fp8_rope_bf16_pack.k_nope_fp8,
|
||||
nope_fp8_rope_bf16_pack.k_rope_bf16,
|
||||
nope_fp8_rope_bf16_pack.scale_k_nope_ue8m0,
|
||||
)
|
||||
|
||||
num_tokens_to_write_nope, nope_dim = k_nope.shape
|
||||
num_tokens_to_write_rope, rope_dim = k_rope.shape
|
||||
num_tokens_to_write_scale, scale_dim = scale_k_nope.shape
|
||||
|
||||
assert (
|
||||
num_tokens_to_write
|
||||
== num_tokens_to_write_nope
|
||||
== num_tokens_to_write_rope
|
||||
== num_tokens_to_write_scale
|
||||
), (
|
||||
f"{num_tokens_to_write=} {num_tokens_to_write_nope=} {num_tokens_to_write_rope=} {num_tokens_to_write_scale=}"
|
||||
)
|
||||
|
||||
assert buf.dtype == torch.uint8
|
||||
assert loc.dtype in [
|
||||
torch.int64,
|
||||
torch.int32,
|
||||
], f"{loc.dtype=}"
|
||||
|
||||
assert k_nope.dtype == fp8_dtype
|
||||
assert k_rope.dtype == torch.bfloat16
|
||||
assert scale_k_nope.dtype == torch.uint8
|
||||
|
||||
assert buf.is_contiguous()
|
||||
assert loc.is_contiguous()
|
||||
assert k_nope.is_contiguous()
|
||||
assert k_rope.is_contiguous()
|
||||
assert scale_k_nope.is_contiguous()
|
||||
|
||||
buf_fp8 = buf.view(fp8_dtype).flatten()
|
||||
buf_bf16 = buf.view(torch.bfloat16).flatten()
|
||||
buf_scale = buf.view(torch.uint8).flatten()
|
||||
|
||||
loc_page_index = loc // page_size
|
||||
loc_token_offset_in_page = loc % page_size
|
||||
|
||||
s_offset_nbytes_in_page = page_size * (nope_dim + rope_dim * 2)
|
||||
|
||||
nope_offset = loc_page_index * buf_numel_per_page + loc_token_offset_in_page * (
|
||||
nope_dim + rope_dim * 2
|
||||
)
|
||||
|
||||
rope_offset = (
|
||||
loc_page_index * buf_numel_per_page // 2
|
||||
+ (loc_token_offset_in_page * (nope_dim + rope_dim * 2) + nope_dim) // 2
|
||||
)
|
||||
|
||||
s_offset = (
|
||||
loc_page_index * buf_numel_per_page
|
||||
+ s_offset_nbytes_in_page
|
||||
+ loc_token_offset_in_page * (scale_dim + 1)
|
||||
)
|
||||
|
||||
for i in range(num_tokens_to_write):
|
||||
buf_fp8[nope_offset[i] : nope_offset[i] + nope_dim] = k_nope[i]
|
||||
buf_bf16[rope_offset[i] : rope_offset[i] + rope_dim] = k_rope[i]
|
||||
buf_scale[s_offset[i] : s_offset[i] + scale_dim] = scale_k_nope[i]
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quant_k_cache_fused_kernel(
|
||||
k_bf16_ptr,
|
||||
k_nope_fp8_ptr,
|
||||
k_rope_bf16_ptr,
|
||||
scale_k_nope_uint8_ptr,
|
||||
k_bf16_stride_0,
|
||||
k_nope_fp8_stride_0,
|
||||
k_rope_bf16_stride_0,
|
||||
scale_stride_0,
|
||||
DIM_NOPE: tl.constexpr,
|
||||
DIM_ROPE: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
NUM_TILES: tl.constexpr,
|
||||
FP8_MIN: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
):
|
||||
token_id = tl.program_id(0).to(tl.int64)
|
||||
tile_id = tl.program_id(1)
|
||||
|
||||
if tile_id == NUM_TILES:
|
||||
rope_range = tl.arange(0, TILE_SIZE)
|
||||
rope_mask = rope_range < DIM_ROPE
|
||||
|
||||
in_rope_offsets = token_id * k_bf16_stride_0 + DIM_NOPE + rope_range
|
||||
rope_data = tl.load(k_bf16_ptr + in_rope_offsets, mask=rope_mask, other=0.0)
|
||||
|
||||
out_rope_offsets = token_id * k_rope_bf16_stride_0 + rope_range
|
||||
tl.store(k_rope_bf16_ptr + out_rope_offsets, rope_data, mask=rope_mask)
|
||||
else:
|
||||
tile_range = tl.arange(0, TILE_SIZE)
|
||||
|
||||
in_tile_offsets = token_id * k_bf16_stride_0 + tile_id * TILE_SIZE + tile_range
|
||||
x_bf16 = tl.load(k_bf16_ptr + in_tile_offsets)
|
||||
x_fp32 = x_bf16.to(tl.float32)
|
||||
|
||||
abs_x = tl.abs(x_fp32)
|
||||
max_abs = tl.max(abs_x)
|
||||
max_abs_clamped = tl.maximum(max_abs, EPS)
|
||||
scale = max_abs_clamped / FP8_MAX
|
||||
|
||||
log2_scale = tl.log2(scale)
|
||||
ceil_log2 = tl.math.ceil(log2_scale)
|
||||
scale_pow2_fp32 = tl.exp2(ceil_log2)
|
||||
scale_inv = 1.0 / scale_pow2_fp32
|
||||
x_scaled = x_fp32 * scale_inv
|
||||
x_fp8 = tl.clamp(x_scaled, FP8_MIN, FP8_MAX).to(k_nope_fp8_ptr.dtype.element_ty)
|
||||
|
||||
out_fp8_offsets = (
|
||||
token_id * k_nope_fp8_stride_0 + tile_id * TILE_SIZE + tile_range
|
||||
)
|
||||
tl.store(k_nope_fp8_ptr + out_fp8_offsets, x_fp8)
|
||||
|
||||
exponent = ceil_log2.to(tl.int32)
|
||||
scale_uint8 = (exponent + 127).to(tl.uint8)
|
||||
|
||||
out_scale_offset = token_id * scale_stride_0 + tile_id
|
||||
tl.store(scale_k_nope_uint8_ptr + out_scale_offset, scale_uint8)
|
||||
|
||||
|
||||
def quant_to_nope_fp8_rope_bf16_pack_triton(
|
||||
k_bf16: torch.Tensor,
|
||||
) -> NopeFp8RopeBf16Pack:
|
||||
assert k_bf16.dtype == torch.bfloat16
|
||||
num_tokens, hidden_dim = k_bf16.shape
|
||||
assert hidden_dim == 512
|
||||
dim_nope = 448
|
||||
dim_rope = 64
|
||||
tile_size = 64
|
||||
num_tiles = dim_nope // tile_size
|
||||
|
||||
k_bf16 = k_bf16.contiguous()
|
||||
|
||||
k_nope_fp8 = torch.empty(
|
||||
(num_tokens, dim_nope), dtype=fp8_dtype, device=k_bf16.device
|
||||
)
|
||||
k_rope_bf16 = torch.empty(
|
||||
(num_tokens, dim_rope), dtype=torch.bfloat16, device=k_bf16.device
|
||||
)
|
||||
scale_k_nope_ue8m0 = torch.empty(
|
||||
(num_tokens, num_tiles), dtype=torch.uint8, device=k_bf16.device
|
||||
)
|
||||
|
||||
fp8_dtype_info = torch.finfo(fp8_dtype)
|
||||
|
||||
grid = (num_tokens, num_tiles + 1)
|
||||
_quant_k_cache_fused_kernel[grid](
|
||||
k_bf16,
|
||||
k_nope_fp8,
|
||||
k_rope_bf16,
|
||||
scale_k_nope_ue8m0,
|
||||
k_bf16.stride(0),
|
||||
k_nope_fp8.stride(0),
|
||||
k_rope_bf16.stride(0),
|
||||
scale_k_nope_ue8m0.stride(0),
|
||||
DIM_NOPE=dim_nope,
|
||||
DIM_ROPE=dim_rope,
|
||||
TILE_SIZE=tile_size,
|
||||
NUM_TILES=num_tiles,
|
||||
FP8_MIN=fp8_dtype_info.min,
|
||||
FP8_MAX=fp8_dtype_info.max,
|
||||
EPS=1e-8,
|
||||
)
|
||||
|
||||
return NopeFp8RopeBf16Pack(
|
||||
k_nope_fp8=k_nope_fp8,
|
||||
k_rope_bf16=k_rope_bf16,
|
||||
scale_k_nope_ue8m0=scale_k_nope_ue8m0,
|
||||
)
|
||||
@@ -4,8 +4,6 @@ Indexer values use per-32 UE8M0 scales; compressed KV uses per-16 E4M3 scales.
|
||||
Both paths round to the E2M1 grid with ties to even.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
FP8_MAX = 448.0
|
||||
@@ -62,57 +60,3 @@ def fake_quant_compressed_kv(x: torch.Tensor) -> torch.Tensor:
|
||||
scaled = (blocks / scale).clamp(-FP4_MAX, FP4_MAX)
|
||||
deq = round_fp4(scaled) * scale
|
||||
return deq.flatten(-2).to(x.dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pure-torch reference of the paged V4.1 fp8 KV cache format read by the sparse
|
||||
# decode kernel (528 B/token, "V41").
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ceil_pow2_scale(x: torch.Tensor) -> torch.Tensor:
|
||||
"""``2 ** ceil(log2(max(x, 1e-4)))`` as fp32, computed on the IEEE bits so
|
||||
that it is exact at (and just above) powers of two."""
|
||||
x = x.float()
|
||||
scale = ceil_pow2(torch.clamp_min(x, 1e-4))
|
||||
# ceil_pow2 works on the bits of a finite value; a NaN or inf amax passes
|
||||
# through (both become the ue8m0 NaN byte, but only the NaN one turns the
|
||||
# whole tile's payload into NaN).
|
||||
return torch.where(torch.isfinite(x), scale, x)
|
||||
|
||||
|
||||
def quantize_k_cache_v41(
|
||||
k: torch.Tensor, page_bytes: Optional[int] = None
|
||||
) -> torch.Tensor:
|
||||
"""``k`` ``[num_pages, page_size, 512]`` -> uint8 ``[num_pages, page_bytes]``
|
||||
pages of the V41 layout: 512 e4m3 per token, then 16 ue8m0 scales per token
|
||||
(one per 32 values), ``scale = 2 ** ceil(log2(max(amax / 448, 1e-4)))``."""
|
||||
num_pages, page_size, d = k.shape
|
||||
assert d == 512
|
||||
x = k.float().view(num_pages, page_size, 16, 32)
|
||||
scale = ceil_pow2_scale(x.abs().amax(dim=-1) / 448.0)
|
||||
data = (x / scale.unsqueeze(-1)).to(torch.float8_e4m3fn).view(torch.uint8)
|
||||
scale_u8 = scale.to(torch.float8_e8m0fnu).view(torch.uint8)
|
||||
raw = page_size * 528
|
||||
if page_bytes is None:
|
||||
page_bytes = -(-raw // 512) * 512
|
||||
assert page_bytes >= raw
|
||||
out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device)
|
||||
out[:, : page_size * 512] = data.reshape(num_pages, page_size * 512)
|
||||
out[:, page_size * 512 : raw] = scale_u8.reshape(num_pages, page_size * 16)
|
||||
return out
|
||||
|
||||
|
||||
def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor:
|
||||
"""Inverse of :func:`quantize_k_cache_v41`: ``[num_pages, page_size, 512]`` bf16."""
|
||||
num_pages = pages.shape[0]
|
||||
pages = pages.view(torch.uint8)
|
||||
data = pages[:, : page_size * 512].reshape(num_pages, page_size, 512)
|
||||
scale = pages[:, page_size * 512 : page_size * 528].reshape(
|
||||
num_pages, page_size, 16
|
||||
)
|
||||
values = data.view(torch.float8_e4m3fn).to(torch.bfloat16)
|
||||
scale_bf16 = scale.view(torch.float8_e8m0fnu).to(torch.bfloat16)
|
||||
return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view(
|
||||
num_pages, page_size, 512
|
||||
)
|
||||
|
||||
@@ -52,13 +52,13 @@ the captured launch sequence.
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import os
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.utils.common import is_gfx1250_supported
|
||||
|
||||
@@ -117,7 +117,7 @@ _FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
|
||||
# End to end on DSv4 / MI355X / c128, n-weighted at matched batch size against
|
||||
# the same config with the plain heuristic: decode step 121.76 -> 114.12 ms
|
||||
# (-7.50 ms, -6.2 %). At c256 the gain is smaller (-5.37 ms of 156.19).
|
||||
_HCA_KV_SPLITS = int(os.environ.get("SGLANG_MLA_HCA_KV_SPLITS", "4"))
|
||||
_HCA_KV_SPLITS = envs.SGLANG_MLA_HCA_KV_SPLITS.get()
|
||||
|
||||
|
||||
def _kv_splits_for_stream(compress_ratio: int) -> int | None:
|
||||
|
||||
@@ -24,7 +24,6 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.decode_attention import _extract_kv_strides
|
||||
from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd
|
||||
from sglang.kernels.ops.attention.score_mod import unpack_aux_tensors
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_cuda, is_gfx95_supported, is_gfx1250_supported, is_hip
|
||||
@@ -1170,43 +1169,6 @@ def extend_attention_fwd(
|
||||
)
|
||||
|
||||
|
||||
def redundant_attention(
|
||||
q_extend,
|
||||
o_extend,
|
||||
k_buffer,
|
||||
v_buffer,
|
||||
b_req_idx,
|
||||
b_start_loc,
|
||||
b_seq_len,
|
||||
b_seq_len_prefix,
|
||||
max_len_in_batch,
|
||||
):
|
||||
total_token_num = k_buffer.shape[0]
|
||||
B, H_Q, D = b_req_idx.shape[0], q_extend.shape[-2], q_extend.shape[-1]
|
||||
q_buffer = torch.empty(
|
||||
(total_token_num, H_Q, D), dtype=q_extend.dtype, device=q_extend.device
|
||||
)
|
||||
|
||||
pt = 0
|
||||
for i in range(B):
|
||||
cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i]
|
||||
pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i]
|
||||
q_buffer[pl:pr] = q_extend[pt : pt + cur_seq_len_extend]
|
||||
pt += cur_seq_len_extend
|
||||
|
||||
o_buffer = torch.empty_like(q_buffer)
|
||||
context_attention_fwd(
|
||||
q_buffer, k_buffer, v_buffer, o_buffer, b_start_loc, b_seq_len, max_len_in_batch
|
||||
)
|
||||
|
||||
pt = 0
|
||||
for i in range(B):
|
||||
cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i]
|
||||
pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i]
|
||||
o_extend[pt : pt + cur_seq_len_extend] = o_buffer[pl:pr]
|
||||
pt += cur_seq_len_extend
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kernel_unified(
|
||||
Q,
|
||||
|
||||
@@ -594,145 +594,6 @@ def gdn_replayssm_exact_fold_kernel(
|
||||
tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gdn_replayssm_circular_commit_kernel(
|
||||
h0, # [num_layers, num_slots, HV, V, K]
|
||||
rawv_cache, # [num_layers, num_slots, HV, L, V]
|
||||
rawk_cache, # [num_layers, num_slots, H, L, K]
|
||||
g_cache, # [num_layers, num_slots, HV, L]
|
||||
beta_cache, # [num_layers, num_slots, HV, L]
|
||||
ssm_state_indices, # [B]
|
||||
write_pos, # [num_slots], already advanced by the accepted length
|
||||
cache_base, # [num_slots]
|
||||
is_flush_flags, # [num_slots]
|
||||
accept_lens, # [B]
|
||||
mamba_track_indices, # [B]
|
||||
mamba_steps_to_track, # [B]
|
||||
stride_state_slot: tl.constexpr,
|
||||
stride_rawv_slot: tl.constexpr,
|
||||
stride_rawk_slot: tl.constexpr,
|
||||
stride_g_slot: tl.constexpr,
|
||||
stride_beta_slot: tl.constexpr,
|
||||
stride_state_layer: tl.constexpr,
|
||||
stride_rawv_layer: tl.constexpr,
|
||||
stride_rawk_layer: tl.constexpr,
|
||||
stride_g_layer: tl.constexpr,
|
||||
stride_beta_layer: tl.constexpr,
|
||||
stride_indices: tl.constexpr,
|
||||
stride_accept: tl.constexpr,
|
||||
stride_track: tl.constexpr,
|
||||
stride_steps: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
MAX_CACHE_LEN: tl.constexpr,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
NULL_BLOCK_ID: tl.constexpr,
|
||||
HAS_TRACK: tl.constexpr,
|
||||
):
|
||||
"""Materialize circular history only at capacity or radix-track boundaries."""
|
||||
i_v = tl.program_id(0)
|
||||
i_n = tl.program_id(1)
|
||||
i_hvl = tl.program_id(2)
|
||||
i_layer = (i_hvl // HV).to(tl.int64)
|
||||
i_hv = i_hvl % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices).to(tl.int64)
|
||||
if state_idx <= NULL_BLOCK_ID:
|
||||
return
|
||||
n_history = tl.load(write_pos + state_idx).to(tl.int32)
|
||||
fold_active = tl.load(is_flush_flags + state_idx) != 0
|
||||
|
||||
if HAS_TRACK:
|
||||
track_idx = tl.load(mamba_track_indices + i_n * stride_track).to(tl.int64)
|
||||
track_step = tl.load(mamba_steps_to_track + i_n * stride_steps).to(tl.int32)
|
||||
has_track = (track_idx > NULL_BLOCK_ID) & (track_step >= 0)
|
||||
n_accepted = tl.load(accept_lens + i_n * stride_accept).to(tl.int32)
|
||||
previous_history = n_history - n_accepted
|
||||
track_pos = previous_history + track_step + 1
|
||||
else:
|
||||
track_idx = NULL_BLOCK_ID
|
||||
has_track = False
|
||||
track_pos = 0
|
||||
|
||||
if (not fold_active) and (not has_track):
|
||||
return
|
||||
replay_len = tl.maximum(tl.where(fold_active, n_history, 0), track_pos)
|
||||
if replay_len <= 0:
|
||||
return
|
||||
|
||||
h0 = h0 + i_layer * stride_state_layer
|
||||
rawv_cache = rawv_cache + i_layer * stride_rawv_layer
|
||||
rawk_cache = rawk_cache + i_layer * stride_rawk_layer
|
||||
g_cache = g_cache + i_layer * stride_g_layer
|
||||
beta_cache = beta_cache + i_layer * stride_beta_layer
|
||||
base = tl.load(cache_base + state_idx).to(tl.int32)
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
mask_k = o_k < K
|
||||
mask_v = o_v < V
|
||||
mask_h = mask_k[:, None] & mask_v[None, :]
|
||||
p_h0 = (
|
||||
h0
|
||||
+ state_idx * stride_state_slot
|
||||
+ i_hv * V * K
|
||||
+ o_v[None, :] * K
|
||||
+ o_k[:, None]
|
||||
)
|
||||
b_h = tl.load(p_h0, mask=mask_h, other=0.0).to(tl.float32)
|
||||
|
||||
for t in range(0, replay_len):
|
||||
phys = ((base + t) & (MAX_CACHE_LEN - 1)).to(tl.int64)
|
||||
b_k = tl.load(
|
||||
rawk_cache
|
||||
+ state_idx * stride_rawk_slot
|
||||
+ (i_h * MAX_CACHE_LEN + phys) * K
|
||||
+ o_k,
|
||||
mask=mask_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_v = tl.load(
|
||||
rawv_cache
|
||||
+ state_idx * stride_rawv_slot
|
||||
+ (i_hv * MAX_CACHE_LEN + phys) * V
|
||||
+ o_v,
|
||||
mask=mask_v,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
b_g = tl.load(
|
||||
g_cache + state_idx * stride_g_slot + i_hv * MAX_CACHE_LEN + phys
|
||||
).to(tl.float32)
|
||||
b_beta = tl.load(
|
||||
beta_cache + state_idx * stride_beta_slot + i_hv * MAX_CACHE_LEN + phys
|
||||
).to(tl.float32)
|
||||
if USE_QK_L2NORM_IN_KERNEL:
|
||||
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
||||
b_h *= tl.exp(b_g)
|
||||
b_v -= tl.sum(b_h * b_k[:, None], 0)
|
||||
b_v *= b_beta
|
||||
b_h += b_k[:, None] * b_v[None, :]
|
||||
|
||||
if HAS_TRACK:
|
||||
if has_track and (t + 1 == track_pos):
|
||||
tl.store(
|
||||
h0
|
||||
+ track_idx * stride_state_slot
|
||||
+ i_hv * V * K
|
||||
+ o_v[None, :] * K
|
||||
+ o_k[:, None],
|
||||
b_h.to(h0.dtype.element_ty),
|
||||
mask=mask_h,
|
||||
)
|
||||
|
||||
if fold_active:
|
||||
tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gdn_replayssm_compact_commit_kernel(
|
||||
h0,
|
||||
@@ -1011,39 +872,6 @@ def _advance_gdn_spec_cursors_kernel(
|
||||
tl.store(is_flush_ptr + blk, next_is_flush, mask=valid)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reset_gdn_replayssm_spec_cursors_kernel(
|
||||
write_pos_ptr,
|
||||
cache_base_ptr,
|
||||
is_flush_ptr,
|
||||
do_reset_ptr,
|
||||
state_batch_indices_ptr,
|
||||
n_rows,
|
||||
stride_sbi: tl.constexpr,
|
||||
stride_reset: tl.constexpr,
|
||||
INIT_FLUSH: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NULL_BLOCK_ID: tl.constexpr,
|
||||
):
|
||||
offs = tl.arange(0, BLOCK)
|
||||
row_mask = offs < n_rows
|
||||
blk = tl.load(
|
||||
state_batch_indices_ptr + offs * stride_sbi, mask=row_mask, other=NULL_BLOCK_ID
|
||||
).to(tl.int64)
|
||||
do_reset = tl.load(do_reset_ptr + offs * stride_reset, mask=row_mask, other=0).to(
|
||||
tl.int32
|
||||
)
|
||||
do = row_mask & (blk > NULL_BLOCK_ID) & (do_reset != 0)
|
||||
|
||||
tl.store(write_pos_ptr + blk, tl.zeros_like(blk).to(tl.int32), mask=do)
|
||||
tl.store(cache_base_ptr + blk, tl.zeros_like(blk).to(tl.int32), mask=do)
|
||||
tl.store(
|
||||
is_flush_ptr + blk,
|
||||
tl.full([BLOCK], INIT_FLUSH, dtype=tl.int8),
|
||||
mask=do,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Python wrappers.
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1541,33 +1369,3 @@ def commit_gdn_replayssm_circular(
|
||||
BLOCK=block,
|
||||
NULL_BLOCK_ID=null_block_id,
|
||||
)
|
||||
|
||||
|
||||
def reset_gdn_replayssm_spec_cursors(
|
||||
write_pos: torch.Tensor,
|
||||
cache_base: torch.Tensor,
|
||||
is_flush: torch.Tensor,
|
||||
do_reset: torch.Tensor, # [n_rows] int/bool 1 for first-decode rows
|
||||
state_batch_indices: torch.Tensor, # [n_rows] int
|
||||
max_cache_len: int,
|
||||
max_spec_len: int,
|
||||
null_block_id: int = 0,
|
||||
):
|
||||
"""Reset the cursors of first-decode rows (prefill->decode handoff)."""
|
||||
n_rows = state_batch_indices.shape[0]
|
||||
BLOCK = triton.next_power_of_2(max(1, n_rows))
|
||||
# Early-flush margin (2 * max_spec_len, strict '>'): mirror _advance_gdn_spec_cursors_kernel.
|
||||
init_flush = 1 if 2 * max_spec_len > max_cache_len else 0
|
||||
_reset_gdn_replayssm_spec_cursors_kernel[(1,)](
|
||||
write_pos,
|
||||
cache_base,
|
||||
is_flush,
|
||||
do_reset,
|
||||
state_batch_indices,
|
||||
n_rows,
|
||||
stride_sbi=state_batch_indices.stride(0),
|
||||
stride_reset=do_reset.stride(0),
|
||||
INIT_FLUSH=init_flush,
|
||||
BLOCK=BLOCK,
|
||||
NULL_BLOCK_ID=null_block_id,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from functools import cache, lru_cache
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
@@ -11,14 +9,9 @@ import torch.nn.functional as F
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
|
||||
try:
|
||||
if os.environ.get("SGLANG_INKLING_FA4_USE_PIP") == "1":
|
||||
# A/B debug escape hatch: route through the pip flash-attn-4 package
|
||||
# (dev's stack). rel_bias is vendored-only, so SHEARED must be 0.
|
||||
from flash_attn.cute import flash_attn_varlen_func as _flash_attn_varlen_func
|
||||
else:
|
||||
from sglang.kernels.ops.attention.flash_attn.cute import (
|
||||
flash_attn_varlen_func as _flash_attn_varlen_func,
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_attn.cute import (
|
||||
flash_attn_varlen_func as _flash_attn_varlen_func,
|
||||
)
|
||||
except Exception as _e: # pragma: no cover
|
||||
_flash_attn_varlen_func = None
|
||||
_flash_attn_import_error = _e
|
||||
@@ -26,14 +19,6 @@ else:
|
||||
_flash_attn_import_error = None
|
||||
|
||||
|
||||
def _takes_out(kernel) -> bool:
|
||||
return kernel is not None and "out" in inspect.signature(kernel).parameters
|
||||
|
||||
|
||||
# Released flash-attn-4 builds may lack the vendored kernel's out=.
|
||||
_kernel_takes_out = _takes_out(_flash_attn_varlen_func)
|
||||
|
||||
|
||||
def is_flash_attention_v4_available() -> bool:
|
||||
return _flash_attn_varlen_func is not None
|
||||
|
||||
@@ -293,11 +278,7 @@ def flash_attn_varlen_func(
|
||||
rel_bias_kwargs["rel_bias_prep_cache"] = rel_bias_prep_cache
|
||||
# The kernel writes into out, except where it pads MLA heads: its output
|
||||
# then has more heads than out, and the result is a new tensor.
|
||||
out_kwargs = (
|
||||
{"out": out}
|
||||
if out is not None and mla_head_padding is None and _kernel_takes_out
|
||||
else {}
|
||||
)
|
||||
out_kwargs = {"out": out} if out is not None and mla_head_padding is None else {}
|
||||
result = _flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=k,
|
||||
|
||||
@@ -3,13 +3,17 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.kernels.ops.attention.fa4_sm120.dispatch import (
|
||||
get_forward_arch,
|
||||
resolve_runtime_policy,
|
||||
try_cached_paged_decode,
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_attention_v4 import (
|
||||
_flash_attn_import_error,
|
||||
_flash_attn_varlen_func,
|
||||
@@ -18,18 +22,6 @@ from sglang.kernels.ops.attention.flash_attention_v4 import (
|
||||
_unpad_mla_result,
|
||||
)
|
||||
|
||||
if os.environ.get("SGLANG_INKLING_FA4_USE_PIP") == "1":
|
||||
# The pip escape hatch deliberately bypasses SGLang-owned SM12x kernels.
|
||||
get_forward_arch = None
|
||||
resolve_runtime_policy = None
|
||||
try_cached_paged_decode = None
|
||||
else:
|
||||
from sglang.kernels.ops.attention.fa4_sm120.dispatch import (
|
||||
get_forward_arch,
|
||||
resolve_runtime_policy,
|
||||
try_cached_paged_decode,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashAttentionV4SM120RuntimePolicy:
|
||||
@@ -44,13 +36,6 @@ def get_flash_attention_v4_sm120_runtime_policy(
|
||||
deterministic: bool,
|
||||
) -> FlashAttentionV4SM120RuntimePolicy:
|
||||
"""Resolve the SM12x FA4 launch policy exposed to SGLang."""
|
||||
if resolve_runtime_policy is None:
|
||||
num_splits = 1 if deterministic or device_capability < (9, 0) else 0
|
||||
return FlashAttentionV4SM120RuntimePolicy(
|
||||
num_splits=num_splits,
|
||||
decode_num_splits=num_splits,
|
||||
decode_uses_static_max_seqlen_k=False,
|
||||
)
|
||||
num_splits, decode_num_splits, decode_uses_static_max_seqlen_k = (
|
||||
resolve_runtime_policy(
|
||||
device_capability=device_capability,
|
||||
@@ -251,7 +236,7 @@ def flash_attn_with_kvcache(
|
||||
(k_cache.shape[0],), cache_seqlens, dtype=torch.int32, device=k_cache.device
|
||||
)
|
||||
|
||||
forward_arch = get_forward_arch(q.device) if get_forward_arch is not None else None
|
||||
forward_arch = get_forward_arch(q.device)
|
||||
if (
|
||||
forward_arch is not None
|
||||
and not return_softmax_lse
|
||||
|
||||
@@ -5,13 +5,11 @@
|
||||
import aiter as _aiter # noqa: F401
|
||||
|
||||
from .kimi_k3_kda_decode import (
|
||||
flydsl_kimi_k3_kda_decode,
|
||||
flydsl_kimi_k3_kda_decode_with_f_b,
|
||||
is_flydsl_kimi_k3_kda_decode_supported,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"flydsl_kimi_k3_kda_decode",
|
||||
"flydsl_kimi_k3_kda_decode_with_f_b",
|
||||
"is_flydsl_kimi_k3_kda_decode_supported",
|
||||
]
|
||||
|
||||
@@ -1,450 +0,0 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
|
||||
|
||||
"""FlyDSL kernel for the fused Kimi-K3 KDA decode path on gfx950."""
|
||||
|
||||
import functools
|
||||
import math
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels.tensor_shim import GTensor
|
||||
from flydsl.expr import range_constexpr
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
_HEADS = 12
|
||||
_DIM = 128
|
||||
_LOG2E = math.log2(math.e)
|
||||
_SCALE = _DIM**-0.5
|
||||
_BLOCK_THREADS = 256
|
||||
_NUM_WARPS = 4
|
||||
_WARP_SIZE = 64
|
||||
_WARP_THREADS_K = 8
|
||||
_VALUES_PER_THREAD_K = 4
|
||||
_WARP_TILE_K = _WARP_THREADS_K * _VALUES_PER_THREAD_K
|
||||
_K_ITERS = _DIM // _WARP_TILE_K
|
||||
_WARP_THREADS_V = _WARP_SIZE // _WARP_THREADS_K
|
||||
_V_GROUP_TILE = _NUM_WARPS * _WARP_THREADS_V
|
||||
_V_ITERS = _DIM // _V_GROUP_TILE
|
||||
_WAVES_PER_EU = 3
|
||||
|
||||
|
||||
@functools.cache
|
||||
def create_kimi_k3_kda_decode_kernel(norm_eps: float, lower_bound: float):
|
||||
"""Build the fixed gfx950 BF16 Kimi-K3 decode specialization."""
|
||||
|
||||
@fx.struct
|
||||
class SharedStorage:
|
||||
q: fx.Array[fx.BFloat16, _DIM, 16]
|
||||
k: fx.Array[fx.BFloat16, _DIM, 16]
|
||||
v: fx.Array[fx.BFloat16, _DIM, 16]
|
||||
recurrent_out: fx.Array[fx.BFloat16, _DIM, 16]
|
||||
norm_partial: fx.Array[fx.Float32, 2, 16]
|
||||
|
||||
@flyc.kernel(
|
||||
name="kimi_k3_kda_decode_bf16_gfx950",
|
||||
known_block_size=[_BLOCK_THREADS, 1, 1],
|
||||
)
|
||||
def kernel(
|
||||
x_mem: fx.Tensor,
|
||||
weight_mem: fx.Tensor,
|
||||
conv_state_mem: fx.Tensor,
|
||||
raw_g_mem: fx.Tensor,
|
||||
raw_beta_mem: fx.Tensor,
|
||||
A_log_mem: fx.Tensor,
|
||||
dt_bias_mem: fx.Tensor,
|
||||
state_mem: fx.Tensor,
|
||||
state_indices_mem: fx.Tensor,
|
||||
output_gate_mem: fx.Tensor,
|
||||
norm_weight_mem: fx.Tensor,
|
||||
out_mem: fx.Tensor,
|
||||
batch_size: fx.Int32,
|
||||
stride_x_token: fx.Int32,
|
||||
stride_weight_channel: fx.Int32,
|
||||
stride_weight_width: fx.Int32,
|
||||
stride_conv_slot: fx.Int32,
|
||||
stride_conv_channel: fx.Int32,
|
||||
stride_conv_width: fx.Int32,
|
||||
stride_g_token: fx.Int32,
|
||||
stride_beta_token: fx.Int32,
|
||||
stride_state_slot: fx.Int32,
|
||||
stride_gate_token: fx.Int32,
|
||||
stride_gate_head: fx.Int32,
|
||||
stride_out_token: fx.Int32,
|
||||
stride_out_head: fx.Int32,
|
||||
):
|
||||
del batch_size
|
||||
|
||||
x = GTensor(x_mem, dtype=T.bf16, shape=(-1,))
|
||||
weight = GTensor(weight_mem, dtype=T.f32, shape=(-1,))
|
||||
conv_state = GTensor(conv_state_mem, dtype=T.bf16, shape=(-1,))
|
||||
raw_g = GTensor(raw_g_mem, dtype=T.bf16, shape=(-1,))
|
||||
raw_beta = GTensor(raw_beta_mem, dtype=T.bf16, shape=(-1,))
|
||||
A_log = GTensor(A_log_mem, dtype=T.f32, shape=(-1,))
|
||||
dt_bias = GTensor(dt_bias_mem, dtype=T.f32, shape=(-1,))
|
||||
state = GTensor(state_mem, dtype=T.f32, shape=(-1,))
|
||||
state_indices = GTensor(state_indices_mem, dtype=T.i32, shape=(-1,))
|
||||
output_gate = GTensor(output_gate_mem, dtype=T.bf16, shape=(-1,))
|
||||
norm_weight = GTensor(norm_weight_mem, dtype=T.bf16, shape=(-1,))
|
||||
out = GTensor(out_mem, dtype=T.bf16, shape=(-1,))
|
||||
|
||||
shared = fx.SharedAllocator().allocate(SharedStorage).peek()
|
||||
q_lds = shared.q.ptr
|
||||
k_lds = shared.k.ptr
|
||||
v_lds = shared.v.ptr
|
||||
out_lds = shared.recurrent_out.ptr
|
||||
norm_lds = shared.norm_partial.ptr
|
||||
|
||||
tid = fx.thread_idx.x
|
||||
block = fx.block_idx.x
|
||||
batch = block // fx.Int32(_HEADS)
|
||||
head = block % fx.Int32(_HEADS)
|
||||
lane = tid % fx.Int32(_WARP_SIZE)
|
||||
warp = tid // fx.Int32(_WARP_SIZE)
|
||||
lane_k = lane % fx.Int32(_WARP_THREADS_K)
|
||||
|
||||
state_idx = fx.Int32(state_indices[batch])
|
||||
valid = state_idx > fx.Int32(0)
|
||||
|
||||
# Branch bodies are closures: a bare `if` would make the AST rewriter
|
||||
# carry the GTensors whose methods the body calls out as scf.if results.
|
||||
def _valid_then():
|
||||
# A workgroup exclusively owns all three convolution channels for
|
||||
# its (batch, head), so every cache entry is shifted exactly once.
|
||||
def _conv_then():
|
||||
channel_local = tid
|
||||
q_channel = head * fx.Int32(_DIM) + channel_local
|
||||
k_channel = (
|
||||
fx.Int32(_HEADS * _DIM) + head * fx.Int32(_DIM) + channel_local
|
||||
)
|
||||
v_channel = (
|
||||
fx.Int32(2 * _HEADS * _DIM) + head * fx.Int32(_DIM) + channel_local
|
||||
)
|
||||
|
||||
def convolve_channel(channel):
|
||||
cs_base = (
|
||||
state_idx * stride_conv_slot + channel * stride_conv_channel
|
||||
)
|
||||
c0 = fx.Float32(conv_state[cs_base])
|
||||
c1 = fx.Float32(conv_state[cs_base + stride_conv_width])
|
||||
c2 = fx.Float32(
|
||||
conv_state[cs_base + fx.Int32(2) * stride_conv_width]
|
||||
)
|
||||
current = fx.BFloat16(x[batch * stride_x_token + channel])
|
||||
current_f32 = fx.Float32(current)
|
||||
w_base = channel * stride_weight_channel
|
||||
acc = c0 * fx.Float32(weight[w_base])
|
||||
acc = acc + c1 * fx.Float32(weight[w_base + stride_weight_width])
|
||||
acc = acc + c2 * fx.Float32(
|
||||
weight[w_base + fx.Int32(2) * stride_weight_width]
|
||||
)
|
||||
acc = acc + current_f32 * fx.Float32(
|
||||
weight[w_base + fx.Int32(3) * stride_weight_width]
|
||||
)
|
||||
silu = acc / (
|
||||
fx.Float32(1.0) + fx.math.exp2(-acc * fx.Float32(_LOG2E))
|
||||
)
|
||||
conv_state.store(
|
||||
cs_base,
|
||||
fx.BFloat16(c1),
|
||||
)
|
||||
conv_state.store(
|
||||
cs_base + stride_conv_width,
|
||||
fx.BFloat16(c2),
|
||||
)
|
||||
conv_state.store(
|
||||
cs_base + fx.Int32(2) * stride_conv_width,
|
||||
current,
|
||||
)
|
||||
return silu.to(fx.BFloat16)
|
||||
|
||||
q_conv = convolve_channel(q_channel)
|
||||
k_conv = convolve_channel(k_channel)
|
||||
v_conv = convolve_channel(v_channel)
|
||||
fx.ptr_store(q_conv, q_lds + tid)
|
||||
fx.ptr_store(k_conv, k_lds + tid)
|
||||
fx.ptr_store(v_conv, v_lds + tid)
|
||||
|
||||
if tid < fx.Int32(_DIM):
|
||||
_conv_then()
|
||||
|
||||
fx.gpu.barrier()
|
||||
|
||||
# Four waves split V into 32-row groups. Eight-lane subgroups
|
||||
# reduce K; each lane issues one aligned f32x4 state transaction.
|
||||
k_vec_start = lane_k * fx.Int32(_VALUES_PER_THREAD_K)
|
||||
global_v_start = warp * fx.Int32(_WARP_THREADS_V) + lane // fx.Int32(
|
||||
_WARP_THREADS_K
|
||||
)
|
||||
vec_f32 = T.vec(_VALUES_PER_THREAD_K, T.f32)
|
||||
vec_bf16 = T.vec(_VALUES_PER_THREAD_K, T.bf16)
|
||||
zero_vec = fx.full(
|
||||
_VALUES_PER_THREAD_K,
|
||||
0.0,
|
||||
fx.Float32,
|
||||
)
|
||||
|
||||
q_vecs = []
|
||||
k_vecs = []
|
||||
decay_vecs = []
|
||||
sum_q_partial = fx.Float32(0.0)
|
||||
sum_k_partial = fx.Float32(0.0)
|
||||
a = fx.math.exp2(fx.Float32(A_log[head]) * fx.Float32(_LOG2E))
|
||||
|
||||
for ki in range_constexpr(_K_ITERS):
|
||||
k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K)
|
||||
q_bf16 = fx.ptr_load(
|
||||
q_lds + k_base,
|
||||
result_type=vec_bf16,
|
||||
)
|
||||
k_bf16 = fx.ptr_load(
|
||||
k_lds + k_base,
|
||||
result_type=vec_bf16,
|
||||
)
|
||||
q_f32 = q_bf16.extf(vec_f32)
|
||||
k_f32 = k_bf16.extf(vec_f32)
|
||||
q_vecs.append(q_f32)
|
||||
k_vecs.append(k_f32)
|
||||
sum_q_vec = q_f32 * q_f32
|
||||
sum_k_vec = k_f32 * k_f32
|
||||
sum_q_partial = sum_q_partial + fx.Vector(sum_q_vec).reduce("add")
|
||||
sum_k_partial = sum_k_partial + fx.Vector(sum_k_vec).reduce("add")
|
||||
|
||||
gate_bf16 = raw_g.vec_load(
|
||||
(batch * stride_g_token + head * fx.Int32(_DIM) + k_base,),
|
||||
_VALUES_PER_THREAD_K,
|
||||
)
|
||||
gate_f32 = gate_bf16.extf(vec_f32)
|
||||
dt = dt_bias.vec_load(
|
||||
(head * fx.Int32(_DIM) + k_base,),
|
||||
_VALUES_PER_THREAD_K,
|
||||
)
|
||||
sigmoid_arg = (gate_f32 + dt) * a
|
||||
gate = fx.Float32(lower_bound) / (
|
||||
fx.Float32(1.0) + fx.math.exp2(-sigmoid_arg * fx.Float32(_LOG2E))
|
||||
)
|
||||
decay_vecs.append(fx.math.exp2(gate * fx.Float32(_LOG2E)))
|
||||
|
||||
width = fx.Int32(_WARP_SIZE)
|
||||
for offset in (1, 2, 4):
|
||||
sum_q_partial = sum_q_partial + fx.gpu.shuffle_xor(
|
||||
sum_q_partial, fx.Int32(offset), width
|
||||
)
|
||||
sum_k_partial = sum_k_partial + fx.gpu.shuffle_xor(
|
||||
sum_k_partial, fx.Int32(offset), width
|
||||
)
|
||||
|
||||
subgroup_leader = (lane // fx.Int32(_WARP_THREADS_K)) * fx.Int32(
|
||||
_WARP_THREADS_K
|
||||
)
|
||||
norm_q = fx.gpu.shuffle_idx(sum_q_partial, subgroup_leader, width)
|
||||
norm_k = fx.gpu.shuffle_idx(sum_k_partial, subgroup_leader, width)
|
||||
inv_q = fx.math.rsqrt(fx.Float32(norm_q) + fx.Float32(1e-6))
|
||||
inv_k = fx.math.rsqrt(fx.Float32(norm_k) + fx.Float32(1e-6))
|
||||
|
||||
for ki in range_constexpr(_K_ITERS):
|
||||
q_vecs[ki] = q_vecs[ki] * fx.Float32(inv_q) * fx.Float32(_SCALE)
|
||||
k_vecs[ki] = k_vecs[ki] * fx.Float32(inv_k)
|
||||
|
||||
dot_kq_vec = zero_vec
|
||||
for ki in range_constexpr(_K_ITERS):
|
||||
dot_kq_vec = fx.fma(k_vecs[ki], q_vecs[ki], dot_kq_vec)
|
||||
dot_kq = fx.Vector(dot_kq_vec).reduce("add")
|
||||
for offset in (1, 2, 4):
|
||||
dot_kq = dot_kq + fx.gpu.shuffle_xor(dot_kq, fx.Int32(offset), width)
|
||||
|
||||
beta_value = fx.Float32(raw_beta[batch * stride_beta_token + head])
|
||||
beta = fx.Float32(1.0) / (
|
||||
fx.Float32(1.0) + fx.math.exp2(-beta_value * fx.Float32(_LOG2E))
|
||||
)
|
||||
state_head_base = state_idx * stride_state_slot + head * fx.Int32(
|
||||
_DIM * _DIM
|
||||
)
|
||||
|
||||
state_vecs = []
|
||||
for vi in range_constexpr(_V_ITERS):
|
||||
global_v = global_v_start + fx.Int32(vi * _V_GROUP_TILE)
|
||||
for ki in range_constexpr(_K_ITERS):
|
||||
k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K)
|
||||
state_off = state_head_base + global_v * fx.Int32(_DIM) + k_base
|
||||
state_vecs.append(
|
||||
state.vec_load(
|
||||
(state_off,),
|
||||
_VALUES_PER_THREAD_K,
|
||||
)
|
||||
)
|
||||
|
||||
for vi in range_constexpr(_V_ITERS):
|
||||
global_v = global_v_start + fx.Int32(vi * _V_GROUP_TILE)
|
||||
sum_hk_vec = zero_vec
|
||||
sum_hq_vec = zero_vec
|
||||
for ki in range_constexpr(_K_ITERS):
|
||||
state_pos = vi * _K_ITERS + ki
|
||||
decayed = state_vecs[state_pos] * decay_vecs[ki]
|
||||
state_vecs[state_pos] = decayed
|
||||
sum_hk_vec = fx.fma(decayed, k_vecs[ki], sum_hk_vec)
|
||||
sum_hq_vec = fx.fma(decayed, q_vecs[ki], sum_hq_vec)
|
||||
|
||||
sum_hk = fx.Vector(sum_hk_vec).reduce("add")
|
||||
sum_hq = fx.Vector(sum_hq_vec).reduce("add")
|
||||
for offset in (1, 2, 4):
|
||||
sum_hk = sum_hk + fx.gpu.shuffle_xor(
|
||||
sum_hk, fx.Int32(offset), width
|
||||
)
|
||||
sum_hq = sum_hq + fx.gpu.shuffle_xor(
|
||||
sum_hq, fx.Int32(offset), width
|
||||
)
|
||||
|
||||
conv_v = fx.Float32(fx.ptr_load(v_lds + global_v))
|
||||
v_new = (conv_v - fx.Float32(sum_hk)) * beta
|
||||
v_new = fx.gpu.shuffle_idx(v_new, subgroup_leader, width)
|
||||
recurrent_value = fx.Float32(sum_hq) + fx.Float32(v_new) * fx.Float32(
|
||||
dot_kq
|
||||
)
|
||||
v_new_vec = fx.full(_VALUES_PER_THREAD_K, v_new, fx.Float32)
|
||||
|
||||
for ki in range_constexpr(_K_ITERS):
|
||||
state_pos = vi * _K_ITERS + ki
|
||||
updated = fx.fma(k_vecs[ki], v_new_vec, state_vecs[state_pos])
|
||||
k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K)
|
||||
state_off = state_head_base + global_v * fx.Int32(_DIM) + k_base
|
||||
state.vec_store(
|
||||
(state_off,),
|
||||
updated,
|
||||
_VALUES_PER_THREAD_K,
|
||||
)
|
||||
|
||||
if lane_k == fx.Int32(0):
|
||||
fx.ptr_store(
|
||||
fx.BFloat16(recurrent_value),
|
||||
out_lds + global_v,
|
||||
)
|
||||
|
||||
fx.gpu.barrier()
|
||||
|
||||
# Preserve the model's BF16 boundary before RMSNorm and gating.
|
||||
def _norm_then():
|
||||
recurrent_bf16 = fx.ptr_load(out_lds + tid)
|
||||
recurrent_f32 = fx.Float32(recurrent_bf16)
|
||||
square = recurrent_f32 * recurrent_f32
|
||||
for offset in (32, 16, 8, 4, 2, 1):
|
||||
square = square + fx.gpu.shuffle_xor(
|
||||
square, fx.Int32(offset), width
|
||||
)
|
||||
if lane == fx.Int32(0):
|
||||
fx.ptr_store(square, norm_lds + warp)
|
||||
|
||||
if tid < fx.Int32(_DIM):
|
||||
_norm_then()
|
||||
|
||||
fx.gpu.barrier()
|
||||
|
||||
def _output_then():
|
||||
norm_sum = fx.Float32(fx.ptr_load(norm_lds))
|
||||
norm_sum = norm_sum + fx.Float32(fx.ptr_load(norm_lds + fx.Int32(1)))
|
||||
inv_rms = fx.math.rsqrt(
|
||||
norm_sum * fx.Float32(1.0 / _DIM) + fx.Float32(norm_eps)
|
||||
)
|
||||
recurrent_f32 = fx.Float32(fx.ptr_load(out_lds + tid))
|
||||
norm_w = fx.Float32(norm_weight[tid])
|
||||
gate_value = fx.Float32(
|
||||
output_gate[
|
||||
batch * stride_gate_token + head * stride_gate_head + tid
|
||||
]
|
||||
)
|
||||
output_sigmoid = fx.Float32(1.0) / (
|
||||
fx.Float32(1.0) + fx.math.exp2(-gate_value * fx.Float32(_LOG2E))
|
||||
)
|
||||
result = recurrent_f32 * inv_rms * norm_w * output_sigmoid
|
||||
out.store(
|
||||
batch * stride_out_token + head * stride_out_head + tid,
|
||||
result.to(fx.BFloat16),
|
||||
)
|
||||
|
||||
if tid < fx.Int32(_DIM):
|
||||
_output_then()
|
||||
|
||||
def _zero_then():
|
||||
out.store(
|
||||
batch * stride_out_token + head * stride_out_head + tid,
|
||||
fx.BFloat16(0.0),
|
||||
)
|
||||
|
||||
if valid:
|
||||
_valid_then()
|
||||
elif tid < fx.Int32(_DIM):
|
||||
_zero_then()
|
||||
|
||||
@flyc.jit
|
||||
def launch(
|
||||
x_mem: fx.Tensor,
|
||||
weight_mem: fx.Tensor,
|
||||
conv_state_mem: fx.Tensor,
|
||||
raw_g_mem: fx.Tensor,
|
||||
raw_beta_mem: fx.Tensor,
|
||||
A_log_mem: fx.Tensor,
|
||||
dt_bias_mem: fx.Tensor,
|
||||
state_mem: fx.Tensor,
|
||||
state_indices_mem: fx.Tensor,
|
||||
output_gate_mem: fx.Tensor,
|
||||
norm_weight_mem: fx.Tensor,
|
||||
out_mem: fx.Tensor,
|
||||
batch_size: fx.Int32,
|
||||
stride_x_token: fx.Int32,
|
||||
stride_weight_channel: fx.Int32,
|
||||
stride_weight_width: fx.Int32,
|
||||
stride_conv_slot: fx.Int32,
|
||||
stride_conv_channel: fx.Int32,
|
||||
stride_conv_width: fx.Int32,
|
||||
stride_g_token: fx.Int32,
|
||||
stride_beta_token: fx.Int32,
|
||||
stride_state_slot: fx.Int32,
|
||||
stride_gate_token: fx.Int32,
|
||||
stride_gate_head: fx.Int32,
|
||||
stride_out_token: fx.Int32,
|
||||
stride_out_head: fx.Int32,
|
||||
stream: fx.Stream = fx.Stream(None), # noqa: B008
|
||||
):
|
||||
kernel(
|
||||
x_mem,
|
||||
weight_mem,
|
||||
conv_state_mem,
|
||||
raw_g_mem,
|
||||
raw_beta_mem,
|
||||
A_log_mem,
|
||||
dt_bias_mem,
|
||||
state_mem,
|
||||
state_indices_mem,
|
||||
output_gate_mem,
|
||||
norm_weight_mem,
|
||||
out_mem,
|
||||
batch_size,
|
||||
stride_x_token,
|
||||
stride_weight_channel,
|
||||
stride_weight_width,
|
||||
stride_conv_slot,
|
||||
stride_conv_channel,
|
||||
stride_conv_width,
|
||||
stride_g_token,
|
||||
stride_beta_token,
|
||||
stride_state_slot,
|
||||
stride_gate_token,
|
||||
stride_gate_head,
|
||||
stride_out_token,
|
||||
stride_out_head,
|
||||
).launch(
|
||||
grid=(batch_size * fx.Int32(_HEADS), 1, 1),
|
||||
block=(_BLOCK_THREADS, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
launch.compile_hints = {
|
||||
"waves_per_eu": _WAVES_PER_EU,
|
||||
"llvm_options": {
|
||||
"amdgpu-expert-scheduling-mode": True,
|
||||
},
|
||||
}
|
||||
return launch
|
||||
|
||||
|
||||
__all__ = ["create_kimi_k3_kda_decode_kernel"]
|
||||
@@ -11,9 +11,6 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
from aiter.ops.flydsl.kernels.tensor_shim import _run_compiled
|
||||
|
||||
from .kernels.kimi_k3_kda_decode import (
|
||||
create_kimi_k3_kda_decode_kernel,
|
||||
)
|
||||
from .kernels.kimi_k3_kda_decode_fb import (
|
||||
create_kimi_k3_kda_decode_fb_kernel,
|
||||
)
|
||||
@@ -127,7 +124,7 @@ def _validate_kda_inputs(
|
||||
norm_weight: torch.Tensor,
|
||||
out: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Validate operands shared by both explicit KDA specializations."""
|
||||
"""Validate the KDA decode operands."""
|
||||
if not is_flydsl_kimi_k3_kda_decode_supported(device):
|
||||
raise RuntimeError(f"`{api_name}` requires a gfx950 GPU.")
|
||||
if batch <= 0:
|
||||
@@ -256,109 +253,6 @@ def _validate_kda_inputs(
|
||||
return out
|
||||
|
||||
|
||||
def flydsl_kimi_k3_kda_decode(
|
||||
x: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
conv_bias: torch.Tensor | None,
|
||||
conv_state: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
lower_bound: float | None,
|
||||
state: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
output_gate: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run fused Kimi-K3 KDA decode on MI350-series GPUs.
|
||||
|
||||
This pure-decode specialization fuses the packed width-4 Q/K/V causal
|
||||
convolution, the FP32 recurrent-state update, and the BF16
|
||||
RMSNorm/sigmoid output gate. Slot zero is reserved: non-positive
|
||||
``state_indices`` produce zero output without modifying either cache.
|
||||
|
||||
The layout is fixed to Kimi-K3 TP8: 12 local heads and 128-dimensional
|
||||
key/value state. Call
|
||||
:func:`is_flydsl_kimi_k3_kda_decode_supported` before dispatching from a
|
||||
model implementation.
|
||||
"""
|
||||
if x.ndim != 2:
|
||||
raise ValueError(f"`x` must have rank 2, got rank {x.ndim}.")
|
||||
if not x.is_cuda:
|
||||
raise ValueError("`x` must be a CUDA tensor.")
|
||||
device = x.device
|
||||
batch = x.shape[0]
|
||||
out = _validate_kda_inputs(
|
||||
api_name="flydsl_kimi_k3_kda_decode",
|
||||
batch_source="x",
|
||||
device=device,
|
||||
batch=batch,
|
||||
x=x,
|
||||
conv_weight=conv_weight,
|
||||
conv_bias=conv_bias,
|
||||
conv_state=conv_state,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
state=state,
|
||||
state_indices=state_indices,
|
||||
output_gate=output_gate,
|
||||
norm_weight=norm_weight,
|
||||
out=out,
|
||||
)
|
||||
_check_same_device((("raw_g", raw_g),), device)
|
||||
_check_tensor(
|
||||
"raw_g",
|
||||
raw_g,
|
||||
shape=(1, batch, _HEADS, _DIM),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
inner_strides=(_DIM, 1),
|
||||
)
|
||||
|
||||
executable = create_kimi_k3_kda_decode_kernel(
|
||||
float(norm_eps),
|
||||
float(lower_bound),
|
||||
)
|
||||
with torch.cuda.device(device):
|
||||
stream = torch.cuda.current_stream(device)
|
||||
_run_compiled(
|
||||
executable,
|
||||
x,
|
||||
conv_weight,
|
||||
conv_state,
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log,
|
||||
dt_bias,
|
||||
state,
|
||||
state_indices,
|
||||
output_gate,
|
||||
norm_weight,
|
||||
out,
|
||||
batch,
|
||||
x.stride(0),
|
||||
conv_weight.stride(0),
|
||||
conv_weight.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
raw_g.stride(1),
|
||||
raw_beta.stride(1),
|
||||
state.stride(0),
|
||||
output_gate.stride(0),
|
||||
output_gate.stride(1),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
stream,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def flydsl_kimi_k3_kda_decode_with_f_b(
|
||||
f_a: torch.Tensor,
|
||||
f_b_weight: torch.Tensor,
|
||||
@@ -471,7 +365,6 @@ def flydsl_kimi_k3_kda_decode_with_f_b(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"flydsl_kimi_k3_kda_decode",
|
||||
"flydsl_kimi_k3_kda_decode_with_f_b",
|
||||
"is_flydsl_kimi_k3_kda_decode_supported",
|
||||
]
|
||||
|
||||
@@ -37,20 +37,12 @@ from sglang.kernels.ops.attention.fla.index import (
|
||||
prepare_chunk_indices,
|
||||
)
|
||||
|
||||
try:
|
||||
from .Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host
|
||||
from .fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM
|
||||
from .fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM
|
||||
from .fuse_k4_only_persistent import make_host_fn as _k4p_make_host
|
||||
from .fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK
|
||||
from .fuse_kernel123_persistent import make_host_function as _fused_make_host
|
||||
except ImportError:
|
||||
from Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host
|
||||
from fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM
|
||||
from fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM
|
||||
from fuse_k4_only_persistent import make_host_fn as _k4p_make_host
|
||||
from fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK
|
||||
from fuse_kernel123_persistent import make_host_function as _fused_make_host
|
||||
from .Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host
|
||||
from .fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM
|
||||
from .fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM
|
||||
from .fuse_k4_only_persistent import make_host_fn as _k4p_make_host
|
||||
from .fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK
|
||||
from .fuse_kernel123_persistent import make_host_function as _fused_make_host
|
||||
|
||||
|
||||
def _ct(t, etype):
|
||||
@@ -219,62 +211,6 @@ _k4p_cache = {}
|
||||
_k4p_tm_ws = {}
|
||||
|
||||
|
||||
# Beta absorption (K1 fusion reverted; K123 stays at 375us baseline).
|
||||
# Two-stage strategy on side stream:
|
||||
# 1) v*beta runs PARALLEL with K123 (no data dep on K123 outputs)
|
||||
# 2) k_scaled*beta runs AFTER K123 on side stream, PARALLEL with akk_inv
|
||||
@torch.compile(fullgraph=True)
|
||||
def _v_absorb(v, beta):
|
||||
return v * beta.unsqueeze(-1).to(v.dtype)
|
||||
|
||||
|
||||
@torch.compile(fullgraph=True)
|
||||
def _ks_absorb(k_scaled, beta):
|
||||
return k_scaled * beta.unsqueeze(-1).to(k_scaled.dtype)
|
||||
|
||||
|
||||
# Side stream cache for v_absorb || K123 overlap.
|
||||
_side_streams = {}
|
||||
|
||||
|
||||
def _get_side_stream(dev):
|
||||
idx = dev.index or 0
|
||||
if idx not in _side_streams:
|
||||
_side_streams[idx] = torch.cuda.Stream(device=dev)
|
||||
return _side_streams[idx]
|
||||
|
||||
|
||||
# ===== Per-section timing (K123 / K4) =====
|
||||
# Set _TIMING_ENABLED=True to record CUDA-event timings for each call.
|
||||
# Times accumulate in _TIMING_STATS; use reset_timings() / get_timings() to manage.
|
||||
_TIMING_ENABLED = False
|
||||
_TIMING_STATS = {"k123_us": 0.0, "k4_us": 0.0, "count": 0}
|
||||
|
||||
|
||||
def enable_timing(enabled=True):
|
||||
"""Enable/disable per-section CUDA event timing in chunk_kda_fwd.
|
||||
Each call adds CUDA event records around K123 and K4 launches and a sync
|
||||
at end of forward, so this slows down execution. Use only for benchmarking."""
|
||||
global _TIMING_ENABLED
|
||||
_TIMING_ENABLED = enabled
|
||||
|
||||
|
||||
def reset_timings():
|
||||
_TIMING_STATS["k123_us"] = 0.0
|
||||
_TIMING_STATS["k4_us"] = 0.0
|
||||
_TIMING_STATS["count"] = 0
|
||||
|
||||
|
||||
def get_timings():
|
||||
"""Returns (k123_avg_us, k4_avg_us, n_calls). Avgs are over all calls since last reset."""
|
||||
n = max(_TIMING_STATS["count"], 1)
|
||||
return (
|
||||
_TIMING_STATS["k123_us"] / n,
|
||||
_TIMING_STATS["k4_us"] / n,
|
||||
_TIMING_STATS["count"],
|
||||
)
|
||||
|
||||
|
||||
# Buffer cache: avoid re-allocating ~67us of intermediate tensors per call.
|
||||
# Also caches cute.Tensor wrappers (saves ~7us each call from from_dlpack).
|
||||
_buf_cache = {}
|
||||
@@ -406,11 +342,6 @@ def _get_buffers(dev, dtype_k, B, T, H, K_dim, V_dim, NT, N_seqs, BT):
|
||||
s_ct = from_dlpack(S_out, assumed_align=16)
|
||||
s_ct.element_type = fp32
|
||||
|
||||
# Cache PyTorch streams once per buf_cache entry — torch.cuda.current_stream
|
||||
# is a torch C call that costs ~2us each invocation.
|
||||
main_stream_cached = torch.cuda.current_stream(dev)
|
||||
side_stream_cached = _get_side_stream(dev)
|
||||
|
||||
cute_wrappers = dict(
|
||||
a_ct=a_ct,
|
||||
aqc_ct=aqc_ct,
|
||||
@@ -424,8 +355,6 @@ def _get_buffers(dev, dtype_k, B, T, H, K_dim, V_dim, NT, N_seqs, BT):
|
||||
akk_in_view=akk_in_view,
|
||||
akk_out_view=akk_out_view,
|
||||
s_ct=s_ct,
|
||||
main_stream=main_stream_cached,
|
||||
side_stream=side_stream_cached,
|
||||
# Filled lazily on first launch — saves cache_key tuple build +
|
||||
# outer dict lookup on subsequent calls.
|
||||
_k123_fns={},
|
||||
@@ -936,18 +865,12 @@ def chunk_kda_fwd(
|
||||
# only pays off for huge multi-seq varlen states (~192MB at N=32); for
|
||||
# serving single-seq engagements the state is <1MB and the side stream
|
||||
# was the last remaining cross-stream surface in this path.
|
||||
main_stream = torch.cuda.current_stream(device)
|
||||
needs_copy = S_in.data_ptr() != S_out.data_ptr()
|
||||
if needs_copy:
|
||||
S_out.copy_(S_in)
|
||||
|
||||
# Beta is fused entirely in akk_inv kernel epilogue (post-inv column-scale).
|
||||
# No host v*beta and no K1 k_scaled*beta any more.
|
||||
if _TIMING_ENABLED:
|
||||
k123_s = torch.cuda.Event(enable_timing=True)
|
||||
k123_e = torch.cuda.Event(enable_timing=True)
|
||||
k123_s.record(stream=main_stream)
|
||||
|
||||
_launch_fused_k123_inv(
|
||||
q,
|
||||
k,
|
||||
@@ -974,9 +897,6 @@ def chunk_kda_fwd(
|
||||
varlen_pure_override=(True if needs_varlen_single_pad else varlen_pure),
|
||||
)
|
||||
|
||||
if _TIMING_ENABLED:
|
||||
k123_e.record(stream=main_stream)
|
||||
|
||||
# ===== K4: persistent kernel (eqlen + varlen via cu_seqlens) =====
|
||||
if is_varlen:
|
||||
# GPU-side cumsum + cache by id(cu_seqlens). No host sync.
|
||||
@@ -987,11 +907,6 @@ def chunk_kda_fwd(
|
||||
|
||||
# State copy now runs on the caller's stream; K4 is stream-ordered after it.
|
||||
|
||||
if _TIMING_ENABLED:
|
||||
k4_s = torch.cuda.Event(enable_timing=True)
|
||||
k4_e = torch.cuda.Event(enable_timing=True)
|
||||
k4_s.record(stream=main_stream)
|
||||
|
||||
_launch_k4_persistent(
|
||||
cute_wrappers,
|
||||
v,
|
||||
@@ -1005,13 +920,6 @@ def chunk_kda_fwd(
|
||||
use_fast_sync=(not is_varlen),
|
||||
)
|
||||
|
||||
if _TIMING_ENABLED:
|
||||
k4_e.record(stream=cute_wrappers["main_stream"])
|
||||
torch.cuda.synchronize(device)
|
||||
_TIMING_STATS["k123_us"] += k123_s.elapsed_time(k123_e) * 1000.0
|
||||
_TIMING_STATS["k4_us"] += k4_s.elapsed_time(k4_e) * 1000.0
|
||||
_TIMING_STATS["count"] += 1
|
||||
|
||||
o = O_flat
|
||||
A_qk_out = A_qk
|
||||
A_kk_out = A_kk
|
||||
|
||||
+4
-49
@@ -36,8 +36,6 @@ State management:
|
||||
- single state_ready_mbar after both passes (TMEM no-overlap constraint)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
import cutlass.cutlass_dsl as _dsl_mod
|
||||
@@ -62,54 +60,11 @@ from cutlass.cute.nvgpu import cpasync, tcgen05
|
||||
from cutlass.cute.nvgpu.tcgen05 import Field, OperandMajorMode, OperandSource
|
||||
from cutlass.cute.runtime import from_dlpack
|
||||
from cutlass.cutlass_dsl import Int32, T
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
# GDN tile scheduler: import from the public FlashInfer release
|
||||
# (`pip install flashinfer-python>=0.6.13`). Fall back to a local
|
||||
# dynamic-kernel-generator checkout only if FlashInfer is unavailable.
|
||||
try:
|
||||
from flashinfer.gdn_kernels.blackwell.gated_delta_net_tile_scheduler import (
|
||||
GDNTileScheduler,
|
||||
GDNTileSchedulerParams,
|
||||
)
|
||||
except ImportError:
|
||||
# Try multiple locations for GDNTileScheduler (workspace layout vs kda_optimized layout)
|
||||
_GDN_PATHS = [
|
||||
# workspace/flash-linear-attention/prefill -> ../../../dynamic-kernel-generator
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"dynamic-kernel-generator",
|
||||
"cutlass_ir",
|
||||
"compiler",
|
||||
"python",
|
||||
"examples",
|
||||
"blackwell",
|
||||
"gated_delta_net",
|
||||
),
|
||||
# scripts/kda_optimized -> ../../dynamic-kernel-generator
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"..",
|
||||
"..",
|
||||
"dynamic-kernel-generator",
|
||||
"cutlass_ir",
|
||||
"compiler",
|
||||
"python",
|
||||
"examples",
|
||||
"blackwell",
|
||||
"gated_delta_net",
|
||||
),
|
||||
]
|
||||
for _p in _GDN_PATHS:
|
||||
if os.path.isdir(_p):
|
||||
sys.path.insert(0, _p)
|
||||
break
|
||||
from gated_delta_net_tile_scheduler import GDNTileSchedulerParams, GDNTileScheduler
|
||||
|
||||
from cutlass.utils import TensorMapManager, TensorMapUpdateMode
|
||||
from flashinfer.gdn_kernels.blackwell.gated_delta_net_tile_scheduler import (
|
||||
GDNTileScheduler,
|
||||
GDNTileSchedulerParams,
|
||||
)
|
||||
|
||||
|
||||
def transform_partitioned_tensor_layout(tensor):
|
||||
|
||||
@@ -29,178 +29,6 @@ class SegLaMeta:
|
||||
mask: Optional[torch.Tensor] = None # Currently not supported
|
||||
|
||||
|
||||
# fused
|
||||
@triton.jit
|
||||
def seg_la_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
S,
|
||||
Out,
|
||||
softmax_scale,
|
||||
stride_q,
|
||||
stride_k,
|
||||
stride_v,
|
||||
stride_s,
|
||||
stride_o,
|
||||
s_offsets,
|
||||
q_offsets,
|
||||
q_lengths,
|
||||
s_scales,
|
||||
decay_scales,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
SPLIT_DIM: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
EVEN: tl.constexpr,
|
||||
DECOUPLE: tl.constexpr,
|
||||
):
|
||||
bid = tl.program_id(0)
|
||||
hid = tl.program_id(1)
|
||||
sid = tl.program_id(2)
|
||||
|
||||
# s_scale is 0 (prefill) or 1 (decode)
|
||||
s_scale = tl.load(s_scales + bid)
|
||||
q_length = tl.load(q_lengths + bid)
|
||||
q_offset = tl.load(q_offsets + bid)
|
||||
s_offset = tl.load(s_offsets + bid)
|
||||
decay_scale = -tl.load(decay_scales + hid)
|
||||
|
||||
offs_b = tl.arange(0, BLOCK)
|
||||
offs_d = tl.arange(0, HEAD_DIM)
|
||||
offs_s = tl.arange(0, SPLIT_DIM)
|
||||
|
||||
if s_offset == -1:
|
||||
return
|
||||
|
||||
q_ptrs = (
|
||||
Q
|
||||
+ q_offset * stride_q
|
||||
+ hid * HEAD_DIM
|
||||
+ (offs_b[:, None] * stride_q + offs_d[None, :])
|
||||
)
|
||||
k_ptrs = (
|
||||
K
|
||||
+ q_offset * stride_k
|
||||
+ hid * HEAD_DIM
|
||||
+ (offs_b[:, None] * stride_k + offs_d[None, :])
|
||||
)
|
||||
v_ptrs = (
|
||||
V
|
||||
+ q_offset * stride_v
|
||||
+ hid * HEAD_DIM
|
||||
+ sid * SPLIT_DIM
|
||||
+ (offs_b[:, None] * stride_v + offs_s[None, :])
|
||||
)
|
||||
out_ptrs = (
|
||||
Out
|
||||
+ q_offset * stride_o
|
||||
+ hid * HEAD_DIM
|
||||
+ sid * SPLIT_DIM
|
||||
+ (offs_b[:, None] * stride_o + offs_s[None, :])
|
||||
)
|
||||
s_ptrs = (
|
||||
S
|
||||
+ s_offset * stride_s
|
||||
+ hid * HEAD_DIM * HEAD_DIM
|
||||
+ sid * SPLIT_DIM
|
||||
+ (offs_d[:, None] * HEAD_DIM + offs_s[None, :])
|
||||
)
|
||||
state = tl.load(s_ptrs, mask=s_scale > 0).to(tl.float32)
|
||||
|
||||
if BLOCK > 1:
|
||||
for n in range(0, q_length, BLOCK):
|
||||
n = tl.multiple_of(n, BLOCK)
|
||||
|
||||
if EVEN:
|
||||
q = tl.load(q_ptrs + n * stride_q).to(tl.float32)
|
||||
k = tl.trans(tl.load(k_ptrs + n * stride_k)).to(tl.float32)
|
||||
v = tl.load(v_ptrs + n * stride_k).to(tl.float32)
|
||||
else:
|
||||
q = tl.load(
|
||||
q_ptrs + n * stride_q,
|
||||
mask=(n + offs_b)[:, None] < q_length,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k = tl.trans(
|
||||
tl.load(
|
||||
k_ptrs + n * stride_k,
|
||||
mask=(n + offs_b)[:, None] < q_length,
|
||||
other=0.0,
|
||||
)
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
v_ptrs + n * stride_k,
|
||||
mask=(n + offs_b)[:, None] < q_length,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
if DECOUPLE:
|
||||
# only work with small scales
|
||||
if EVEN:
|
||||
b = BLOCK
|
||||
else:
|
||||
b = min(BLOCK, q_length - n)
|
||||
b_offs = b - 1 - offs_b
|
||||
|
||||
edb = tl.exp(decay_scale * b_offs)
|
||||
decays = tl.where(b_offs >= 0, edb, 0)
|
||||
inv_decays = tl.where(b_offs >= 0, 1 / edb, 0)
|
||||
|
||||
q = q * inv_decays[:, None]
|
||||
k = k * decays[None, :]
|
||||
qk = tl.dot(q, k) * softmax_scale
|
||||
qk = tl.where(offs_b[None, :] <= offs_b[:, None], qk, 0.0)
|
||||
o = tl.dot(qk, v)
|
||||
|
||||
block_decay = tl.exp(decay_scale * b)
|
||||
block_decay_plus = block_decay * softmax_scale
|
||||
o = tl.dot(q, state) * block_decay_plus + o
|
||||
|
||||
state = state * block_decay + tl.dot(k, v)
|
||||
else:
|
||||
qk = tl.dot(q, k) * softmax_scale
|
||||
decays = tl.exp(decay_scale * (offs_b[:, None] - offs_b[None, :]))
|
||||
decays = tl.where(offs_b[None, :] <= offs_b[:, None], decays, 0.0)
|
||||
qk *= decays
|
||||
o = tl.dot(qk, v)
|
||||
|
||||
decay_arr = tl.exp(decay_scale * (offs_b[:, None] + 1)) * softmax_scale
|
||||
o = tl.dot(q * decay_arr, state, acc=o)
|
||||
|
||||
if EVEN:
|
||||
b = BLOCK
|
||||
else:
|
||||
b = min(BLOCK, q_length - n)
|
||||
b_offs = b - 1 - offs_b
|
||||
b_offs = tl.where(b_offs >= 0, b_offs, 10000)
|
||||
decays = tl.exp(decay_scale * b_offs)
|
||||
block_decay = tl.exp(decay_scale * b)
|
||||
state = state * block_decay + tl.dot(k * decays[None, :], v)
|
||||
|
||||
if EVEN:
|
||||
tl.store(out_ptrs + n * stride_o, o.to(Out.dtype.element_ty))
|
||||
else:
|
||||
tl.store(
|
||||
out_ptrs + n * stride_o,
|
||||
o.to(Out.dtype.element_ty),
|
||||
mask=(n + offs_b)[:, None] < q_length,
|
||||
)
|
||||
|
||||
tl.store(s_ptrs, state.to(S.dtype.element_ty))
|
||||
|
||||
else:
|
||||
q = tl.trans(tl.load(q_ptrs)).to(tl.float32) * softmax_scale
|
||||
k = tl.trans(tl.load(k_ptrs)).to(tl.float32)
|
||||
v = tl.load(v_ptrs).to(tl.float32)
|
||||
state = state * tl.exp(decay_scale) + k * v
|
||||
|
||||
o = tl.sum(q * state, axis=0, keep_dims=True)
|
||||
|
||||
tl.store(out_ptrs, o.to(Out.dtype.element_ty))
|
||||
|
||||
tl.store(s_ptrs, state.to(S.dtype.element_ty))
|
||||
|
||||
|
||||
# used for prefilling
|
||||
@triton.jit
|
||||
def seg_la_p_kernel(
|
||||
@@ -691,7 +519,6 @@ def seg_la_fwd(
|
||||
track_lens=None,
|
||||
track_state_indices=None,
|
||||
softmax_scale=None,
|
||||
decouple=False,
|
||||
):
|
||||
length, qo_heads, HEAD_DIM = q.shape
|
||||
_, kv_heads, _ = k.shape
|
||||
@@ -706,7 +533,6 @@ def seg_la_fwd(
|
||||
|
||||
if MAX_LENGTH > 1:
|
||||
# prefill with partitioning q/k/v
|
||||
# BLOCK should <= 64 with decouple
|
||||
K_SPLIT_DIM = 32
|
||||
V_SPLIT_DIM = 32 if bs <= 2 else 64
|
||||
|
||||
@@ -884,61 +710,4 @@ def seg_la_fwd(
|
||||
else:
|
||||
o = tmp[0]
|
||||
|
||||
# if fallback:
|
||||
# # prefill/decode with partitioning v only
|
||||
# o = torch.empty(q.shape, device=q.device, dtype=q.dtype)
|
||||
# if MAX_LENGTH == 1:
|
||||
# # decode
|
||||
# BLOCK = 1
|
||||
# EVEN = False
|
||||
# SPLIT_DIM = 32
|
||||
# num_warps = 8
|
||||
# num_stages = 2
|
||||
# num_dim_block = HEAD_DIM // SPLIT_DIM
|
||||
# grid = (batch, kv_heads, num_dim_block)
|
||||
# else:
|
||||
# # prefill
|
||||
# if decouple:
|
||||
# BLOCK = 64
|
||||
# SPLIT_DIM = 16
|
||||
# else:
|
||||
# BLOCK = HEAD_DIM
|
||||
# SPLIT_DIM = 32
|
||||
# # EVEN = all([x % BLOCK == 0 for x in meta.qls])
|
||||
# EVEN = False
|
||||
# num_warps = 8
|
||||
# num_stages = 2
|
||||
# # prop = torch.cuda.get_device_properties(q.device.index)
|
||||
# # arch = prop.major * 10 + prop.minor
|
||||
# # if arch not in (80, 90):
|
||||
# # num_stages = 1
|
||||
|
||||
# num_dim_block = HEAD_DIM // SPLIT_DIM
|
||||
# grid = (batch, kv_heads, num_dim_block)
|
||||
|
||||
# seg_la_kernel[grid](
|
||||
# q,
|
||||
# k,
|
||||
# v,
|
||||
# s,
|
||||
# o,
|
||||
# softmax_scale,
|
||||
# q.stride(0),
|
||||
# k.stride(0),
|
||||
# v.stride(0),
|
||||
# s.stride(0),
|
||||
# o.stride(0),
|
||||
# meta.s_offsets,
|
||||
# meta.q_offsets,
|
||||
# meta.q_lengths,
|
||||
# meta.s_scales,
|
||||
# decay_scales,
|
||||
# HEAD_DIM=HEAD_DIM,
|
||||
# SPLIT_DIM=SPLIT_DIM,
|
||||
# BLOCK=BLOCK,
|
||||
# EVEN=EVEN,
|
||||
# DECOUPLE=decouple,
|
||||
# num_warps=num_warps,
|
||||
# num_stages=num_stages
|
||||
# )
|
||||
return o
|
||||
|
||||
@@ -4,10 +4,11 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
_is_hip = is_hip()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip
|
||||
|
||||
# =============================================================================
|
||||
# Fused kernel — reads INTERLEAVED input format
|
||||
|
||||
@@ -15,6 +15,7 @@ HIP-graph-capture safe (no host sync).
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
@@ -24,6 +25,8 @@ import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.cuda_wrapper import find_loaded_library
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
_LOG2E = 1.4426950408889634
|
||||
|
||||
@@ -205,10 +208,9 @@ def asm_kernel_available() -> bool:
|
||||
def _disable(reason):
|
||||
global _disabled_reason
|
||||
_disabled_reason = reason
|
||||
print(
|
||||
logger.warning(
|
||||
f"[vattn_asm] disabling the gfx950 asm attention kernel, falling back to "
|
||||
f"the Triton path: {reason}",
|
||||
flush=True,
|
||||
f"the Triton path: {reason}"
|
||||
)
|
||||
raise AsmKernelUnavailable(reason)
|
||||
|
||||
@@ -264,10 +266,9 @@ def _get_kernel(hq):
|
||||
except RuntimeError as e:
|
||||
_disable(f"loading {co} failed: {e}")
|
||||
_kernels[hq] = kern
|
||||
print(
|
||||
logger.info(
|
||||
f"[vattn_asm] gfx950 asm verify-attention kernel active "
|
||||
f"(in-tree loader, GQA ratio {hq}, built {co})",
|
||||
flush=True,
|
||||
f"(in-tree loader, GQA ratio {hq}, built {co})"
|
||||
)
|
||||
return kern
|
||||
|
||||
|
||||
@@ -79,11 +79,13 @@ def can_use_fused_qk_complex_rope(
|
||||
device: torch.device,
|
||||
freqs_cis: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Select the fused path before projecting the vision Q/K tensors."""
|
||||
"""Select the fused path before projecting the vision Q/K tensors.
|
||||
|
||||
``freqs_cis`` sharing the activations' device is the kernel's contract,
|
||||
which :func:`apply_fused_qk_complex_rope` asserts, not a dispatch choice.
|
||||
"""
|
||||
if not (
|
||||
device.type == "cuda"
|
||||
and freqs_cis.is_cuda
|
||||
and device == freqs_cis.device
|
||||
and dtype in (torch.bfloat16, torch.float16)
|
||||
and freqs_cis.dtype == torch.complex64
|
||||
):
|
||||
@@ -115,7 +117,7 @@ def apply_fused_qk_complex_rope(
|
||||
f"freqs={freqs_cis.shape}/{freqs_cis.dtype}/{freqs_cis.device}"
|
||||
)
|
||||
|
||||
assert q.device == k.device and q.dtype == k.dtype
|
||||
assert q.device == k.device == freqs_cis.device and q.dtype == k.dtype
|
||||
assert q.shape == k.shape and q.ndim >= 3
|
||||
assert q.shape[-1] % 2 == 0
|
||||
assert freqs_cis.shape == q.shape[:-2] + (q.shape[-1] // 2,)
|
||||
|
||||
@@ -6,7 +6,12 @@ records metadata; importing this group does not load the implementations.
|
||||
"""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
from sglang.kernels.spec import (
|
||||
CUDA_ONLY,
|
||||
CapabilityRequirement,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
|
||||
__all__ = []
|
||||
|
||||
@@ -16,7 +21,6 @@ register_kernel(
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.communication.all_reduce_fusion:moe_finalize_shared_gate_all_reduce",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
|
||||
description="BF16 routed finalize, FP32 shared gate, and clustered all-reduce.",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -39,7 +43,7 @@ for _mod, _fn in [
|
||||
op=f"communication.{_fn}",
|
||||
backend=KernelBackend.JIT,
|
||||
target=f"sglang.kernels.ops.communication.{_mod}:{_fn}",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
@@ -24,7 +24,6 @@ def _jit_inkling_all_reduce_module(dtype: torch.dtype, world_size: int) -> Modul
|
||||
*args,
|
||||
cuda_files=["inkling/inkling_all_reduce.cuh"],
|
||||
cuda_wrappers=[
|
||||
("two_shot_all_reduce", f"inkling_two_shot_all_reduce<{args}>"),
|
||||
("two_shot_all_reduce_fused", f"inkling_two_shot_all_reduce_fused<{args}>"),
|
||||
("multimem_one_shot_fused", f"inkling_multimem_one_shot_fused<{args}>"),
|
||||
("multimem_full_oneshot", f"inkling_multimem_full_oneshot<{args}>"),
|
||||
@@ -143,32 +142,6 @@ def compile_inkling_all_reduce(dtype: torch.dtype, world_size: int) -> None:
|
||||
_jit_inkling_all_reduce_module(dtype, world_size)
|
||||
|
||||
|
||||
def inkling_two_shot_all_reduce(
|
||||
buffer: torch.Tensor,
|
||||
peer_ptrs_dev: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
num_items: int,
|
||||
) -> None:
|
||||
"""Two-shot all-reduce in place over ``num_items`` elements of the symm buffer.
|
||||
|
||||
Args:
|
||||
buffer: this rank's symm buffer (1D, contiguous, bf16), sliced to
|
||||
``num_items``; used for device/dtype validation. The producer must
|
||||
have already written this rank's shard into it.
|
||||
peer_ptrs_dev: ``hdl.buffer_ptrs_dev`` -- device address of the array of
|
||||
``world_size`` peer buffer base pointers.
|
||||
rank: this rank within the TP group.
|
||||
world_size: TP world size (compile-time template arg).
|
||||
num_items: number of elements to reduce (multiple of 8 for bf16).
|
||||
|
||||
The caller is responsible for ``hdl.barrier()`` before (producers done) and
|
||||
after (result visible) this call.
|
||||
"""
|
||||
module = _jit_inkling_all_reduce_module(buffer.dtype, world_size)
|
||||
module.two_shot_all_reduce(buffer, peer_ptrs_dev, rank, num_items)
|
||||
|
||||
|
||||
def inkling_two_shot_all_reduce_fused(
|
||||
buffer: torch.Tensor,
|
||||
data_ptrs_dev: int,
|
||||
|
||||
@@ -1,30 +1,9 @@
|
||||
"""Multi-process / multi-GPU launching utilities (torchrun-based).
|
||||
|
||||
Shared `multigpu_launch` helper that both `sglang.test.kernels.utils` and
|
||||
`sglang.kernels.jit.benchmark.utils` build their domain-specific entry points on
|
||||
top of (`multigpu_pytest_main`, `multigpu_bench_main`).
|
||||
|
||||
When a script that calls one of those wrappers is run with plain `python`, the
|
||||
launcher relaunches the same file under `torchrun` once for each `N` in
|
||||
`num_gpus`. When the inner workers run (identified by an env_key being set),
|
||||
the same launcher calls `inner()` on every rank, silences stdout on non-zero
|
||||
ranks, and exits with its return code.
|
||||
"""
|
||||
"""Shutdown helper for custom all-reduce communicators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Callable, List, NoReturn, Optional, Sequence
|
||||
|
||||
import psutil
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from typing import Any
|
||||
|
||||
|
||||
def register_comm_cleanup(comm: Any) -> None:
|
||||
@@ -52,166 +31,3 @@ def register_comm_cleanup(comm: Any) -> None:
|
||||
pass
|
||||
|
||||
atexit.register(_safe_close)
|
||||
|
||||
|
||||
def _kill_pgroup(pgid: int) -> None:
|
||||
try:
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def _kill_descendants(pid: int) -> None:
|
||||
"""Snapshot every descendant of `pid` *now* and SIGKILL them all.
|
||||
|
||||
Must be called BEFORE the direct child (torchrun) dies -- once it does,
|
||||
its workers get reparented to init and we lose them via the process tree.
|
||||
"""
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
except psutil.NoSuchProcess:
|
||||
return
|
||||
descendants = root.children(recursive=True)
|
||||
for proc in descendants:
|
||||
try:
|
||||
proc.kill()
|
||||
except psutil.Error:
|
||||
# NoSuchProcess (already gone) or AccessDenied -- nothing to do.
|
||||
pass
|
||||
psutil.wait_procs(descendants, timeout=5)
|
||||
|
||||
|
||||
def _extract_num_gpus_override(
|
||||
argv: list[str],
|
||||
) -> tuple[list[int] | None, list[str]]:
|
||||
"""Pop `--num-gpu(s)` flags out of `argv` and return them separately.
|
||||
|
||||
Accepts `--num-gpu N`, `--num-gpu=N`, `--num-gpus ...`, and comma-separated
|
||||
lists like `--num-gpu 2,4,8`. May be repeated.
|
||||
"""
|
||||
override: list[int] = []
|
||||
remaining: list[str] = []
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
a = argv[i]
|
||||
if a in ("--num-gpu", "--num-gpus"):
|
||||
if i + 1 >= len(argv):
|
||||
raise ValueError(f"missing value for {a} (expected e.g. `{a} 2,4`)")
|
||||
override.extend(int(x) for x in argv[i + 1].split(","))
|
||||
i += 2
|
||||
elif a.startswith("--num-gpu=") or a.startswith("--num-gpus="):
|
||||
_, val = a.split("=", 1)
|
||||
override.extend(int(x) for x in val.split(","))
|
||||
i += 1
|
||||
else:
|
||||
remaining.append(a)
|
||||
i += 1
|
||||
return (override if override else None), remaining
|
||||
|
||||
|
||||
def multigpu_launch(
|
||||
name: str,
|
||||
file: str,
|
||||
num_gpus: Sequence[int],
|
||||
env_key: str,
|
||||
inner: Callable[[], int],
|
||||
kind: str,
|
||||
pre_launch_fn: Optional[Callable[[List[int]], None]] = None,
|
||||
timeout: Optional[int] = None,
|
||||
) -> NoReturn | None:
|
||||
"""Shared torchrun-based launcher.
|
||||
|
||||
See module docstring. `name` is the caller's `__name__`; `file` is its
|
||||
`__file__`. `env_key` is a unique string per kind (test/benchmark) used to
|
||||
detect the inside-torchrun state. `inner` returns an exit code.
|
||||
|
||||
`pre_launch_fn`, if given, runs once in the outer process *before* any
|
||||
torchrun child is spawned. It receives the list of world sizes that will
|
||||
actually be launched (already filtered against the host's GPU count and
|
||||
any ``--num-gpu`` override). Use it for parallel JIT precompilation so the
|
||||
on-disk kernel cache is warm by the time the torchrun children import
|
||||
their kernels.
|
||||
|
||||
`timeout`, if given, bounds each per-world-size torchrun invocation (in
|
||||
seconds). On expiry the child's whole process group is killed and the
|
||||
launcher exits non-zero. `None` (the default) waits indefinitely.
|
||||
"""
|
||||
pid_key = env_key + "_PID"
|
||||
if env_key in os.environ:
|
||||
assert pid_key in os.environ
|
||||
if name != "__main__":
|
||||
return
|
||||
rank = int(os.environ["LOCAL_RANK"])
|
||||
if rank != 0:
|
||||
sys.stdout = open(os.devnull, "w")
|
||||
torch.cuda.set_device(rank)
|
||||
return sys.exit(inner())
|
||||
assert pid_key not in os.environ
|
||||
if name != "__main__":
|
||||
return logger.warning(
|
||||
f"{file} can not directly run with `pytest`. "
|
||||
"Use `python` to invoke it, which will internally relaunch it "
|
||||
"under torchrun for each requested number of GPUs."
|
||||
)
|
||||
num_devices = torch.cuda.device_count()
|
||||
override, forwarded_args = _extract_num_gpus_override(sys.argv[1:])
|
||||
if override is not None:
|
||||
logger.info(f"--num-gpu override: running only with {override}")
|
||||
num_gpus = override
|
||||
for N in num_gpus:
|
||||
if N <= 1 or N > num_devices:
|
||||
raise ValueError(
|
||||
f"Invalid number of GPUs requested: {N} (available: {num_devices})"
|
||||
)
|
||||
os.environ[env_key] = "1"
|
||||
os.environ[pid_key] = str(os.getpid())
|
||||
os.environ.setdefault("OMP_NUM_THREADS", "1")
|
||||
os.environ.setdefault("GLOO_SOCKET_IFNAME", "lo") # single-machine setup
|
||||
# Unbuffered child stdout: when a worker is killed on timeout, pytest's
|
||||
# block-buffered progress output is otherwise lost or flushed out of
|
||||
# order into the CI log, making it impossible to tell which test hung.
|
||||
os.environ.setdefault("PYTHONUNBUFFERED", "1")
|
||||
signal.signal(signal.SIGINT, signal.default_int_handler)
|
||||
runnable: List[int] = []
|
||||
for N in sorted(num_gpus):
|
||||
assert N > 1
|
||||
if N > num_devices:
|
||||
logger.warning(f"Skipping {kind} with {N} GPUs ({num_devices} available)")
|
||||
continue
|
||||
runnable.append(N)
|
||||
if pre_launch_fn is not None and runnable:
|
||||
logger.info(f"Running pre-launch hook for world sizes {runnable}")
|
||||
pre_launch_fn(runnable)
|
||||
for N in runnable:
|
||||
logger.info(f"Running {kind} with {N} GPUs")
|
||||
cmd = [
|
||||
"torchrun",
|
||||
"--nproc_per_node",
|
||||
str(N),
|
||||
"--local-addr",
|
||||
"127.0.0.1",
|
||||
file,
|
||||
]
|
||||
cmd += forwarded_args
|
||||
proc = subprocess.Popen(cmd, start_new_session=True)
|
||||
pgid = proc.pid
|
||||
returncode = -1
|
||||
timed_out = False
|
||||
try:
|
||||
returncode = proc.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
timed_out = True
|
||||
finally:
|
||||
_kill_descendants(os.getpid())
|
||||
_kill_pgroup(pgid)
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
if timed_out:
|
||||
logger.error(f"{kind} (nproc={N}) timed out after {timeout} seconds")
|
||||
sys.exit(1)
|
||||
if returncode != 0:
|
||||
logger.error(f"{kind} failed with {N} GPUs (exit {returncode})")
|
||||
sys.exit(returncode)
|
||||
logger.info(f"All {kind}s passed")
|
||||
|
||||
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.communication.all_reduce import Communicator
|
||||
|
||||
|
||||
_PRIMITIVES: Final = ["all_reduce", "all_gather", "reduce_scatter"]
|
||||
_PRIMITIVES: Final = ["all_gather"]
|
||||
|
||||
|
||||
def get_multicast_ptr(tensor: torch.Tensor) -> int:
|
||||
@@ -61,17 +61,6 @@ def _jit_push_module(dtype: torch.dtype, world_size: int) -> Module:
|
||||
|
||||
# `residual` on any of these is folded into the reduction; it may be shaped like
|
||||
# this rank's shard or like the whole tensor, of which this rank's slice is taken.
|
||||
def all_reduce_push(
|
||||
comm: Communicator,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
_jit_push_module(input.dtype, comm.world_size).all_reduce(
|
||||
comm, input, output, residual
|
||||
)
|
||||
|
||||
|
||||
def all_gather_push(
|
||||
comm: Communicator,
|
||||
input: torch.Tensor,
|
||||
@@ -83,39 +72,6 @@ def all_gather_push(
|
||||
)
|
||||
|
||||
|
||||
def reduce_scatter_push(
|
||||
comm: Communicator,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
_jit_push_module(input.dtype, comm.world_size).reduce_scatter(
|
||||
comm, input, output, residual
|
||||
)
|
||||
|
||||
|
||||
def all_reduce_pull(
|
||||
comm: Communicator,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
*,
|
||||
in_mc_ptr: int = 0,
|
||||
out_mc_ptr: int = 0,
|
||||
num_unroll=4,
|
||||
num_blocks_hint: int = 0,
|
||||
) -> None:
|
||||
_jit_pull_module(input.dtype, num_unroll).all_reduce(
|
||||
comm,
|
||||
input,
|
||||
output,
|
||||
residual,
|
||||
in_mc_ptr or get_multicast_ptr(input),
|
||||
out_mc_ptr or get_multicast_ptr(output),
|
||||
num_blocks_hint,
|
||||
)
|
||||
|
||||
|
||||
def all_gather_pull(
|
||||
comm: Communicator,
|
||||
input: torch.Tensor,
|
||||
@@ -134,23 +90,3 @@ def all_gather_pull(
|
||||
out_mc_ptr or get_multicast_ptr(output),
|
||||
num_blocks_hint,
|
||||
)
|
||||
|
||||
|
||||
def reduce_scatter_pull(
|
||||
comm: Communicator,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
*,
|
||||
in_mc_ptr: int = 0,
|
||||
num_unroll=4,
|
||||
num_blocks_hint: int = 0,
|
||||
) -> None:
|
||||
_jit_pull_module(input.dtype, num_unroll).reduce_scatter(
|
||||
comm,
|
||||
input,
|
||||
output,
|
||||
residual,
|
||||
in_mc_ptr or get_multicast_ptr(input),
|
||||
num_blocks_hint,
|
||||
)
|
||||
|
||||
@@ -117,20 +117,6 @@ def get_dispatch(
|
||||
)
|
||||
|
||||
|
||||
def get_tuning(
|
||||
kind: str,
|
||||
world_size: int,
|
||||
hidden_size: int,
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
) -> Optional[Tuning]:
|
||||
"""Compatibility helper for callers that only support staging push."""
|
||||
dispatch = get_dispatch(kind, world_size, hidden_size, num_tokens, device)
|
||||
if dispatch is None or dispatch.strategy != "push":
|
||||
return None
|
||||
return dispatch.tuning
|
||||
|
||||
|
||||
def get_fusion_dispatch(
|
||||
kind: str,
|
||||
world_size: int,
|
||||
|
||||
@@ -30,12 +30,7 @@ from __future__ import annotations
|
||||
from typing import Any
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import (
|
||||
CapabilityRequirement,
|
||||
FormatSignature,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
_CUDA = frozenset({CapabilityRequirement.CUDA})
|
||||
_CUDA_SM90_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))})
|
||||
@@ -49,654 +44,556 @@ _CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10,
|
||||
# ``scale_residual_norm_scale_shift`` in Triton, CuTe-DSL and FlyDSL) are
|
||||
# inventory: callers name the one they want via ``select_kernel``.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
|
||||
(
|
||||
"diffusion.residual_gate_fp32",
|
||||
KernelBackend.TRITON,
|
||||
"modulate.residual_gate_fp32:residual_gate_fp32",
|
||||
_CUDA,
|
||||
"Residual gating with separate FP32 product and sum rounding.",
|
||||
),
|
||||
(
|
||||
"diffusion.matrix_rope",
|
||||
KernelBackend.TRITON,
|
||||
"rope.matrix_rope:apply_matrix_rope",
|
||||
_CUDA,
|
||||
"Interleaved FP32 matrix RoPE with separate product and sum rounding.",
|
||||
),
|
||||
(
|
||||
"diffusion.fp8_rowwise",
|
||||
KernelBackend.TRITON,
|
||||
"quantization.fp8_rowwise_triton:fp8_rowwise",
|
||||
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
|
||||
"FLUX.3 dynamic rowwise E4M3 quantization and zero row padding.",
|
||||
),
|
||||
(
|
||||
"diffusion.gelu_tanh_cat",
|
||||
KernelBackend.JIT,
|
||||
"activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat",
|
||||
_CUDA,
|
||||
"BF16 tanh GELU plus channel concatenation.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_cast_before_weight",
|
||||
KernelBackend.TRITON,
|
||||
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_bitexact",
|
||||
_CUDA,
|
||||
"MOSSImage RMSNorm with ATen-ordered reduction and BF16 rounding before gamma.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_cast_before_weight_modulate",
|
||||
KernelBackend.TRITON,
|
||||
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_modulate_bitexact",
|
||||
_CUDA,
|
||||
"MOSSImage ATen-ordered RMSNorm with exact BF16 modulation.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_cast_before_weight_rope",
|
||||
KernelBackend.TRITON,
|
||||
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_rope_bitexact",
|
||||
_CUDA,
|
||||
"MOSSImage ATen-ordered head RMSNorm with exact rotate-half RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.apply_group_norm_silu",
|
||||
KernelBackend.TRITON,
|
||||
"norm.group_norm_silu:apply_group_norm_silu",
|
||||
_CUDA,
|
||||
"Fused GroupNorm + SiLU.",
|
||||
),
|
||||
(
|
||||
"diffusion.group_norm_silu_4d",
|
||||
KernelBackend.TRITON,
|
||||
"norm.group_norm_silu_twopass_triton:group_norm_silu_4d",
|
||||
_CUDA,
|
||||
"Channels-last two-pass GroupNorm(+SiLU), 4D.",
|
||||
),
|
||||
(
|
||||
"diffusion.group_norm_silu_rows",
|
||||
KernelBackend.TRITON,
|
||||
"norm.group_norm_silu_twopass_triton:group_norm_silu_rows",
|
||||
_CUDA,
|
||||
"Channels-last two-pass GroupNorm(+SiLU) over (N, L, C) rows.",
|
||||
),
|
||||
(
|
||||
"diffusion.wan_rmsnorm_silu",
|
||||
KernelBackend.TRITON,
|
||||
"norm.wan_rmsnorm_silu_triton:wan_rmsnorm_silu",
|
||||
_CUDA,
|
||||
"Wan VAE channels_last_3d RMSNorm + SiLU.",
|
||||
),
|
||||
(
|
||||
"diffusion.wan_norm_silu_post",
|
||||
KernelBackend.JIT,
|
||||
"norm.wan_norm_silu_post:wan_norm_silu_post",
|
||||
_CUDA,
|
||||
"Wan VAE FP32 normalization post-ops with the native denominator.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_scale_shift",
|
||||
KernelBackend.TRITON,
|
||||
"norm.rmsnorm_scale_shift_bitexact:fused_rmsnorm_scale_shift_bitexact",
|
||||
_CUDA,
|
||||
"Bit-exact RMSNorm + adaLN scale/shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.scale_residual_norm_scale_shift",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_scale_residual_norm_scale_shift",
|
||||
_CUDA_SM100_PLUS,
|
||||
"KDA B200 native CUDA residual + LayerNorm + scale/shift (#27392).",
|
||||
),
|
||||
(
|
||||
"diffusion.scale_residual_norm_scale_shift",
|
||||
KernelBackend.TRITON,
|
||||
"norm.rmsnorm_scale_shift_bitexact:fused_scale_residual_rmsnorm_scale_shift_bitexact",
|
||||
_CUDA,
|
||||
"Bit-exact residual-gate add + RMSNorm + scale/shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.scale_residual_norm_scale_shift",
|
||||
KernelBackend.CUTE_DSL,
|
||||
"norm.scale_residual_norm_cutedsl:fused_scale_residual_norm_scale_shift",
|
||||
_CUDA,
|
||||
"CuTe-DSL residual + norm + scale/shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.scale_residual_norm_scale_shift",
|
||||
KernelBackend.FLYDSL,
|
||||
"norm.fused_residual_norm_flydsl:flydsl_fused_residual_norm_scale_shift",
|
||||
_HIP,
|
||||
"FlyDSL (ROCm gfx950) residual + norm + scale/shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.scale_residual_norm_scale_shift_nvfp4",
|
||||
KernelBackend.JIT,
|
||||
"sglang.kernels.kda_kernels.norm_scale_shift_jit:try_fused_scale_residual_norm_scale_shift_nvfp4",
|
||||
_CUDA,
|
||||
"Qwen residual LayerNorm/modulation + NVFP4 quantization.",
|
||||
),
|
||||
(
|
||||
"diffusion.norm_scale_shift",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_norm_scale_shift",
|
||||
_CUDA_SM100_PLUS,
|
||||
"KDA B200 native CUDA LayerNorm + scale/shift (#27392).",
|
||||
),
|
||||
(
|
||||
"diffusion.norm_scale_shift",
|
||||
KernelBackend.CUTE_DSL,
|
||||
"norm.scale_residual_norm_cutedsl:fused_norm_scale_shift",
|
||||
_CUDA,
|
||||
"CuTe-DSL norm + scale/shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.norm_scale_shift",
|
||||
KernelBackend.FLYDSL,
|
||||
"norm.fused_residual_norm_flydsl:flydsl_norm_scale_shift",
|
||||
_HIP,
|
||||
"FlyDSL (ROCm gfx950) norm + scale/shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.layernorm_modulate",
|
||||
KernelBackend.TRITON,
|
||||
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate",
|
||||
_CUDA,
|
||||
"Bit-exact LayerNorm + adaLN modulate.",
|
||||
),
|
||||
(
|
||||
"diffusion.qk_head_layernorm",
|
||||
KernelBackend.TRITON,
|
||||
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_qk_head_layernorm",
|
||||
_CUDA,
|
||||
"Bit-exact per-head LayerNorm for q/k.",
|
||||
),
|
||||
(
|
||||
"diffusion.qk_rmsnorm_native",
|
||||
KernelBackend.TRITON,
|
||||
"norm.zimage_qk_rmsnorm_triton:zimage_qk_rmsnorm_native",
|
||||
_CUDA,
|
||||
"Z-Image bf16-native per-head QK RMSNorm.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_scale",
|
||||
KernelBackend.TRITON,
|
||||
"norm.native_bf16_rmsnorm_triton:rmsnorm_scale",
|
||||
_CUDA,
|
||||
"BF16-native RMSNorm * scale.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_tanh_residual",
|
||||
KernelBackend.TRITON,
|
||||
"norm.native_bf16_rmsnorm_triton:rmsnorm_tanh_residual",
|
||||
_CUDA,
|
||||
"BF16-native x + tanh(gate) * RMSNorm(y).",
|
||||
),
|
||||
(
|
||||
"diffusion.modulate_scale_shift",
|
||||
KernelBackend.JIT,
|
||||
"modulate.modulate_scale_shift_jit:modulate_scale_shift",
|
||||
_CUDA,
|
||||
"Bit-exact adaLN modulate x * (1 + scale) + shift.",
|
||||
),
|
||||
(
|
||||
"diffusion.residual_gate_add",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.residual_gate_add_jit:residual_gate_add",
|
||||
_CUDA,
|
||||
"KDA native CUDA residual + gate * update (#29361).",
|
||||
),
|
||||
(
|
||||
"diffusion.residual_gate_add2",
|
||||
KernelBackend.JIT,
|
||||
"modulate.residual_gate_add2_jit:residual_gate_add2",
|
||||
_CUDA,
|
||||
"Fused residual + gate * (update_a + update_b) for two-branch blocks.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux3_fp8_linear",
|
||||
KernelBackend.FLASHINFER,
|
||||
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_linear",
|
||||
_CUDA_SM10X,
|
||||
"FLUX.3 rowwise-FP8 PrimsTS GEMM.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux3_fp8_linear_swiglu",
|
||||
KernelBackend.FLASHINFER,
|
||||
"gemm.flux3_fp8_prims_ts:flux3_fused_mlp_up",
|
||||
_CUDA_SM10X,
|
||||
"FLUX.3 rowwise-FP8 PrimsTS GEMM with SwiGLU epilogue.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux3_fp8_qkv_qknorm_rope",
|
||||
KernelBackend.FLASHINFER,
|
||||
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_qkv_qknorm_rope",
|
||||
_CUDA_SM10X,
|
||||
"FLUX.3 rowwise-FP8 PrimsTS QKV GEMM with QK-norm/RoPE epilogue.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux3_nvfp4_linear",
|
||||
KernelBackend.FLASHINFER,
|
||||
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear",
|
||||
_CUDA_SM10X,
|
||||
"FLUX.3 NVFP4 PrimsTS GEMM.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux3_nvfp4_linear_swiglu",
|
||||
KernelBackend.FLASHINFER,
|
||||
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear_swiglu",
|
||||
_CUDA_SM10X,
|
||||
"FLUX.3 NVFP4 PrimsTS GEMM with SwiGLU epilogue.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux3_nvfp4_qkv_qknorm_rope",
|
||||
KernelBackend.FLASHINFER,
|
||||
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_qkv_qknorm_rope",
|
||||
_CUDA_SM10X,
|
||||
"FLUX.3 NVFP4 PrimsTS QKV GEMM with QK-norm/RoPE epilogue.",
|
||||
),
|
||||
(
|
||||
"diffusion.timestep_embedding",
|
||||
KernelBackend.JIT,
|
||||
"modulate.timestep_embedding_jit:timestep_embedding",
|
||||
_CUDA,
|
||||
"Sinusoidal timestep embedding.",
|
||||
),
|
||||
(
|
||||
"diffusion.temb_table_slices",
|
||||
KernelBackend.TRITON,
|
||||
"modulate.wan_temb_table_slices_triton:fused_temb_table_slices",
|
||||
_CUDA,
|
||||
"Contiguous adaLN slices for Wan2.2-TI2V.",
|
||||
),
|
||||
(
|
||||
"diffusion.ltx2_ada_values",
|
||||
KernelBackend.TRITON,
|
||||
"modulate.ltx2_ada_values_triton:ltx2_ada_values9",
|
||||
_CUDA,
|
||||
"LTX-2 nine-way adaLN value split.",
|
||||
),
|
||||
(
|
||||
"diffusion.fused_inplace_qknorm_rope",
|
||||
KernelBackend.JIT,
|
||||
"rope.qknorm_rope_jit:fused_inplace_qknorm_rope",
|
||||
_CUDA,
|
||||
"Fused in-place QK RMS-norm + RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.vsa_block_sparse_sm100",
|
||||
KernelBackend.JIT,
|
||||
"attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100",
|
||||
frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}),
|
||||
"FastVideo's warp-specialized tcgen05 block-sparse VSA forward (64-token tiles).",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_rmsnorm",
|
||||
KernelBackend.TRITON,
|
||||
"norm.h3_vae_fused_norm:h3_vae_rmsnorm",
|
||||
_CUDA,
|
||||
"FP32-accumulation RMSNorm for the MiniMax-H3 video VAE decoder.",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_scale_add_rmsnorm",
|
||||
KernelBackend.TRITON,
|
||||
"norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm",
|
||||
_CUDA,
|
||||
"Scaled residual add plus the following RMSNorm for the H3 VAE decoder.",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_qk_rmsnorm_rope",
|
||||
KernelBackend.TRITON,
|
||||
"norm.h3_vae_fused_norm:h3_vae_qk_rmsnorm_rope",
|
||||
_CUDA,
|
||||
"Affine-free Q/K RMSNorm plus partial NeoX RoPE for the H3 VAE decoder.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux2_layernorm_modulate_fp8_quant",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate_fp8_quant_raw",
|
||||
_CUDA,
|
||||
"KDA-generated FLUX.2 LayerNorm + adaLN modulation + static FP8 quantization.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux2_qkv_epilogue",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit:try_fused_flux2_qkv_epilogue",
|
||||
_CUDA,
|
||||
"KDA-generated FLUX.2 QK RMS-norm + RoPE + joint QKV packing.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux2_token_cat_fp8",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton:try_flux2_token_cat_fp8",
|
||||
_CUDA,
|
||||
"KDA-generated FLUX.2 token concatenation + static FP8 quantization.",
|
||||
),
|
||||
(
|
||||
"diffusion.qwen_qkv_epilogue",
|
||||
KernelBackend.JIT,
|
||||
"rope.qwen_qkv_epilogue_jit:try_fused_qwen_qkv_epilogue",
|
||||
_CUDA_SM90_PLUS,
|
||||
"Qwen-Image QK RMS-norm, RoPE, and joint QKV writes.",
|
||||
),
|
||||
(
|
||||
"diffusion.ltx2_qknorm_split_rope",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit:ltx2_qknorm_split_rope_cuda",
|
||||
_CUDA_SM100_PLUS,
|
||||
"KDA native CUDA LTX-2 QK-norm + split RoPE (#29708).",
|
||||
),
|
||||
(
|
||||
"diffusion.ltx25_decoder_rope",
|
||||
KernelBackend.JIT,
|
||||
"rope.ltx25_decoder_rope_jit:fused_ltx25_decoder_rope",
|
||||
_CUDA,
|
||||
"Paired LTX-2.5 decoder 3D RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.rope_rotate_half_fp32",
|
||||
KernelBackend.TRITON,
|
||||
"rope.rope_rotate_half_fp32:fused_rope_rotate_half_fp32",
|
||||
_CUDA,
|
||||
"Paired split-half RoPE with FP32 arithmetic and one output cast.",
|
||||
),
|
||||
(
|
||||
"diffusion.rope_rotate_half",
|
||||
KernelBackend.TRITON,
|
||||
"rope.rope_rotate_half_bitexact:fused_rope_rotate_half_bitexact",
|
||||
_CUDA,
|
||||
"Bit-exact rotate-half RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.interleaved_rope_fp64",
|
||||
KernelBackend.JIT,
|
||||
"rope.interleaved_rope_fp64_jit:fused_interleaved_rope_fp64",
|
||||
_CUDA,
|
||||
"Paired interleaved RoPE with fp64 Diffusers semantics.",
|
||||
),
|
||||
(
|
||||
"diffusion.helios_qk_rope",
|
||||
KernelBackend.JIT,
|
||||
"rope.helios_qk_rope_jit:fused_inplace_helios_qk_rope",
|
||||
_CUDA,
|
||||
"Paired in-place Helios transposed Q/K RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.complex_rope",
|
||||
KernelBackend.TRITON,
|
||||
"rope.complex_rope_triton:fused_complex_rope",
|
||||
_CUDA,
|
||||
"Paired RoPE preserving PyTorch complex64 multiplication rounding.",
|
||||
),
|
||||
(
|
||||
"diffusion.hunyuan_qkv_rope_pack",
|
||||
KernelBackend.TRITON,
|
||||
"rope.hunyuan_qkv_pack_triton:hunyuan_qkv_rope_pack",
|
||||
_CUDA,
|
||||
"HunyuanVideo QKV pack + RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.joint_qkv_cat",
|
||||
KernelBackend.TRITON,
|
||||
"layout.joint_qkv_cat_triton:joint_qkv_cat",
|
||||
_CUDA,
|
||||
"Concatenate image/text QKV views into joint attention inputs.",
|
||||
),
|
||||
(
|
||||
"diffusion.rmsnorm_preserve_reduction",
|
||||
KernelBackend.TRITON,
|
||||
"norm.rmsnorm_preserve_reduction:rmsnorm_preserve_reduction",
|
||||
_CUDA,
|
||||
"Cast-before-weight RMSNorm preserving the native FP32 mean reduction.",
|
||||
),
|
||||
(
|
||||
"diffusion.silu_mul",
|
||||
KernelBackend.TRITON,
|
||||
"activation.silu_mul_bitexact:fused_silu_mul_bitexact",
|
||||
_CUDA,
|
||||
"Bit-exact silu(a) * b for split-projection SwiGLU.",
|
||||
),
|
||||
(
|
||||
"diffusion.situ_softcap_mul",
|
||||
KernelBackend.TRITON,
|
||||
"activation.situ_softcap_mul_bitexact:fused_situ_softcap_mul_bitexact",
|
||||
_CUDA,
|
||||
"Bit-exact bounded SiTU for MOSSImage FFN projections.",
|
||||
),
|
||||
(
|
||||
"diffusion.bias_silu",
|
||||
KernelBackend.TRITON,
|
||||
"activation.sana_conv_post_triton:fused_bias_silu",
|
||||
_CUDA,
|
||||
"Bit-exact conv bias + SiLU (Sana GLUMB).",
|
||||
),
|
||||
(
|
||||
"diffusion.bias_glu",
|
||||
KernelBackend.TRITON,
|
||||
"activation.sana_conv_post_triton:fused_bias_glu",
|
||||
_CUDA,
|
||||
"Bit-exact conv bias + GLU (Sana GLUMB).",
|
||||
),
|
||||
(
|
||||
"diffusion.linear_gelu_tanh",
|
||||
KernelBackend.AOT,
|
||||
"sites.fused_linear_gelu_site:fused_linear_gelu_tanh",
|
||||
_CUDA,
|
||||
"Linear + tanh-GELU via the cublasLt epilogue.",
|
||||
),
|
||||
(
|
||||
"diffusion.sparse_linear_attn_fwd",
|
||||
KernelBackend.TRITON,
|
||||
"attention.sparse_linear_attn_triton:_attn_fwd",
|
||||
_CUDA,
|
||||
"Sparse linear attention forward.",
|
||||
),
|
||||
(
|
||||
"diffusion.bigdn",
|
||||
KernelBackend.TRITON,
|
||||
"attention.sana_wm_gdn_triton:fused_bigdn_func",
|
||||
_CUDA,
|
||||
"Sana-WM bidirectional gated delta-net.",
|
||||
),
|
||||
(
|
||||
"diffusion.fused_qknorm_rope_out_of_place",
|
||||
KernelBackend.JIT,
|
||||
"rope.qknorm_rope_jit:fused_qknorm_rope_out_of_place",
|
||||
_CUDA,
|
||||
"Out-of-place fused QK-norm + RoPE (raw q/k preserved).",
|
||||
),
|
||||
(
|
||||
"diffusion.qknorm_complex_rope_cuda",
|
||||
KernelBackend.JIT,
|
||||
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda",
|
||||
_CUDA,
|
||||
"Bit-exact 128-wide RMSNorm + complex RoPE (Qwen-Image 2.1), CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.qknorm_complex_rope_pack_",
|
||||
KernelBackend.JIT,
|
||||
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_",
|
||||
_CUDA,
|
||||
"Q/K RMSNorm + complex RoPE with prefix K/V packing into [B, P+S, H, 128] buffers, CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.channel_rmsnorm_finish_silu",
|
||||
KernelBackend.JIT,
|
||||
"norm.channel_rmsnorm_finish_silu_jit:channel_rmsnorm_finish_silu",
|
||||
_CUDA,
|
||||
"Bit-exact channel-first RMSNorm finish + SiLU on aten's fp32 channel norm (Qwen-Image 2.1 VAE), CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.channel_rmsnorm_silu_nhwc",
|
||||
KernelBackend.JIT,
|
||||
"norm.channel_rmsnorm_silu_nhwc_jit:channel_rmsnorm_silu_nhwc",
|
||||
_CUDA,
|
||||
"Channels-last channel RMSNorm + SiLU with an in-warp reduction (Qwen-Image 2.1 VAE, quality-gated), CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.bias_residual_add",
|
||||
KernelBackend.JIT,
|
||||
"norm.bias_residual_add_jit:bias_residual_add",
|
||||
_CUDA,
|
||||
"Bit-exact conv bias + residual add in one pass for channels_last or NCHW bf16 tensors, CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_delta_factors",
|
||||
KernelBackend.JIT,
|
||||
"attention.vdn_delta_factors_jit:vdn_delta_factors",
|
||||
_CUDA,
|
||||
"VDN-H3 delta rule: fused (I + A)^-1 -> transition / injection (fp32, head_dim 128).",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_temporal_conv_act",
|
||||
KernelBackend.TRITON,
|
||||
"attention.vdn_linear_branch_triton:vdn_temporal_conv_act",
|
||||
_CUDA,
|
||||
"VDN-H3 linear branch: 5-tap temporal conv + SiLU + L2 norm.",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_silu_l2norm",
|
||||
KernelBackend.TRITON,
|
||||
"attention.vdn_linear_branch_triton:vdn_silu_l2norm",
|
||||
_CUDA,
|
||||
"VDN-H3 linear branch: SiLU + L2 norm over head_dim.",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_frame_stats_prep",
|
||||
KernelBackend.TRITON,
|
||||
"attention.vdn_linear_branch_triton:vdn_frame_stats_prep",
|
||||
_CUDA,
|
||||
"VDN-H3 linear branch: frame-statistics GEMM operands in one pass.",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_gather_linear_state",
|
||||
KernelBackend.TRITON,
|
||||
"attention.vdn_linear_branch_triton:vdn_gather_linear_state",
|
||||
_CUDA,
|
||||
"VDN-H3 linear branch: alpha-bridged boundary gather in one pass.",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_linear_epilogue",
|
||||
KernelBackend.TRITON,
|
||||
"attention.vdn_linear_branch_triton:vdn_linear_epilogue",
|
||||
_CUDA,
|
||||
"VDN-H3 linear branch: RMSNorm * gate readout epilogue.",
|
||||
),
|
||||
(
|
||||
"diffusion.mxfp8_quantize_swizzled",
|
||||
KernelBackend.TRITON,
|
||||
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
|
||||
_CUDA,
|
||||
"bf16 -> MXFP8 (e4m3, block-32 E8M0 scales in the cuBLASLt swizzled layout).",
|
||||
),
|
||||
(
|
||||
"diffusion.silu_mul_mxfp8",
|
||||
KernelBackend.TRITON,
|
||||
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
|
||||
_CUDA,
|
||||
"SwiGLU + MXFP8 quant for the online mxfp8 fc2 input.",
|
||||
),
|
||||
(
|
||||
"diffusion.indexed_scale_shift_mxfp8_",
|
||||
KernelBackend.TRITON,
|
||||
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
|
||||
_CUDA,
|
||||
"Indexed adaLN modulation + MXFP8 quant for the online mxfp8 qkv/fc1 inputs.",
|
||||
),
|
||||
(
|
||||
"diffusion.group_limited_topk",
|
||||
KernelBackend.TRITON,
|
||||
"routing.group_limited_topk_triton:group_limited_topk",
|
||||
_CUDA,
|
||||
"LingBot Video group-limited MoE top-k expert selection.",
|
||||
),
|
||||
(
|
||||
"diffusion.usp_merge_heads",
|
||||
KernelBackend.JIT,
|
||||
"layout.usp_relayout_jit:usp_merge_heads",
|
||||
_CUDA,
|
||||
"USP all-to-all output head merge.",
|
||||
),
|
||||
(
|
||||
"diffusion.pack_qkv_destination_major",
|
||||
KernelBackend.TRITON,
|
||||
"layout.ulysses_qkv_triton:pack_qkv_destination_major",
|
||||
_CUDA,
|
||||
"Ulysses destination-major QKV pack.",
|
||||
),
|
||||
(
|
||||
"diffusion.varlen_pack_qkv",
|
||||
KernelBackend.TRITON,
|
||||
"layout.varlen_pack_pad_triton:fused_pack_qkv",
|
||||
_CUDA,
|
||||
"Varlen gather of Q/K/V at valid positions.",
|
||||
),
|
||||
(
|
||||
"diffusion.varlen_pack_segmented_qkv",
|
||||
KernelBackend.TRITON,
|
||||
"layout.varlen_pack_pad_triton:fused_pack_segmented_qkv",
|
||||
_CUDA,
|
||||
"Varlen gather from a virtual prefix/main Q/K/V sequence.",
|
||||
),
|
||||
(
|
||||
"diffusion.varlen_scatter_to_padded",
|
||||
KernelBackend.TRITON,
|
||||
"layout.varlen_pack_pad_triton:fused_scatter_to_padded",
|
||||
_CUDA,
|
||||
"Varlen scatter back to the dense layout.",
|
||||
),
|
||||
(
|
||||
"diffusion.causal_conv3d_cat_pad",
|
||||
KernelBackend.KDA,
|
||||
"sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit:fused_causal_conv3d_cat_pad_cuda",
|
||||
_CUDA,
|
||||
"KDA native CUDA causal Conv3d cat + pad (#29281).",
|
||||
),
|
||||
(
|
||||
"diffusion.causal_conv3d_cat_pad",
|
||||
KernelBackend.TRITON,
|
||||
"layout.causal_conv3d_cat_pad_triton:fused_causal_conv3d_cat_pad",
|
||||
_CUDA,
|
||||
"Causal Conv3d cat + pad (Triton).",
|
||||
),
|
||||
(
|
||||
"diffusion.cat_pad_channels_last_3d",
|
||||
KernelBackend.TRITON,
|
||||
"layout.wan_causal_cache_triton:cat_pad_channels_last_3d",
|
||||
_CUDA,
|
||||
"Wan causal VAE cat + pad in channels_last_3d.",
|
||||
),
|
||||
(
|
||||
"diffusion.dup_up3d_add",
|
||||
KernelBackend.TRITON,
|
||||
"layout.wan_causal_cache_triton:dup_up3d_add",
|
||||
_CUDA,
|
||||
"Wan causal VAE main + DupUp3D(src).",
|
||||
),
|
||||
(
|
||||
"diffusion.conv_bias_epilogue",
|
||||
KernelBackend.TRITON,
|
||||
"modulate.conv_bias_epilogue_triton:conv_bias_epilogue",
|
||||
_CUDA,
|
||||
"Bit-exact conv bias (+ residual) epilogue on channels_last outputs.",
|
||||
),
|
||||
(
|
||||
"diffusion.nearest_upsample_nhwc",
|
||||
KernelBackend.TRITON,
|
||||
"layout.nearest_upsample_nhwc_triton:nearest_upsample_nhwc",
|
||||
_CUDA,
|
||||
"Wan-family VAE channels_last integer-factor nearest upsample.",
|
||||
),
|
||||
(
|
||||
"diffusion.flux2_token_cat_nvfp4",
|
||||
KernelBackend.JIT,
|
||||
"layout.flux2_token_cat_nvfp4_jit:try_flux2_token_cat_nvfp4",
|
||||
_CUDA,
|
||||
"FLUX.2 single-block token concatenation + NVFP4 quantization.",
|
||||
),
|
||||
)
|
||||
|
||||
for _op, _backend, _target, _caps, _description in _SPECS:
|
||||
for _op, _backend, _target, _caps in _SPECS:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=_op,
|
||||
@@ -707,8 +604,6 @@ for _op, _backend, _target, _caps, _description in _SPECS:
|
||||
else f"sglang.kernels.ops.diffusion.{_target}"
|
||||
),
|
||||
capabilities=_caps,
|
||||
format_signature=FormatSignature(description=_description),
|
||||
description=_description,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -786,6 +681,12 @@ _EXPORTS: dict[str, str] = {
|
||||
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
||||
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
|
||||
"zimage_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
|
||||
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
# adaLN modulation, gating and timestep conditioning
|
||||
"indexed_gate_bf16": "modulate.indexed_modulation_triton",
|
||||
"indexed_gate_bf16_": "modulate.indexed_modulation_triton",
|
||||
@@ -906,80 +807,31 @@ _EXPORTS: dict[str, str] = {
|
||||
"BitExactFusionGate": "sites.bitexact_gate",
|
||||
"flashinfer_rmsnorm_diagnostic_hint": "sites.bitexact_gate",
|
||||
"tensors_equal": "sites.bitexact_gate",
|
||||
"fused_gate_rmsnorm_active": "sites.fused_gate_rmsnorm_site",
|
||||
"FLUX2_NVFP4_SWIGLU_QUANT": "sites.flux2_nvfp4_swiglu_quant_site",
|
||||
"FUSED_GATE_RMSNORM": "sites.fused_gate_rmsnorm_site",
|
||||
"fused_rmsnorm_scale": "sites.fused_gate_rmsnorm_site",
|
||||
"fused_rmsnorm_tanh_residual": "sites.fused_gate_rmsnorm_site",
|
||||
"mark_fused_gate_rmsnorm_site": "sites.fused_gate_rmsnorm_site",
|
||||
"mount_fused_gate_rmsnorm": "sites.fused_gate_rmsnorm_site",
|
||||
"unmount_fused_gate_rmsnorm": "sites.fused_gate_rmsnorm_site",
|
||||
"FUSED_LINEAR_GELU": "sites.fused_linear_gelu_site",
|
||||
"can_use_linear_gelu": "sites.fused_linear_gelu_site",
|
||||
"fused_gelu_active": "sites.fused_linear_gelu_site",
|
||||
"fused_linear_gelu_tanh": "sites.fused_linear_gelu_site",
|
||||
"mark_fused_gelu_site": "sites.fused_linear_gelu_site",
|
||||
"mount_fused_linear_gelu": "sites.fused_linear_gelu_site",
|
||||
"unmount_fused_linear_gelu": "sites.fused_linear_gelu_site",
|
||||
"flux2_nvfp4_swiglu_quant_active": "sites.flux2_nvfp4_swiglu_quant_site",
|
||||
"mark_flux2_nvfp4_swiglu_quant_site": "sites.flux2_nvfp4_swiglu_quant_site",
|
||||
"mount_flux2_nvfp4_swiglu_quant": "sites.flux2_nvfp4_swiglu_quant_site",
|
||||
"unmount_flux2_nvfp4_swiglu_quant": "sites.flux2_nvfp4_swiglu_quant_site",
|
||||
"mark_nvfp4_bias_gelu_site": "sites.nvfp4_bias_gelu_site",
|
||||
"mount_nvfp4_bias_gelu": "sites.nvfp4_bias_gelu_site",
|
||||
"nvfp4_bias_gelu_active": "sites.nvfp4_bias_gelu_site",
|
||||
"unmount_nvfp4_bias_gelu": "sites.nvfp4_bias_gelu_site",
|
||||
"mark_qwen_image_added_qkv_site": "sites.qwen_image_added_qkv_site",
|
||||
"mount_qwen_image_added_qkv": "sites.qwen_image_added_qkv_site",
|
||||
"qwen_image_added_qkv_active": "sites.qwen_image_added_qkv_site",
|
||||
"unmount_qwen_image_added_qkv": "sites.qwen_image_added_qkv_site",
|
||||
"FUSED_LN_MODULATE": "sites.fused_ln_modulate_site",
|
||||
"can_use_ln_modulate": "sites.fused_ln_modulate_site",
|
||||
"fused_ln_modulate": "sites.fused_ln_modulate_site",
|
||||
"fused_ln_modulate_active": "sites.fused_ln_modulate_site",
|
||||
"mark_fused_ln_modulate_site": "sites.fused_ln_modulate_site",
|
||||
"mount_fused_ln_modulate": "sites.fused_ln_modulate_site",
|
||||
"unmount_fused_ln_modulate": "sites.fused_ln_modulate_site",
|
||||
"mark_hunyuan_qknorm_site": "sites.hunyuan_qknorm_site",
|
||||
"mount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
||||
"HELIOS_GATED_RESIDUAL": "sites.gated_residual_site",
|
||||
"LINGBOT_VIDEO_GATED_RESIDUAL": "sites.gated_residual_site",
|
||||
"try_gated_residual": "sites.gated_residual_site",
|
||||
"HUNYUAN_QKNORM": "sites.hunyuan_qknorm_site",
|
||||
"try_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
||||
"unmount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
||||
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"ltx2_rms_norm_modulate_active": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"mark_ltx2_rms_norm_modulate_site": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"mount_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"unmount_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"ltx2_qknorm_split_rope_active": "sites.ltx2_qknorm_split_rope_site",
|
||||
"mark_ltx2_qknorm_split_rope_site": "sites.ltx2_qknorm_split_rope_site",
|
||||
"mount_ltx2_qknorm_split_rope": "sites.ltx2_qknorm_split_rope_site",
|
||||
"unmount_ltx2_qknorm_split_rope": "sites.ltx2_qknorm_split_rope_site",
|
||||
"lingbot_video_rmsnorm_active": "sites.lingbot_video_rmsnorm_site",
|
||||
"mark_lingbot_video_rmsnorm_site": "sites.lingbot_video_rmsnorm_site",
|
||||
"mount_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
|
||||
"LINGBOT_VIDEO_RMSNORM": "sites.lingbot_video_rmsnorm_site",
|
||||
"try_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
|
||||
"unmount_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
|
||||
"helios_gated_residual_active": "sites.helios_gated_residual_site",
|
||||
"mark_helios_gated_residual_site": "sites.helios_gated_residual_site",
|
||||
"mount_helios_gated_residual": "sites.helios_gated_residual_site",
|
||||
"try_helios_gated_residual": "sites.helios_gated_residual_site",
|
||||
"unmount_helios_gated_residual": "sites.helios_gated_residual_site",
|
||||
"lingbot_video_gated_residual_active": "sites.lingbot_video_gated_residual_site",
|
||||
"mark_lingbot_video_gated_residual_site": "sites.lingbot_video_gated_residual_site",
|
||||
"mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site",
|
||||
"mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site",
|
||||
"minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site",
|
||||
"mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
|
||||
"unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site",
|
||||
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
||||
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
||||
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
|
||||
"h3_vae_qk_rmsnorm_rope": "norm.h3_vae_fused_norm",
|
||||
"mark_sana_video_linear_attention_site": "sites.sana_video_linear_attention_site",
|
||||
"mount_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
|
||||
"sana_video_linear_attention_active": "sites.sana_video_linear_attention_site",
|
||||
"LTX2_QKNORM_SPLIT_ROPE": "sites.ltx2_qknorm_split_rope_site",
|
||||
"LTX2_RMS_NORM_MODULATE": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"MINIMAX_H3_NORM_MODULATE": "sites.minimax_h3_norm_modulate_site",
|
||||
"NVFP4_BIAS_GELU": "sites.nvfp4_bias_gelu_site",
|
||||
"QWEN_IMAGE_ADDED_QKV": "sites.qwen_image_added_qkv_site",
|
||||
"SANA_VIDEO_LINEAR_ATTENTION": "sites.sana_video_linear_attention_site",
|
||||
"try_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
|
||||
"unmount_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
|
||||
"QualityGatedFusion": "sites.quality_gate",
|
||||
# JIT C++/CUDA extensions (not kernels, not in the registry)
|
||||
"interpolate": "ext.hunyuan3d_rasterizer",
|
||||
|
||||
@@ -166,7 +166,7 @@ _CHUNKWISE_TUNING: dict[tuple[str, str], _ChunkwiseCfg] = {
|
||||
# nw=16 OOMs the 102 KB SRAM cap at TF32 (needs 131 KB); nw=8 fits and is
|
||||
# within noise. The D-tile path (auto-enabled on spark, see
|
||||
# `_pick_phase_b_d_splits`) is ~2.6× faster at TF32 / ~13% at IEEE — these
|
||||
# baseline params only apply when PHASE_B_D_SPLITS=1 is forced.
|
||||
# baseline params only apply where that picks d_splits=1.
|
||||
B=_PhaseCfg(nw=8, use_acc=False, ns=1),
|
||||
C=_PhaseCfg(nw=8, BS=16, ns=1), # binding constraint: M.fp32 64 KB + Q stage
|
||||
),
|
||||
@@ -1206,20 +1206,7 @@ def _pick_phase_b_d_splits(BLOCK_D: int, dot_precision: int = 0):
|
||||
`d_splits>1` → `_phase_b_dtile_kernel` with nw/ns/acc overrides.
|
||||
Per-arch optima from a 96-config sweep (2026-04-29, T=11 B=1 IEEE) and a
|
||||
multi-arch bf16/TF32 sweep (2026-05-06, F=11 S=920); see inline cfg comments.
|
||||
Env overrides: PHASE_B_D_SPLITS, PHASE_B_DTILE_NW/NS/ACC (1=True / 0=False).
|
||||
"""
|
||||
import os
|
||||
|
||||
env_d = os.environ.get("PHASE_B_D_SPLITS", None)
|
||||
if env_d is not None:
|
||||
d = int(env_d)
|
||||
if d < 1 or BLOCK_D % d != 0:
|
||||
return (1, None, None, None)
|
||||
nw = int(os.environ.get("PHASE_B_DTILE_NW", "0")) or None
|
||||
ns = int(os.environ.get("PHASE_B_DTILE_NS", "0")) or None
|
||||
acc_env = os.environ.get("PHASE_B_DTILE_ACC", None)
|
||||
acc = bool(int(acc_env)) if acc_env is not None else None
|
||||
return (d, nw, ns, acc)
|
||||
try:
|
||||
import torch
|
||||
|
||||
@@ -1617,18 +1604,6 @@ def fused_bigdn_bidi_chunkwise(
|
||||
return out
|
||||
|
||||
|
||||
def _default_dot_prec() -> int:
|
||||
try:
|
||||
from sglang.kernels.ops.diffusion.attention.sana_wm_gdn_triton import (
|
||||
_resolve_launch_config,
|
||||
)
|
||||
|
||||
_, dot_prec, _, _ = _resolve_launch_config()
|
||||
return dot_prec
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _cam_identity_tables(
|
||||
*,
|
||||
B: int,
|
||||
@@ -1683,7 +1658,7 @@ def cam_scan_bidi_chunkwise(
|
||||
assert decay.shape == (B, H, F)
|
||||
|
||||
if dot_precision is None:
|
||||
dot_precision = _default_dot_prec()
|
||||
dot_precision = 0
|
||||
|
||||
qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype)
|
||||
qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2))
|
||||
|
||||
@@ -14,85 +14,16 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""Inference-side helpers for the bidirectional fused GDN path.
|
||||
|
||||
Precision knob: env var ``FUSED_GDN_PRECISION`` or ``PRECISION_OVERRIDE``:
|
||||
0=IEEE fp32 dots, 1=TF32, 2=bf16 TC + fp32 state [default], 3=bf16 TC + bf16 state.
|
||||
"""
|
||||
"""Inference-side helpers for the bidirectional fused GDN path."""
|
||||
|
||||
# ruff: noqa: E501
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
# =====================================================================
|
||||
# GPU-adaptive kernel config
|
||||
# =====================================================================
|
||||
|
||||
|
||||
def _get_kernel_config() -> dict:
|
||||
"""Return optimal kernel parameters for the current GPU.
|
||||
|
||||
STATE_FP32 (fp32 state_prev) needs ~128KB SRAM (H100 228KB), vs ~96KB for
|
||||
bf16 state_prev (fits GB10's 101KB).
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 4, "STATE_FP32": False}
|
||||
smem = torch.cuda.get_device_properties(0).shared_memory_per_multiprocessor
|
||||
state_fp32 = smem >= 150 * 1024 # H100 (228KB) yes, GB10 (101KB) no
|
||||
return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 8, "STATE_FP32": state_fp32}
|
||||
|
||||
|
||||
_KCFG = None
|
||||
|
||||
|
||||
def _kcfg():
|
||||
global _KCFG
|
||||
if _KCFG is None:
|
||||
_KCFG = _get_kernel_config()
|
||||
return _KCFG
|
||||
|
||||
|
||||
# precision=0 → IEEE fp32 dots + fp32 state (DOT_PRECISION=2, STATE_FP32=1)
|
||||
# precision=1 → TF32 dots + fp32 state (DOT_PRECISION=1, STATE_FP32=1)
|
||||
# precision=2 → bf16 dots + fp32 state (DOT_PRECISION=0, STATE_FP32=1) [default]
|
||||
# precision=3 → bf16 dots + bf16 state (DOT_PRECISION=0, STATE_FP32=0)
|
||||
def _precision_params(precision: int) -> tuple:
|
||||
if precision == 0:
|
||||
return 2, True
|
||||
elif precision == 1:
|
||||
return 1, True
|
||||
elif precision == 3:
|
||||
return 0, False
|
||||
else: # default
|
||||
return 0, True
|
||||
|
||||
|
||||
_env_prec = os.environ.get("FUSED_GDN_PRECISION", None)
|
||||
PRECISION_OVERRIDE: int | None = int(_env_prec) if _env_prec is not None else None
|
||||
|
||||
|
||||
def _resolve_launch_config() -> tuple:
|
||||
"""Returns (prec, dot_prec, state_fp32, num_warps).
|
||||
|
||||
Uses ``PRECISION_OVERRIDE`` when set, else ``_kcfg()`` (per-GPU SRAM).
|
||||
num_warps clamped to 4 when dots run on fp32 operands (more registers).
|
||||
"""
|
||||
cfg = _kcfg()
|
||||
prec = PRECISION_OVERRIDE if PRECISION_OVERRIDE is not None else 2
|
||||
dot_prec, state_fp32 = _precision_params(prec)
|
||||
if PRECISION_OVERRIDE is None:
|
||||
state_fp32 = cfg["STATE_FP32"]
|
||||
nw = cfg["num_warps"]
|
||||
if dot_prec >= 1:
|
||||
nw = min(nw, 4)
|
||||
return prec, dot_prec, state_fp32, nw
|
||||
|
||||
|
||||
def prepare_rope_tables(
|
||||
rotary_emb, N: int, D: int, device
|
||||
|
||||
@@ -33,11 +33,16 @@ from typing import Any, Callable
|
||||
import torch
|
||||
from torch.utils._python_dispatch import TorchDispatchMode
|
||||
from torch.utils._pytree import tree_leaves
|
||||
from triton import knobs
|
||||
from triton.knobs import HookChain
|
||||
from triton.runtime import driver
|
||||
from triton.runtime.jit import JITFunction
|
||||
|
||||
try:
|
||||
from triton import knobs
|
||||
from triton.knobs import HookChain
|
||||
except ImportError:
|
||||
# Vendor Triton builds may not expose the upstream launch-hook API.
|
||||
knobs = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ENABLED = os.environ.get(
|
||||
@@ -162,6 +167,9 @@ class _Plan:
|
||||
|
||||
def _launch_hooked() -> bool:
|
||||
"""A profiler such as proton hooks Triton launches; those go through the runner."""
|
||||
if knobs is None:
|
||||
# Keep the regular runner when its hooks cannot be inspected safely.
|
||||
return True
|
||||
return any(
|
||||
hook is not None and not (isinstance(hook, HookChain) and not hook.calls)
|
||||
for hook in (knobs.runtime.launch_enter_hook, knobs.runtime.launch_exit_hook)
|
||||
|
||||
@@ -27,8 +27,6 @@ from typing import Callable, TypeVar
|
||||
|
||||
F = TypeVar("F", bound=Callable)
|
||||
|
||||
_CUDA_LIKE = frozenset({"cuda", "hip"})
|
||||
|
||||
|
||||
def platform_key() -> str:
|
||||
"""Return the live device family: ``cuda``/``hip``/``npu``/``mps``/``musa``/``xpu``/``cpu``.
|
||||
@@ -72,11 +70,6 @@ def is_xpu() -> bool:
|
||||
return current_platform.is_xpu()
|
||||
|
||||
|
||||
def has_triton() -> bool:
|
||||
"""True when the live device runs the Triton implementations."""
|
||||
return platform_key() in _CUDA_LIKE
|
||||
|
||||
|
||||
def lazy_fallback(kind: str, name: str) -> Callable:
|
||||
"""Name a fallback without importing its module.
|
||||
|
||||
|
||||
@@ -7,9 +7,6 @@ promotes the residual to FP32, so later layers carry an FP32 residual and an
|
||||
FP16 attention/FFN update. ``h3_vae_scale_add_rmsnorm`` does the add and the
|
||||
next RMSNorm in one launch: the sum stays in registers for the variance, and
|
||||
the stored residual matches the eager FP32 add.
|
||||
|
||||
``h3_vae_qk_rmsnorm_rope`` is the affine-free head-64 path: RMSNorm, round to
|
||||
the activation dtype, then partial NeoX RoPE on the first 48 channels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -79,59 +76,6 @@ def _scale_add_rmsnorm_kernel(
|
||||
tl.store(out_ptr + row * D + cols, acc * rstd * weight, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qk_rmsnorm_rope_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
qo_ptr,
|
||||
ko_ptr,
|
||||
cos_ptr,
|
||||
sin_ptr,
|
||||
stride_qb,
|
||||
stride_qs,
|
||||
stride_qh,
|
||||
stride_kb,
|
||||
stride_ks,
|
||||
stride_kh,
|
||||
seq_len,
|
||||
n_heads,
|
||||
eps,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
heads = n_heads
|
||||
token = row // heads
|
||||
head = row % heads
|
||||
seq = token % seq_len
|
||||
batch = token // seq_len
|
||||
cols = tl.arange(0, 64)
|
||||
rope_mask = cols < 48
|
||||
q_base = q_ptr + batch * stride_qb + seq * stride_qs + head * stride_qh
|
||||
k_base = k_ptr + batch * stride_kb + seq * stride_ks + head * stride_kh
|
||||
q = tl.load(q_base + cols).to(tl.float32)
|
||||
k = tl.load(k_base + cols).to(tl.float32)
|
||||
q = (q * tl.rsqrt(tl.sum(q * q, axis=0) / 64 + eps)).to(tl.float16)
|
||||
k = (k * tl.rsqrt(tl.sum(k * k, axis=0) / 64 + eps)).to(tl.float16)
|
||||
cos = tl.load(cos_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
|
||||
sin = tl.load(sin_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16)
|
||||
# NeoX rotate-half: [-x2, x1] on the first 48 channels.
|
||||
src = tl.where(cols < 24, cols + 24, cols - 24)
|
||||
sign = tl.where(cols < 24, -1.0, 1.0)
|
||||
q_rot = (tl.gather(q.to(tl.float32), src, 0) * sign).to(tl.float16)
|
||||
k_rot = (tl.gather(k.to(tl.float32), src, 0) * sign).to(tl.float16)
|
||||
yq = (
|
||||
(q.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
+ (q_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
).to(tl.float16)
|
||||
yk = (
|
||||
(k.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
+ (k_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32)
|
||||
).to(tl.float16)
|
||||
yq = tl.where(rope_mask, yq, q)
|
||||
yk = tl.where(rope_mask, yk, k)
|
||||
tl.store(qo_ptr + row * 64 + cols, yq)
|
||||
tl.store(ko_ptr + row * 64 + cols, yk)
|
||||
|
||||
|
||||
def _out_kind(dtype: torch.dtype) -> int:
|
||||
if dtype == torch.float32:
|
||||
return 0
|
||||
@@ -239,61 +183,3 @@ def h3_vae_scale_add_rmsnorm(
|
||||
if inplace:
|
||||
return residual, normed.view(out_shape)
|
||||
return residual_out, normed.view(out_shape)
|
||||
|
||||
|
||||
def h3_vae_qk_rmsnorm_rope(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
eps: float,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""Affine-free RMSNorm plus partial NeoX RoPE for head 64 / rotary 48.
|
||||
|
||||
Returns None when the tensors are outside that contract so the caller can
|
||||
keep the unfused norm and RoPE.
|
||||
"""
|
||||
if (
|
||||
not query.is_cuda
|
||||
or query.shape != key.shape
|
||||
or query.dtype != key.dtype
|
||||
or query.dtype != torch.float16
|
||||
or query.dim() != 4
|
||||
or query.shape[-1] != 64
|
||||
or query.stride(-1) != 1
|
||||
or key.stride(-1) != 1
|
||||
or cos.shape != sin.shape
|
||||
or cos.dim() != 2
|
||||
or cos.shape[-1] != 48
|
||||
or cos.shape[0] != query.shape[1]
|
||||
or cos.device != query.device
|
||||
or sin.device != query.device
|
||||
or key.device != query.device
|
||||
):
|
||||
return None
|
||||
batch, seq_len, n_heads, _ = query.shape
|
||||
rows = batch * seq_len * n_heads
|
||||
if rows == 0:
|
||||
return None
|
||||
query_out = torch.empty(rows, 64, device=query.device, dtype=query.dtype)
|
||||
key_out = torch.empty(rows, 64, device=key.device, dtype=key.dtype)
|
||||
_qk_rmsnorm_rope_kernel[(rows,)](
|
||||
query,
|
||||
key,
|
||||
query_out,
|
||||
key_out,
|
||||
cos.contiguous(),
|
||||
sin.contiguous(),
|
||||
query.stride(0),
|
||||
query.stride(1),
|
||||
query.stride(2),
|
||||
key.stride(0),
|
||||
key.stride(1),
|
||||
key.stride(2),
|
||||
seq_len,
|
||||
n_heads,
|
||||
float(eps),
|
||||
num_warps=1,
|
||||
)
|
||||
view = (batch, seq_len, n_heads, 64)
|
||||
return query_out.view(view), key_out.view(view)
|
||||
|
||||
@@ -79,9 +79,9 @@ class BitExactFusionGate:
|
||||
|
||||
The first-sight check runs the eager reference chain plus a host sync, so
|
||||
it must never happen inside ``torch.compile`` tracing or CUDA graph
|
||||
capture. Once-for-all callers get both guards from
|
||||
:meth:`can_attempt_once`; per-signature callers must keep their own
|
||||
compile/capture checks next to the signature lookup (see FLUX / Sana).
|
||||
capture. :meth:`run` applies both guards in either mode; hand-written
|
||||
sites get them from :meth:`can_attempt_once` (once-for-all) or keep their
|
||||
own compile/capture checks next to the signature lookup (FLUX / Sana).
|
||||
"""
|
||||
|
||||
__slots__ = ("name", "disabled", "verified", "verified_sigs")
|
||||
@@ -114,14 +114,49 @@ class BitExactFusionGate:
|
||||
return False
|
||||
if self.verified:
|
||||
return True
|
||||
# First-sight verify runs the eager reference chain and a host sync:
|
||||
# attempt neither inside compile tracing nor CUDA graph capture (the
|
||||
# sync would abort the capture; BCG then blocks the signature). Once
|
||||
# verified, the fused kernel runs alone and is compile/capture-safe.
|
||||
if torch.compiler.is_compiling():
|
||||
return False
|
||||
return not (
|
||||
torch.cuda.is_available() and torch.cuda.is_current_stream_capturing()
|
||||
return _can_verify_now()
|
||||
|
||||
def run(
|
||||
self,
|
||||
fused: Callable[[], T],
|
||||
reference: Callable[[], T],
|
||||
*,
|
||||
sig: Any = None,
|
||||
equal: EqualFn | None = None,
|
||||
logger: logging.Logger | None = None,
|
||||
diagnostic_hint: DiagnosticHintFn | None = None,
|
||||
reraise: tuple[type[Exception], ...] = (),
|
||||
) -> T:
|
||||
"""Return ``fused()`` once it has matched ``reference()`` bit-for-bit.
|
||||
|
||||
The caller has already checked that the fast path applies. An
|
||||
unverified ``sig`` is verified on this call unless it is inside compile
|
||||
tracing or CUDA graph capture, where ``reference()`` runs instead. A
|
||||
launch failure (other than ``reraise``) or a mismatch disables the
|
||||
gate and returns ``reference()``.
|
||||
"""
|
||||
if self.disabled:
|
||||
return reference()
|
||||
sigs = self.verified_sigs
|
||||
verified = self.verified if sigs is None else sig in sigs
|
||||
if not verified and not _can_verify_now():
|
||||
return reference()
|
||||
try:
|
||||
out = fused()
|
||||
except reraise:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.on_exception(exc, logger=logger)
|
||||
return reference()
|
||||
if verified:
|
||||
return out
|
||||
return self.accept_or_fallback(
|
||||
out,
|
||||
reference(),
|
||||
sig=sig,
|
||||
equal=equal,
|
||||
logger=logger,
|
||||
diagnostic_hint=diagnostic_hint,
|
||||
)
|
||||
|
||||
def on_exception(
|
||||
@@ -145,7 +180,6 @@ class BitExactFusionGate:
|
||||
sig: Any = None,
|
||||
equal: EqualFn | None = None,
|
||||
logger: logging.Logger | None = None,
|
||||
mismatch_msg: str | None = None,
|
||||
diagnostic_hint: DiagnosticHintFn | None = None,
|
||||
) -> T:
|
||||
"""Return ``out`` when bit-exact; otherwise disable and return ``ref``."""
|
||||
@@ -156,10 +190,6 @@ class BitExactFusionGate:
|
||||
self.mark_verified(sig)
|
||||
return out
|
||||
if logger is not None:
|
||||
message = mismatch_msg or (
|
||||
f"{self.name} fast path is not bit-exact against this "
|
||||
"platform's reference dispatch; falling back to eager"
|
||||
)
|
||||
details = (
|
||||
"Correctness is preserved because the eager reference output "
|
||||
"is used. A platform-specific reference kernel or reduction-order "
|
||||
@@ -172,11 +202,24 @@ class BitExactFusionGate:
|
||||
diagnostic_details = None
|
||||
if diagnostic_details:
|
||||
details = f"{details}. {diagnostic_details.rstrip('.')}"
|
||||
logger.warning_once(f"{message.rstrip('.')}. {details}.")
|
||||
logger.warning_once(
|
||||
f"{self.name} fast path is not bit-exact against this platform's "
|
||||
f"reference dispatch; falling back to eager. {details}."
|
||||
)
|
||||
self.disable()
|
||||
return ref
|
||||
|
||||
|
||||
def _can_verify_now() -> bool:
|
||||
# First-sight verify runs the eager reference chain and a host sync:
|
||||
# attempt neither inside compile tracing nor CUDA graph capture (the
|
||||
# sync would abort the capture; BCG then blocks the signature). Once
|
||||
# verified, the fused kernel runs alone and is compile/capture-safe.
|
||||
if torch.compiler.is_compiling():
|
||||
return False
|
||||
return not (torch.cuda.is_available() and torch.cuda.is_current_stream_capturing())
|
||||
|
||||
|
||||
def tensors_equal(a: Any, b: Any) -> bool:
|
||||
"""``torch.equal`` for a tensor or a sequence of tensors."""
|
||||
if isinstance(a, torch.Tensor):
|
||||
|
||||
@@ -8,29 +8,10 @@ Keep it disabled below the approximate tier and mount it only for
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
FLUX2_NVFP4_SWIGLU_QUANT = QualityGatedFusion(
|
||||
name="FLUX.2 NVFP4 FC1+SwiGLU+quant",
|
||||
marker_attr="_sgl_flux2_nvfp4_swiglu_quant_site",
|
||||
enabled_attr="_sgl_flux2_nvfp4_swiglu_quant_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_flux2_nvfp4_swiglu_quant_site(module: nn.Module) -> None:
|
||||
"""Mark an eligible FLUX.2 feed-forward site, disabled by default."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def flux2_nvfp4_swiglu_quant_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_flux2_nvfp4_swiglu_quant(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root)
|
||||
|
||||
|
||||
def unmount_flux2_nvfp4_swiglu_quant(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
@@ -12,9 +12,8 @@ native-bf16 reference RMSNorm. Ideogram's reference norm is ``F.rms_norm``
|
||||
(fp32 internal statistics), so the fused path is numerically close (the norm
|
||||
statistics round through bf16) but **not bit-exact**. Following the
|
||||
fused-linear-GELU precedent, sites are therefore mounted only for
|
||||
``quality="high"`` requests via :func:`mount_fused_gate_rmsnorm` /
|
||||
:func:`unmount_fused_gate_rmsnorm` at batch boundaries; the default
|
||||
``"exact"`` path keeps the unmodified reference chain bit-for-bit.
|
||||
``quality="high"`` requests at batch boundaries; the default ``"exact"`` path
|
||||
keeps the unmodified reference chain bit-for-bit.
|
||||
|
||||
Mounting is all-or-nothing per transformer: if any marked site fails the
|
||||
static guards (non-bf16 norm weight, hidden size above the kernel limit, ...)
|
||||
@@ -23,7 +22,6 @@ every site on that transformer stays on the reference path.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from importlib import import_module
|
||||
|
||||
import torch
|
||||
@@ -31,17 +29,6 @@ import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Attributes of the site protocol (set by ``mark_fused_gate_rmsnorm_site``).
|
||||
_SITE_NORM_ATTRS = "_sgl_fused_gate_rmsnorm_norm_attrs"
|
||||
_SITE_ENABLED_ATTR = "_sgl_fused_gate_rmsnorm_enabled"
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="fused gate RMSNorm",
|
||||
marker_attr=_SITE_NORM_ATTRS,
|
||||
enabled_attr=_SITE_ENABLED_ATTR,
|
||||
)
|
||||
|
||||
# The Triton kernels mask a single block over the hidden dim.
|
||||
_MAX_HIDDEN_SIZE = 8192
|
||||
|
||||
@@ -78,7 +65,7 @@ def _static_reject_reason(site: nn.Module) -> str | None:
|
||||
import_module("triton")
|
||||
except ImportError:
|
||||
return "triton unavailable"
|
||||
for attr in _FUSION.metadata(site, ()):
|
||||
for attr in FUSED_GATE_RMSNORM.metadata(site, ()):
|
||||
norm = getattr(site, attr, None)
|
||||
weight = getattr(norm, "weight", None)
|
||||
if weight is None or weight.dim() != 1:
|
||||
@@ -90,33 +77,10 @@ def _static_reject_reason(site: nn.Module) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def mark_fused_gate_rmsnorm_site(module: nn.Module, norm_attrs: tuple[str, ...]):
|
||||
"""Declare ``module`` as a fused RMSNorm modulate/gate site.
|
||||
|
||||
``norm_attrs`` names the site's RMSNorm submodules (checked by the static
|
||||
guards at mount time). The site starts unmounted
|
||||
(``_sgl_fused_gate_rmsnorm_enabled = False``): the module's forward must
|
||||
keep the reference path bit-exact until :func:`mount_fused_gate_rmsnorm`
|
||||
enables it.
|
||||
"""
|
||||
_FUSION.mark(module, tuple(norm_attrs))
|
||||
|
||||
|
||||
def fused_gate_rmsnorm_active(module: nn.Module) -> bool:
|
||||
"""Whether the quality-gated fused path is mounted on ``module``."""
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_fused_gate_rmsnorm(root: nn.Module) -> bool:
|
||||
"""Enable the fused kernels on every marked site under ``root``.
|
||||
|
||||
All-or-nothing: if any marked site fails the static guards, every site is
|
||||
left (or reset) on the reference path and False is returned. Returns False
|
||||
as well when ``root`` has no marked sites.
|
||||
"""
|
||||
return _FUSION.mount(root, reject_reason=_static_reject_reason, logger=logger)
|
||||
|
||||
|
||||
def unmount_fused_gate_rmsnorm(root: nn.Module) -> None:
|
||||
"""Reset every marked site under ``root`` to the bit-exact reference path."""
|
||||
_FUSION.unmount(root)
|
||||
# A site's marker metadata is the tuple of its RMSNorm submodule names.
|
||||
FUSED_GATE_RMSNORM = QualityGatedFusion(
|
||||
name="fused gate RMSNorm",
|
||||
marker_attr="_sgl_fused_gate_rmsnorm_norm_attrs",
|
||||
enabled_attr="_sgl_fused_gate_rmsnorm_enabled",
|
||||
reject_reason=_static_reject_reason,
|
||||
)
|
||||
|
||||
@@ -11,10 +11,9 @@ reference only at bf16/fp16 rounding-order level -- close, but not bit-exact.
|
||||
|
||||
Because it is not bit-exact, the fused path is **mounted only for
|
||||
``quality="high"`` requests** (see ``SamplingParams.quality``): model code marks
|
||||
its GELU up-projection sites with :func:`mark_fused_gelu_site` (default: off,
|
||||
reference path, bit-exact), and the denoising stage calls
|
||||
:func:`mount_fused_linear_gelu` / :func:`unmount_fused_linear_gelu` at batch
|
||||
boundaries. Mounting is all-or-nothing per transformer: if any marked site
|
||||
its GELU up-projection sites with ``FUSED_LINEAR_GELU.mark`` (default: off,
|
||||
reference path, bit-exact), and the denoising stage mounts / unmounts
|
||||
``FUSED_LINEAR_GELU`` at batch boundaries. Mounting is all-or-nothing per transformer: if any marked site
|
||||
fails the static guards (quantized weights, missing bias, non-half dtype, ...)
|
||||
no site on that transformer is fused.
|
||||
|
||||
@@ -25,7 +24,6 @@ single opaque op under ``torch.compile`` -- no graph break.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import weakref
|
||||
from typing import Any
|
||||
|
||||
@@ -36,22 +34,11 @@ from sglang.kernels.jit.utils import get_jit_cuda_arch
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ``torch._addmm_activation`` is the (private but stable) entry point to the
|
||||
# cublasLt GEMM+bias+activation epilogue. Guard for builds where it is absent
|
||||
# so the reference path is always available.
|
||||
_HAS_ADDMM_ACTIVATION = hasattr(torch, "_addmm_activation")
|
||||
|
||||
# Attributes of the site protocol (set by ``mark_fused_gelu_site``).
|
||||
_SITE_LINEAR_ATTR = "_sgl_fused_gelu_linear_attr"
|
||||
_SITE_ENABLED_ATTR = "_sgl_fused_gelu_enabled"
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="fused linear+GELU",
|
||||
marker_attr=_SITE_LINEAR_ATTR,
|
||||
enabled_attr=_SITE_ENABLED_ATTR,
|
||||
)
|
||||
|
||||
|
||||
def _fused_linear_gelu_tanh_fake(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor
|
||||
@@ -157,39 +144,26 @@ def can_use_linear_gelu(linear: Any, x: torch.Tensor) -> bool:
|
||||
return verdict
|
||||
|
||||
|
||||
def mark_fused_gelu_site(module: nn.Module, linear_attr: str) -> None:
|
||||
"""Declare ``module`` as a tanh-GELU up-projection fusion site.
|
||||
|
||||
``getattr(module, linear_attr)`` must be the up-projection linear whose
|
||||
output feeds ``F.gelu(..., approximate="tanh")``. The site starts unmounted
|
||||
(``_sgl_fused_gelu_enabled = False``): the module's forward must keep the
|
||||
reference path bit-exact until :func:`mount_fused_linear_gelu` enables it.
|
||||
"""
|
||||
_FUSION.mark(module, linear_attr)
|
||||
|
||||
|
||||
def fused_gelu_active(module: nn.Module) -> bool:
|
||||
"""Whether the quality-gated fused path is mounted on ``module``."""
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
linear = getattr(site, _FUSION.metadata(site), None)
|
||||
linear = getattr(site, FUSED_LINEAR_GELU.metadata(site), None)
|
||||
return "missing linear" if linear is None else _static_reject_reason(linear)
|
||||
|
||||
|
||||
def mount_fused_linear_gelu(root: nn.Module) -> bool:
|
||||
"""Enable the fused epilogue on every marked site under ``root``.
|
||||
class _LinearGeluFusion(QualityGatedFusion):
|
||||
def mount(self, root: nn.Module) -> bool:
|
||||
_VERDICTS.clear()
|
||||
return super().mount(root)
|
||||
|
||||
All-or-nothing: if any marked site fails the static guards, every site is
|
||||
left (or reset) on the reference path and False is returned. Returns False
|
||||
as well when ``root`` has no marked sites.
|
||||
"""
|
||||
_VERDICTS.clear()
|
||||
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
|
||||
def unmount(self, root: nn.Module) -> None:
|
||||
_VERDICTS.clear()
|
||||
super().unmount(root)
|
||||
|
||||
|
||||
def unmount_fused_linear_gelu(root: nn.Module) -> None:
|
||||
"""Reset every marked site under ``root`` to the bit-exact reference path."""
|
||||
_VERDICTS.clear()
|
||||
_FUSION.unmount(root)
|
||||
# A site's marker metadata names its up-projection linear, whose output feeds
|
||||
# ``F.gelu(..., approximate="tanh")``.
|
||||
FUSED_LINEAR_GELU = _LinearGeluFusion(
|
||||
name="fused linear+GELU",
|
||||
marker_attr="_sgl_fused_gelu_linear_attr",
|
||||
enabled_attr="_sgl_fused_gelu_enabled",
|
||||
reject_reason=_site_reject_reason,
|
||||
)
|
||||
|
||||
@@ -8,48 +8,27 @@ the result is not bit-exact vs the reference (half-precision rounding-order
|
||||
differences only).
|
||||
|
||||
Because it is not bit-exact the fold is opt-in per batch: model code marks
|
||||
its LN+modulate sites with :func:`mark_fused_ln_modulate_site` (default off,
|
||||
reference path), and the denoising stage calls
|
||||
:func:`mount_fused_ln_modulate` / :func:`unmount_fused_ln_modulate` at batch
|
||||
boundaries for ``quality="high"`` requests.
|
||||
its LN+modulate sites with ``FUSED_LN_MODULATE.mark`` (default off,
|
||||
reference path), and the denoising stage mounts / unmounts
|
||||
``FUSED_LN_MODULATE`` at batch boundaries for ``quality="high"`` requests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
_SITE_MARKER_ATTR = "_sgl_fused_ln_modulate_site"
|
||||
_SITE_ENABLED_ATTR = "_sgl_fused_ln_modulate_enabled"
|
||||
_FUSION = QualityGatedFusion(
|
||||
FUSED_LN_MODULATE = QualityGatedFusion(
|
||||
name="fused LN+modulate",
|
||||
marker_attr=_SITE_MARKER_ATTR,
|
||||
enabled_attr=_SITE_ENABLED_ATTR,
|
||||
marker_attr="_sgl_fused_ln_modulate_site",
|
||||
enabled_attr="_sgl_fused_ln_modulate_enabled",
|
||||
)
|
||||
|
||||
_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
|
||||
|
||||
|
||||
def mark_fused_ln_modulate_site(module: nn.Module) -> None:
|
||||
"""Mark ``module`` as an LN+modulate fusion site (mounted off)."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def fused_ln_modulate_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_fused_ln_modulate(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root)
|
||||
|
||||
|
||||
def unmount_fused_ln_modulate(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
|
||||
def can_use_ln_modulate(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> bool:
|
||||
|
||||
+14
-29
@@ -1,8 +1,8 @@
|
||||
"""LingBot Video per-token gated-residual fusion, gated by request quality.
|
||||
"""Per-token gated-residual fusion for Helios and LingBot Video, gated by quality.
|
||||
|
||||
Each LingBot block applies ``residual + (gate * update).to(residual.dtype)`` at
|
||||
the attention and FFN updates, where ``update`` (post-norm) and the per-token
|
||||
``gate`` (``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. The shared
|
||||
Each block applies ``residual + (gate * update).to(residual.dtype)`` at its
|
||||
attention and FFN updates, where ``update`` and the per-token ``gate``
|
||||
(``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. The shared
|
||||
``residual_gate_add`` kernel computes ``residual + update * gate`` in a single
|
||||
pass but requires one dtype, so the gate and update are first cast to BF16.
|
||||
That reordering of the FP32 multiply is numerically equivalent only at
|
||||
@@ -13,51 +13,36 @@ half-precision rounding level (not bit-exact), so the fusion is opt-in:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
HELIOS_GATED_RESIDUAL = QualityGatedFusion(
|
||||
name="Helios per-token gated residual",
|
||||
marker_attr="_sgl_helios_gated_residual_site",
|
||||
enabled_attr="_sgl_helios_gated_residual_enabled",
|
||||
)
|
||||
LINGBOT_VIDEO_GATED_RESIDUAL = QualityGatedFusion(
|
||||
name="LingBot Video per-token gated residual",
|
||||
marker_attr="_sgl_lingbot_video_gated_residual_site",
|
||||
enabled_attr="_sgl_lingbot_video_gated_residual_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_lingbot_video_gated_residual_site(module: nn.Module) -> None:
|
||||
"""Mark a LingBot block; it starts on the reference path."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def lingbot_video_gated_residual_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_lingbot_video_gated_residual(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, logger=logger)
|
||||
|
||||
|
||||
def unmount_lingbot_video_gated_residual(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
|
||||
def try_lingbot_video_gated_residual(
|
||||
def try_gated_residual(
|
||||
fusion: QualityGatedFusion,
|
||||
site: nn.Module,
|
||||
residual: torch.Tensor,
|
||||
update: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
"""Return the fused ``residual + update * gate`` when the site is enabled.
|
||||
"""Return the fused ``residual + update * gate`` when ``site`` is enabled.
|
||||
|
||||
Returns ``None`` (caller runs the reference path) when the site is off or
|
||||
the tensors are not eligible for the per-token fast path.
|
||||
"""
|
||||
if not _FUSION.is_enabled(site):
|
||||
if not fusion.is_enabled(site):
|
||||
return None
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_residual_gate_add_cuda,
|
||||
@@ -1,72 +0,0 @@
|
||||
"""Helios per-token gated-residual fusion, gated by request quality.
|
||||
|
||||
Each Helios block applies ``residual + (gate * update).to(residual.dtype)`` at
|
||||
the self-attention and FFN updates, where ``update`` (post-attn / post-FFN) and
|
||||
the per-token ``gate`` (``[B, S, 1]``) stay in FP32 while ``residual`` is BF16.
|
||||
The shared ``residual_gate_add`` kernel computes ``residual + update * gate``
|
||||
in a single pass but requires one dtype, so the gate and update are first cast
|
||||
to BF16. That reordering of the FP32 multiply is numerically equivalent only at
|
||||
half-precision rounding level (not bit-exact), so the fusion is opt-in:
|
||||
``quality="lossless"`` and ``quality="high"`` mount it, while the default
|
||||
``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="Helios per-token gated residual",
|
||||
marker_attr="_sgl_helios_gated_residual_site",
|
||||
enabled_attr="_sgl_helios_gated_residual_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_helios_gated_residual_site(module: nn.Module) -> None:
|
||||
"""Mark a Helios block; it starts on the reference path."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def helios_gated_residual_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_helios_gated_residual(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, logger=logger)
|
||||
|
||||
|
||||
def unmount_helios_gated_residual(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
|
||||
def try_helios_gated_residual(
|
||||
site: nn.Module,
|
||||
residual: torch.Tensor,
|
||||
update: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
"""Return the fused ``residual + update * gate`` when the site is enabled.
|
||||
|
||||
Returns ``None`` (caller runs the reference path) when the site is off or
|
||||
the tensors are not eligible for the per-token fast path.
|
||||
"""
|
||||
if not _FUSION.is_enabled(site):
|
||||
return None
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_residual_gate_add_cuda,
|
||||
residual_gate_add,
|
||||
)
|
||||
|
||||
if residual.dtype != update.dtype or residual.dtype != gate.dtype:
|
||||
update = update.to(residual.dtype)
|
||||
gate = gate.to(residual.dtype)
|
||||
if not can_use_residual_gate_add_cuda(residual, update, gate):
|
||||
return None
|
||||
return residual_gate_add(residual, update, gate)
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
@@ -10,14 +9,6 @@ import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="HunyuanVideo strided QK RMSNorm",
|
||||
marker_attr="_sgl_hunyuan_qknorm_site",
|
||||
enabled_attr="_sgl_hunyuan_qknorm_enabled",
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _get_qk_rmsnorm_cute():
|
||||
@@ -30,26 +21,18 @@ def _get_qk_rmsnorm_cute():
|
||||
return qk_rmsnorm_cute
|
||||
|
||||
|
||||
def mark_hunyuan_qknorm_site(module: nn.Module) -> None:
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def _site_reject_reason(_site: nn.Module) -> str | None:
|
||||
if _get_qk_rmsnorm_cute() is None:
|
||||
return "FlashInfer CuTe QK RMSNorm unavailable"
|
||||
return None
|
||||
|
||||
|
||||
def mount_hunyuan_qknorm(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(
|
||||
root,
|
||||
reject_reason=_site_reject_reason,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
|
||||
def unmount_hunyuan_qknorm(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
HUNYUAN_QKNORM = QualityGatedFusion(
|
||||
name="HunyuanVideo strided QK RMSNorm",
|
||||
marker_attr="_sgl_hunyuan_qknorm_site",
|
||||
enabled_attr="_sgl_hunyuan_qknorm_enabled",
|
||||
reject_reason=_site_reject_reason,
|
||||
)
|
||||
|
||||
|
||||
def try_hunyuan_qknorm(
|
||||
@@ -62,7 +45,7 @@ def try_hunyuan_qknorm(
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""Normalize strided Hunyuan Q/K views without materializing inputs."""
|
||||
if not (
|
||||
_FUSION.is_enabled(site)
|
||||
HUNYUAN_QKNORM.is_enabled(site)
|
||||
and not torch.compiler.is_compiling()
|
||||
and q.is_cuda
|
||||
and q.dtype == torch.bfloat16
|
||||
|
||||
@@ -13,7 +13,6 @@ remains unchanged.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from importlib import import_module
|
||||
|
||||
import torch
|
||||
@@ -21,23 +20,6 @@ import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="LingBot Video fused RMSNorm",
|
||||
marker_attr="_sgl_lingbot_video_rmsnorm_site",
|
||||
enabled_attr="_sgl_lingbot_video_rmsnorm_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_lingbot_video_rmsnorm_site(module: nn.Module) -> None:
|
||||
"""Mark a LingBot RMSNorm module; it starts on the reference path."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def lingbot_video_rmsnorm_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
try:
|
||||
@@ -54,12 +36,12 @@ def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def mount_lingbot_video_rmsnorm(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
|
||||
|
||||
|
||||
def unmount_lingbot_video_rmsnorm(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
LINGBOT_VIDEO_RMSNORM = QualityGatedFusion(
|
||||
name="LingBot Video fused RMSNorm",
|
||||
marker_attr="_sgl_lingbot_video_rmsnorm_site",
|
||||
enabled_attr="_sgl_lingbot_video_rmsnorm_enabled",
|
||||
reject_reason=_site_reject_reason,
|
||||
)
|
||||
|
||||
|
||||
def try_lingbot_video_rmsnorm(
|
||||
@@ -70,7 +52,7 @@ def try_lingbot_video_rmsnorm(
|
||||
) -> torch.Tensor | None:
|
||||
"""Return the quality-gated RMSNorm result, or ``None`` to fall back."""
|
||||
if not (
|
||||
_FUSION.is_enabled(site)
|
||||
LINGBOT_VIDEO_RMSNORM.is_enabled(site)
|
||||
and hidden_states.is_cuda
|
||||
and hidden_states.dtype in (torch.float16, torch.bfloat16)
|
||||
and hidden_states.is_contiguous()
|
||||
|
||||
@@ -8,28 +8,10 @@ kernel fusions.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
LTX2_QKNORM_SPLIT_ROPE = QualityGatedFusion(
|
||||
name="LTX-2 Hopper QKNorm+split-RoPE",
|
||||
marker_attr="_sgl_ltx2_qknorm_split_rope_site",
|
||||
enabled_attr="_sgl_ltx2_qknorm_split_rope_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_ltx2_qknorm_split_rope_site(module: nn.Module) -> None:
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def ltx2_qknorm_split_rope_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_ltx2_qknorm_split_rope(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root)
|
||||
|
||||
|
||||
def unmount_ltx2_qknorm_split_rope(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
@@ -10,51 +10,30 @@ The fused kernel reproduces the RMSNorm math via ``rsqrt.approx`` (the
|
||||
flashinfer CuTe form), which differs from aten's refined ``rsqrtf`` by at
|
||||
most one bf16 ULP on a small fraction of elements. It is therefore *not*
|
||||
bit-exact vs the eager reference, so the fold is opt-in per batch: model code
|
||||
marks its adaLN sites with :func:`mark_ltx2_rms_norm_modulate_site` (default
|
||||
off, reference path) and the denoising stage calls
|
||||
:func:`mount_ltx2_rms_norm_modulate` / :func:`unmount_ltx2_rms_norm_modulate`
|
||||
at batch boundaries for ``quality="high"`` requests.
|
||||
marks its adaLN sites with ``LTX2_RMS_NORM_MODULATE.mark`` (default off,
|
||||
reference path) and the denoising stage mounts / unmounts
|
||||
``LTX2_RMS_NORM_MODULATE`` at batch boundaries for ``quality="high"`` requests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.norm.rmsnorm_scale_shift_bitexact import (
|
||||
fused_rmsnorm_scale_shift_bitexact,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
_SITE_MARKER_ATTR = "_sgl_ltx2_rms_norm_modulate_site"
|
||||
_SITE_ENABLED_ATTR = "_sgl_ltx2_rms_norm_modulate_enabled"
|
||||
_FUSION = QualityGatedFusion(
|
||||
LTX2_RMS_NORM_MODULATE = QualityGatedFusion(
|
||||
name="LTX-2 RMSNorm+modulate",
|
||||
marker_attr=_SITE_MARKER_ATTR,
|
||||
enabled_attr=_SITE_ENABLED_ATTR,
|
||||
marker_attr="_sgl_ltx2_rms_norm_modulate_site",
|
||||
enabled_attr="_sgl_ltx2_rms_norm_modulate_enabled",
|
||||
)
|
||||
|
||||
# ``RMSNormNoWeight`` applies no scale, so a ones weight reproduces it exactly.
|
||||
_ONES_WEIGHT_CACHE: dict[tuple[torch.device, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
def mark_ltx2_rms_norm_modulate_site(module: nn.Module) -> None:
|
||||
"""Mark ``module`` as an LTX-2 RMSNorm+modulate fusion site (mounted off)."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def ltx2_rms_norm_modulate_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_ltx2_rms_norm_modulate(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root)
|
||||
|
||||
|
||||
def unmount_ltx2_rms_norm_modulate(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
|
||||
def _ones_weight(x: torch.Tensor) -> torch.Tensor:
|
||||
key = (x.device, int(x.shape[-1]))
|
||||
w = _ONES_WEIGHT_CACHE.get(key)
|
||||
|
||||
@@ -9,33 +9,10 @@ mount it, while ``"exact"`` keeps the reference chain bit-for-bit.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
MINIMAX_H3_NORM_MODULATE = QualityGatedFusion(
|
||||
name="MiniMax-H3 fused RMSNorm + AdaLN",
|
||||
marker_attr="_sgl_minimax_h3_norm_modulate_site",
|
||||
enabled_attr="_sgl_minimax_h3_norm_modulate_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None:
|
||||
"""Mark an H3 DiT block; it starts on the reference path."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def minimax_h3_norm_modulate_active(module: nn.Module) -> bool:
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, logger=logger)
|
||||
|
||||
|
||||
def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
|
||||
@@ -8,30 +8,10 @@ only for the existing ``quality="high"`` contract.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="Wan NVFP4 bias+GELU",
|
||||
marker_attr="_sgl_nvfp4_bias_gelu_site",
|
||||
enabled_attr="_sgl_nvfp4_bias_gelu_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_nvfp4_bias_gelu_site(module: nn.Module) -> None:
|
||||
"""Mark an MLP whose ``fc_in`` bias can be deferred for this fusion."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def nvfp4_bias_gelu_active(module: nn.Module) -> bool:
|
||||
"""Whether the quality-gated fusion is mounted on ``module``."""
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
if not getattr(site, "fuse_bias_gelu_tanh", False):
|
||||
@@ -51,17 +31,27 @@ def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def mount_nvfp4_bias_gelu(root: nn.Module) -> bool:
|
||||
"""Enable every eligible marked site under ``root``."""
|
||||
sites = list(_FUSION.iter_sites(root))
|
||||
mounted = _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
|
||||
for site in sites:
|
||||
site.fc_in.skip_bias_add = mounted
|
||||
return mounted
|
||||
class _NVFP4BiasGeluFusion(QualityGatedFusion):
|
||||
"""A mounted site defers its ``fc_in`` bias add to the fused GELU."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def mount(self, root: nn.Module) -> bool:
|
||||
mounted = super().mount(root)
|
||||
for site in self.iter_sites(root):
|
||||
site.fc_in.skip_bias_add = mounted
|
||||
return mounted
|
||||
|
||||
def unmount(self, root: nn.Module) -> None:
|
||||
super().unmount(root)
|
||||
for site in self.iter_sites(root):
|
||||
site.fc_in.skip_bias_add = False
|
||||
|
||||
|
||||
def unmount_nvfp4_bias_gelu(root: nn.Module) -> None:
|
||||
"""Restore every marked site to its original linear+GELU path."""
|
||||
_FUSION.unmount(root)
|
||||
for site in _FUSION.iter_sites(root):
|
||||
site.fc_in.skip_bias_add = False
|
||||
# Marks an MLP whose ``fc_in`` bias can be deferred for this fusion.
|
||||
NVFP4_BIAS_GELU = _NVFP4BiasGeluFusion(
|
||||
name="Wan NVFP4 bias+GELU",
|
||||
marker_attr="_sgl_nvfp4_bias_gelu_site",
|
||||
enabled_attr="_sgl_nvfp4_bias_gelu_enabled",
|
||||
reject_reason=_site_reject_reason,
|
||||
)
|
||||
|
||||
@@ -8,6 +8,8 @@ from typing import Any
|
||||
|
||||
from torch import nn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RejectReason = Callable[[nn.Module], str | None]
|
||||
|
||||
|
||||
@@ -16,15 +18,24 @@ class QualityGatedFusion:
|
||||
|
||||
The marker metadata describes the model attribute(s) owned by a site. The
|
||||
enabled flag remains a plain module attribute so compiled model forwards
|
||||
can read it without depending on this helper object.
|
||||
can read it without depending on this helper object. ``reject_reason``
|
||||
holds the static guards :meth:`mount` checks on every marked site.
|
||||
"""
|
||||
|
||||
__slots__ = ("enabled_attr", "marker_attr", "name")
|
||||
__slots__ = ("enabled_attr", "marker_attr", "name", "reject_reason")
|
||||
|
||||
def __init__(self, *, name: str, marker_attr: str, enabled_attr: str) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
marker_attr: str,
|
||||
enabled_attr: str,
|
||||
reject_reason: RejectReason | None = None,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.marker_attr = marker_attr
|
||||
self.enabled_attr = enabled_attr
|
||||
self.reject_reason = reject_reason
|
||||
|
||||
def mark(self, module: nn.Module, metadata: Any = True) -> None:
|
||||
setattr(module, self.marker_attr, metadata)
|
||||
@@ -41,32 +52,25 @@ class QualityGatedFusion:
|
||||
if hasattr(module, self.marker_attr):
|
||||
yield module
|
||||
|
||||
def mount(
|
||||
self,
|
||||
root: nn.Module,
|
||||
*,
|
||||
reject_reason: RejectReason | None = None,
|
||||
logger: logging.Logger | None = None,
|
||||
) -> bool:
|
||||
def mount(self, root: nn.Module) -> bool:
|
||||
"""Enable every eligible site, or leave the whole family disabled."""
|
||||
sites = list(self.iter_sites(root))
|
||||
if not sites:
|
||||
return False
|
||||
|
||||
if reject_reason is not None:
|
||||
if self.reject_reason is not None:
|
||||
for site in sites:
|
||||
reason = reject_reason(site)
|
||||
reason = self.reject_reason(site)
|
||||
if reason is None:
|
||||
continue
|
||||
self._set_enabled(sites, False)
|
||||
if logger is not None:
|
||||
logger.info(
|
||||
"%s: %s site failed static guards (%s); keeping the "
|
||||
"whole model on the reference path",
|
||||
self.name,
|
||||
type(site).__name__,
|
||||
reason,
|
||||
)
|
||||
logger.info(
|
||||
"%s: %s site failed static guards (%s); keeping the "
|
||||
"whole model on the reference path",
|
||||
self.name,
|
||||
type(site).__name__,
|
||||
reason,
|
||||
)
|
||||
return False
|
||||
|
||||
self._set_enabled(sites, True)
|
||||
|
||||
@@ -8,30 +8,10 @@ slices independently. ``quality="high"`` mounts the single-GEMM path.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="Qwen-Image fused added-QKV",
|
||||
marker_attr="_sgl_qwen_image_added_qkv_site",
|
||||
enabled_attr="_sgl_qwen_image_added_qkv_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_qwen_image_added_qkv_site(module: nn.Module) -> None:
|
||||
"""Mark an unquantized Qwen-Image attention site; it starts unmounted."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def qwen_image_added_qkv_active(module: nn.Module) -> bool:
|
||||
"""Whether the request-scoped packed added-QKV GEMM is mounted."""
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
linear = getattr(site, "to_added_qkv", None)
|
||||
@@ -44,9 +24,9 @@ def _site_reject_reason(site: nn.Module) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def mount_qwen_image_added_qkv(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
|
||||
|
||||
|
||||
def unmount_qwen_image_added_qkv(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
QWEN_IMAGE_ADDED_QKV = QualityGatedFusion(
|
||||
name="Qwen-Image fused added-QKV",
|
||||
marker_attr="_sgl_qwen_image_added_qkv_site",
|
||||
enabled_attr="_sgl_qwen_image_added_qkv_enabled",
|
||||
reject_reason=_site_reject_reason,
|
||||
)
|
||||
|
||||
@@ -13,31 +13,11 @@ eligible; unsupported dtypes and layouts fall back to the reference path.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FUSION = QualityGatedFusion(
|
||||
name="SANA-Video BF16-input linear attention",
|
||||
marker_attr="_sgl_sana_video_linear_attention_site",
|
||||
enabled_attr="_sgl_sana_video_linear_attention_enabled",
|
||||
)
|
||||
|
||||
|
||||
def mark_sana_video_linear_attention_site(module: nn.Module) -> None:
|
||||
"""Mark a SANA-Video linear-attention module; it starts unmounted."""
|
||||
_FUSION.mark(module)
|
||||
|
||||
|
||||
def sana_video_linear_attention_active(module: nn.Module) -> bool:
|
||||
"""Whether the request-scoped BF16-input path is mounted on ``module``."""
|
||||
return _FUSION.is_enabled(module)
|
||||
|
||||
|
||||
def _site_reject_reason(_site: nn.Module) -> str | None:
|
||||
if torch.version.cuda is None:
|
||||
@@ -45,12 +25,12 @@ def _site_reject_reason(_site: nn.Module) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def mount_sana_video_linear_attention(root: nn.Module) -> bool:
|
||||
return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger)
|
||||
|
||||
|
||||
def unmount_sana_video_linear_attention(root: nn.Module) -> None:
|
||||
_FUSION.unmount(root)
|
||||
SANA_VIDEO_LINEAR_ATTENTION = QualityGatedFusion(
|
||||
name="SANA-Video BF16-input linear attention",
|
||||
marker_attr="_sgl_sana_video_linear_attention_site",
|
||||
enabled_attr="_sgl_sana_video_linear_attention_enabled",
|
||||
reject_reason=_site_reject_reason,
|
||||
)
|
||||
|
||||
|
||||
def try_sana_video_linear_attention(
|
||||
@@ -62,7 +42,7 @@ def try_sana_video_linear_attention(
|
||||
) -> torch.Tensor | None:
|
||||
"""Return the quality-gated attention result, or ``None`` to fall back."""
|
||||
if not (
|
||||
_FUSION.is_enabled(site)
|
||||
SANA_VIDEO_LINEAR_ATTENTION.is_enabled(site)
|
||||
and query_rotate.is_cuda
|
||||
and query_rotate.dtype == torch.bfloat16
|
||||
and key_rotate.dtype == query_rotate.dtype
|
||||
|
||||
@@ -9,7 +9,12 @@ they logically belong to.
|
||||
"""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
from sglang.kernels.spec import (
|
||||
CUDA_ONLY,
|
||||
CapabilityRequirement,
|
||||
KernelBackend,
|
||||
KernelSpec,
|
||||
)
|
||||
|
||||
__all__ = []
|
||||
|
||||
@@ -20,7 +25,6 @@ for _fn in ("hc_combine_gate", "hc_combine_apply", "hc_combine_apply_norm"):
|
||||
backend=KernelBackend.JIT,
|
||||
target=f"sglang.kernels.ops.elementwise.hc_combine_decode:{_fn}",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
|
||||
description="BF16 HC decode with four branches of width 2560 and PDL.",
|
||||
)
|
||||
)
|
||||
del _fn
|
||||
@@ -32,7 +36,7 @@ register_kernel(
|
||||
op="elementwise.fused_qwen4_gate_value",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_value",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -40,7 +44,7 @@ register_kernel(
|
||||
op="elementwise.fused_qwen4_gate_reduce",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_reduce",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
@@ -48,6 +52,6 @@ register_kernel(
|
||||
op="elementwise.row_scale_bf16",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.elementwise.row_scale:row_scale_bf16",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user