[Perf] dsv41: Serve token-interleaved V4.1 KV caches with sgl-flash-mla (#45893)
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
kpham-sgl
Claude Opus 5.5
parent
49933325cc
commit
e83f585ae6
@@ -76,6 +76,7 @@ dependencies = [
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.3",
|
||||
"sgl-flash-mla==1.0.0",
|
||||
"sglang-kernel==0.4.10",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -162,7 +162,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
if (out_loc <= 0) return;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
@@ -191,7 +191,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
|
||||
if (out_loc <= 0) return;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
|
||||
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
|
||||
// FP8 over the whole row, RoPE included; the layout selects the scaling.
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, data);
|
||||
}
|
||||
|
||||
@@ -232,7 +232,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
|
||||
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
@@ -258,7 +258,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
|
||||
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
|
||||
// FP8 over the whole row, RoPE included; the layout selects the scaling.
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
|
||||
@@ -16,9 +16,8 @@
|
||||
// critical path, and it cannot be hoisted out because the schedule depends on
|
||||
// the per-request `topk_length` of the step being replayed.
|
||||
//
|
||||
// This kernel produces the same output bit for bit; the shared-memory layout
|
||||
// matches FlashMLA's exactly because the idle-tail fill below reads one entry
|
||||
// past `first_block_idx_shared`, as FlashMLA does.
|
||||
// This kernel produces the same output bit for bit as the legacy scheduler in
|
||||
// sgl-flash-mla 1.0.0, idle-tail partitions included.
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -180,9 +179,8 @@ __global__ void __launch_bounds__(kBlockSize) flashmla_sched_meta_kernel(__grid_
|
||||
const int lb_last = last_block_idx_shared[b - 1];
|
||||
const int sk_last = seqlens_k_shared[b - 1];
|
||||
const int end_block_idx = (sk_last == 0) ? 0 : lb_last + 1;
|
||||
// FlashMLA reads first_block_idx_shared[batch_size] for these, one past the
|
||||
// end of that array, which aliases last_block_idx_shared[0].
|
||||
const int begin_block_idx = last_block_idx_shared[0];
|
||||
// `now_block` is 0 once every request is consumed.
|
||||
const int begin_block_idx = 0;
|
||||
const int is_last_req_splitted = (end_block_idx != lb_last + 1) && (sk_last != 0);
|
||||
for (int i = first_idle_part_shared + threadIdx.x; i < p.num_sm_parts; i += kBlockSize) {
|
||||
int* meta = out_shared + i * kMetaInts;
|
||||
|
||||
@@ -84,7 +84,7 @@ __global__ void fused_store_flashmla_cache(const __grid_constant__ FusedStoreCac
|
||||
/// Elements per thread of the V4.1 store (`512 / vec` threads per token): 4 for fp8 rows, 2 for
|
||||
/// fp4, whose per-element IEEE divisions spread better over more threads (measured on B200, bs 1..512).
|
||||
constexpr uint32_t v41_store_vec_size(deepseek_v4::KVLayout layout) {
|
||||
return layout == deepseek_v4::KVLayout::V41 ? 4 : 2;
|
||||
return deepseek_v4::kv_encoding(layout) == deepseek_v4::KVLayout::V41 ? 4 : 2;
|
||||
}
|
||||
|
||||
template <
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
#endif
|
||||
|
||||
// Paged fp8 / fp4 KV cache layouts read by the d_qk = 512 sparse MLA decode
|
||||
// kernels. A page block is `page_size` data rows followed by `page_size` scale
|
||||
// rows, so the scale region starts at byte `page_size * kDataBytes`:
|
||||
// kernels. A block-split page is `page_size` data rows followed by `page_size`
|
||||
// scale rows, so the scale region starts at byte `page_size * kDataBytes`:
|
||||
//
|
||||
// V4 584 B/token: 448 e4m3 + 64 bf16 (RoPE) data, 7 ue8m0 scales + 1 pad,
|
||||
// one scale per 64 e4m3 values.
|
||||
@@ -26,16 +26,27 @@
|
||||
// V41_FP4 288 B/token: 512 e2m1 data packed two per byte (even index in the
|
||||
// low nibble), 32 e4m3 scales, one per 16 values.
|
||||
//
|
||||
// The reader requires the rows of a page to be contiguous and the page stride
|
||||
// to be a multiple of kPageAlign (its TMA row stride), which is what
|
||||
// kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is
|
||||
// `sglang.test.kernels.deepseek_v4.k_cache_reference`.
|
||||
// V41_NEW / V41_FP4_NEW (FlashMLA's 2026.09.30 kernels) store the V41 / V41_FP4
|
||||
// bytes interleaved: each token is its data immediately followed by its scales,
|
||||
// 528 / 288 B apart, with no page padding.
|
||||
//
|
||||
// The block-split reader requires the rows of a page to be contiguous and the
|
||||
// page stride to be a multiple of kPageAlign (its TMA row stride), which is what
|
||||
// kv_page_bytes pads to. The pure-torch reference of the V41 quantizers is
|
||||
// `sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference`.
|
||||
|
||||
namespace sglang {
|
||||
|
||||
namespace deepseek_v4 {
|
||||
|
||||
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 };
|
||||
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3, V41_NEW = 4, V41_FP4_NEW = 5 };
|
||||
|
||||
/// The token encoding of a layout: V41_NEW / V41_FP4_NEW store the V41 / V41_FP4 bytes.
|
||||
constexpr KVLayout kv_encoding(KVLayout layout) {
|
||||
if (layout == KVLayout::V41_NEW) return KVLayout::V41;
|
||||
if (layout == KVLayout::V41_FP4_NEW) return KVLayout::V41_FP4;
|
||||
return layout;
|
||||
}
|
||||
|
||||
template <KVLayout kLayout>
|
||||
struct KVLayoutTraits;
|
||||
@@ -46,6 +57,7 @@ struct KVLayoutTraits<KVLayout::V4> {
|
||||
static constexpr int64_t kScaleBytes = 8;
|
||||
static constexpr int64_t kTileSize = 64;
|
||||
static constexpr int64_t kPageAlign = 576;
|
||||
static constexpr bool kInterleaved = false;
|
||||
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
|
||||
};
|
||||
|
||||
@@ -55,6 +67,7 @@ struct KVLayoutTraits<KVLayout::V41> {
|
||||
static constexpr int64_t kScaleBytes = 16;
|
||||
static constexpr int64_t kTileSize = 32;
|
||||
static constexpr int64_t kPageAlign = 512;
|
||||
static constexpr bool kInterleaved = false;
|
||||
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
|
||||
};
|
||||
|
||||
@@ -64,6 +77,7 @@ struct KVLayoutTraits<KVLayout::V41_FP4> {
|
||||
static constexpr int64_t kScaleBytes = 32;
|
||||
static constexpr int64_t kTileSize = 16;
|
||||
static constexpr int64_t kPageAlign = 256;
|
||||
static constexpr bool kInterleaved = false;
|
||||
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
|
||||
};
|
||||
|
||||
@@ -73,9 +87,22 @@ struct KVLayoutTraits<KVLayout::UNIFORM_FP8> {
|
||||
static constexpr int64_t kScaleBytes = 0;
|
||||
static constexpr int64_t kTileSize = 512;
|
||||
static constexpr int64_t kPageAlign = 512;
|
||||
static constexpr bool kInterleaved = false;
|
||||
static constexpr int64_t kBytesPerToken = kDataBytes;
|
||||
};
|
||||
|
||||
template <KVLayout kLayout>
|
||||
struct InterleavedKVLayoutTraits : KVLayoutTraits<kv_encoding(kLayout)> {
|
||||
static constexpr bool kInterleaved = true;
|
||||
static constexpr int64_t kPageAlign = KVLayoutTraits<kv_encoding(kLayout)>::kBytesPerToken;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct KVLayoutTraits<KVLayout::V41_NEW> : InterleavedKVLayoutTraits<KVLayout::V41_NEW> {};
|
||||
|
||||
template <>
|
||||
struct KVLayoutTraits<KVLayout::V41_FP4_NEW> : InterleavedKVLayoutTraits<KVLayout::V41_FP4_NEW> {};
|
||||
|
||||
/// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride.
|
||||
template <KVLayout kLayout>
|
||||
constexpr int64_t kv_page_bytes(int64_t page_size) {
|
||||
@@ -83,14 +110,16 @@ constexpr int64_t kv_page_bytes(int64_t page_size) {
|
||||
return (page_size * Traits::kBytesPerToken + Traits::kPageAlign - 1) / Traits::kPageAlign * Traits::kPageAlign;
|
||||
}
|
||||
|
||||
/// Addressing of a paged cache: `1 << kPageBits` data rows then as many scale rows, padded to kPageAlign.
|
||||
/// Addressing of a paged cache of `1 << kPageBits` tokens per page, padded to kPageAlign:
|
||||
/// slot `i`'s data at `i * kDataStride`, its scales at `kScaleBase + i * kScaleStride`.
|
||||
template <KVLayout kLayout, uint32_t kPageBits>
|
||||
struct PagedKV {
|
||||
using Traits = KVLayoutTraits<kLayout>;
|
||||
static constexpr int64_t kPageSize = int64_t{1} << kPageBits;
|
||||
static constexpr int64_t kPageBytes = kv_page_bytes<kLayout>(kPageSize);
|
||||
/// Byte offset of the scale rows inside a page.
|
||||
static constexpr int64_t kScaleBase = Traits::kDataBytes << kPageBits;
|
||||
static constexpr int64_t kDataStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kDataBytes;
|
||||
static constexpr int64_t kScaleBase = Traits::kInterleaved ? Traits::kDataBytes : Traits::kDataBytes << kPageBits;
|
||||
static constexpr int64_t kScaleStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kScaleBytes;
|
||||
|
||||
template <typename LocT>
|
||||
static constexpr LocT page_of(LocT loc) {
|
||||
@@ -103,11 +132,11 @@ struct PagedKV {
|
||||
/// Byte offsets of token `loc`'s data row and scale row from the cache base.
|
||||
template <typename LocT>
|
||||
static constexpr int64_t data_offset(LocT loc) {
|
||||
return page_of(loc) * kPageBytes + slot_of(loc) * Traits::kDataBytes;
|
||||
return page_of(loc) * kPageBytes + slot_of(loc) * kDataStride;
|
||||
}
|
||||
template <typename LocT>
|
||||
static constexpr int64_t scale_offset(LocT loc) {
|
||||
return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * Traits::kScaleBytes;
|
||||
return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * kScaleStride;
|
||||
}
|
||||
|
||||
struct Row {
|
||||
@@ -118,7 +147,7 @@ struct PagedKV {
|
||||
template <typename LocT>
|
||||
SGL_DEVICE static Row row(uint8_t* cache, LocT loc) {
|
||||
uint8_t* page = cache + page_of(loc) * kPageBytes;
|
||||
return {page + slot_of(loc) * Traits::kDataBytes, page + kScaleBase + slot_of(loc) * Traits::kScaleBytes};
|
||||
return {page + slot_of(loc) * kDataStride, page + kScaleBase + slot_of(loc) * kScaleStride};
|
||||
}
|
||||
};
|
||||
|
||||
@@ -213,7 +242,7 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
|
||||
template <KVLayout kLayout, uint32_t kVecSize>
|
||||
SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const float (&v)[kVecSize]) {
|
||||
static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller");
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
if constexpr (kv_encoding(kLayout) == KVLayout::V41) {
|
||||
store_row_fp8(data_row, scale_row, tx, v);
|
||||
} else if constexpr (kLayout == KVLayout::UNIFORM_FP8) {
|
||||
device::AlignedVector<fp8x2_e4m3_t, kVecSize / 2> out;
|
||||
|
||||
@@ -7,7 +7,7 @@ rather than into the grid, the online softmax stays in registers, and ``V`` is
|
||||
read as ``kv[:, :d_v]``, so there are no partials and no merge pass.
|
||||
``dsa/triton_sparse_mla.py`` is the gfx950-only FP8 sibling of this bf16 module.
|
||||
|
||||
Interface matches ``sgl_kernel.flash_mla.flash_mla_sparse_fwd``::
|
||||
Interface matches ``sglang.kernels.ops.attention.flash_mla_interface.flash_mla_sparse_fwd``::
|
||||
|
||||
q [T, H, 576] bf16 (absorbed MLA: 512 nope + 64 rope)
|
||||
kv [S, 576] bf16 (V is kv[:, :512])
|
||||
|
||||
@@ -102,13 +102,13 @@ def dequantize_k_cache_paged_v41(
|
||||
out: Optional[torch.Tensor] = None,
|
||||
layout: KVLayout = KVLayout.V41,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize a V4.1 paged cache (fp8 ``V41`` or fp4 ``V41_FP4``) for a list
|
||||
"""Dequantize a V4.1 paged cache (fp8 ``V41`` / ``V41_NEW`` or fp4 ``V41_FP4`` / ``V41_FP4_NEW``) for a list
|
||||
of token IDs into ``(num_tokens, 1, 512)`` bf16.
|
||||
|
||||
Bit-exact with the pure-torch dequantizer of these formats.
|
||||
"""
|
||||
layout = KVLayout.parse(layout)
|
||||
assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout
|
||||
assert layout.encoding in (KVLayout.V41, KVLayout.V41_FP4), layout
|
||||
if is_hip_runtime():
|
||||
supported = is_gfx95_supported()
|
||||
else:
|
||||
@@ -143,9 +143,10 @@ def dequantize_k_cache_paged_v41(
|
||||
|
||||
kernel = (
|
||||
_dequantize_k_cache_paged_v41_fp8_kernel
|
||||
if layout is KVLayout.V41
|
||||
if layout.encoding is KVLayout.V41
|
||||
else _dequantize_k_cache_paged_v41_fp4_kernel
|
||||
)
|
||||
data_stride, scale_base, scale_stride = layout.row_addressing(page_size)
|
||||
kernel[(num_tokens,)](
|
||||
out,
|
||||
buf_fp8,
|
||||
@@ -157,7 +158,9 @@ def dequantize_k_cache_paged_v41(
|
||||
DATA_BYTES=layout.data_bytes,
|
||||
SCALE_BYTES=layout.scale_bytes,
|
||||
TILE_SIZE=layout.tile_size,
|
||||
S_OFFSET_BYTES=layout.scale_offset(page_size),
|
||||
DATA_STRIDE=data_stride,
|
||||
SCALE_BASE=scale_base,
|
||||
SCALE_STRIDE=scale_stride,
|
||||
IS_HIP=is_hip_runtime(),
|
||||
)
|
||||
return out
|
||||
@@ -329,7 +332,9 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
|
||||
DATA_BYTES: tl.constexpr,
|
||||
SCALE_BYTES: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
S_OFFSET_BYTES: tl.constexpr,
|
||||
DATA_STRIDE: tl.constexpr,
|
||||
SCALE_BASE: tl.constexpr,
|
||||
SCALE_STRIDE: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values).
|
||||
@@ -339,8 +344,8 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
|
||||
page_idx = loc // PAGE_SIZE
|
||||
in_page = loc % PAGE_SIZE
|
||||
page_byte_base = page_idx * BYTES_PER_PAGE
|
||||
token_data_base = page_byte_base + in_page * DATA_BYTES
|
||||
token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES
|
||||
token_data_base = page_byte_base + in_page * DATA_STRIDE
|
||||
token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE
|
||||
|
||||
offs = tl.arange(0, DATA_BYTES)
|
||||
vals = tl.load(buf_fp8_ptr + token_data_base + offs).to(tl.float32)
|
||||
@@ -364,7 +369,9 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
|
||||
DATA_BYTES: tl.constexpr,
|
||||
SCALE_BYTES: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
S_OFFSET_BYTES: tl.constexpr,
|
||||
DATA_STRIDE: tl.constexpr,
|
||||
SCALE_BASE: tl.constexpr,
|
||||
SCALE_STRIDE: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble),
|
||||
@@ -375,8 +382,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
|
||||
page_idx = loc // PAGE_SIZE
|
||||
in_page = loc % PAGE_SIZE
|
||||
page_byte_base = page_idx * BYTES_PER_PAGE
|
||||
token_data_base = page_byte_base + in_page * DATA_BYTES
|
||||
token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES
|
||||
token_data_base = page_byte_base + in_page * DATA_STRIDE
|
||||
token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE
|
||||
|
||||
boffs = tl.arange(0, DATA_BYTES)
|
||||
packed = tl.load(buf_uint8_ptr + token_data_base + boffs)
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""Paged fp8 / fp4 KV cache layouts of the DeepSeek-V4 family sparse MLA decode kernels.
|
||||
|
||||
A page block stores ``page_size`` data rows followed by ``page_size`` scale rows.
|
||||
The reader selects the format from the bytes per token (the last dim of the
|
||||
``(num_pages, page_size, 1, bytes_per_token)`` view) and requires the page
|
||||
stride to be a multiple of its TMA row stride, which :meth:`KVLayout.page_bytes`
|
||||
pads to. Mirrors ``sgl_kernel/deepseek_v4/kv_layout.cuh``.
|
||||
Block-split layouts store a page as ``page_size`` data rows followed by
|
||||
``page_size`` scale rows, and the reader requires the page stride to be a
|
||||
multiple of its TMA row stride, which :meth:`KVLayout.page_bytes` pads to. The
|
||||
``_NEW`` layouts (FlashMLA's 2026.09.30 kernels) interleave them instead: each
|
||||
token is its data immediately followed by its scales, with no page padding.
|
||||
Mirrors ``sgl_kernel/deepseek_v4/kv_layout.cuh``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import functools
|
||||
from typing import Union
|
||||
|
||||
|
||||
@@ -22,62 +24,94 @@ class KVLayout(str, enum.Enum):
|
||||
V41_FP4 = "v41_fp4"
|
||||
# TRT-LLM only: 512 e4m3 values with a fixed external unit scale.
|
||||
UNIFORM_FP8 = "uniform_fp8"
|
||||
# V41 / V41_FP4 encodings, each token's scales right after its data.
|
||||
V41_NEW = "v41_new"
|
||||
V41_FP4_NEW = "v41_fp4_new"
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def is_interleaved(self) -> bool:
|
||||
return self in (KVLayout.V41_NEW, KVLayout.V41_FP4_NEW)
|
||||
|
||||
@functools.cached_property
|
||||
def encoding(self) -> KVLayout:
|
||||
"""The block-split layout storing a token with the same bytes."""
|
||||
return _INTERLEAVED_ENCODING.get(self, self)
|
||||
|
||||
def with_interleaved(self, interleaved: bool) -> KVLayout:
|
||||
"""This layout's encoding in the block-split or the interleaved arrangement."""
|
||||
if not interleaved:
|
||||
return self.encoding
|
||||
for new, old in _INTERLEAVED_ENCODING.items():
|
||||
if old is self.encoding:
|
||||
return new
|
||||
raise ValueError(f"{self} has no interleaved arrangement")
|
||||
|
||||
@functools.cached_property
|
||||
def data_bytes(self) -> int:
|
||||
return {
|
||||
KVLayout.V4: 576,
|
||||
KVLayout.V41: 512,
|
||||
KVLayout.V41_NEW: 512,
|
||||
KVLayout.V41_FP4: 256,
|
||||
KVLayout.V41_FP4_NEW: 256,
|
||||
KVLayout.UNIFORM_FP8: 512,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def scale_bytes(self) -> int:
|
||||
return {
|
||||
KVLayout.V4: 8,
|
||||
KVLayout.V41: 16,
|
||||
KVLayout.V41_NEW: 16,
|
||||
KVLayout.V41_FP4: 32,
|
||||
KVLayout.V41_FP4_NEW: 32,
|
||||
KVLayout.UNIFORM_FP8: 0,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def tile_size(self) -> int:
|
||||
"""Values sharing one scale."""
|
||||
return {
|
||||
KVLayout.V4: 64,
|
||||
KVLayout.V41: 32,
|
||||
KVLayout.V41_NEW: 32,
|
||||
KVLayout.V41_FP4: 16,
|
||||
KVLayout.V41_FP4_NEW: 16,
|
||||
KVLayout.UNIFORM_FP8: 512,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def bytes_per_token(self) -> int:
|
||||
return self.data_bytes + self.scale_bytes
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def page_align(self) -> int:
|
||||
"""Unit the page stride is padded to: the reader's TMA row stride."""
|
||||
return {
|
||||
KVLayout.V4: 576,
|
||||
KVLayout.V41: 512,
|
||||
KVLayout.V41_NEW: 528,
|
||||
KVLayout.V41_FP4: 256,
|
||||
KVLayout.V41_FP4_NEW: 288,
|
||||
KVLayout.UNIFORM_FP8: 512,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def is_fp4(self) -> bool:
|
||||
return self is KVLayout.V41_FP4
|
||||
return self.encoding is KVLayout.V41_FP4
|
||||
|
||||
def page_bytes(self, page_size: int) -> int:
|
||||
raw = page_size * self.bytes_per_token
|
||||
return -(-raw // self.page_align) * self.page_align
|
||||
|
||||
def scale_offset(self, page_size: int) -> int:
|
||||
"""Byte offset of the scale rows inside a page."""
|
||||
return page_size * self.data_bytes
|
||||
def row_addressing(self, page_size: int) -> tuple[int, int, int]:
|
||||
"""(data row stride, scale base, scale row stride) inside a page: slot
|
||||
``i``'s data is at ``i * data_stride``, its scales at ``scale_base + i * scale_stride``."""
|
||||
if self.is_interleaved:
|
||||
return self.bytes_per_token, self.data_bytes, self.bytes_per_token
|
||||
return self.data_bytes, page_size * self.data_bytes, self.scale_bytes
|
||||
|
||||
@property
|
||||
@functools.cached_property
|
||||
def cpp_name(self) -> str:
|
||||
"""The C++ enumerator, for JIT template arguments. Bare, because it is
|
||||
also part of the JIT module name; the headers `using enum` it in."""
|
||||
@@ -90,7 +124,17 @@ class KVLayout(str, enum.Enum):
|
||||
return cls(str(value).lower())
|
||||
|
||||
|
||||
_INTERLEAVED_ENCODING = {
|
||||
KVLayout.V41_NEW: KVLayout.V41,
|
||||
KVLayout.V41_FP4_NEW: KVLayout.V41_FP4,
|
||||
}
|
||||
|
||||
|
||||
def is_valid_kv_layout_pair(kv: KVLayout, extra_kv: KVLayout) -> bool:
|
||||
"""The (main, extra) cache pairs the decode kernel accepts: identical layouts,
|
||||
or the fp4 extra cache next to a V4.1 fp8 main cache."""
|
||||
return extra_kv is kv or (kv is KVLayout.V41 and extra_kv is KVLayout.V41_FP4)
|
||||
or the fp4 extra cache next to a V4.1 fp8 main cache of the same arrangement."""
|
||||
return extra_kv is kv or (
|
||||
kv.encoding is KVLayout.V41
|
||||
and extra_kv.encoding is KVLayout.V41_FP4
|
||||
and kv.is_interleaved == extra_kv.is_interleaved
|
||||
)
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""FlashMLA from the ``sgl-flash-mla`` wheel, behind the call signatures of the
|
||||
FlashMLA ops sgl-kernel used to ship (``sgl_kernel.flash_mla`` and
|
||||
``sgl_kernel.flash_mla_mega``).
|
||||
|
||||
The wheel carries two kernel sets. Everything the fork had before FlashMLA's
|
||||
2026.09.30 release (SM90 dense decode, the V3.2 / V3.2-no-RoPE / V4 formats,
|
||||
the block-split V4.1 caches and their MegaAttention) is its legacy side, the
|
||||
same sources sgl-kernel built, so ``use_legacy`` defaults to True. Only the
|
||||
token-interleaved V4.1 caches (``KVLayout.V41_NEW`` / ``V41_FP4_NEW``) take
|
||||
``use_legacy=False``. CUDA only: HIP and XPU keep their own sgl-kernel ops.
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
import flash_mla
|
||||
import torch
|
||||
from flash_mla import fused_norm_rope_attn_rope_cast as _mega
|
||||
from flash_mla.flash_mla_interface import FlashMLASchedMeta
|
||||
|
||||
# 512 fp8 values + 4 fp32 scales; the V4.1 formats have the same size, which is
|
||||
# why the tensor-metadata API (it predates V4.1) reads such a cache as V3.2.
|
||||
_V32_NO_ROPE_BYTES_PER_TOKEN = 528
|
||||
|
||||
# The only RoPE / output-cast configuration MegaAttention accepts:
|
||||
# both kernel sets assert every one of these values.
|
||||
_MEGA_FIXED_PARAMS = dict(
|
||||
is_rope_neox_style=False,
|
||||
rope_dim=64,
|
||||
num_per_channels=32,
|
||||
use_tma_aligned_col_major_sf=True,
|
||||
round_sf=True,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
|
||||
# The current (use_legacy=False) decode kernel splits the KV only past this
|
||||
# many main + extra indices; below it it schedules nothing and leaves the
|
||||
# metadata empty. Mirrors `enable_split_kv` in FlashMLA's csrc/api/sparse_decode.cpp.
|
||||
SPLIT_KV_MIN_TOPK = 641
|
||||
|
||||
|
||||
def get_mla_metadata(
|
||||
cache_seqlens: Optional[torch.Tensor] = None,
|
||||
num_q_tokens_per_head_k: Optional[int] = None,
|
||||
num_heads_k: Optional[int] = None,
|
||||
num_heads_q: Optional[int] = None,
|
||||
is_fp8_kvcache: bool = False,
|
||||
topk: Optional[int] = None,
|
||||
) -> Tuple[Union[torch.Tensor, FlashMLASchedMeta], Optional[torch.Tensor]]:
|
||||
"""Without ``cache_seqlens``, an empty ``FlashMLASchedMeta`` that the first
|
||||
decode call fills; with it, the legacy split-KV schedule as tensors."""
|
||||
if cache_seqlens is None:
|
||||
return flash_mla.get_mla_metadata()[0], None
|
||||
assert num_q_tokens_per_head_k is not None
|
||||
assert num_heads_k is not None
|
||||
return flash_mla.get_mla_decoding_metadata(
|
||||
cache_seqlens,
|
||||
num_q_tokens_per_head_k,
|
||||
num_heads_k,
|
||||
num_heads_q,
|
||||
is_fp8_kvcache,
|
||||
topk,
|
||||
use_legacy=True,
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_with_kvcache(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
block_table: Optional[torch.Tensor],
|
||||
cache_seqlens: Optional[torch.Tensor],
|
||||
head_dim_v: int,
|
||||
tile_scheduler_metadata: Union[torch.Tensor, FlashMLASchedMeta],
|
||||
num_splits: Optional[torch.Tensor] = None,
|
||||
softmax_scale: Optional[float] = None,
|
||||
causal: bool = False,
|
||||
descale_q: Optional[torch.Tensor] = None,
|
||||
descale_k: Optional[torch.Tensor] = None,
|
||||
is_fp8_kvcache: bool = False,
|
||||
indices: Optional[torch.Tensor] = None,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
extra_k_cache: Optional[torch.Tensor] = None,
|
||||
extra_indices_in_kvcache: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
extra_topk_length: Optional[torch.Tensor] = None,
|
||||
kv_format: Optional[str] = None,
|
||||
*,
|
||||
use_legacy: bool = True,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Paged MLA decode. ``tile_scheduler_metadata`` is a ``FlashMLASchedMeta``,
|
||||
or the schedule tensors from ``get_mla_metadata(cache_seqlens, ...)`` with
|
||||
``num_splits``, which ride on a fresh ``FlashMLASchedMeta`` (legacy only:
|
||||
V3.2 sparse, dense bf16, or dense fp8 when ``q`` is fp8)."""
|
||||
if softmax_scale is None:
|
||||
softmax_scale = q.shape[-1] ** (-0.5)
|
||||
if isinstance(tile_scheduler_metadata, FlashMLASchedMeta):
|
||||
assert num_splits is None, "num_splits must be None with FlashMLASchedMeta"
|
||||
sched_meta = tile_scheduler_metadata
|
||||
else:
|
||||
assert use_legacy, (
|
||||
"tensor scheduling metadata exists only for the legacy kernels"
|
||||
)
|
||||
assert num_splits is not None
|
||||
assert block_table is not None and cache_seqlens is not None
|
||||
assert all(
|
||||
x is None for x in (attn_sink, extra_k_cache, extra_indices_in_kvcache)
|
||||
), "attn_sink and extra caches need FlashMLASchedMeta"
|
||||
assert (descale_q is None) == (descale_k is None), (
|
||||
"descale_q and descale_k should be both None or both not None"
|
||||
)
|
||||
sched_meta = FlashMLASchedMeta(
|
||||
tile_scheduler_metadata=tile_scheduler_metadata, num_splits=num_splits
|
||||
)
|
||||
if indices is None:
|
||||
is_fp8_kvcache = q.element_size() == 1
|
||||
elif (
|
||||
kv_format is None
|
||||
and q.shape[-1] == 512
|
||||
and k_cache.shape[-1] == _V32_NO_ROPE_BYTES_PER_TOKEN
|
||||
):
|
||||
kv_format = "V32_NO_ROPE"
|
||||
return flash_mla.flash_mla_with_kvcache(
|
||||
q=q,
|
||||
k_cache=k_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
head_dim_v=head_dim_v,
|
||||
tile_scheduler_metadata=sched_meta,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
is_fp8_kvcache=is_fp8_kvcache,
|
||||
indices=indices,
|
||||
attn_sink=attn_sink,
|
||||
extra_k_cache=extra_k_cache,
|
||||
extra_indices_in_kvcache=extra_indices_in_kvcache,
|
||||
topk_length=topk_length,
|
||||
extra_topk_length=extra_topk_length,
|
||||
use_legacy=use_legacy,
|
||||
kv_format=kv_format,
|
||||
descale_q=descale_q,
|
||||
descale_k=descale_k,
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_sparse_fwd(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
sm_scale: float,
|
||||
d_v: int = 512,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
use_legacy: bool = True,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Sparse prefill over a bf16 ``kv``: (output, max_logits, lse). The legacy
|
||||
kernel's statistics are base 2, as sgl-kernel returned them; the current
|
||||
kernel's are natural-log."""
|
||||
return flash_mla.flash_mla_sparse_fwd(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
sm_scale,
|
||||
d_v,
|
||||
attn_sink,
|
||||
topk_length,
|
||||
use_legacy=use_legacy,
|
||||
return_log2_lse=True if use_legacy else None,
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_mega_prefill(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
*,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
enable_q_norm: bool = False,
|
||||
rms_norm_eps: float = 0.0,
|
||||
use_legacy: bool = True,
|
||||
) -> Tuple[torch.Tensor, ...]:
|
||||
"""MegaAttention prefill over a bf16 ``kv``: (out_fp8, out_sf, max_logits, lse)."""
|
||||
return _mega.prefill(
|
||||
token_positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
n_wv_group=q.shape[1] // 8,
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
sm_scale=softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=topk_length,
|
||||
enable_q_norm=enable_q_norm,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
use_legacy=use_legacy,
|
||||
**_MEGA_FIXED_PARAMS,
|
||||
)
|
||||
|
||||
|
||||
def flash_mla_mega_decode(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
*,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
extra_kv: Optional[torch.Tensor] = None,
|
||||
extra_indices: Optional[torch.Tensor] = None,
|
||||
extra_topk_length: Optional[torch.Tensor] = None,
|
||||
enable_q_norm: bool = False,
|
||||
rms_norm_eps: float = 0.0,
|
||||
use_legacy: bool = True,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""MegaAttention decode over a paged cache: (out_fp8, out_sf, lse)."""
|
||||
return _mega.decode(
|
||||
token_positions=positions,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
n_wv_group=q.shape[1] // 8,
|
||||
q=q,
|
||||
k_cache=kv,
|
||||
indices_in_kvcache=indices,
|
||||
sm_scale=softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=topk_length,
|
||||
extra_k_cache=extra_kv,
|
||||
extra_indices_in_kvcache=extra_indices,
|
||||
extra_topk_length=extra_topk_length,
|
||||
enable_q_norm=enable_q_norm,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
use_legacy=use_legacy,
|
||||
**_MEGA_FIXED_PARAMS,
|
||||
)
|
||||
@@ -4,6 +4,17 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _row_offset(loc, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE):
|
||||
# Byte `col` of token `loc`'s data-then-scales, from its page start; padding is untouched.
|
||||
slot = loc % PAGE
|
||||
return tl.where(
|
||||
col < DATA,
|
||||
slot * DATA_STRIDE + col,
|
||||
SCALE_BASE + slot * SCALE_STRIDE + col - DATA,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _copy_packed_rows(
|
||||
src,
|
||||
@@ -17,23 +28,20 @@ def _copy_packed_rows(
|
||||
PAGE: tl.constexpr,
|
||||
DATA: tl.constexpr,
|
||||
SCALE: tl.constexpr,
|
||||
DATA_STRIDE: tl.constexpr,
|
||||
SCALE_BASE: tl.constexpr,
|
||||
SCALE_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
source = tl.load(src_loc + row * SRC_LOC_STRIDE).to(tl.int64)
|
||||
target = tl.load(dst_loc + row * DST_LOC_STRIDE).to(tl.int64)
|
||||
col = tl.arange(0, BLOCK)
|
||||
# Each page contains all data rows, followed by all scale rows. Its
|
||||
# physical stride may also contain alignment padding, which is untouched.
|
||||
source_offset = tl.where(
|
||||
col < DATA,
|
||||
(source % PAGE) * DATA + col,
|
||||
PAGE * DATA + (source % PAGE) * SCALE + col - DATA,
|
||||
source_offset = _row_offset(
|
||||
source, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
|
||||
)
|
||||
target_offset = tl.where(
|
||||
col < DATA,
|
||||
(target % PAGE) * DATA + col,
|
||||
PAGE * DATA + (target % PAGE) * SCALE + col - DATA,
|
||||
target_offset = _row_offset(
|
||||
target, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
|
||||
)
|
||||
value = tl.load(
|
||||
src + source // PAGE * SRC_STRIDE + source_offset,
|
||||
@@ -60,6 +68,7 @@ def copy_packed_rows(src, dst, src_loc, dst_loc, *, page_size, layout):
|
||||
page_size,
|
||||
layout.data_bytes,
|
||||
layout.scale_bytes,
|
||||
*layout.row_addressing(page_size),
|
||||
triton.next_power_of_2(layout.bytes_per_token),
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -80,6 +89,9 @@ def _commit_packed_rows(
|
||||
CAPACITY: tl.constexpr,
|
||||
DATA: tl.constexpr,
|
||||
SCALE: tl.constexpr,
|
||||
DATA_STRIDE: tl.constexpr,
|
||||
SCALE_BASE: tl.constexpr,
|
||||
SCALE_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
@@ -89,15 +101,11 @@ def _commit_packed_rows(
|
||||
position = tl.load(pos + row).to(tl.int64)
|
||||
target = tl.load(req + row).to(tl.int64) * CAPACITY + position % CAPACITY
|
||||
col = tl.arange(0, BLOCK)
|
||||
source_offset = tl.where(
|
||||
col < DATA,
|
||||
(source % PAGE) * DATA + col,
|
||||
PAGE * DATA + (source % PAGE) * SCALE + col - DATA,
|
||||
source_offset = _row_offset(
|
||||
source, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
|
||||
)
|
||||
target_offset = tl.where(
|
||||
col < DATA,
|
||||
(target % PAGE) * DATA + col,
|
||||
PAGE * DATA + (target % PAGE) * SCALE + col - DATA,
|
||||
target_offset = _row_offset(
|
||||
target, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
|
||||
)
|
||||
value = tl.load(
|
||||
src + source // PAGE * SRC_STRIDE + source_offset,
|
||||
@@ -127,6 +135,7 @@ def commit_packed_rows(src, dst, tags, window, *, page_size, capacity, layout):
|
||||
capacity,
|
||||
layout.data_bytes,
|
||||
layout.scale_bytes,
|
||||
*layout.row_addressing(page_size),
|
||||
triton.next_power_of_2(layout.bytes_per_token),
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
@@ -36,7 +36,7 @@ def _default_dsv4_attention_backend(server_args: ServerArgs) -> str:
|
||||
or cfg.device != "cuda"
|
||||
or not is_sm100_supported()
|
||||
or cfg.kv_cache_dtype != "fp8_e4m3"
|
||||
or (layout.is_set() and layout.get().lower() not in ("auto", "v41"))
|
||||
or (layout.is_set() and layout.get().lower() not in ("auto", "v41", "v41_new"))
|
||||
):
|
||||
return "flashmla"
|
||||
return "flashmla_mega"
|
||||
|
||||
@@ -1707,7 +1707,8 @@ class Envs:
|
||||
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
|
||||
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
|
||||
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
|
||||
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
|
||||
# fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for
|
||||
# FlashMLA's 2026.09.30 kernels) or "auto" (v41 on SM100, v4 elsewhere).
|
||||
SGLANG_DSV4_KV_LAYOUT = EnvStr("v4")
|
||||
# Compressed-cache layout under "v41": "auto" (fp4 for the fp4-rounded
|
||||
# ratio-1 / ratio-2 latents, fp8 for ratios 4 / 128), "fp8" or "fp4" for all.
|
||||
|
||||
@@ -146,8 +146,7 @@ from sglang.srt.utils import ceil_align, is_cuda, is_xpu
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import FlashMLASchedMeta
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -210,9 +209,9 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T:
|
||||
def _create_flashmla_metadata():
|
||||
if get_platform().is_sm120 or _is_xpu:
|
||||
return None
|
||||
import sgl_kernel.flash_mla as flash_mla
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata
|
||||
|
||||
return flash_mla.get_mla_metadata()[0]
|
||||
return get_mla_metadata()[0]
|
||||
|
||||
|
||||
# FlashMLA's head64 sm100 decode scheduling constants; not exported, so they hold
|
||||
@@ -273,6 +272,13 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
flashmla_metadata.num_splits = num_splits
|
||||
|
||||
|
||||
def _flashmla_v41_splits_kv(
|
||||
indices: torch.Tensor, extra_indices: Optional[torch.Tensor], min_topk: int
|
||||
) -> bool:
|
||||
extra_topk = 0 if extra_indices is None else extra_indices.shape[-1]
|
||||
return indices.shape[-1] + extra_topk >= min_topk
|
||||
|
||||
|
||||
def _low_ratio_source_projections(
|
||||
layer, x, q_lora, positions, bufs, run_compressor, run_indexer
|
||||
):
|
||||
@@ -1237,6 +1243,17 @@ class DeepseekV4AttnBackend(
|
||||
|
||||
self.req_to_token_pool = model_runner.req_to_token_pool
|
||||
self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool
|
||||
if is_cuda():
|
||||
from sglang.kernels.ops.attention import flash_mla_interface
|
||||
|
||||
self._swa_interleaved = (
|
||||
self.token_to_kv_pool.get_swa_key_layout().is_interleaved
|
||||
)
|
||||
self._flash_mla_with_kvcache = functools.partial(
|
||||
flash_mla_interface.flash_mla_with_kvcache,
|
||||
use_legacy=not self._swa_interleaved,
|
||||
)
|
||||
self._flashmla_split_kv_min_topk = flash_mla_interface.SPLIT_KV_MIN_TOPK
|
||||
# The C4 state ring is addressed per SWA page, so a page holds whole windows.
|
||||
assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0
|
||||
self.hisparse_coordinator = model_runner.hisparse_coordinator
|
||||
@@ -3781,7 +3798,7 @@ class DeepseekV4AttnBackend(
|
||||
loc=slots,
|
||||
cache_k=layer.indexer.index_keys(latent, freqs),
|
||||
)
|
||||
if pool.get_extra_key_layout(layer.layer_id) is KVLayout.V41_FP4:
|
||||
if pool.get_extra_key_layout(layer.layer_id).is_fp4:
|
||||
# The fp4 cache stores e2m1 codes: the kernel rotates the tail and
|
||||
# rounds once, with no fake quantization in between.
|
||||
pool.set_extra_key_buffer_fused(
|
||||
@@ -4290,9 +4307,16 @@ class DeepseekV4AttnBackend(
|
||||
if _is_xpu:
|
||||
from sgl_kernel import flash_mla_with_kvcache
|
||||
else:
|
||||
from sgl_kernel.flash_mla import flash_mla_with_kvcache
|
||||
flash_mla_with_kvcache = self._flash_mla_with_kvcache
|
||||
|
||||
if self.is_dsv41:
|
||||
if self.is_dsv41 and (
|
||||
not self._swa_interleaved
|
||||
or _flashmla_v41_splits_kv(
|
||||
swa_page_indices,
|
||||
extra_indices,
|
||||
self._flashmla_split_kv_min_topk,
|
||||
)
|
||||
):
|
||||
_maybe_precompute_flashmla_sched_meta(
|
||||
flashmla_metadata,
|
||||
q=q,
|
||||
@@ -4339,7 +4363,9 @@ class DeepseekV4AttnBackend(
|
||||
if _is_xpu:
|
||||
from sgl_kernel import flash_mla_sparse_fwd
|
||||
else:
|
||||
from sgl_kernel.flash_mla import flash_mla_sparse_fwd
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import (
|
||||
flash_mla_sparse_fwd,
|
||||
)
|
||||
|
||||
# q is (b, 1, h_q, d_qk); flash_mla_sparse_fwd takes (s_q, h_q, d_qk).
|
||||
q_flat = q.squeeze(1)
|
||||
|
||||
@@ -116,8 +116,6 @@ from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout
|
||||
from sglang.srt.utils import ceil_align, is_gfx95_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
FP4DecodeWorkspace,
|
||||
FP4KWriteMetadata,
|
||||
@@ -125,6 +123,7 @@ if TYPE_CHECKING:
|
||||
FP4RowgroupDecodeWorkspace,
|
||||
FP4RowgroupPrefillWorkspace,
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import FlashMLASchedMeta
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
@@ -244,9 +243,9 @@ def _create_flashmla_metadata():
|
||||
|
||||
if is_hip():
|
||||
return None
|
||||
import sgl_kernel.flash_mla as flash_mla
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata
|
||||
|
||||
return flash_mla.get_mla_metadata()[0]
|
||||
return get_mla_metadata()[0]
|
||||
|
||||
|
||||
def _create_dummy_paged_compress_data(compress_ratio: int):
|
||||
|
||||
@@ -78,9 +78,9 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
|
||||
extra_indices: Optional[torch.Tensor],
|
||||
extra_topk_lengths: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
from sgl_kernel.flash_mla_mega import flash_mla_mega_decode
|
||||
from sglang.kernels.ops.attention import flash_mla_interface
|
||||
|
||||
data, scale, _ = flash_mla_mega_decode(
|
||||
data, scale, _ = flash_mla_interface.flash_mla_mega_decode(
|
||||
q,
|
||||
swa_k_cache,
|
||||
swa_page_indices,
|
||||
@@ -92,6 +92,7 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
|
||||
extra_kv=extra_k_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_topk_length=extra_topk_lengths,
|
||||
use_legacy=not self._swa_interleaved,
|
||||
)
|
||||
return data, scale
|
||||
|
||||
@@ -106,9 +107,9 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
|
||||
attn_sink: torch.Tensor,
|
||||
topk_length: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill
|
||||
from sglang.kernels.ops.attention import flash_mla_interface
|
||||
|
||||
data, scale, _, _ = flash_mla_mega_prefill(
|
||||
data, scale, _, _ = flash_mla_interface.flash_mla_mega_prefill(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
@@ -117,5 +118,6 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
|
||||
self.softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=topk_length,
|
||||
use_legacy=not self._swa_interleaved,
|
||||
)
|
||||
return data, scale
|
||||
|
||||
@@ -2812,7 +2812,9 @@ class DeepseekSparseAttnBackend(
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
from sgl_kernel.flash_mla import flash_mla_sparse_fwd
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import (
|
||||
flash_mla_sparse_fwd,
|
||||
)
|
||||
|
||||
# FlashMLA sparse kernel requires num_heads to be a multiple of 64 (Hopper) or 128 (Blackwell)
|
||||
# When using TP, num_heads might be smaller (e.g., 256//8=32)
|
||||
@@ -3314,7 +3316,9 @@ class DeepseekSparseAttnBackend(
|
||||
metadata: DSAMetadata,
|
||||
page_table_1,
|
||||
) -> torch.Tensor:
|
||||
from sgl_kernel.flash_mla import flash_mla_with_kvcache
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import (
|
||||
flash_mla_with_kvcache,
|
||||
)
|
||||
|
||||
cache_seqlens = metadata.dsa_cache_seqlens_int32
|
||||
assert metadata.flashmla_metadata is not None
|
||||
@@ -4113,7 +4117,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
|
||||
def _compute_flashmla_metadata(self, cache_seqlens: torch.Tensor, seq_len_q: int):
|
||||
from sgl_kernel.flash_mla import get_mla_metadata
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata
|
||||
|
||||
num_heads_q = self.flashmla_kv_num_q_heads
|
||||
|
||||
|
||||
@@ -10,8 +10,11 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import triton
|
||||
from sgl_kernel.flash_mla import flash_mla_with_kvcache, get_mla_metadata
|
||||
|
||||
from sglang.kernels.ops.attention.flash_mla_interface import (
|
||||
flash_mla_with_kvcache,
|
||||
get_mla_metadata,
|
||||
)
|
||||
from sglang.kernels.ops.attention.kv_indices import (
|
||||
create_flashmla_kv_indices_triton,
|
||||
get_num_kv_index_blocks_flashmla,
|
||||
|
||||
@@ -203,7 +203,7 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
|
||||
backend if backend in _HIP_BACKENDS else None
|
||||
)
|
||||
else:
|
||||
import sgl_kernel.flash_mla as flash_mla
|
||||
from sglang.kernels.ops.attention import flash_mla_interface
|
||||
|
||||
if backend == "comparison":
|
||||
pack_ref, pack_fast_via_tester = flash_mla_with_kvcache_entrypoint(
|
||||
@@ -238,7 +238,7 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
|
||||
return aiter_sparse_decode_fwd(**kwargs)
|
||||
|
||||
if backend == "kernel":
|
||||
return flash_mla.flash_mla_with_kvcache(**kwargs)
|
||||
return flash_mla_interface.flash_mla_with_kvcache(**kwargs)
|
||||
|
||||
raise NotImplementedError(f"unknown backend: {backend!r}")
|
||||
|
||||
|
||||
@@ -189,12 +189,10 @@ def resolve_compressed_kv_layout(
|
||||
if kv_layout is KVLayout.V4:
|
||||
assert option in (None, "fp8"), "the V4 main cache only pairs with V4 caches"
|
||||
return KVLayout.V4
|
||||
assert kv_layout is KVLayout.V41, f"{kv_layout} is not a main-cache layout"
|
||||
if option == "fp8":
|
||||
return KVLayout.V41
|
||||
if option == "fp4":
|
||||
return KVLayout.V41_FP4
|
||||
return KVLayout.V41_FP4 if compress_ratio in (1, 2) else KVLayout.V41
|
||||
assert kv_layout.encoding is KVLayout.V41, f"{kv_layout} is not a main-cache layout"
|
||||
use_fp4 = option == "fp4" or (option is None and compress_ratio in (1, 2))
|
||||
encoding = KVLayout.V41_FP4 if use_fp4 else KVLayout.V41
|
||||
return encoding.with_interleaved(kv_layout.is_interleaved)
|
||||
|
||||
|
||||
def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
@@ -203,7 +201,7 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
AITER attention."""
|
||||
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
|
||||
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
|
||||
if get_exec().kernel.dsv4_attn_backend == "flashmla_mega":
|
||||
if get_exec().kernel.dsv4_attn_backend == "flashmla_mega" and mode != "v41_new":
|
||||
mode = "v41"
|
||||
if get_exec().kernel.dsv4_attn_backend == "trtllm":
|
||||
assert mode in ("auto", "v4") and option in ("auto", "fp8"), (
|
||||
@@ -212,7 +210,7 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
return KVLayout.V4, None
|
||||
if mode == "v4":
|
||||
return KVLayout.V4, None if option == "auto" else option
|
||||
assert mode in ("v41", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}"
|
||||
assert mode in ("v41", "v41_new", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}"
|
||||
is_sm100 = (
|
||||
torch.cuda.is_available()
|
||||
and torch.version.cuda is not None
|
||||
@@ -224,15 +222,16 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
)
|
||||
if mode == "auto" and not supported:
|
||||
return KVLayout.V4, None
|
||||
if not supported:
|
||||
if not supported or mode == "v41_new":
|
||||
raise ValueError(
|
||||
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention"
|
||||
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention, "
|
||||
"which reads the block-split v41 layout"
|
||||
)
|
||||
return KVLayout.V41, option
|
||||
if mode == "auto" and not is_sm100:
|
||||
return KVLayout.V4, None
|
||||
assert is_sm100, "the V4.1 KV cache layouts need an SM100 / SM103 GPU"
|
||||
return KVLayout.V41, option
|
||||
return (KVLayout.V41_NEW if mode == "v41_new" else KVLayout.V41), option
|
||||
|
||||
|
||||
class DeepSeekV4SingleKVPool(KVCache):
|
||||
@@ -995,6 +994,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
assert self.kv_layout in (
|
||||
KVLayout.V4,
|
||||
KVLayout.V41,
|
||||
KVLayout.V41_NEW,
|
||||
), f"{self.kv_layout} is only valid for a compressed (extra) cache"
|
||||
self.compressed_kv_layout_option = compressed_kv_layout
|
||||
c4_logical_size = c128_size * 32
|
||||
@@ -2265,7 +2265,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
_, compress_layer_id, compress_kv_pool = self.layer_mapping[layer_id]
|
||||
assert compress_kv_pool is not None
|
||||
if freqs_cis is not None:
|
||||
assert compress_kv_pool.kv_layout is KVLayout.V41_FP4, (
|
||||
assert compress_kv_pool.kv_layout.is_fp4, (
|
||||
"in-kernel RoPE is for the fp4 cache; fp8 caches take the finished value"
|
||||
)
|
||||
return compress_kv_pool.set_key_buffer_fused(
|
||||
|
||||
@@ -560,7 +560,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
graph_supported_backend_types.append(FlashAttentionBackend)
|
||||
# FlashMLA is CUDA-only; import lazily so CPU builds don't pull
|
||||
# sgl_kernel.flash_mla at import time.
|
||||
# FlashMLA at import time.
|
||||
from sglang.srt.layers.attention.flashmla_backend import FlashMLABackend
|
||||
|
||||
graph_supported_backend_types.append(FlashMLABackend)
|
||||
|
||||
@@ -145,3 +145,39 @@ def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor:
|
||||
return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view(
|
||||
num_pages, page_size, 512
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The interleaved arrangement of the same bytes (V41_NEW / V41_FP4_NEW): each
|
||||
# token is its data immediately followed by its scales, pages unpadded.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def interleave_pages(
|
||||
pages: torch.Tensor, page_size: int, data_bytes: int, scale_bytes: int
|
||||
) -> torch.Tensor:
|
||||
"""Block-split pages ``[num_pages, >= page_size * (data + scale)]`` uint8 ->
|
||||
interleaved ``[num_pages, page_size * (data + scale)]``."""
|
||||
num_pages = pages.shape[0]
|
||||
split = page_size * data_bytes
|
||||
data = pages[:, :split].reshape(num_pages, page_size, data_bytes)
|
||||
scale = pages[:, split : split + page_size * scale_bytes].reshape(
|
||||
num_pages, page_size, scale_bytes
|
||||
)
|
||||
return torch.cat([data, scale], dim=-1).reshape(num_pages, -1)
|
||||
|
||||
|
||||
def deinterleave_pages(
|
||||
pages: torch.Tensor, page_size: int, data_bytes: int, scale_bytes: int
|
||||
) -> torch.Tensor:
|
||||
"""Inverse of :func:`interleave_pages`, without page padding."""
|
||||
num_pages = pages.shape[0]
|
||||
tokens = pages.view(torch.uint8)[:, : page_size * (data_bytes + scale_bytes)]
|
||||
tokens = tokens.reshape(num_pages, page_size, data_bytes + scale_bytes)
|
||||
return torch.cat(
|
||||
[
|
||||
tokens[..., :data_bytes].reshape(num_pages, -1),
|
||||
tokens[..., data_bytes:].reshape(num_pages, -1),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
@@ -1128,13 +1128,6 @@ def dsa_impl_capability(impl: str) -> tuple[bool, str]:
|
||||
major, minor = _torch.cuda.get_device_capability()
|
||||
|
||||
if impl == "flashmla_sparse" or impl == "flashmla_kv":
|
||||
try:
|
||||
from sgl_kernel.flash_mla import ( # noqa: F401
|
||||
flash_mla_sparse_fwd,
|
||||
flash_mla_with_kvcache,
|
||||
)
|
||||
except ImportError as exc:
|
||||
return False, f"sgl_kernel.flash_mla unavailable: {exc}"
|
||||
if major < 9:
|
||||
return False, f"{impl} requires SM>=9.0, got SM{major}.x"
|
||||
return True, ""
|
||||
|
||||
Reference in New Issue
Block a user