[Perf] dsv41: Serve token-interleaved V4.1 KV caches with sgl-flash-mla (#45893)

Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ziyi Xu
2026-10-22 22:02:49 -07:00
committed by GitHub
co-authored by kpham-sgl Claude Opus 5.5
parent 49933325cc
commit e83f585ae6
36 changed files with 965 additions and 251 deletions
+1
View File
@@ -76,6 +76,7 @@ dependencies = [
"setproctitle",
"sgl-deep-ep==0.1.2.post1",
"sgl-deep-gemm==0.2.3",
"sgl-flash-mla==1.0.0",
"sglang-kernel==0.4.10",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
@@ -162,7 +162,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
}
}
if constexpr (kLayout == KVLayout::V41_FP4) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) {
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
if (out_loc <= 0) return;
const auto kv_row = Paged::row(params.kvcache, out_loc);
@@ -191,7 +191,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
if (out_loc <= 0) return;
const auto kv_row = Paged::row(params.kvcache, out_loc);
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
// FP8 over the whole row, RoPE included; the layout selects the scaling.
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, data);
}
@@ -232,7 +232,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
if constexpr (kLayout == KVLayout::V41_FP4) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) {
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
const auto kv_row = Paged::row(params.kvcache, out_loc);
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
@@ -258,7 +258,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
const auto kv_row = Paged::row(params.kvcache, out_loc);
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
// FP8 over the whole row, RoPE included; the layout selects the scaling.
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
}
@@ -16,9 +16,8 @@
// critical path, and it cannot be hoisted out because the schedule depends on
// the per-request `topk_length` of the step being replayed.
//
// This kernel produces the same output bit for bit; the shared-memory layout
// matches FlashMLA's exactly because the idle-tail fill below reads one entry
// past `first_block_idx_shared`, as FlashMLA does.
// This kernel produces the same output bit for bit as the legacy scheduler in
// sgl-flash-mla 1.0.0, idle-tail partitions included.
namespace sglang {
@@ -180,9 +179,8 @@ __global__ void __launch_bounds__(kBlockSize) flashmla_sched_meta_kernel(__grid_
const int lb_last = last_block_idx_shared[b - 1];
const int sk_last = seqlens_k_shared[b - 1];
const int end_block_idx = (sk_last == 0) ? 0 : lb_last + 1;
// FlashMLA reads first_block_idx_shared[batch_size] for these, one past the
// end of that array, which aliases last_block_idx_shared[0].
const int begin_block_idx = last_block_idx_shared[0];
// `now_block` is 0 once every request is consumed.
const int begin_block_idx = 0;
const int is_last_req_splitted = (end_block_idx != lb_last + 1) && (sk_last != 0);
for (int i = first_idle_part_shared + threadIdx.x; i < p.num_sm_parts; i += kBlockSize) {
int* meta = out_shared + i * kMetaInts;
@@ -84,7 +84,7 @@ __global__ void fused_store_flashmla_cache(const __grid_constant__ FusedStoreCac
/// Elements per thread of the V4.1 store (`512 / vec` threads per token): 4 for fp8 rows, 2 for
/// fp4, whose per-element IEEE divisions spread better over more threads (measured on B200, bs 1..512).
constexpr uint32_t v41_store_vec_size(deepseek_v4::KVLayout layout) {
return layout == deepseek_v4::KVLayout::V41 ? 4 : 2;
return deepseek_v4::kv_encoding(layout) == deepseek_v4::KVLayout::V41 ? 4 : 2;
}
template <
@@ -16,8 +16,8 @@
#endif
// Paged fp8 / fp4 KV cache layouts read by the d_qk = 512 sparse MLA decode
// kernels. A page block is `page_size` data rows followed by `page_size` scale
// rows, so the scale region starts at byte `page_size * kDataBytes`:
// kernels. A block-split page is `page_size` data rows followed by `page_size`
// scale rows, so the scale region starts at byte `page_size * kDataBytes`:
//
// V4 584 B/token: 448 e4m3 + 64 bf16 (RoPE) data, 7 ue8m0 scales + 1 pad,
// one scale per 64 e4m3 values.
@@ -26,16 +26,27 @@
// V41_FP4 288 B/token: 512 e2m1 data packed two per byte (even index in the
// low nibble), 32 e4m3 scales, one per 16 values.
//
// The reader requires the rows of a page to be contiguous and the page stride
// to be a multiple of kPageAlign (its TMA row stride), which is what
// kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is
// `sglang.test.kernels.deepseek_v4.k_cache_reference`.
// V41_NEW / V41_FP4_NEW (FlashMLA's 2026.09.30 kernels) store the V41 / V41_FP4
// bytes interleaved: each token is its data immediately followed by its scales,
// 528 / 288 B apart, with no page padding.
//
// The block-split reader requires the rows of a page to be contiguous and the
// page stride to be a multiple of kPageAlign (its TMA row stride), which is what
// kv_page_bytes pads to. The pure-torch reference of the V41 quantizers is
// `sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference`.
namespace sglang {
namespace deepseek_v4 {
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 };
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3, V41_NEW = 4, V41_FP4_NEW = 5 };
/// The token encoding of a layout: V41_NEW / V41_FP4_NEW store the V41 / V41_FP4 bytes.
constexpr KVLayout kv_encoding(KVLayout layout) {
if (layout == KVLayout::V41_NEW) return KVLayout::V41;
if (layout == KVLayout::V41_FP4_NEW) return KVLayout::V41_FP4;
return layout;
}
template <KVLayout kLayout>
struct KVLayoutTraits;
@@ -46,6 +57,7 @@ struct KVLayoutTraits<KVLayout::V4> {
static constexpr int64_t kScaleBytes = 8;
static constexpr int64_t kTileSize = 64;
static constexpr int64_t kPageAlign = 576;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};
@@ -55,6 +67,7 @@ struct KVLayoutTraits<KVLayout::V41> {
static constexpr int64_t kScaleBytes = 16;
static constexpr int64_t kTileSize = 32;
static constexpr int64_t kPageAlign = 512;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};
@@ -64,6 +77,7 @@ struct KVLayoutTraits<KVLayout::V41_FP4> {
static constexpr int64_t kScaleBytes = 32;
static constexpr int64_t kTileSize = 16;
static constexpr int64_t kPageAlign = 256;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};
@@ -73,9 +87,22 @@ struct KVLayoutTraits<KVLayout::UNIFORM_FP8> {
static constexpr int64_t kScaleBytes = 0;
static constexpr int64_t kTileSize = 512;
static constexpr int64_t kPageAlign = 512;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes;
};
template <KVLayout kLayout>
struct InterleavedKVLayoutTraits : KVLayoutTraits<kv_encoding(kLayout)> {
static constexpr bool kInterleaved = true;
static constexpr int64_t kPageAlign = KVLayoutTraits<kv_encoding(kLayout)>::kBytesPerToken;
};
template <>
struct KVLayoutTraits<KVLayout::V41_NEW> : InterleavedKVLayoutTraits<KVLayout::V41_NEW> {};
template <>
struct KVLayoutTraits<KVLayout::V41_FP4_NEW> : InterleavedKVLayoutTraits<KVLayout::V41_FP4_NEW> {};
/// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride.
template <KVLayout kLayout>
constexpr int64_t kv_page_bytes(int64_t page_size) {
@@ -83,14 +110,16 @@ constexpr int64_t kv_page_bytes(int64_t page_size) {
return (page_size * Traits::kBytesPerToken + Traits::kPageAlign - 1) / Traits::kPageAlign * Traits::kPageAlign;
}
/// Addressing of a paged cache: `1 << kPageBits` data rows then as many scale rows, padded to kPageAlign.
/// Addressing of a paged cache of `1 << kPageBits` tokens per page, padded to kPageAlign:
/// slot `i`'s data at `i * kDataStride`, its scales at `kScaleBase + i * kScaleStride`.
template <KVLayout kLayout, uint32_t kPageBits>
struct PagedKV {
using Traits = KVLayoutTraits<kLayout>;
static constexpr int64_t kPageSize = int64_t{1} << kPageBits;
static constexpr int64_t kPageBytes = kv_page_bytes<kLayout>(kPageSize);
/// Byte offset of the scale rows inside a page.
static constexpr int64_t kScaleBase = Traits::kDataBytes << kPageBits;
static constexpr int64_t kDataStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kDataBytes;
static constexpr int64_t kScaleBase = Traits::kInterleaved ? Traits::kDataBytes : Traits::kDataBytes << kPageBits;
static constexpr int64_t kScaleStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kScaleBytes;
template <typename LocT>
static constexpr LocT page_of(LocT loc) {
@@ -103,11 +132,11 @@ struct PagedKV {
/// Byte offsets of token `loc`'s data row and scale row from the cache base.
template <typename LocT>
static constexpr int64_t data_offset(LocT loc) {
return page_of(loc) * kPageBytes + slot_of(loc) * Traits::kDataBytes;
return page_of(loc) * kPageBytes + slot_of(loc) * kDataStride;
}
template <typename LocT>
static constexpr int64_t scale_offset(LocT loc) {
return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * Traits::kScaleBytes;
return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * kScaleStride;
}
struct Row {
@@ -118,7 +147,7 @@ struct PagedKV {
template <typename LocT>
SGL_DEVICE static Row row(uint8_t* cache, LocT loc) {
uint8_t* page = cache + page_of(loc) * kPageBytes;
return {page + slot_of(loc) * Traits::kDataBytes, page + kScaleBase + slot_of(loc) * Traits::kScaleBytes};
return {page + slot_of(loc) * kDataStride, page + kScaleBase + slot_of(loc) * kScaleStride};
}
};
@@ -213,7 +242,7 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
template <KVLayout kLayout, uint32_t kVecSize>
SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const float (&v)[kVecSize]) {
static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller");
if constexpr (kLayout == KVLayout::V41) {
if constexpr (kv_encoding(kLayout) == KVLayout::V41) {
store_row_fp8(data_row, scale_row, tx, v);
} else if constexpr (kLayout == KVLayout::UNIFORM_FP8) {
device::AlignedVector<fp8x2_e4m3_t, kVecSize / 2> out;
@@ -7,7 +7,7 @@ rather than into the grid, the online softmax stays in registers, and ``V`` is
read as ``kv[:, :d_v]``, so there are no partials and no merge pass.
``dsa/triton_sparse_mla.py`` is the gfx950-only FP8 sibling of this bf16 module.
Interface matches ``sgl_kernel.flash_mla.flash_mla_sparse_fwd``::
Interface matches ``sglang.kernels.ops.attention.flash_mla_interface.flash_mla_sparse_fwd``::
q [T, H, 576] bf16 (absorbed MLA: 512 nope + 64 rope)
kv [S, 576] bf16 (V is kv[:, :512])
@@ -102,13 +102,13 @@ def dequantize_k_cache_paged_v41(
out: Optional[torch.Tensor] = None,
layout: KVLayout = KVLayout.V41,
) -> torch.Tensor:
"""Dequantize a V4.1 paged cache (fp8 ``V41`` or fp4 ``V41_FP4``) for a list
"""Dequantize a V4.1 paged cache (fp8 ``V41`` / ``V41_NEW`` or fp4 ``V41_FP4`` / ``V41_FP4_NEW``) for a list
of token IDs into ``(num_tokens, 1, 512)`` bf16.
Bit-exact with the pure-torch dequantizer of these formats.
"""
layout = KVLayout.parse(layout)
assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout
assert layout.encoding in (KVLayout.V41, KVLayout.V41_FP4), layout
if is_hip_runtime():
supported = is_gfx95_supported()
else:
@@ -143,9 +143,10 @@ def dequantize_k_cache_paged_v41(
kernel = (
_dequantize_k_cache_paged_v41_fp8_kernel
if layout is KVLayout.V41
if layout.encoding is KVLayout.V41
else _dequantize_k_cache_paged_v41_fp4_kernel
)
data_stride, scale_base, scale_stride = layout.row_addressing(page_size)
kernel[(num_tokens,)](
out,
buf_fp8,
@@ -157,7 +158,9 @@ def dequantize_k_cache_paged_v41(
DATA_BYTES=layout.data_bytes,
SCALE_BYTES=layout.scale_bytes,
TILE_SIZE=layout.tile_size,
S_OFFSET_BYTES=layout.scale_offset(page_size),
DATA_STRIDE=data_stride,
SCALE_BASE=scale_base,
SCALE_STRIDE=scale_stride,
IS_HIP=is_hip_runtime(),
)
return out
@@ -329,7 +332,9 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
DATA_BYTES: tl.constexpr,
SCALE_BYTES: tl.constexpr,
TILE_SIZE: tl.constexpr,
S_OFFSET_BYTES: tl.constexpr,
DATA_STRIDE: tl.constexpr,
SCALE_BASE: tl.constexpr,
SCALE_STRIDE: tl.constexpr,
IS_HIP: tl.constexpr,
):
# V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values).
@@ -339,8 +344,8 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
page_idx = loc // PAGE_SIZE
in_page = loc % PAGE_SIZE
page_byte_base = page_idx * BYTES_PER_PAGE
token_data_base = page_byte_base + in_page * DATA_BYTES
token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES
token_data_base = page_byte_base + in_page * DATA_STRIDE
token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE
offs = tl.arange(0, DATA_BYTES)
vals = tl.load(buf_fp8_ptr + token_data_base + offs).to(tl.float32)
@@ -364,7 +369,9 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
DATA_BYTES: tl.constexpr,
SCALE_BYTES: tl.constexpr,
TILE_SIZE: tl.constexpr,
S_OFFSET_BYTES: tl.constexpr,
DATA_STRIDE: tl.constexpr,
SCALE_BASE: tl.constexpr,
SCALE_STRIDE: tl.constexpr,
IS_HIP: tl.constexpr,
):
# V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble),
@@ -375,8 +382,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
page_idx = loc // PAGE_SIZE
in_page = loc % PAGE_SIZE
page_byte_base = page_idx * BYTES_PER_PAGE
token_data_base = page_byte_base + in_page * DATA_BYTES
token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES
token_data_base = page_byte_base + in_page * DATA_STRIDE
token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE
boffs = tl.arange(0, DATA_BYTES)
packed = tl.load(buf_uint8_ptr + token_data_base + boffs)
@@ -1,15 +1,17 @@
"""Paged fp8 / fp4 KV cache layouts of the DeepSeek-V4 family sparse MLA decode kernels.
A page block stores ``page_size`` data rows followed by ``page_size`` scale rows.
The reader selects the format from the bytes per token (the last dim of the
``(num_pages, page_size, 1, bytes_per_token)`` view) and requires the page
stride to be a multiple of its TMA row stride, which :meth:`KVLayout.page_bytes`
pads to. Mirrors ``sgl_kernel/deepseek_v4/kv_layout.cuh``.
Block-split layouts store a page as ``page_size`` data rows followed by
``page_size`` scale rows, and the reader requires the page stride to be a
multiple of its TMA row stride, which :meth:`KVLayout.page_bytes` pads to. The
``_NEW`` layouts (FlashMLA's 2026.09.30 kernels) interleave them instead: each
token is its data immediately followed by its scales, with no page padding.
Mirrors ``sgl_kernel/deepseek_v4/kv_layout.cuh``.
"""
from __future__ import annotations
import enum
import functools
from typing import Union
@@ -22,62 +24,94 @@ class KVLayout(str, enum.Enum):
V41_FP4 = "v41_fp4"
# TRT-LLM only: 512 e4m3 values with a fixed external unit scale.
UNIFORM_FP8 = "uniform_fp8"
# V41 / V41_FP4 encodings, each token's scales right after its data.
V41_NEW = "v41_new"
V41_FP4_NEW = "v41_fp4_new"
@property
@functools.cached_property
def is_interleaved(self) -> bool:
return self in (KVLayout.V41_NEW, KVLayout.V41_FP4_NEW)
@functools.cached_property
def encoding(self) -> KVLayout:
"""The block-split layout storing a token with the same bytes."""
return _INTERLEAVED_ENCODING.get(self, self)
def with_interleaved(self, interleaved: bool) -> KVLayout:
"""This layout's encoding in the block-split or the interleaved arrangement."""
if not interleaved:
return self.encoding
for new, old in _INTERLEAVED_ENCODING.items():
if old is self.encoding:
return new
raise ValueError(f"{self} has no interleaved arrangement")
@functools.cached_property
def data_bytes(self) -> int:
return {
KVLayout.V4: 576,
KVLayout.V41: 512,
KVLayout.V41_NEW: 512,
KVLayout.V41_FP4: 256,
KVLayout.V41_FP4_NEW: 256,
KVLayout.UNIFORM_FP8: 512,
}[self]
@property
@functools.cached_property
def scale_bytes(self) -> int:
return {
KVLayout.V4: 8,
KVLayout.V41: 16,
KVLayout.V41_NEW: 16,
KVLayout.V41_FP4: 32,
KVLayout.V41_FP4_NEW: 32,
KVLayout.UNIFORM_FP8: 0,
}[self]
@property
@functools.cached_property
def tile_size(self) -> int:
"""Values sharing one scale."""
return {
KVLayout.V4: 64,
KVLayout.V41: 32,
KVLayout.V41_NEW: 32,
KVLayout.V41_FP4: 16,
KVLayout.V41_FP4_NEW: 16,
KVLayout.UNIFORM_FP8: 512,
}[self]
@property
@functools.cached_property
def bytes_per_token(self) -> int:
return self.data_bytes + self.scale_bytes
@property
@functools.cached_property
def page_align(self) -> int:
"""Unit the page stride is padded to: the reader's TMA row stride."""
return {
KVLayout.V4: 576,
KVLayout.V41: 512,
KVLayout.V41_NEW: 528,
KVLayout.V41_FP4: 256,
KVLayout.V41_FP4_NEW: 288,
KVLayout.UNIFORM_FP8: 512,
}[self]
@property
@functools.cached_property
def is_fp4(self) -> bool:
return self is KVLayout.V41_FP4
return self.encoding is KVLayout.V41_FP4
def page_bytes(self, page_size: int) -> int:
raw = page_size * self.bytes_per_token
return -(-raw // self.page_align) * self.page_align
def scale_offset(self, page_size: int) -> int:
"""Byte offset of the scale rows inside a page."""
return page_size * self.data_bytes
def row_addressing(self, page_size: int) -> tuple[int, int, int]:
"""(data row stride, scale base, scale row stride) inside a page: slot
``i``'s data is at ``i * data_stride``, its scales at ``scale_base + i * scale_stride``."""
if self.is_interleaved:
return self.bytes_per_token, self.data_bytes, self.bytes_per_token
return self.data_bytes, page_size * self.data_bytes, self.scale_bytes
@property
@functools.cached_property
def cpp_name(self) -> str:
"""The C++ enumerator, for JIT template arguments. Bare, because it is
also part of the JIT module name; the headers `using enum` it in."""
@@ -90,7 +124,17 @@ class KVLayout(str, enum.Enum):
return cls(str(value).lower())
_INTERLEAVED_ENCODING = {
KVLayout.V41_NEW: KVLayout.V41,
KVLayout.V41_FP4_NEW: KVLayout.V41_FP4,
}
def is_valid_kv_layout_pair(kv: KVLayout, extra_kv: KVLayout) -> bool:
"""The (main, extra) cache pairs the decode kernel accepts: identical layouts,
or the fp4 extra cache next to a V4.1 fp8 main cache."""
return extra_kv is kv or (kv is KVLayout.V41 and extra_kv is KVLayout.V41_FP4)
or the fp4 extra cache next to a V4.1 fp8 main cache of the same arrangement."""
return extra_kv is kv or (
kv.encoding is KVLayout.V41
and extra_kv.encoding is KVLayout.V41_FP4
and kv.is_interleaved == extra_kv.is_interleaved
)
@@ -0,0 +1,238 @@
"""FlashMLA from the ``sgl-flash-mla`` wheel, behind the call signatures of the
FlashMLA ops sgl-kernel used to ship (``sgl_kernel.flash_mla`` and
``sgl_kernel.flash_mla_mega``).
The wheel carries two kernel sets. Everything the fork had before FlashMLA's
2026.09.30 release (SM90 dense decode, the V3.2 / V3.2-no-RoPE / V4 formats,
the block-split V4.1 caches and their MegaAttention) is its legacy side, the
same sources sgl-kernel built, so ``use_legacy`` defaults to True. Only the
token-interleaved V4.1 caches (``KVLayout.V41_NEW`` / ``V41_FP4_NEW``) take
``use_legacy=False``. CUDA only: HIP and XPU keep their own sgl-kernel ops.
"""
from typing import Optional, Tuple, Union
import flash_mla
import torch
from flash_mla import fused_norm_rope_attn_rope_cast as _mega
from flash_mla.flash_mla_interface import FlashMLASchedMeta
# 512 fp8 values + 4 fp32 scales; the V4.1 formats have the same size, which is
# why the tensor-metadata API (it predates V4.1) reads such a cache as V3.2.
_V32_NO_ROPE_BYTES_PER_TOKEN = 528
# The only RoPE / output-cast configuration MegaAttention accepts:
# both kernel sets assert every one of these values.
_MEGA_FIXED_PARAMS = dict(
is_rope_neox_style=False,
rope_dim=64,
num_per_channels=32,
use_tma_aligned_col_major_sf=True,
round_sf=True,
use_packed_ue8m0=True,
)
# The current (use_legacy=False) decode kernel splits the KV only past this
# many main + extra indices; below it it schedules nothing and leaves the
# metadata empty. Mirrors `enable_split_kv` in FlashMLA's csrc/api/sparse_decode.cpp.
SPLIT_KV_MIN_TOPK = 641
def get_mla_metadata(
cache_seqlens: Optional[torch.Tensor] = None,
num_q_tokens_per_head_k: Optional[int] = None,
num_heads_k: Optional[int] = None,
num_heads_q: Optional[int] = None,
is_fp8_kvcache: bool = False,
topk: Optional[int] = None,
) -> Tuple[Union[torch.Tensor, FlashMLASchedMeta], Optional[torch.Tensor]]:
"""Without ``cache_seqlens``, an empty ``FlashMLASchedMeta`` that the first
decode call fills; with it, the legacy split-KV schedule as tensors."""
if cache_seqlens is None:
return flash_mla.get_mla_metadata()[0], None
assert num_q_tokens_per_head_k is not None
assert num_heads_k is not None
return flash_mla.get_mla_decoding_metadata(
cache_seqlens,
num_q_tokens_per_head_k,
num_heads_k,
num_heads_q,
is_fp8_kvcache,
topk,
use_legacy=True,
)
def flash_mla_with_kvcache(
q: torch.Tensor,
k_cache: torch.Tensor,
block_table: Optional[torch.Tensor],
cache_seqlens: Optional[torch.Tensor],
head_dim_v: int,
tile_scheduler_metadata: Union[torch.Tensor, FlashMLASchedMeta],
num_splits: Optional[torch.Tensor] = None,
softmax_scale: Optional[float] = None,
causal: bool = False,
descale_q: Optional[torch.Tensor] = None,
descale_k: Optional[torch.Tensor] = None,
is_fp8_kvcache: bool = False,
indices: Optional[torch.Tensor] = None,
attn_sink: Optional[torch.Tensor] = None,
extra_k_cache: Optional[torch.Tensor] = None,
extra_indices_in_kvcache: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
extra_topk_length: Optional[torch.Tensor] = None,
kv_format: Optional[str] = None,
*,
use_legacy: bool = True,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Paged MLA decode. ``tile_scheduler_metadata`` is a ``FlashMLASchedMeta``,
or the schedule tensors from ``get_mla_metadata(cache_seqlens, ...)`` with
``num_splits``, which ride on a fresh ``FlashMLASchedMeta`` (legacy only:
V3.2 sparse, dense bf16, or dense fp8 when ``q`` is fp8)."""
if softmax_scale is None:
softmax_scale = q.shape[-1] ** (-0.5)
if isinstance(tile_scheduler_metadata, FlashMLASchedMeta):
assert num_splits is None, "num_splits must be None with FlashMLASchedMeta"
sched_meta = tile_scheduler_metadata
else:
assert use_legacy, (
"tensor scheduling metadata exists only for the legacy kernels"
)
assert num_splits is not None
assert block_table is not None and cache_seqlens is not None
assert all(
x is None for x in (attn_sink, extra_k_cache, extra_indices_in_kvcache)
), "attn_sink and extra caches need FlashMLASchedMeta"
assert (descale_q is None) == (descale_k is None), (
"descale_q and descale_k should be both None or both not None"
)
sched_meta = FlashMLASchedMeta(
tile_scheduler_metadata=tile_scheduler_metadata, num_splits=num_splits
)
if indices is None:
is_fp8_kvcache = q.element_size() == 1
elif (
kv_format is None
and q.shape[-1] == 512
and k_cache.shape[-1] == _V32_NO_ROPE_BYTES_PER_TOKEN
):
kv_format = "V32_NO_ROPE"
return flash_mla.flash_mla_with_kvcache(
q=q,
k_cache=k_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
head_dim_v=head_dim_v,
tile_scheduler_metadata=sched_meta,
softmax_scale=softmax_scale,
causal=causal,
is_fp8_kvcache=is_fp8_kvcache,
indices=indices,
attn_sink=attn_sink,
extra_k_cache=extra_k_cache,
extra_indices_in_kvcache=extra_indices_in_kvcache,
topk_length=topk_length,
extra_topk_length=extra_topk_length,
use_legacy=use_legacy,
kv_format=kv_format,
descale_q=descale_q,
descale_k=descale_k,
)
def flash_mla_sparse_fwd(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
sm_scale: float,
d_v: int = 512,
attn_sink: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
*,
use_legacy: bool = True,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Sparse prefill over a bf16 ``kv``: (output, max_logits, lse). The legacy
kernel's statistics are base 2, as sgl-kernel returned them; the current
kernel's are natural-log."""
return flash_mla.flash_mla_sparse_fwd(
q,
kv,
indices,
sm_scale,
d_v,
attn_sink,
topk_length,
use_legacy=use_legacy,
return_log2_lse=True if use_legacy else None,
)
def flash_mla_mega_prefill(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
softmax_scale: float,
*,
attn_sink: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
enable_q_norm: bool = False,
rms_norm_eps: float = 0.0,
use_legacy: bool = True,
) -> Tuple[torch.Tensor, ...]:
"""MegaAttention prefill over a bf16 ``kv``: (out_fp8, out_sf, max_logits, lse)."""
return _mega.prefill(
token_positions=positions,
cos_sin_cache=cos_sin_cache,
n_wv_group=q.shape[1] // 8,
q=q,
kv=kv,
indices=indices,
sm_scale=softmax_scale,
attn_sink=attn_sink,
topk_length=topk_length,
enable_q_norm=enable_q_norm,
rms_norm_eps=rms_norm_eps,
use_legacy=use_legacy,
**_MEGA_FIXED_PARAMS,
)
def flash_mla_mega_decode(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
softmax_scale: float,
*,
attn_sink: Optional[torch.Tensor] = None,
topk_length: Optional[torch.Tensor] = None,
extra_kv: Optional[torch.Tensor] = None,
extra_indices: Optional[torch.Tensor] = None,
extra_topk_length: Optional[torch.Tensor] = None,
enable_q_norm: bool = False,
rms_norm_eps: float = 0.0,
use_legacy: bool = True,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""MegaAttention decode over a paged cache: (out_fp8, out_sf, lse)."""
return _mega.decode(
token_positions=positions,
cos_sin_cache=cos_sin_cache,
n_wv_group=q.shape[1] // 8,
q=q,
k_cache=kv,
indices_in_kvcache=indices,
sm_scale=softmax_scale,
attn_sink=attn_sink,
topk_length=topk_length,
extra_k_cache=extra_kv,
extra_indices_in_kvcache=extra_indices,
extra_topk_length=extra_topk_length,
enable_q_norm=enable_q_norm,
rms_norm_eps=rms_norm_eps,
use_legacy=use_legacy,
**_MEGA_FIXED_PARAMS,
)
@@ -4,6 +4,17 @@ import triton
import triton.language as tl
@triton.jit
def _row_offset(loc, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE):
# Byte `col` of token `loc`'s data-then-scales, from its page start; padding is untouched.
slot = loc % PAGE
return tl.where(
col < DATA,
slot * DATA_STRIDE + col,
SCALE_BASE + slot * SCALE_STRIDE + col - DATA,
)
@triton.jit
def _copy_packed_rows(
src,
@@ -17,23 +28,20 @@ def _copy_packed_rows(
PAGE: tl.constexpr,
DATA: tl.constexpr,
SCALE: tl.constexpr,
DATA_STRIDE: tl.constexpr,
SCALE_BASE: tl.constexpr,
SCALE_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
source = tl.load(src_loc + row * SRC_LOC_STRIDE).to(tl.int64)
target = tl.load(dst_loc + row * DST_LOC_STRIDE).to(tl.int64)
col = tl.arange(0, BLOCK)
# Each page contains all data rows, followed by all scale rows. Its
# physical stride may also contain alignment padding, which is untouched.
source_offset = tl.where(
col < DATA,
(source % PAGE) * DATA + col,
PAGE * DATA + (source % PAGE) * SCALE + col - DATA,
source_offset = _row_offset(
source, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
)
target_offset = tl.where(
col < DATA,
(target % PAGE) * DATA + col,
PAGE * DATA + (target % PAGE) * SCALE + col - DATA,
target_offset = _row_offset(
target, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
)
value = tl.load(
src + source // PAGE * SRC_STRIDE + source_offset,
@@ -60,6 +68,7 @@ def copy_packed_rows(src, dst, src_loc, dst_loc, *, page_size, layout):
page_size,
layout.data_bytes,
layout.scale_bytes,
*layout.row_addressing(page_size),
triton.next_power_of_2(layout.bytes_per_token),
num_warps=4,
)
@@ -80,6 +89,9 @@ def _commit_packed_rows(
CAPACITY: tl.constexpr,
DATA: tl.constexpr,
SCALE: tl.constexpr,
DATA_STRIDE: tl.constexpr,
SCALE_BASE: tl.constexpr,
SCALE_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
@@ -89,15 +101,11 @@ def _commit_packed_rows(
position = tl.load(pos + row).to(tl.int64)
target = tl.load(req + row).to(tl.int64) * CAPACITY + position % CAPACITY
col = tl.arange(0, BLOCK)
source_offset = tl.where(
col < DATA,
(source % PAGE) * DATA + col,
PAGE * DATA + (source % PAGE) * SCALE + col - DATA,
source_offset = _row_offset(
source, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
)
target_offset = tl.where(
col < DATA,
(target % PAGE) * DATA + col,
PAGE * DATA + (target % PAGE) * SCALE + col - DATA,
target_offset = _row_offset(
target, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE
)
value = tl.load(
src + source // PAGE * SRC_STRIDE + source_offset,
@@ -127,6 +135,7 @@ def commit_packed_rows(src, dst, tags, window, *, page_size, capacity, layout):
capacity,
layout.data_bytes,
layout.scale_bytes,
*layout.row_addressing(page_size),
triton.next_power_of_2(layout.bytes_per_token),
num_warps=4,
)
@@ -36,7 +36,7 @@ def _default_dsv4_attention_backend(server_args: ServerArgs) -> str:
or cfg.device != "cuda"
or not is_sm100_supported()
or cfg.kv_cache_dtype != "fp8_e4m3"
or (layout.is_set() and layout.get().lower() not in ("auto", "v41"))
or (layout.is_set() and layout.get().lower() not in ("auto", "v41", "v41_new"))
):
return "flashmla"
return "flashmla_mega"
+2 -1
View File
@@ -1707,7 +1707,8 @@ class Envs:
SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048)
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
# fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for
# FlashMLA's 2026.09.30 kernels) or "auto" (v41 on SM100, v4 elsewhere).
SGLANG_DSV4_KV_LAYOUT = EnvStr("v4")
# Compressed-cache layout under "v41": "auto" (fp4 for the fp4-rounded
# ratio-1 / ratio-2 latents, fp8 for ratios 4 / 128), "fp8" or "fp4" for all.
@@ -146,8 +146,7 @@ from sglang.srt.utils import ceil_align, is_cuda, is_xpu
from sglang.srt.utils.common import async_h2d
if TYPE_CHECKING:
from sgl_kernel.flash_mla import FlashMLASchedMeta
from sglang.kernels.ops.attention.flash_mla_interface import FlashMLASchedMeta
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -210,9 +209,9 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T:
def _create_flashmla_metadata():
if get_platform().is_sm120 or _is_xpu:
return None
import sgl_kernel.flash_mla as flash_mla
from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata
return flash_mla.get_mla_metadata()[0]
return get_mla_metadata()[0]
# FlashMLA's head64 sm100 decode scheduling constants; not exported, so they hold
@@ -273,6 +272,13 @@ def _maybe_precompute_flashmla_sched_meta(
flashmla_metadata.num_splits = num_splits
def _flashmla_v41_splits_kv(
indices: torch.Tensor, extra_indices: Optional[torch.Tensor], min_topk: int
) -> bool:
extra_topk = 0 if extra_indices is None else extra_indices.shape[-1]
return indices.shape[-1] + extra_topk >= min_topk
def _low_ratio_source_projections(
layer, x, q_lora, positions, bufs, run_compressor, run_indexer
):
@@ -1237,6 +1243,17 @@ class DeepseekV4AttnBackend(
self.req_to_token_pool = model_runner.req_to_token_pool
self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool
if is_cuda():
from sglang.kernels.ops.attention import flash_mla_interface
self._swa_interleaved = (
self.token_to_kv_pool.get_swa_key_layout().is_interleaved
)
self._flash_mla_with_kvcache = functools.partial(
flash_mla_interface.flash_mla_with_kvcache,
use_legacy=not self._swa_interleaved,
)
self._flashmla_split_kv_min_topk = flash_mla_interface.SPLIT_KV_MIN_TOPK
# The C4 state ring is addressed per SWA page, so a page holds whole windows.
assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0
self.hisparse_coordinator = model_runner.hisparse_coordinator
@@ -3781,7 +3798,7 @@ class DeepseekV4AttnBackend(
loc=slots,
cache_k=layer.indexer.index_keys(latent, freqs),
)
if pool.get_extra_key_layout(layer.layer_id) is KVLayout.V41_FP4:
if pool.get_extra_key_layout(layer.layer_id).is_fp4:
# The fp4 cache stores e2m1 codes: the kernel rotates the tail and
# rounds once, with no fake quantization in between.
pool.set_extra_key_buffer_fused(
@@ -4290,9 +4307,16 @@ class DeepseekV4AttnBackend(
if _is_xpu:
from sgl_kernel import flash_mla_with_kvcache
else:
from sgl_kernel.flash_mla import flash_mla_with_kvcache
flash_mla_with_kvcache = self._flash_mla_with_kvcache
if self.is_dsv41:
if self.is_dsv41 and (
not self._swa_interleaved
or _flashmla_v41_splits_kv(
swa_page_indices,
extra_indices,
self._flashmla_split_kv_min_topk,
)
):
_maybe_precompute_flashmla_sched_meta(
flashmla_metadata,
q=q,
@@ -4339,7 +4363,9 @@ class DeepseekV4AttnBackend(
if _is_xpu:
from sgl_kernel import flash_mla_sparse_fwd
else:
from sgl_kernel.flash_mla import flash_mla_sparse_fwd
from sglang.kernels.ops.attention.flash_mla_interface import (
flash_mla_sparse_fwd,
)
# q is (b, 1, h_q, d_qk); flash_mla_sparse_fwd takes (s_q, h_q, d_qk).
q_flat = q.squeeze(1)
@@ -116,8 +116,6 @@ from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout
from sglang.srt.utils import ceil_align, is_gfx95_supported
if TYPE_CHECKING:
from sgl_kernel.flash_mla import FlashMLASchedMeta
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
FP4DecodeWorkspace,
FP4KWriteMetadata,
@@ -125,6 +123,7 @@ if TYPE_CHECKING:
FP4RowgroupDecodeWorkspace,
FP4RowgroupPrefillWorkspace,
)
from sglang.kernels.ops.attention.flash_mla_interface import FlashMLASchedMeta
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -244,9 +243,9 @@ def _create_flashmla_metadata():
if is_hip():
return None
import sgl_kernel.flash_mla as flash_mla
from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata
return flash_mla.get_mla_metadata()[0]
return get_mla_metadata()[0]
def _create_dummy_paged_compress_data(compress_ratio: int):
@@ -78,9 +78,9 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
extra_indices: Optional[torch.Tensor],
extra_topk_lengths: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
from sgl_kernel.flash_mla_mega import flash_mla_mega_decode
from sglang.kernels.ops.attention import flash_mla_interface
data, scale, _ = flash_mla_mega_decode(
data, scale, _ = flash_mla_interface.flash_mla_mega_decode(
q,
swa_k_cache,
swa_page_indices,
@@ -92,6 +92,7 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
extra_kv=extra_k_cache,
extra_indices=extra_indices,
extra_topk_length=extra_topk_lengths,
use_legacy=not self._swa_interleaved,
)
return data, scale
@@ -106,9 +107,9 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
attn_sink: torch.Tensor,
topk_length: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill
from sglang.kernels.ops.attention import flash_mla_interface
data, scale, _, _ = flash_mla_mega_prefill(
data, scale, _, _ = flash_mla_interface.flash_mla_mega_prefill(
q,
kv,
indices,
@@ -117,5 +118,6 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
self.softmax_scale,
attn_sink=attn_sink,
topk_length=topk_length,
use_legacy=not self._swa_interleaved,
)
return data, scale
@@ -2812,7 +2812,9 @@ class DeepseekSparseAttnBackend(
topk_length: Optional[torch.Tensor] = None,
attn_sink: Optional[torch.Tensor] = None,
) -> torch.Tensor:
from sgl_kernel.flash_mla import flash_mla_sparse_fwd
from sglang.kernels.ops.attention.flash_mla_interface import (
flash_mla_sparse_fwd,
)
# FlashMLA sparse kernel requires num_heads to be a multiple of 64 (Hopper) or 128 (Blackwell)
# When using TP, num_heads might be smaller (e.g., 256//8=32)
@@ -3314,7 +3316,9 @@ class DeepseekSparseAttnBackend(
metadata: DSAMetadata,
page_table_1,
) -> torch.Tensor:
from sgl_kernel.flash_mla import flash_mla_with_kvcache
from sglang.kernels.ops.attention.flash_mla_interface import (
flash_mla_with_kvcache,
)
cache_seqlens = metadata.dsa_cache_seqlens_int32
assert metadata.flashmla_metadata is not None
@@ -4113,7 +4117,7 @@ class DeepseekSparseAttnBackend(
)
def _compute_flashmla_metadata(self, cache_seqlens: torch.Tensor, seq_len_q: int):
from sgl_kernel.flash_mla import get_mla_metadata
from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata
num_heads_q = self.flashmla_kv_num_q_heads
@@ -10,8 +10,11 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union
import torch
import triton
from sgl_kernel.flash_mla import flash_mla_with_kvcache, get_mla_metadata
from sglang.kernels.ops.attention.flash_mla_interface import (
flash_mla_with_kvcache,
get_mla_metadata,
)
from sglang.kernels.ops.attention.kv_indices import (
create_flashmla_kv_indices_triton,
get_num_kv_index_blocks_flashmla,
@@ -203,7 +203,7 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
backend if backend in _HIP_BACKENDS else None
)
else:
import sgl_kernel.flash_mla as flash_mla
from sglang.kernels.ops.attention import flash_mla_interface
if backend == "comparison":
pack_ref, pack_fast_via_tester = flash_mla_with_kvcache_entrypoint(
@@ -238,7 +238,7 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
return aiter_sparse_decode_fwd(**kwargs)
if backend == "kernel":
return flash_mla.flash_mla_with_kvcache(**kwargs)
return flash_mla_interface.flash_mla_with_kvcache(**kwargs)
raise NotImplementedError(f"unknown backend: {backend!r}")
@@ -189,12 +189,10 @@ def resolve_compressed_kv_layout(
if kv_layout is KVLayout.V4:
assert option in (None, "fp8"), "the V4 main cache only pairs with V4 caches"
return KVLayout.V4
assert kv_layout is KVLayout.V41, f"{kv_layout} is not a main-cache layout"
if option == "fp8":
return KVLayout.V41
if option == "fp4":
return KVLayout.V41_FP4
return KVLayout.V41_FP4 if compress_ratio in (1, 2) else KVLayout.V41
assert kv_layout.encoding is KVLayout.V41, f"{kv_layout} is not a main-cache layout"
use_fp4 = option == "fp4" or (option is None and compress_ratio in (1, 2))
encoding = KVLayout.V41_FP4 if use_fp4 else KVLayout.V41
return encoding.with_interleaved(kv_layout.is_interleaved)
def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
@@ -203,7 +201,7 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
AITER attention."""
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
if get_exec().kernel.dsv4_attn_backend == "flashmla_mega":
if get_exec().kernel.dsv4_attn_backend == "flashmla_mega" and mode != "v41_new":
mode = "v41"
if get_exec().kernel.dsv4_attn_backend == "trtllm":
assert mode in ("auto", "v4") and option in ("auto", "fp8"), (
@@ -212,7 +210,7 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
return KVLayout.V4, None
if mode == "v4":
return KVLayout.V4, None if option == "auto" else option
assert mode in ("v41", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}"
assert mode in ("v41", "v41_new", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}"
is_sm100 = (
torch.cuda.is_available()
and torch.version.cuda is not None
@@ -224,15 +222,16 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
)
if mode == "auto" and not supported:
return KVLayout.V4, None
if not supported:
if not supported or mode == "v41_new":
raise ValueError(
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention"
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention, "
"which reads the block-split v41 layout"
)
return KVLayout.V41, option
if mode == "auto" and not is_sm100:
return KVLayout.V4, None
assert is_sm100, "the V4.1 KV cache layouts need an SM100 / SM103 GPU"
return KVLayout.V41, option
return (KVLayout.V41_NEW if mode == "v41_new" else KVLayout.V41), option
class DeepSeekV4SingleKVPool(KVCache):
@@ -995,6 +994,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
assert self.kv_layout in (
KVLayout.V4,
KVLayout.V41,
KVLayout.V41_NEW,
), f"{self.kv_layout} is only valid for a compressed (extra) cache"
self.compressed_kv_layout_option = compressed_kv_layout
c4_logical_size = c128_size * 32
@@ -2265,7 +2265,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
_, compress_layer_id, compress_kv_pool = self.layer_mapping[layer_id]
assert compress_kv_pool is not None
if freqs_cis is not None:
assert compress_kv_pool.kv_layout is KVLayout.V41_FP4, (
assert compress_kv_pool.kv_layout.is_fp4, (
"in-kernel RoPE is for the fp4 cache; fp8 caches take the finished value"
)
return compress_kv_pool.set_key_buffer_fused(
@@ -560,7 +560,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
graph_supported_backend_types.append(FlashAttentionBackend)
# FlashMLA is CUDA-only; import lazily so CPU builds don't pull
# sgl_kernel.flash_mla at import time.
# FlashMLA at import time.
from sglang.srt.layers.attention.flashmla_backend import FlashMLABackend
graph_supported_backend_types.append(FlashMLABackend)
@@ -145,3 +145,39 @@ def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor:
return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view(
num_pages, page_size, 512
)
# ---------------------------------------------------------------------------
# The interleaved arrangement of the same bytes (V41_NEW / V41_FP4_NEW): each
# token is its data immediately followed by its scales, pages unpadded.
# ---------------------------------------------------------------------------
def interleave_pages(
pages: torch.Tensor, page_size: int, data_bytes: int, scale_bytes: int
) -> torch.Tensor:
"""Block-split pages ``[num_pages, >= page_size * (data + scale)]`` uint8 ->
interleaved ``[num_pages, page_size * (data + scale)]``."""
num_pages = pages.shape[0]
split = page_size * data_bytes
data = pages[:, :split].reshape(num_pages, page_size, data_bytes)
scale = pages[:, split : split + page_size * scale_bytes].reshape(
num_pages, page_size, scale_bytes
)
return torch.cat([data, scale], dim=-1).reshape(num_pages, -1)
def deinterleave_pages(
pages: torch.Tensor, page_size: int, data_bytes: int, scale_bytes: int
) -> torch.Tensor:
"""Inverse of :func:`interleave_pages`, without page padding."""
num_pages = pages.shape[0]
tokens = pages.view(torch.uint8)[:, : page_size * (data_bytes + scale_bytes)]
tokens = tokens.reshape(num_pages, page_size, data_bytes + scale_bytes)
return torch.cat(
[
tokens[..., :data_bytes].reshape(num_pages, -1),
tokens[..., data_bytes:].reshape(num_pages, -1),
],
dim=-1,
)
@@ -1128,13 +1128,6 @@ def dsa_impl_capability(impl: str) -> tuple[bool, str]:
major, minor = _torch.cuda.get_device_capability()
if impl == "flashmla_sparse" or impl == "flashmla_kv":
try:
from sgl_kernel.flash_mla import ( # noqa: F401
flash_mla_sparse_fwd,
flash_mla_with_kvcache,
)
except ImportError as exc:
return False, f"sgl_kernel.flash_mla unavailable: {exc}"
if major < 9:
return False, f"{impl} requires SM>=9.0, got SM{major}.x"
return True, ""