diff --git a/python/pyproject.toml b/python/pyproject.toml index 51f18793f3..f9e3c5b904 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -76,6 +76,7 @@ dependencies = [ "setproctitle", "sgl-deep-ep==0.1.2.post1", "sgl-deep-gemm==0.2.3", + "sgl-flash-mla==1.0.0", "sglang-kernel==0.4.10", "smg-grpc-servicer>=0.9.0", "soundfile==0.13.1", diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh index 3e3511c955..8a0d848f58 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh @@ -162,7 +162,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel( } } - if constexpr (kLayout == KVLayout::V41_FP4) { + if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) { // The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant. if (out_loc <= 0) return; const auto kv_row = Paged::row(params.kvcache, out_loc); @@ -191,7 +191,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel( if (out_loc <= 0) return; const auto kv_row = Paged::row(params.kvcache, out_loc); - if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) { + if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) { // FP8 over the whole row, RoPE included; the layout selects the scaling. return deepseek_v4::v41::store_row(kv_row.data, kv_row.scale, tx, data); } diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh index 175fbb7627..a9ae1322f9 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh @@ -232,7 +232,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( } const int32_t out_loc = static_cast(raw_out_loc) >> 1; - if constexpr (kLayout == KVLayout::V41_FP4) { + if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) { // The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant. const auto kv_row = Paged::row(params.kvcache, out_loc); return deepseek_v4::v41::store_row(kv_row.data, kv_row.scale, tx, staged); @@ -258,7 +258,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( // `raw_out_loc / ratio`; ratio 2 makes it a shift. const auto kv_row = Paged::row(params.kvcache, out_loc); - if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) { + if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) { // FP8 over the whole row, RoPE included; the layout selects the scaling. return deepseek_v4::v41::store_row(kv_row.data, kv_row.scale, tx, staged); } diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/flashmla_sched_meta.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/flashmla_sched_meta.cuh index b41821b2d0..dd73c04b75 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/flashmla_sched_meta.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/flashmla_sched_meta.cuh @@ -16,9 +16,8 @@ // critical path, and it cannot be hoisted out because the schedule depends on // the per-request `topk_length` of the step being replayed. // -// This kernel produces the same output bit for bit; the shared-memory layout -// matches FlashMLA's exactly because the idle-tail fill below reads one entry -// past `first_block_idx_shared`, as FlashMLA does. +// This kernel produces the same output bit for bit as the legacy scheduler in +// sgl-flash-mla 1.0.0, idle-tail partitions included. namespace sglang { @@ -180,9 +179,8 @@ __global__ void __launch_bounds__(kBlockSize) flashmla_sched_meta_kernel(__grid_ const int lb_last = last_block_idx_shared[b - 1]; const int sk_last = seqlens_k_shared[b - 1]; const int end_block_idx = (sk_last == 0) ? 0 : lb_last + 1; - // FlashMLA reads first_block_idx_shared[batch_size] for these, one past the - // end of that array, which aliases last_block_idx_shared[0]. - const int begin_block_idx = last_block_idx_shared[0]; + // `now_block` is 0 once every request is consumed. + const int begin_block_idx = 0; const int is_last_req_splitted = (end_block_idx != lb_last + 1) && (sk_last != 0); for (int i = first_idle_part_shared + threadIdx.x; i < p.num_sm_parts; i += kBlockSize) { int* meta = out_shared + i * kMetaInts; diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh index 37f13be3d6..f7b69f7c41 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh @@ -84,7 +84,7 @@ __global__ void fused_store_flashmla_cache(const __grid_constant__ FusedStoreCac /// Elements per thread of the V4.1 store (`512 / vec` threads per token): 4 for fp8 rows, 2 for /// fp4, whose per-element IEEE divisions spread better over more threads (measured on B200, bs 1..512). constexpr uint32_t v41_store_vec_size(deepseek_v4::KVLayout layout) { - return layout == deepseek_v4::KVLayout::V41 ? 4 : 2; + return deepseek_v4::kv_encoding(layout) == deepseek_v4::KVLayout::V41 ? 4 : 2; } template < diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh index 1dbddd91b2..150f53fa6f 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh @@ -16,8 +16,8 @@ #endif // Paged fp8 / fp4 KV cache layouts read by the d_qk = 512 sparse MLA decode -// kernels. A page block is `page_size` data rows followed by `page_size` scale -// rows, so the scale region starts at byte `page_size * kDataBytes`: +// kernels. A block-split page is `page_size` data rows followed by `page_size` +// scale rows, so the scale region starts at byte `page_size * kDataBytes`: // // V4 584 B/token: 448 e4m3 + 64 bf16 (RoPE) data, 7 ue8m0 scales + 1 pad, // one scale per 64 e4m3 values. @@ -26,16 +26,27 @@ // V41_FP4 288 B/token: 512 e2m1 data packed two per byte (even index in the // low nibble), 32 e4m3 scales, one per 16 values. // -// The reader requires the rows of a page to be contiguous and the page stride -// to be a multiple of kPageAlign (its TMA row stride), which is what -// kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is -// `sglang.test.kernels.deepseek_v4.k_cache_reference`. +// V41_NEW / V41_FP4_NEW (FlashMLA's 2026.09.30 kernels) store the V41 / V41_FP4 +// bytes interleaved: each token is its data immediately followed by its scales, +// 528 / 288 B apart, with no page padding. +// +// The block-split reader requires the rows of a page to be contiguous and the +// page stride to be a multiple of kPageAlign (its TMA row stride), which is what +// kv_page_bytes pads to. The pure-torch reference of the V41 quantizers is +// `sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference`. namespace sglang { namespace deepseek_v4 { -enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 }; +enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3, V41_NEW = 4, V41_FP4_NEW = 5 }; + +/// The token encoding of a layout: V41_NEW / V41_FP4_NEW store the V41 / V41_FP4 bytes. +constexpr KVLayout kv_encoding(KVLayout layout) { + if (layout == KVLayout::V41_NEW) return KVLayout::V41; + if (layout == KVLayout::V41_FP4_NEW) return KVLayout::V41_FP4; + return layout; +} template struct KVLayoutTraits; @@ -46,6 +57,7 @@ struct KVLayoutTraits { static constexpr int64_t kScaleBytes = 8; static constexpr int64_t kTileSize = 64; static constexpr int64_t kPageAlign = 576; + static constexpr bool kInterleaved = false; static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes; }; @@ -55,6 +67,7 @@ struct KVLayoutTraits { static constexpr int64_t kScaleBytes = 16; static constexpr int64_t kTileSize = 32; static constexpr int64_t kPageAlign = 512; + static constexpr bool kInterleaved = false; static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes; }; @@ -64,6 +77,7 @@ struct KVLayoutTraits { static constexpr int64_t kScaleBytes = 32; static constexpr int64_t kTileSize = 16; static constexpr int64_t kPageAlign = 256; + static constexpr bool kInterleaved = false; static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes; }; @@ -73,9 +87,22 @@ struct KVLayoutTraits { static constexpr int64_t kScaleBytes = 0; static constexpr int64_t kTileSize = 512; static constexpr int64_t kPageAlign = 512; + static constexpr bool kInterleaved = false; static constexpr int64_t kBytesPerToken = kDataBytes; }; +template +struct InterleavedKVLayoutTraits : KVLayoutTraits { + static constexpr bool kInterleaved = true; + static constexpr int64_t kPageAlign = KVLayoutTraits::kBytesPerToken; +}; + +template <> +struct KVLayoutTraits : InterleavedKVLayoutTraits {}; + +template <> +struct KVLayoutTraits : InterleavedKVLayoutTraits {}; + /// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride. template constexpr int64_t kv_page_bytes(int64_t page_size) { @@ -83,14 +110,16 @@ constexpr int64_t kv_page_bytes(int64_t page_size) { return (page_size * Traits::kBytesPerToken + Traits::kPageAlign - 1) / Traits::kPageAlign * Traits::kPageAlign; } -/// Addressing of a paged cache: `1 << kPageBits` data rows then as many scale rows, padded to kPageAlign. +/// Addressing of a paged cache of `1 << kPageBits` tokens per page, padded to kPageAlign: +/// slot `i`'s data at `i * kDataStride`, its scales at `kScaleBase + i * kScaleStride`. template struct PagedKV { using Traits = KVLayoutTraits; static constexpr int64_t kPageSize = int64_t{1} << kPageBits; static constexpr int64_t kPageBytes = kv_page_bytes(kPageSize); - /// Byte offset of the scale rows inside a page. - static constexpr int64_t kScaleBase = Traits::kDataBytes << kPageBits; + static constexpr int64_t kDataStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kDataBytes; + static constexpr int64_t kScaleBase = Traits::kInterleaved ? Traits::kDataBytes : Traits::kDataBytes << kPageBits; + static constexpr int64_t kScaleStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kScaleBytes; template static constexpr LocT page_of(LocT loc) { @@ -103,11 +132,11 @@ struct PagedKV { /// Byte offsets of token `loc`'s data row and scale row from the cache base. template static constexpr int64_t data_offset(LocT loc) { - return page_of(loc) * kPageBytes + slot_of(loc) * Traits::kDataBytes; + return page_of(loc) * kPageBytes + slot_of(loc) * kDataStride; } template static constexpr int64_t scale_offset(LocT loc) { - return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * Traits::kScaleBytes; + return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * kScaleStride; } struct Row { @@ -118,7 +147,7 @@ struct PagedKV { template SGL_DEVICE static Row row(uint8_t* cache, LocT loc) { uint8_t* page = cache + page_of(loc) * kPageBytes; - return {page + slot_of(loc) * Traits::kDataBytes, page + kScaleBase + slot_of(loc) * Traits::kScaleBytes}; + return {page + slot_of(loc) * kDataStride, page + kScaleBase + slot_of(loc) * kScaleStride}; } }; @@ -213,7 +242,7 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx template SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const float (&v)[kVecSize]) { static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller"); - if constexpr (kLayout == KVLayout::V41) { + if constexpr (kv_encoding(kLayout) == KVLayout::V41) { store_row_fp8(data_row, scale_row, tx, v); } else if constexpr (kLayout == KVLayout::UNIFORM_FP8) { device::AlignedVector out; diff --git a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py index 7d52e5bb1f..dd42f77dca 100644 --- a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py +++ b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py @@ -7,7 +7,7 @@ rather than into the grid, the online softmax stays in registers, and ``V`` is read as ``kv[:, :d_v]``, so there are no partials and no merge pass. ``dsa/triton_sparse_mla.py`` is the gfx950-only FP8 sibling of this bf16 module. -Interface matches ``sgl_kernel.flash_mla.flash_mla_sparse_fwd``:: +Interface matches ``sglang.kernels.ops.attention.flash_mla_interface.flash_mla_sparse_fwd``:: q [T, H, 576] bf16 (absorbed MLA: 512 nope + 64 rope) kv [S, 576] bf16 (V is kv[:, :512]) diff --git a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py index b4afb2f732..f32cfc3584 100644 --- a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py @@ -102,13 +102,13 @@ def dequantize_k_cache_paged_v41( out: Optional[torch.Tensor] = None, layout: KVLayout = KVLayout.V41, ) -> torch.Tensor: - """Dequantize a V4.1 paged cache (fp8 ``V41`` or fp4 ``V41_FP4``) for a list + """Dequantize a V4.1 paged cache (fp8 ``V41`` / ``V41_NEW`` or fp4 ``V41_FP4`` / ``V41_FP4_NEW``) for a list of token IDs into ``(num_tokens, 1, 512)`` bf16. Bit-exact with the pure-torch dequantizer of these formats. """ layout = KVLayout.parse(layout) - assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout + assert layout.encoding in (KVLayout.V41, KVLayout.V41_FP4), layout if is_hip_runtime(): supported = is_gfx95_supported() else: @@ -143,9 +143,10 @@ def dequantize_k_cache_paged_v41( kernel = ( _dequantize_k_cache_paged_v41_fp8_kernel - if layout is KVLayout.V41 + if layout.encoding is KVLayout.V41 else _dequantize_k_cache_paged_v41_fp4_kernel ) + data_stride, scale_base, scale_stride = layout.row_addressing(page_size) kernel[(num_tokens,)]( out, buf_fp8, @@ -157,7 +158,9 @@ def dequantize_k_cache_paged_v41( DATA_BYTES=layout.data_bytes, SCALE_BYTES=layout.scale_bytes, TILE_SIZE=layout.tile_size, - S_OFFSET_BYTES=layout.scale_offset(page_size), + DATA_STRIDE=data_stride, + SCALE_BASE=scale_base, + SCALE_STRIDE=scale_stride, IS_HIP=is_hip_runtime(), ) return out @@ -329,7 +332,9 @@ def _dequantize_k_cache_paged_v41_fp8_kernel( DATA_BYTES: tl.constexpr, SCALE_BYTES: tl.constexpr, TILE_SIZE: tl.constexpr, - S_OFFSET_BYTES: tl.constexpr, + DATA_STRIDE: tl.constexpr, + SCALE_BASE: tl.constexpr, + SCALE_STRIDE: tl.constexpr, IS_HIP: tl.constexpr, ): # V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values). @@ -339,8 +344,8 @@ def _dequantize_k_cache_paged_v41_fp8_kernel( page_idx = loc // PAGE_SIZE in_page = loc % PAGE_SIZE page_byte_base = page_idx * BYTES_PER_PAGE - token_data_base = page_byte_base + in_page * DATA_BYTES - token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES + token_data_base = page_byte_base + in_page * DATA_STRIDE + token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE offs = tl.arange(0, DATA_BYTES) vals = tl.load(buf_fp8_ptr + token_data_base + offs).to(tl.float32) @@ -364,7 +369,9 @@ def _dequantize_k_cache_paged_v41_fp4_kernel( DATA_BYTES: tl.constexpr, SCALE_BYTES: tl.constexpr, TILE_SIZE: tl.constexpr, - S_OFFSET_BYTES: tl.constexpr, + DATA_STRIDE: tl.constexpr, + SCALE_BASE: tl.constexpr, + SCALE_STRIDE: tl.constexpr, IS_HIP: tl.constexpr, ): # V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble), @@ -375,8 +382,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel( page_idx = loc // PAGE_SIZE in_page = loc % PAGE_SIZE page_byte_base = page_idx * BYTES_PER_PAGE - token_data_base = page_byte_base + in_page * DATA_BYTES - token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES + token_data_base = page_byte_base + in_page * DATA_STRIDE + token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE boffs = tl.arange(0, DATA_BYTES) packed = tl.load(buf_uint8_ptr + token_data_base + boffs) diff --git a/python/sglang/kernels/ops/attention/dsv4/kv_layout.py b/python/sglang/kernels/ops/attention/dsv4/kv_layout.py index 73a53ea412..f674dd1397 100644 --- a/python/sglang/kernels/ops/attention/dsv4/kv_layout.py +++ b/python/sglang/kernels/ops/attention/dsv4/kv_layout.py @@ -1,15 +1,17 @@ """Paged fp8 / fp4 KV cache layouts of the DeepSeek-V4 family sparse MLA decode kernels. -A page block stores ``page_size`` data rows followed by ``page_size`` scale rows. -The reader selects the format from the bytes per token (the last dim of the -``(num_pages, page_size, 1, bytes_per_token)`` view) and requires the page -stride to be a multiple of its TMA row stride, which :meth:`KVLayout.page_bytes` -pads to. Mirrors ``sgl_kernel/deepseek_v4/kv_layout.cuh``. +Block-split layouts store a page as ``page_size`` data rows followed by +``page_size`` scale rows, and the reader requires the page stride to be a +multiple of its TMA row stride, which :meth:`KVLayout.page_bytes` pads to. The +``_NEW`` layouts (FlashMLA's 2026.09.30 kernels) interleave them instead: each +token is its data immediately followed by its scales, with no page padding. +Mirrors ``sgl_kernel/deepseek_v4/kv_layout.cuh``. """ from __future__ import annotations import enum +import functools from typing import Union @@ -22,62 +24,94 @@ class KVLayout(str, enum.Enum): V41_FP4 = "v41_fp4" # TRT-LLM only: 512 e4m3 values with a fixed external unit scale. UNIFORM_FP8 = "uniform_fp8" + # V41 / V41_FP4 encodings, each token's scales right after its data. + V41_NEW = "v41_new" + V41_FP4_NEW = "v41_fp4_new" - @property + @functools.cached_property + def is_interleaved(self) -> bool: + return self in (KVLayout.V41_NEW, KVLayout.V41_FP4_NEW) + + @functools.cached_property + def encoding(self) -> KVLayout: + """The block-split layout storing a token with the same bytes.""" + return _INTERLEAVED_ENCODING.get(self, self) + + def with_interleaved(self, interleaved: bool) -> KVLayout: + """This layout's encoding in the block-split or the interleaved arrangement.""" + if not interleaved: + return self.encoding + for new, old in _INTERLEAVED_ENCODING.items(): + if old is self.encoding: + return new + raise ValueError(f"{self} has no interleaved arrangement") + + @functools.cached_property def data_bytes(self) -> int: return { KVLayout.V4: 576, KVLayout.V41: 512, + KVLayout.V41_NEW: 512, KVLayout.V41_FP4: 256, + KVLayout.V41_FP4_NEW: 256, KVLayout.UNIFORM_FP8: 512, }[self] - @property + @functools.cached_property def scale_bytes(self) -> int: return { KVLayout.V4: 8, KVLayout.V41: 16, + KVLayout.V41_NEW: 16, KVLayout.V41_FP4: 32, + KVLayout.V41_FP4_NEW: 32, KVLayout.UNIFORM_FP8: 0, }[self] - @property + @functools.cached_property def tile_size(self) -> int: """Values sharing one scale.""" return { KVLayout.V4: 64, KVLayout.V41: 32, + KVLayout.V41_NEW: 32, KVLayout.V41_FP4: 16, + KVLayout.V41_FP4_NEW: 16, KVLayout.UNIFORM_FP8: 512, }[self] - @property + @functools.cached_property def bytes_per_token(self) -> int: return self.data_bytes + self.scale_bytes - @property + @functools.cached_property def page_align(self) -> int: """Unit the page stride is padded to: the reader's TMA row stride.""" return { KVLayout.V4: 576, KVLayout.V41: 512, + KVLayout.V41_NEW: 528, KVLayout.V41_FP4: 256, + KVLayout.V41_FP4_NEW: 288, KVLayout.UNIFORM_FP8: 512, }[self] - @property + @functools.cached_property def is_fp4(self) -> bool: - return self is KVLayout.V41_FP4 + return self.encoding is KVLayout.V41_FP4 def page_bytes(self, page_size: int) -> int: raw = page_size * self.bytes_per_token return -(-raw // self.page_align) * self.page_align - def scale_offset(self, page_size: int) -> int: - """Byte offset of the scale rows inside a page.""" - return page_size * self.data_bytes + def row_addressing(self, page_size: int) -> tuple[int, int, int]: + """(data row stride, scale base, scale row stride) inside a page: slot + ``i``'s data is at ``i * data_stride``, its scales at ``scale_base + i * scale_stride``.""" + if self.is_interleaved: + return self.bytes_per_token, self.data_bytes, self.bytes_per_token + return self.data_bytes, page_size * self.data_bytes, self.scale_bytes - @property + @functools.cached_property def cpp_name(self) -> str: """The C++ enumerator, for JIT template arguments. Bare, because it is also part of the JIT module name; the headers `using enum` it in.""" @@ -90,7 +124,17 @@ class KVLayout(str, enum.Enum): return cls(str(value).lower()) +_INTERLEAVED_ENCODING = { + KVLayout.V41_NEW: KVLayout.V41, + KVLayout.V41_FP4_NEW: KVLayout.V41_FP4, +} + + def is_valid_kv_layout_pair(kv: KVLayout, extra_kv: KVLayout) -> bool: """The (main, extra) cache pairs the decode kernel accepts: identical layouts, - or the fp4 extra cache next to a V4.1 fp8 main cache.""" - return extra_kv is kv or (kv is KVLayout.V41 and extra_kv is KVLayout.V41_FP4) + or the fp4 extra cache next to a V4.1 fp8 main cache of the same arrangement.""" + return extra_kv is kv or ( + kv.encoding is KVLayout.V41 + and extra_kv.encoding is KVLayout.V41_FP4 + and kv.is_interleaved == extra_kv.is_interleaved + ) diff --git a/python/sglang/kernels/ops/attention/flash_mla_interface.py b/python/sglang/kernels/ops/attention/flash_mla_interface.py new file mode 100644 index 0000000000..7d8520c3c1 --- /dev/null +++ b/python/sglang/kernels/ops/attention/flash_mla_interface.py @@ -0,0 +1,238 @@ +"""FlashMLA from the ``sgl-flash-mla`` wheel, behind the call signatures of the +FlashMLA ops sgl-kernel used to ship (``sgl_kernel.flash_mla`` and +``sgl_kernel.flash_mla_mega``). + +The wheel carries two kernel sets. Everything the fork had before FlashMLA's +2026.09.30 release (SM90 dense decode, the V3.2 / V3.2-no-RoPE / V4 formats, +the block-split V4.1 caches and their MegaAttention) is its legacy side, the +same sources sgl-kernel built, so ``use_legacy`` defaults to True. Only the +token-interleaved V4.1 caches (``KVLayout.V41_NEW`` / ``V41_FP4_NEW``) take +``use_legacy=False``. CUDA only: HIP and XPU keep their own sgl-kernel ops. +""" + +from typing import Optional, Tuple, Union + +import flash_mla +import torch +from flash_mla import fused_norm_rope_attn_rope_cast as _mega +from flash_mla.flash_mla_interface import FlashMLASchedMeta + +# 512 fp8 values + 4 fp32 scales; the V4.1 formats have the same size, which is +# why the tensor-metadata API (it predates V4.1) reads such a cache as V3.2. +_V32_NO_ROPE_BYTES_PER_TOKEN = 528 + +# The only RoPE / output-cast configuration MegaAttention accepts: +# both kernel sets assert every one of these values. +_MEGA_FIXED_PARAMS = dict( + is_rope_neox_style=False, + rope_dim=64, + num_per_channels=32, + use_tma_aligned_col_major_sf=True, + round_sf=True, + use_packed_ue8m0=True, +) + +# The current (use_legacy=False) decode kernel splits the KV only past this +# many main + extra indices; below it it schedules nothing and leaves the +# metadata empty. Mirrors `enable_split_kv` in FlashMLA's csrc/api/sparse_decode.cpp. +SPLIT_KV_MIN_TOPK = 641 + + +def get_mla_metadata( + cache_seqlens: Optional[torch.Tensor] = None, + num_q_tokens_per_head_k: Optional[int] = None, + num_heads_k: Optional[int] = None, + num_heads_q: Optional[int] = None, + is_fp8_kvcache: bool = False, + topk: Optional[int] = None, +) -> Tuple[Union[torch.Tensor, FlashMLASchedMeta], Optional[torch.Tensor]]: + """Without ``cache_seqlens``, an empty ``FlashMLASchedMeta`` that the first + decode call fills; with it, the legacy split-KV schedule as tensors.""" + if cache_seqlens is None: + return flash_mla.get_mla_metadata()[0], None + assert num_q_tokens_per_head_k is not None + assert num_heads_k is not None + return flash_mla.get_mla_decoding_metadata( + cache_seqlens, + num_q_tokens_per_head_k, + num_heads_k, + num_heads_q, + is_fp8_kvcache, + topk, + use_legacy=True, + ) + + +def flash_mla_with_kvcache( + q: torch.Tensor, + k_cache: torch.Tensor, + block_table: Optional[torch.Tensor], + cache_seqlens: Optional[torch.Tensor], + head_dim_v: int, + tile_scheduler_metadata: Union[torch.Tensor, FlashMLASchedMeta], + num_splits: Optional[torch.Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + descale_q: Optional[torch.Tensor] = None, + descale_k: Optional[torch.Tensor] = None, + is_fp8_kvcache: bool = False, + indices: Optional[torch.Tensor] = None, + attn_sink: Optional[torch.Tensor] = None, + extra_k_cache: Optional[torch.Tensor] = None, + extra_indices_in_kvcache: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, + kv_format: Optional[str] = None, + *, + use_legacy: bool = True, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Paged MLA decode. ``tile_scheduler_metadata`` is a ``FlashMLASchedMeta``, + or the schedule tensors from ``get_mla_metadata(cache_seqlens, ...)`` with + ``num_splits``, which ride on a fresh ``FlashMLASchedMeta`` (legacy only: + V3.2 sparse, dense bf16, or dense fp8 when ``q`` is fp8).""" + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + if isinstance(tile_scheduler_metadata, FlashMLASchedMeta): + assert num_splits is None, "num_splits must be None with FlashMLASchedMeta" + sched_meta = tile_scheduler_metadata + else: + assert use_legacy, ( + "tensor scheduling metadata exists only for the legacy kernels" + ) + assert num_splits is not None + assert block_table is not None and cache_seqlens is not None + assert all( + x is None for x in (attn_sink, extra_k_cache, extra_indices_in_kvcache) + ), "attn_sink and extra caches need FlashMLASchedMeta" + assert (descale_q is None) == (descale_k is None), ( + "descale_q and descale_k should be both None or both not None" + ) + sched_meta = FlashMLASchedMeta( + tile_scheduler_metadata=tile_scheduler_metadata, num_splits=num_splits + ) + if indices is None: + is_fp8_kvcache = q.element_size() == 1 + elif ( + kv_format is None + and q.shape[-1] == 512 + and k_cache.shape[-1] == _V32_NO_ROPE_BYTES_PER_TOKEN + ): + kv_format = "V32_NO_ROPE" + return flash_mla.flash_mla_with_kvcache( + q=q, + k_cache=k_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + head_dim_v=head_dim_v, + tile_scheduler_metadata=sched_meta, + softmax_scale=softmax_scale, + causal=causal, + is_fp8_kvcache=is_fp8_kvcache, + indices=indices, + attn_sink=attn_sink, + extra_k_cache=extra_k_cache, + extra_indices_in_kvcache=extra_indices_in_kvcache, + topk_length=topk_length, + extra_topk_length=extra_topk_length, + use_legacy=use_legacy, + kv_format=kv_format, + descale_q=descale_q, + descale_k=descale_k, + ) + + +def flash_mla_sparse_fwd( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + sm_scale: float, + d_v: int = 512, + attn_sink: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + *, + use_legacy: bool = True, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Sparse prefill over a bf16 ``kv``: (output, max_logits, lse). The legacy + kernel's statistics are base 2, as sgl-kernel returned them; the current + kernel's are natural-log.""" + return flash_mla.flash_mla_sparse_fwd( + q, + kv, + indices, + sm_scale, + d_v, + attn_sink, + topk_length, + use_legacy=use_legacy, + return_log2_lse=True if use_legacy else None, + ) + + +def flash_mla_mega_prefill( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + softmax_scale: float, + *, + attn_sink: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + enable_q_norm: bool = False, + rms_norm_eps: float = 0.0, + use_legacy: bool = True, +) -> Tuple[torch.Tensor, ...]: + """MegaAttention prefill over a bf16 ``kv``: (out_fp8, out_sf, max_logits, lse).""" + return _mega.prefill( + token_positions=positions, + cos_sin_cache=cos_sin_cache, + n_wv_group=q.shape[1] // 8, + q=q, + kv=kv, + indices=indices, + sm_scale=softmax_scale, + attn_sink=attn_sink, + topk_length=topk_length, + enable_q_norm=enable_q_norm, + rms_norm_eps=rms_norm_eps, + use_legacy=use_legacy, + **_MEGA_FIXED_PARAMS, + ) + + +def flash_mla_mega_decode( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + softmax_scale: float, + *, + attn_sink: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + extra_kv: Optional[torch.Tensor] = None, + extra_indices: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, + enable_q_norm: bool = False, + rms_norm_eps: float = 0.0, + use_legacy: bool = True, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """MegaAttention decode over a paged cache: (out_fp8, out_sf, lse).""" + return _mega.decode( + token_positions=positions, + cos_sin_cache=cos_sin_cache, + n_wv_group=q.shape[1] // 8, + q=q, + k_cache=kv, + indices_in_kvcache=indices, + sm_scale=softmax_scale, + attn_sink=attn_sink, + topk_length=topk_length, + extra_k_cache=extra_kv, + extra_indices_in_kvcache=extra_indices, + extra_topk_length=extra_topk_length, + enable_q_norm=enable_q_norm, + rms_norm_eps=rms_norm_eps, + use_legacy=use_legacy, + **_MEGA_FIXED_PARAMS, + ) diff --git a/python/sglang/kernels/ops/kvcache/dsv4_request_window_copy.py b/python/sglang/kernels/ops/kvcache/dsv4_request_window_copy.py index ea2ae28c2e..b38b4548e1 100644 --- a/python/sglang/kernels/ops/kvcache/dsv4_request_window_copy.py +++ b/python/sglang/kernels/ops/kvcache/dsv4_request_window_copy.py @@ -4,6 +4,17 @@ import triton import triton.language as tl +@triton.jit +def _row_offset(loc, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE): + # Byte `col` of token `loc`'s data-then-scales, from its page start; padding is untouched. + slot = loc % PAGE + return tl.where( + col < DATA, + slot * DATA_STRIDE + col, + SCALE_BASE + slot * SCALE_STRIDE + col - DATA, + ) + + @triton.jit def _copy_packed_rows( src, @@ -17,23 +28,20 @@ def _copy_packed_rows( PAGE: tl.constexpr, DATA: tl.constexpr, SCALE: tl.constexpr, + DATA_STRIDE: tl.constexpr, + SCALE_BASE: tl.constexpr, + SCALE_STRIDE: tl.constexpr, BLOCK: tl.constexpr, ): row = tl.program_id(0) source = tl.load(src_loc + row * SRC_LOC_STRIDE).to(tl.int64) target = tl.load(dst_loc + row * DST_LOC_STRIDE).to(tl.int64) col = tl.arange(0, BLOCK) - # Each page contains all data rows, followed by all scale rows. Its - # physical stride may also contain alignment padding, which is untouched. - source_offset = tl.where( - col < DATA, - (source % PAGE) * DATA + col, - PAGE * DATA + (source % PAGE) * SCALE + col - DATA, + source_offset = _row_offset( + source, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE ) - target_offset = tl.where( - col < DATA, - (target % PAGE) * DATA + col, - PAGE * DATA + (target % PAGE) * SCALE + col - DATA, + target_offset = _row_offset( + target, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE ) value = tl.load( src + source // PAGE * SRC_STRIDE + source_offset, @@ -60,6 +68,7 @@ def copy_packed_rows(src, dst, src_loc, dst_loc, *, page_size, layout): page_size, layout.data_bytes, layout.scale_bytes, + *layout.row_addressing(page_size), triton.next_power_of_2(layout.bytes_per_token), num_warps=4, ) @@ -80,6 +89,9 @@ def _commit_packed_rows( CAPACITY: tl.constexpr, DATA: tl.constexpr, SCALE: tl.constexpr, + DATA_STRIDE: tl.constexpr, + SCALE_BASE: tl.constexpr, + SCALE_STRIDE: tl.constexpr, BLOCK: tl.constexpr, ): row = tl.program_id(0) @@ -89,15 +101,11 @@ def _commit_packed_rows( position = tl.load(pos + row).to(tl.int64) target = tl.load(req + row).to(tl.int64) * CAPACITY + position % CAPACITY col = tl.arange(0, BLOCK) - source_offset = tl.where( - col < DATA, - (source % PAGE) * DATA + col, - PAGE * DATA + (source % PAGE) * SCALE + col - DATA, + source_offset = _row_offset( + source, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE ) - target_offset = tl.where( - col < DATA, - (target % PAGE) * DATA + col, - PAGE * DATA + (target % PAGE) * SCALE + col - DATA, + target_offset = _row_offset( + target, col, PAGE, DATA, DATA_STRIDE, SCALE_BASE, SCALE_STRIDE ) value = tl.load( src + source // PAGE * SRC_STRIDE + source_offset, @@ -127,6 +135,7 @@ def commit_packed_rows(src, dst, tags, window, *, page_size, capacity, layout): capacity, layout.data_bytes, layout.scale_bytes, + *layout.row_addressing(page_size), triton.next_power_of_2(layout.bytes_per_token), num_warps=4, ) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 5ad5c455ad..ab1cb5cf7e 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -36,7 +36,7 @@ def _default_dsv4_attention_backend(server_args: ServerArgs) -> str: or cfg.device != "cuda" or not is_sm100_supported() or cfg.kv_cache_dtype != "fp8_e4m3" - or (layout.is_set() and layout.get().lower() not in ("auto", "v41")) + or (layout.is_set() and layout.get().lower() not in ("auto", "v41", "v41_new")) ): return "flashmla" return "flashmla_mega" diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 3d4ff75513..bb935db677 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1707,7 +1707,8 @@ class Envs: SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048) # Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every # GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or - # fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere). + # fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for + # FlashMLA's 2026.09.30 kernels) or "auto" (v41 on SM100, v4 elsewhere). SGLANG_DSV4_KV_LAYOUT = EnvStr("v4") # Compressed-cache layout under "v41": "auto" (fp4 for the fp4-rounded # ratio-1 / ratio-2 latents, fp8 for ratios 4 / 128), "fp8" or "fp4" for all. diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 4ac94e2ebd..8895de7f51 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -146,8 +146,7 @@ from sglang.srt.utils import ceil_align, is_cuda, is_xpu from sglang.srt.utils.common import async_h2d if TYPE_CHECKING: - from sgl_kernel.flash_mla import FlashMLASchedMeta - + from sglang.kernels.ops.attention.flash_mla_interface import FlashMLASchedMeta from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.model_executor.model_runner import ModelRunner @@ -210,9 +209,9 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T: def _create_flashmla_metadata(): if get_platform().is_sm120 or _is_xpu: return None - import sgl_kernel.flash_mla as flash_mla + from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata - return flash_mla.get_mla_metadata()[0] + return get_mla_metadata()[0] # FlashMLA's head64 sm100 decode scheduling constants; not exported, so they hold @@ -273,6 +272,13 @@ def _maybe_precompute_flashmla_sched_meta( flashmla_metadata.num_splits = num_splits +def _flashmla_v41_splits_kv( + indices: torch.Tensor, extra_indices: Optional[torch.Tensor], min_topk: int +) -> bool: + extra_topk = 0 if extra_indices is None else extra_indices.shape[-1] + return indices.shape[-1] + extra_topk >= min_topk + + def _low_ratio_source_projections( layer, x, q_lora, positions, bufs, run_compressor, run_indexer ): @@ -1237,6 +1243,17 @@ class DeepseekV4AttnBackend( self.req_to_token_pool = model_runner.req_to_token_pool self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool + if is_cuda(): + from sglang.kernels.ops.attention import flash_mla_interface + + self._swa_interleaved = ( + self.token_to_kv_pool.get_swa_key_layout().is_interleaved + ) + self._flash_mla_with_kvcache = functools.partial( + flash_mla_interface.flash_mla_with_kvcache, + use_legacy=not self._swa_interleaved, + ) + self._flashmla_split_kv_min_topk = flash_mla_interface.SPLIT_KV_MIN_TOPK # The C4 state ring is addressed per SWA page, so a page holds whole windows. assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0 self.hisparse_coordinator = model_runner.hisparse_coordinator @@ -3781,7 +3798,7 @@ class DeepseekV4AttnBackend( loc=slots, cache_k=layer.indexer.index_keys(latent, freqs), ) - if pool.get_extra_key_layout(layer.layer_id) is KVLayout.V41_FP4: + if pool.get_extra_key_layout(layer.layer_id).is_fp4: # The fp4 cache stores e2m1 codes: the kernel rotates the tail and # rounds once, with no fake quantization in between. pool.set_extra_key_buffer_fused( @@ -4290,9 +4307,16 @@ class DeepseekV4AttnBackend( if _is_xpu: from sgl_kernel import flash_mla_with_kvcache else: - from sgl_kernel.flash_mla import flash_mla_with_kvcache + flash_mla_with_kvcache = self._flash_mla_with_kvcache - if self.is_dsv41: + if self.is_dsv41 and ( + not self._swa_interleaved + or _flashmla_v41_splits_kv( + swa_page_indices, + extra_indices, + self._flashmla_split_kv_min_topk, + ) + ): _maybe_precompute_flashmla_sched_meta( flashmla_metadata, q=q, @@ -4339,7 +4363,9 @@ class DeepseekV4AttnBackend( if _is_xpu: from sgl_kernel import flash_mla_sparse_fwd else: - from sgl_kernel.flash_mla import flash_mla_sparse_fwd + from sglang.kernels.ops.attention.flash_mla_interface import ( + flash_mla_sparse_fwd, + ) # q is (b, 1, h_q, d_qk); flash_mla_sparse_fwd takes (s_q, h_q, d_qk). q_flat = q.squeeze(1) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 18e26c5672..b8d053e197 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -116,8 +116,6 @@ from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout from sglang.srt.utils import ceil_align, is_gfx95_supported if TYPE_CHECKING: - from sgl_kernel.flash_mla import FlashMLASchedMeta - from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( FP4DecodeWorkspace, FP4KWriteMetadata, @@ -125,6 +123,7 @@ if TYPE_CHECKING: FP4RowgroupDecodeWorkspace, FP4RowgroupPrefillWorkspace, ) + from sglang.kernels.ops.attention.flash_mla_interface import FlashMLASchedMeta from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.model_executor.model_runner import ModelRunner @@ -244,9 +243,9 @@ def _create_flashmla_metadata(): if is_hip(): return None - import sgl_kernel.flash_mla as flash_mla + from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata - return flash_mla.get_mla_metadata()[0] + return get_mla_metadata()[0] def _create_dummy_paged_compress_data(compress_ratio: int): diff --git a/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py index 25f1bd227b..d9c1a46775 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py @@ -78,9 +78,9 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend): extra_indices: Optional[torch.Tensor], extra_topk_lengths: Optional[torch.Tensor], ) -> Tuple[torch.Tensor, torch.Tensor]: - from sgl_kernel.flash_mla_mega import flash_mla_mega_decode + from sglang.kernels.ops.attention import flash_mla_interface - data, scale, _ = flash_mla_mega_decode( + data, scale, _ = flash_mla_interface.flash_mla_mega_decode( q, swa_k_cache, swa_page_indices, @@ -92,6 +92,7 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend): extra_kv=extra_k_cache, extra_indices=extra_indices, extra_topk_length=extra_topk_lengths, + use_legacy=not self._swa_interleaved, ) return data, scale @@ -106,9 +107,9 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend): attn_sink: torch.Tensor, topk_length: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill + from sglang.kernels.ops.attention import flash_mla_interface - data, scale, _, _ = flash_mla_mega_prefill( + data, scale, _, _ = flash_mla_interface.flash_mla_mega_prefill( q, kv, indices, @@ -117,5 +118,6 @@ class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend): self.softmax_scale, attn_sink=attn_sink, topk_length=topk_length, + use_legacy=not self._swa_interleaved, ) return data, scale diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 739721a759..1ff1b028bd 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -2812,7 +2812,9 @@ class DeepseekSparseAttnBackend( topk_length: Optional[torch.Tensor] = None, attn_sink: Optional[torch.Tensor] = None, ) -> torch.Tensor: - from sgl_kernel.flash_mla import flash_mla_sparse_fwd + from sglang.kernels.ops.attention.flash_mla_interface import ( + flash_mla_sparse_fwd, + ) # FlashMLA sparse kernel requires num_heads to be a multiple of 64 (Hopper) or 128 (Blackwell) # When using TP, num_heads might be smaller (e.g., 256//8=32) @@ -3314,7 +3316,9 @@ class DeepseekSparseAttnBackend( metadata: DSAMetadata, page_table_1, ) -> torch.Tensor: - from sgl_kernel.flash_mla import flash_mla_with_kvcache + from sglang.kernels.ops.attention.flash_mla_interface import ( + flash_mla_with_kvcache, + ) cache_seqlens = metadata.dsa_cache_seqlens_int32 assert metadata.flashmla_metadata is not None @@ -4113,7 +4117,7 @@ class DeepseekSparseAttnBackend( ) def _compute_flashmla_metadata(self, cache_seqlens: torch.Tensor, seq_len_q: int): - from sgl_kernel.flash_mla import get_mla_metadata + from sglang.kernels.ops.attention.flash_mla_interface import get_mla_metadata num_heads_q = self.flashmla_kv_num_q_heads diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index 4dbc30a2bd..72a2674fa6 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -10,8 +10,11 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union import torch import triton -from sgl_kernel.flash_mla import flash_mla_with_kvcache, get_mla_metadata +from sglang.kernels.ops.attention.flash_mla_interface import ( + flash_mla_with_kvcache, + get_mla_metadata, +) from sglang.kernels.ops.attention.kv_indices import ( create_flashmla_kv_indices_triton, get_num_kv_index_blocks_flashmla, diff --git a/python/sglang/srt/layers/attention/hip_flash_mla.py b/python/sglang/srt/layers/attention/hip_flash_mla.py index e6342d6982..6e24f75a47 100644 --- a/python/sglang/srt/layers/attention/hip_flash_mla.py +++ b/python/sglang/srt/layers/attention/hip_flash_mla.py @@ -203,7 +203,7 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): backend if backend in _HIP_BACKENDS else None ) else: - import sgl_kernel.flash_mla as flash_mla + from sglang.kernels.ops.attention import flash_mla_interface if backend == "comparison": pack_ref, pack_fast_via_tester = flash_mla_with_kvcache_entrypoint( @@ -238,7 +238,7 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): return aiter_sparse_decode_fwd(**kwargs) if backend == "kernel": - return flash_mla.flash_mla_with_kvcache(**kwargs) + return flash_mla_interface.flash_mla_with_kvcache(**kwargs) raise NotImplementedError(f"unknown backend: {backend!r}") diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 9f6f404018..bc8f19fa2f 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -189,12 +189,10 @@ def resolve_compressed_kv_layout( if kv_layout is KVLayout.V4: assert option in (None, "fp8"), "the V4 main cache only pairs with V4 caches" return KVLayout.V4 - assert kv_layout is KVLayout.V41, f"{kv_layout} is not a main-cache layout" - if option == "fp8": - return KVLayout.V41 - if option == "fp4": - return KVLayout.V41_FP4 - return KVLayout.V41_FP4 if compress_ratio in (1, 2) else KVLayout.V41 + assert kv_layout.encoding is KVLayout.V41, f"{kv_layout} is not a main-cache layout" + use_fp4 = option == "fp4" or (option is None and compress_ratio in (1, 2)) + encoding = KVLayout.V41_FP4 if use_fp4 else KVLayout.V41 + return encoding.with_interleaved(kv_layout.is_interleaved) def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: @@ -203,7 +201,7 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: AITER attention.""" mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower() option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower() - if get_exec().kernel.dsv4_attn_backend == "flashmla_mega": + if get_exec().kernel.dsv4_attn_backend == "flashmla_mega" and mode != "v41_new": mode = "v41" if get_exec().kernel.dsv4_attn_backend == "trtllm": assert mode in ("auto", "v4") and option in ("auto", "fp8"), ( @@ -212,7 +210,7 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: return KVLayout.V4, None if mode == "v4": return KVLayout.V4, None if option == "auto" else option - assert mode in ("v41", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}" + assert mode in ("v41", "v41_new", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}" is_sm100 = ( torch.cuda.is_available() and torch.version.cuda is not None @@ -224,15 +222,16 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: ) if mode == "auto" and not supported: return KVLayout.V4, None - if not supported: + if not supported or mode == "v41_new": raise ValueError( - "V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention" + "V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention, " + "which reads the block-split v41 layout" ) return KVLayout.V41, option if mode == "auto" and not is_sm100: return KVLayout.V4, None assert is_sm100, "the V4.1 KV cache layouts need an SM100 / SM103 GPU" - return KVLayout.V41, option + return (KVLayout.V41_NEW if mode == "v41_new" else KVLayout.V41), option class DeepSeekV4SingleKVPool(KVCache): @@ -995,6 +994,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): assert self.kv_layout in ( KVLayout.V4, KVLayout.V41, + KVLayout.V41_NEW, ), f"{self.kv_layout} is only valid for a compressed (extra) cache" self.compressed_kv_layout_option = compressed_kv_layout c4_logical_size = c128_size * 32 @@ -2265,7 +2265,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): _, compress_layer_id, compress_kv_pool = self.layer_mapping[layer_id] assert compress_kv_pool is not None if freqs_cis is not None: - assert compress_kv_pool.kv_layout is KVLayout.V41_FP4, ( + assert compress_kv_pool.kv_layout.is_fp4, ( "in-kernel RoPE is for the fp4 cache; fp8 caches take the finished value" ) return compress_kv_pool.set_key_buffer_fused( diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index b61ceefea5..2158580fc3 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -560,7 +560,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): graph_supported_backend_types.append(FlashAttentionBackend) # FlashMLA is CUDA-only; import lazily so CPU builds don't pull - # sgl_kernel.flash_mla at import time. + # FlashMLA at import time. from sglang.srt.layers.attention.flashmla_backend import FlashMLABackend graph_supported_backend_types.append(FlashMLABackend) diff --git a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py index 4727cc2fbb..78452819bc 100644 --- a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py +++ b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py @@ -145,3 +145,39 @@ def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor: return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view( num_pages, page_size, 512 ) + + +# --------------------------------------------------------------------------- +# The interleaved arrangement of the same bytes (V41_NEW / V41_FP4_NEW): each +# token is its data immediately followed by its scales, pages unpadded. +# --------------------------------------------------------------------------- + + +def interleave_pages( + pages: torch.Tensor, page_size: int, data_bytes: int, scale_bytes: int +) -> torch.Tensor: + """Block-split pages ``[num_pages, >= page_size * (data + scale)]`` uint8 -> + interleaved ``[num_pages, page_size * (data + scale)]``.""" + num_pages = pages.shape[0] + split = page_size * data_bytes + data = pages[:, :split].reshape(num_pages, page_size, data_bytes) + scale = pages[:, split : split + page_size * scale_bytes].reshape( + num_pages, page_size, scale_bytes + ) + return torch.cat([data, scale], dim=-1).reshape(num_pages, -1) + + +def deinterleave_pages( + pages: torch.Tensor, page_size: int, data_bytes: int, scale_bytes: int +) -> torch.Tensor: + """Inverse of :func:`interleave_pages`, without page padding.""" + num_pages = pages.shape[0] + tokens = pages.view(torch.uint8)[:, : page_size * (data_bytes + scale_bytes)] + tokens = tokens.reshape(num_pages, page_size, data_bytes + scale_bytes) + return torch.cat( + [ + tokens[..., :data_bytes].reshape(num_pages, -1), + tokens[..., data_bytes:].reshape(num_pages, -1), + ], + dim=-1, + ) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index 8063e9d644..dd77de6d80 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -1128,13 +1128,6 @@ def dsa_impl_capability(impl: str) -> tuple[bool, str]: major, minor = _torch.cuda.get_device_capability() if impl == "flashmla_sparse" or impl == "flashmla_kv": - try: - from sgl_kernel.flash_mla import ( # noqa: F401 - flash_mla_sparse_fwd, - flash_mla_with_kvcache, - ) - except ImportError as exc: - return False, f"sgl_kernel.flash_mla unavailable: {exc}" if major < 9: return False, f"{impl} requires SM>=9.0, got SM{major}.x" return True, "" diff --git a/test/manual/test_dsv41_ragged_prefill.py b/test/manual/test_dsv41_ragged_prefill.py index 04b466b51e..a38582f441 100644 --- a/test/manual/test_dsv41_ragged_prefill.py +++ b/test/manual/test_dsv41_ragged_prefill.py @@ -55,7 +55,9 @@ def rectangular_gather(cache, page_table, ratio, page_size): @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestRaggedPrefill(unittest.TestCase): def test_gather_and_attention(self): - from sgl_kernel.flash_mla import flash_mla_sparse_fwd + from sglang.kernels.ops.attention.flash_mla_interface import ( + flash_mla_sparse_fwd, + ) torch.manual_seed(42) device = "cuda" diff --git a/test/registered/attention/unittests/dsa/README.md b/test/registered/attention/unittests/dsa/README.md index 9cbc560e63..c33f605730 100644 --- a/test/registered/attention/unittests/dsa/README.md +++ b/test/registered/attention/unittests/dsa/README.md @@ -27,8 +27,8 @@ hardware/SDK. The variant tests live in `test_dsa.py` as | Impl | Prefill | Decode | CG decode | Hardware gate (test box: H200 SM9.0) | |---|---|---|---|---| -| `flashmla_sparse` | ✓ | ✓ | ✓ | SM>=9.0 + `sgl_kernel.flash_mla` | -| `flashmla_kv` | ✓ | ✓ | ✓ | SM>=9.0 + `sgl_kernel.flash_mla` | +| `flashmla_sparse` | ✓ | ✓ | ✓ | SM>=9.0 + `sgl-flash-mla` | +| `flashmla_kv` | ✓ | ✓ | ✓ | SM>=9.0 + `sgl-flash-mla` | | `fa3` | ✓ | ✓ | ✓ | SM>=9.0 + `sglang.kernels.ops.attention.flash_attention` | | `tilelang` | ✓ (topk=2048 dedicated fixture) | ✓ (topk=2048 dedicated fixture) | skipped: not yet wired into CG runner | `tilelang_sparse_fwd` asserts `topk == 2048`; the topk=2048 fixture instance (`build_dsa_sparse_attention_fixture(..., index_topk=2048)`) is used by `test_sparse_tilelang_prefill_case` / `test_sparse_tilelang_decode_case`. The default-topk impl-variant matrix still skips tilelang with the same reason. **SM10.x container gate**: on Blackwell the tilelang JIT generates `wait_wgmma` which the container's MMA template doesn't ship (`KNOWN_FAILURES.md §2`); `dsa_impl_capability("tilelang")` skips on `major >= 10`. Set `SGLANG_TEST_DSA_TILELANG_FORCE=1` to override after re-imaging. | | `trtllm` | skipped: SM<10 | skipped: SM<10 | skipped: SM<10 | TRT-LLM Gen FMHA/MLA requires Blackwell (SM>=10.0). | diff --git a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py index 98f563d8d3..1dee10e2fc 100644 --- a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py +++ b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py @@ -12,7 +12,6 @@ extra-K softmax reference. Compressor math correctness (i.e. verifying the gate+norm+rotate compression itself) is a deferred follow-up. """ -import importlib.util import unittest from types import SimpleNamespace from unittest import mock @@ -20,13 +19,6 @@ from unittest import mock import torch from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.test.test_utils import CustomTestCase - -_FLASH_MLA_AVAILABLE = ( - importlib.util.find_spec("sgl_kernel") is not None - and importlib.util.find_spec("sgl_kernel.flash_mla") is not None -) - from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.attention_unittest.attention_methods.dsv4_attention import ( # noqa: E402 DSV4_PAGE_SIZE, @@ -46,13 +38,13 @@ from sglang.test.kits.attention_unittest.runner_modes.speculative_draft_runner i from sglang.test.kits.attention_unittest.runner_modes.speculative_target_verify_runner import ( # noqa: E402 run_dsv4_eagle_verify_cuda_graph_case, ) +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=14, stage="base-b", runner_config="4-gpu-b200") register_cuda_ci(est_time=13, stage="base-b", runner_config="1-gpu-large") @unittest.skipIf(not torch.cuda.is_available(), "CUDA is required") -@unittest.skipIf(not _FLASH_MLA_AVAILABLE, "flash_mla is required for DSV4 SWA") class TestDSV4AttentionBackendCorrectness(CustomTestCase): CASES = make_dsv4_cases("dsv4") CUDA_GRAPH_DECODE_CASES = ( diff --git a/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py b/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py index 00c5cdda51..444267fad4 100644 --- a/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py +++ b/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py @@ -15,14 +15,10 @@ from sglang.kernels.jit.benchmark.utils import run_benchmark_no_cudagraph from sglang.kernels.ops.attention.dsa.triton_sparse_mla_prefill import ( sparse_mla_prefill, ) +from sglang.kernels.ops.attention.flash_mla_interface import flash_mla_sparse_fwd from sglang.test.ci.ci_register import register_cuda_ci from sglang.utils import is_in_ci -try: - from sgl_kernel.flash_mla import flash_mla_sparse_fwd -except ImportError: - flash_mla_sparse_fwd = None - try: # Same form as the kernel under test, but reachable only on gfx950. Kept as # a provider so the comparison against the closest sibling is visible here @@ -38,8 +34,9 @@ except ImportError: def _flashmla_runnable() -> bool: # The op refuses anything but SM90a / SM100f at call time, so gate on the # device rather than on the import succeeding. - return flash_mla_sparse_fwd is not None and ( - torch.cuda.is_available() and torch.cuda.get_device_capability()[0] in (9, 10) + return torch.cuda.is_available() and torch.cuda.get_device_capability()[0] in ( + 9, + 10, ) @@ -142,7 +139,7 @@ def bench_dsa_triton_sparse_mla_prefill( if provider == "flashmla_sparse": if not HAS_FLASHMLA: - raise RuntimeError("sgl_kernel.flash_mla.flash_mla_sparse_fwd unavailable") + raise RuntimeError("FlashMLA's flash_mla_sparse_fwd needs SM90 or SM100") kv_3d = kv.unsqueeze(1) idx_3d = indices.unsqueeze(1) # The kernel requires num_heads % 64 on SM90 and % 128 on SM100+, so the diff --git a/test/registered/kernels/benchmark/attention/bench_sparse_mla_q8kv8_prefill_sm90.py b/test/registered/kernels/benchmark/attention/bench_sparse_mla_q8kv8_prefill_sm90.py index e18964e0ee..346b177f9a 100644 --- a/test/registered/kernels/benchmark/attention/bench_sparse_mla_q8kv8_prefill_sm90.py +++ b/test/registered/kernels/benchmark/attention/bench_sparse_mla_q8kv8_prefill_sm90.py @@ -7,6 +7,7 @@ import triton import triton.testing from sglang.kernels.jit.benchmark.utils import run_benchmark_no_cudagraph +from sglang.kernels.ops.attention.flash_mla_interface import flash_mla_sparse_fwd from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import ( sparse_mla_q8kv8_prefill_fwd, ) @@ -14,14 +15,6 @@ from sglang.srt.utils import is_sm90_supported from sglang.test.ci.ci_register import register_cuda_ci from sglang.utils import is_in_ci -try: - from sgl_kernel.flash_mla import flash_mla_sparse_fwd - - HAS_Q16_FLASHMLA = True -except ImportError: - flash_mla_sparse_fwd = None - HAS_Q16_FLASHMLA = False - register_cuda_ci( est_time=120, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" ) @@ -49,13 +42,9 @@ else: # DeepSeek NSA E2E does not pass a per-head attention sink into sparse MLA, so # sink-enabled timings are kernel feature coverage rather than E2E proxy data. -LINE_VALS = ["q8_fp8_jit"] -LINE_NAMES = ["Q8 FP8 JIT"] -STYLES = [("blue", "-")] -if HAS_Q16_FLASHMLA: - LINE_VALS.insert(0, "q16_bf16_flashmla") - LINE_NAMES.insert(0, "Q16 BF16 FlashMLA") - STYLES.insert(0, ("orange", "--")) +LINE_VALS = ["q16_bf16_flashmla", "q8_fp8_jit"] +LINE_NAMES = ["Q16 BF16 FlashMLA", "Q8 FP8 JIT"] +STYLES = [("orange", "--"), ("blue", "-")] def _sm90_available() -> bool: @@ -123,10 +112,6 @@ def bench_sparse_mla_q8kv8_prefill_sm90( s_q: int, s_kv: int, h_q: int, d_qk: int, topk: int, provider: str ): if provider == "q16_bf16_flashmla": - if not HAS_Q16_FLASHMLA: - raise RuntimeError( - "sgl_kernel.flash_mla.flash_mla_sparse_fwd is not available" - ) q, kv, indices, sm_scale = _make_q16_inputs(s_q, s_kv, h_q, d_qk, topk) def fn(): diff --git a/test/registered/kernels/ops/attention/dsv4/test_flashmla_v41.py b/test/registered/kernels/ops/attention/dsv4/test_flashmla_v41.py new file mode 100644 index 0000000000..67c38119bd --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_flashmla_v41.py @@ -0,0 +1,301 @@ +"""The current FlashMLA V4.1 kernels on the interleaved caches against the +legacy block-split kernels on the same tokens, and the schedule handling sglang +adds around them (precomputed metadata, reuse across layers, CUDA graphs).""" + +import unittest + +import torch + +from sglang.kernels.ops.attention import flash_mla_interface +from sglang.kernels.ops.attention.flash_mla_interface import SPLIT_KV_MIN_TOPK +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference import ( + dequantize_k_cache_v41, + dequantize_k_cache_v41_fp4, + interleave_pages, + quantize_k_cache_v41, + quantize_k_cache_v41_fp4, +) +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +D = 512 +H = 64 +SM_SCALE = D**-0.5 + + +def paged_view(raw, page_size, bytes_per_token): + return raw[:, : page_size * bytes_per_token].view( + raw.shape[0], page_size, 1, bytes_per_token + ) + + +def unpack_mega_output(data, scale): + tokens, groups, hidden = data.shape + sf = scale.contiguous().view(torch.uint8).float() + values = data.float().view(tokens, groups, hidden // 32, 32) + return (values * torch.exp2(sf - 127).unsqueeze(-1)).view(tokens, groups, hidden) + + +def reference_attention(q, k_rows, valid, sink): + logits = torch.einsum("nhd,ntd->nht", q.float(), k_rows.float()) * SM_SCALE + logits = logits.masked_fill(~valid[:, None, :], float("-inf")) + m = torch.maximum(logits.amax(-1), sink[None, :]) + p = torch.exp(logits - m[..., None]) + denom = p.sum(-1) + torch.exp(sink[None, :] - m) + return torch.einsum("nht,ntd->nhd", p, k_rows.float()) / denom[..., None] + + +class Case: + """An fp8 main cache and an fp4 extra cache, block-split and interleaved.""" + + def __init__(self, n, topk, extra_topk, seed, page=64, extra_page=64): + g = torch.Generator(device="cuda").manual_seed(seed) + self.n, self.topk, self.extra_topk = n, topk, extra_topk + main_pages, extra_pages = 6, 9 + k = torch.randn(main_pages, page, D, generator=g, device="cuda") / 4 + ek = torch.randn(extra_pages, extra_page, D, generator=g, device="cuda") / 4 + main_split = quantize_k_cache_v41(k.bfloat16()) + extra_split = quantize_k_cache_v41_fp4(ek.bfloat16()) + self.main_k = dequantize_k_cache_v41(main_split, page).view(-1, D) + self.extra_k = dequantize_k_cache_v41_fp4(extra_split, extra_page).view(-1, D) + self.split = ( + paged_view(main_split, page, 528), + paged_view(extra_split, extra_page, 288), + ) + self.interleaved = ( + paged_view(interleave_pages(main_split, page, 512, 16), page, 528), + paged_view( + interleave_pages(extra_split, extra_page, 256, 32), extra_page, 288 + ), + ) + self.indices = torch.randint( + 0, + main_pages * page, + (n, topk), + generator=g, + device="cuda", + dtype=torch.int32, + ) + self.extra_indices = torch.randint( + 0, + extra_pages * extra_page, + (n, extra_topk), + generator=g, + device="cuda", + dtype=torch.int32, + ) + self.extra_indices[0, 0] = -1 + self.lengths = torch.randint( + 1, topk + 1, (n,), generator=g, device="cuda", dtype=torch.int32 + ) + self.extra_lengths = torch.randint( + 0, extra_topk + 1, (n,), generator=g, device="cuda", dtype=torch.int32 + ) + self.sink = torch.randn(H, generator=g, device="cuda") + self.q = (torch.randn(n, H, D, generator=g, device="cuda") / 2).bfloat16() + + def reference(self): + valid = torch.arange(self.topk, device="cuda")[None] < self.lengths[:, None] + evalid = ( + torch.arange(self.extra_topk, device="cuda")[None] + < self.extra_lengths[:, None] + ) & (self.extra_indices >= 0) + k_rows = torch.cat( + [ + self.main_k[self.indices.long()], + self.extra_k[self.extra_indices.clamp_min(0).long()], + ], + dim=1, + ) + return reference_attention( + self.q, k_rows, torch.cat([valid, evalid], dim=1), self.sink + ) + + def decode(self, sched_meta=None, *, use_legacy=False): + main, extra = self.split if use_legacy else self.interleaved + return flash_mla_interface.flash_mla_with_kvcache( + q=self.q[:, None], + k_cache=main, + head_dim_v=D, + block_table=None, + cache_seqlens=None, + tile_scheduler_metadata=( + flash_mla_interface.get_mla_metadata()[0] + if sched_meta is None + else sched_meta + ), + softmax_scale=SM_SCALE, + is_fp8_kvcache=True, + indices=self.indices[:, None], + topk_length=self.lengths, + attn_sink=self.sink, + extra_k_cache=extra, + extra_indices_in_kvcache=self.extra_indices[:, None], + extra_topk_length=self.extra_lengths, + use_legacy=use_legacy, + )[0][:, 0] + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10, + "the current FlashMLA V4.1 kernels need SM100/SM103", +) +class TestFlashMLAV41(CustomTestCase): + # The kernel splits the KV only past 640 indices: the first three shapes + # run unsplit, the rest take the split-KV schedule. + SHAPES = ( + (5, 128, 512), + (64, 128, 64), + (1, 128, 128), + (16, 128, 1024), + (32, 512, 512), + (1, 128, 1024), + ) + SPLIT = (16, 128, 1024) + + def test_sparse_decode_matches_reference(self): + for n, topk, extra_topk in self.SHAPES: + with self.subTest(n=n, topk=topk, extra_topk=extra_topk): + c = Case(n, topk, extra_topk, seed=n + topk) + torch.testing.assert_close( + c.decode().float(), c.reference(), atol=2e-2, rtol=2e-2 + ) + + def test_sparse_decode_matches_block_split_kernel(self): + for n, topk, extra_topk in self.SHAPES: + with self.subTest(n=n, topk=topk, extra_topk=extra_topk): + c = Case(n, topk, extra_topk, seed=7 * n + topk) + torch.testing.assert_close( + c.decode(), c.decode(use_legacy=True), atol=1e-2, rtol=1e-2 + ) + + def test_precomputed_schedule_matches_own_schedule(self): + """The backend's `flashmla_sched_meta` precompute feeds the new kernel.""" + from sglang.srt.layers.attention.deepseek_v4_backend import ( + _maybe_precompute_flashmla_sched_meta, + ) + + for n, topk, extra_topk in self.SHAPES: + with self.subTest(n=n, topk=topk, extra_topk=extra_topk): + c = Case(n, topk, extra_topk, seed=11 * n + topk) + meta = flash_mla_interface.get_mla_metadata()[0] + _maybe_precompute_flashmla_sched_meta( + meta, + q=c.q[:, None], + indices=c.indices[:, None], + topk_length=c.lengths, + extra_indices=c.extra_indices[:, None], + extra_topk_length=c.extra_lengths, + ) + self.assertIsNotNone(meta.tile_scheduler_metadata) + self.assertTrue(torch.equal(c.decode(meta), c.decode())) + + def test_backend_precomputes_only_split_schedules(self): + from sglang.srt.layers.attention.deepseek_v4_backend import ( + _flashmla_v41_splits_kv, + ) + + for n, topk, extra_topk in self.SHAPES: + with self.subTest(n=n, topk=topk, extra_topk=extra_topk): + c = Case(n, topk, extra_topk, seed=n) + meta = flash_mla_interface.get_mla_metadata()[0] + c.decode(meta) + self.assertEqual( + meta.tile_scheduler_metadata is not None, + _flashmla_v41_splits_kv( + c.indices, c.extra_indices, SPLIT_KV_MIN_TOPK + ), + ) + + def test_schedule_is_shared_across_layers(self): + """The first call fills the backend's metadata object and later layers reuse it.""" + c = Case(*self.SPLIT, seed=13) + meta = flash_mla_interface.get_mla_metadata()[0] + first = c.decode(meta) + self.assertTrue(meta.have_initialized) + schedule = meta.tile_scheduler_metadata + self.assertIsNotNone(schedule) + second = c.decode(meta) + self.assertIs(meta.tile_scheduler_metadata, schedule) + self.assertTrue(torch.equal(first, second)) + + def test_cuda_graph_replay(self): + """A graph captured with an empty schedule recomputes it on replay, for + new topk lengths, the way the backend's captured decode does.""" + for shape in ((8, 128, 512), (8, 128, 1024)): + with self.subTest(shape=shape): + c = Case(*shape, seed=17) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + c.decode() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = c.decode(flash_mla_interface.get_mla_metadata()[0]) + fresh = Case(*shape, seed=19) + for name in ( + "q", + "indices", + "extra_indices", + "lengths", + "extra_lengths", + ): + getattr(c, name).copy_(getattr(fresh, name)) + graph.replay() + torch.cuda.synchronize() + self.assertTrue(torch.equal(out, c.decode())) + + def test_mega_decode_matches_block_split_kernel(self): + from sglang.srt.layers.attention.deepseek_v4_mega_backend import pack_query + + for page in (64, 256): + with self.subTest(page=page): + c = Case(48, 128, 512, seed=5 + page, page=page) + q = pack_query(c.q, H) + positions = torch.randint( + 0, 4096, (c.n,), device="cuda", dtype=torch.int32 + ) + angle = ( + torch.arange(4096, device="cuda")[:, None] + / torch.arange(1, 33, device="cuda")[None] + ) + table = torch.cat((angle.cos(), angle.sin()), dim=-1).float() + kw = dict( + attn_sink=c.sink, + topk_length=c.lengths, + extra_indices=c.extra_indices, + extra_topk_length=c.extra_lengths, + ) + new = flash_mla_interface.flash_mla_mega_decode( + q, + c.interleaved[0], + c.indices, + positions, + table, + SM_SCALE, + extra_kv=c.interleaved[1], + use_legacy=False, + **kw, + ) + old = flash_mla_interface.flash_mla_mega_decode( + q, + c.split[0], + c.indices, + positions, + table, + SM_SCALE, + extra_kv=c.split[1], + **kw, + ) + torch.testing.assert_close(new[2], old[2], atol=1e-3, rtol=1e-3) + got = unpack_mega_output(new[0], new[1]) + want = unpack_mega_output(old[0], old[1]) + rel = (got - want).norm() / want.norm() + self.assertLess(rel.item(), 1e-2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py b/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py index 2e023ad19f..6682a06da4 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py +++ b/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py @@ -3,8 +3,8 @@ import unittest from unittest.mock import patch import torch -from sgl_kernel.flash_mla_mega import flash_mla_mega_decode +from sglang.kernels.ops.attention.flash_mla_interface import flash_mla_mega_decode from sglang.srt.layers.attention.deepseek_v4_mega_backend import ( interleave_output_columns, interleave_query_rows, diff --git a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py index c51c24bfd5..010b888c7a 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py +++ b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py @@ -39,16 +39,51 @@ register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b # canonicalization and the query rope in the K launch exist only on HIP register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") -REFERENCE = { - KVLayout.V41: tq.quantize_k_cache_v41, - KVLayout.V41_FP4: tq.quantize_k_cache_v41_fp4, +V41_LAYOUTS = ( + KVLayout.V41, + KVLayout.V41_FP4, + KVLayout.V41_NEW, + KVLayout.V41_FP4_NEW, +) +_QUANT = { + KVLayout.V41: (tq.quantize_k_cache_v41, tq.dequantize_k_cache_v41), + KVLayout.V41_FP4: (tq.quantize_k_cache_v41_fp4, tq.dequantize_k_cache_v41_fp4), } -DEQUANT = { - KVLayout.V41: tq.dequantize_k_cache_v41, - KVLayout.V41_FP4: tq.dequantize_k_cache_v41_fp4, + + +def _interleaved(layout, quant, dequant): + """The block-split oracle pair, moved to the interleaved arrangement.""" + shape = (layout.data_bytes, layout.scale_bytes) + + def quant_new(k, page_bytes=None): + pages = tq.interleave_pages(quant(k), k.shape[1], *shape) + out = torch.zeros( + pages.shape[0], + page_bytes or layout.page_bytes(k.shape[1]), + dtype=torch.uint8, + device=pages.device, + ) + out[:, : pages.shape[1]] = pages + return out + + def dequant_new(pages, page_size): + return dequant(tq.deinterleave_pages(pages, page_size, *shape), page_size) + + return quant_new, dequant_new + + +CASES = { + layout: ( + _interleaved(layout, *_QUANT[layout.encoding]) + if layout.is_interleaved + else _QUANT[layout] + ) + for layout in V41_LAYOUTS } +REFERENCE = {layout: quant for layout, (quant, _) in CASES.items()} +DEQUANT = {layout: dequant for layout, (_, dequant) in CASES.items()} # One quantization step, relative: e4m3 has 3 mantissa bits, e2m1 one. -ONE_CODE_RTOL = {KVLayout.V41: 0.13, KVLayout.V41_FP4: 0.51} +ONE_CODE_RTOL = {layout: 0.51 if layout.is_fp4 else 0.13 for layout in V41_LAYOUTS} def rope_tail( @@ -69,14 +104,13 @@ def token_rows( """The (data row, scale row) bytes of the tokens at locs.""" locs = locs.long() page, offset = locs // page_size, locs % page_size + data_stride, scale_base, scale_stride = layout.row_addressing(page_size) data_cols = torch.arange(layout.data_bytes, device=pages.device) scale_cols = torch.arange(layout.scale_bytes, device=pages.device) - data = pages[page[:, None], offset[:, None] * layout.data_bytes + data_cols] + data = pages[page[:, None], offset[:, None] * data_stride + data_cols] scale = pages[ page[:, None], - layout.scale_offset(page_size) - + offset[:, None] * layout.scale_bytes - + scale_cols, + scale_base + offset[:, None] * scale_stride + scale_cols, ] return data, scale @@ -157,7 +191,7 @@ class TestV41KVStore(CustomTestCase): """Ragged page fills: a random subset of slots is written, the rest stays zero.""" g = torch.Generator(device="cuda").manual_seed(0) - for layout in (KVLayout.V41, KVLayout.V41_FP4): + for layout in V41_LAYOUTS: # A full page and a padded 2-token page, each with one index dtype # (the layout, the pad and the dtype are the three template axes). for page_size, num_pages, n, idx_dtype in ( @@ -197,7 +231,7 @@ class TestV41KVStore(CustomTestCase): g = torch.Generator(device="cuda").manual_seed(1) page_size, num_pages, n = 64, 5, 200 - for layout in (KVLayout.V41, KVLayout.V41_FP4): + for layout in V41_LAYOUTS: with self.subTest(layout=layout.name): x = random_rows(n, g) angles = torch.randn(n, 32, generator=g, device="cuda") @@ -225,10 +259,8 @@ class TestV41KVStore(CustomTestCase): layout, page_size, num_pages, locs, rotated, cache.shape[1] ) self.assert_tokens_equal(cache, ref, layout, page_size, locs) - if layout is KVLayout.V41_FP4: - deq = tq.dequantize_k_cache_v41_fp4(cache, page_size).view(-1, 512)[ - locs - ] + if layout.is_fp4: + deq = DEQUANT[layout](cache, page_size).view(-1, 512)[locs] self.assertTrue(torch.equal(deq, fake_quant_compressed_kv(rotated))) def test_boundary_tiles(self): @@ -260,7 +292,7 @@ class TestV41KVStore(CustomTestCase): n = x.shape[0] page_size = 16 locs = torch.arange(n, device="cuda", dtype=torch.int32) - for layout in (KVLayout.V41, KVLayout.V41_FP4): + for layout in V41_LAYOUTS: with self.subTest(layout=layout.name): cache = torch.zeros( 1, layout.page_bytes(page_size), dtype=torch.uint8, device="cuda" @@ -295,7 +327,7 @@ class TestV41KVStore(CustomTestCase): ) locs[7] = -1 # a row without a write target is skipped valid = locs >= 0 - for layout in (KVLayout.V41, KVLayout.V41_FP4): + for layout in V41_LAYOUTS: with self.subTest(layout=layout.name, rows="exact-norm"): # x = +-2^k per row with eps = 0 normalizes to +-1 exactly, so the # normed row is sign * weight and the reference is exact. @@ -376,7 +408,7 @@ class TestV41KVStore(CustomTestCase): angles = torch.randn(4096, 32, generator=g, device="cuda") freqs = torch.polar(torch.ones_like(angles), angles) n, page_size, num_pages = 150, 64, 8 - for layout in (KVLayout.V41, KVLayout.V41_FP4): + for layout in V41_LAYOUTS: seq_lens = ( torch.randint( 1, 1000, (n,), generator=g, device="cuda", dtype=torch.int64 @@ -479,18 +511,14 @@ class TestV41KVStore(CustomTestCase): self.assert_rows_close(cache, ref, layout, page_size, out_loc[valid]) -CASES = { - KVLayout.V41: (tq.quantize_k_cache_v41, tq.dequantize_k_cache_v41), - KVLayout.V41_FP4: (tq.quantize_k_cache_v41_fp4, tq.dequantize_k_cache_v41_fp4), -} - - def bits(t: torch.Tensor) -> torch.Tensor: """bf16 as int16, so that -0.0 and NaN payloads compare exactly.""" return t.contiguous().view(torch.int16) -@unittest.skipUnless(is_hip() and is_gfx95_supported(), "requires gfx950") +@unittest.skipUnless( + _v41_store_kernels_available(), "the V4.1 dequant kernels need gfx950 or SM100" +) class TestV41KVDequant(CustomTestCase): def _gather_ref(self, dequant, pages, page_size, ids): return dequant(pages, page_size).view(-1, 512)[ids.long()].unsqueeze(1) @@ -542,7 +570,7 @@ class TestV41KVDequant(CustomTestCase): ) # The fp4 cache dequantizes to the model's fake-quantized value # (compared by value: the fake quant maps an exact -0.0 to +0.0). - if layout is KVLayout.V41_FP4: + if layout.is_fp4: expect = fake_quant_compressed_kv( k.view(-1, 512)[ids.long()] ).unsqueeze(1) @@ -563,13 +591,14 @@ class TestV41KVDequant(CustomTestCase): dtype=torch.uint8, device="cuda", ) - if layout is KVLayout.V41: - lo = layout.scale_offset(page_size) - hi = lo + page_size * layout.scale_bytes - pages[:, lo:hi] = torch.randint( + if not layout.is_fp4: + _, scale_base, scale_stride = layout.row_addressing(page_size) + cols = scale_base + torch.arange(page_size)[:, None] * scale_stride + cols = (cols + torch.arange(layout.scale_bytes)).flatten().cuda() + pages[:, cols] = torch.randint( 100, 140, - (num_pages, hi - lo), + (num_pages, cols.numel()), generator=g, dtype=torch.uint8, device="cuda", diff --git a/test/registered/kernels/ops/attention/test_dsa_sink_pad_cache.py b/test/registered/kernels/ops/attention/test_dsa_sink_pad_cache.py index 2af86eb97d..6901243616 100644 --- a/test/registered/kernels/ops/attention/test_dsa_sink_pad_cache.py +++ b/test/registered/kernels/ops/attention/test_dsa_sink_pad_cache.py @@ -1,9 +1,9 @@ import sys import pytest -import sgl_kernel.flash_mla as flash_mla import torch +from sglang.kernels.ops.attention import flash_mla_interface from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend from sglang.test.ci.ci_register import register_cuda_ci @@ -23,7 +23,9 @@ def test_sparse_flashmla_sink_padding_refreshes_reused_buffer(monkeypatch): q = kwargs["q"] return q.new_zeros((*q.shape[:2], kwargs["d_v"])), None, None - monkeypatch.setattr(flash_mla, "flash_mla_sparse_fwd", capture_flash_mla_sparse_fwd) + monkeypatch.setattr( + flash_mla_interface, "flash_mla_sparse_fwd", capture_flash_mla_sparse_fwd + ) q = torch.zeros((1, 64, 8), device="cuda") kv_cache = torch.zeros((1, 1, 8), device="cuda") diff --git a/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py b/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py index 7f3f997e30..89c7683b5c 100644 --- a/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py +++ b/test/registered/kernels/ops/attention/test_flashmla_sched_meta.py @@ -30,7 +30,7 @@ def _cache(): def _call(kv, b, s_q, topk, topk_length, *, extra=None, meta=None): - import sgl_kernel.flash_mla as flash_mla + from sglang.kernels.ops.attention import flash_mla_interface g = torch.Generator(device="cuda").manual_seed(b * 7 + s_q * 13 + topk) q = torch.randn( @@ -48,10 +48,10 @@ def _call(kv, b, s_q, topk, topk_length, *, extra=None, meta=None): extra_indices_in_kvcache=extra_indices, extra_topk_length=extra_topk_length, ) - sched = flash_mla.FlashMLASchedMeta() + sched = flash_mla_interface.FlashMLASchedMeta() if meta is not None: sched.tile_scheduler_metadata, sched.num_splits = meta - out, lse = flash_mla.flash_mla_with_kvcache( + out, lse = flash_mla_interface.flash_mla_with_kvcache( q, kv, None, @@ -180,7 +180,7 @@ def test_schedule_at_shared_memory_boundary(offset): expected = [] for i in range(parts): begin, end = min(i * per_part, b), min((i + 1) * per_part, b) - expected.append([begin, end - 1, int(begin == b), 2, 0, 0, 0]) + expected.append([begin, end - 1, 0, 2, 0, 0, 0]) torch.testing.assert_close( meta[:, DEFINED].cpu(), torch.tensor(expected, dtype=torch.int32) ) diff --git a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py index 357c7f95fa..df528a94cf 100644 --- a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py +++ b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py @@ -437,36 +437,26 @@ def _patched_sparse_kernels( ) return out, meta, meta - sgl_kernel_pkg = types.ModuleType("sgl_kernel") - flash_mla_mod = types.ModuleType("sgl_kernel.flash_mla") - flash_mla_mod.flash_mla_sparse_fwd = fake_flash_mla_sparse_fwd - sgl_kernel_pkg.flash_mla = flash_mla_mod - q8_module_name = "sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90" q8_mod = types.ModuleType(q8_module_name) q8_mod.sparse_mla_q8kv8_prefill_fwd = fake_sparse_mla_q8kv8_prefill_fwd q8_mod.q8kv8_padded_num_heads = q8kv8_padded_num_heads q8_mod.cast_q_fp8_for_q8kv8_prefill = cast_q_fp8_for_q8kv8_prefill - old_modules = { - name: sys.modules.get(name) - for name in ( - "sgl_kernel", - "sgl_kernel.flash_mla", - q8_module_name, - ) - } - sys.modules["sgl_kernel"] = sgl_kernel_pkg - sys.modules["sgl_kernel.flash_mla"] = flash_mla_mod + from sglang.kernels.ops.attention import flash_mla_interface + + old_q8_module = sys.modules.get(q8_module_name) + old_flash_mla_sparse_fwd = flash_mla_interface.flash_mla_sparse_fwd + flash_mla_interface.flash_mla_sparse_fwd = fake_flash_mla_sparse_fwd sys.modules[q8_module_name] = q8_mod try: yield finally: - for name, old_value in old_modules.items(): - if old_value is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = old_value + flash_mla_interface.flash_mla_sparse_fwd = old_flash_mla_sparse_fwd + if old_q8_module is None: + sys.modules.pop(q8_module_name, None) + else: + sys.modules[q8_module_name] = old_q8_module @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is not available") @@ -729,11 +719,10 @@ def test_q8kv8_sparse_prefill_real_kernel_repeated_launch_stable(): @pytest.mark.parametrize("ratio", [0, 1, 2, 4, 128]) @pytest.mark.parametrize("scenario", ["first", "next", "replay", "tail", "large"]) def test_request_window_sparse_prefill_matches_paged_attention(ratio, scenario): - from sgl_kernel.flash_mla import ( + from sglang.kernels.ops.attention.flash_mla_interface import ( FlashMLASchedMeta, flash_mla_with_kvcache, ) - from sglang.srt.mem_cache.dsv41_request_window import window_layout from sglang.srt.runtime_context import get_platform diff --git a/test/registered/kernels/ops/kvcache/test_dsv41_request_window_copy.py b/test/registered/kernels/ops/kvcache/test_dsv41_request_window_copy.py index 122d0140b5..b43a1999fd 100644 --- a/test/registered/kernels/ops/kvcache/test_dsv41_request_window_copy.py +++ b/test/registered/kernels/ops/kvcache/test_dsv41_request_window_copy.py @@ -21,10 +21,14 @@ def reference(src, dst, source, target, page, layout): # Independent scalar row slicing also checks every untouched padding byte. result = dst.clone() for s, d in zip(source.tolist(), target.tolist()): - for width, base in ( - (layout.data_bytes, 0), - (layout.scale_bytes, page * layout.data_bytes), - ): + if layout.is_interleaved: + rows = ((layout.bytes_per_token, 0),) + else: + rows = ( + (layout.data_bytes, 0), + (layout.scale_bytes, page * layout.data_bytes), + ) + for width, base in rows: result[ d // page, base + d % page * width : base + (d % page + 1) * width ] = src[s // page, base + s % page * width : base + (s % page + 1) * width] diff --git a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py index f3ba97165b..64a07e8673 100644 --- a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py +++ b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py @@ -328,38 +328,61 @@ class TestV41KVPoolLayouts(CustomTestCase): self.assertEqual(view.stride(0), pool.bytes_per_page_padded) def test_v41_pool_buffers(self): - for option, expect in ((None, KVLayout.V41_FP4), ("fp8", KVLayout.V41)): - with self.subTest(compressed=option): - pool = self.make_pool([0, 0, 2, 1, 1], [2, 3], KVLayout.V41, option) - self.assert_kernel_requirements(pool.swa_kv_pool, KVLayout.V41) - self.assertEqual(pool.get_swa_key_bytes_per_token(), 528) - for ratio in (1, 2): - layer_id = pool.sources_by_ratio[ratio][0] - self.assertIs(pool.get_extra_key_layout(layer_id), expect) - self.assertEqual( - pool.get_extra_key_bytes_per_token(layer_id), - expect.bytes_per_token, - ) - self.assertTrue(is_valid_kv_layout_pair(pool.kv_layout, expect)) - self.assert_kernel_requirements(pool.kv_pools[ratio], expect) - # A pool of the fp4 layout cannot be the main cache. - with self.assertRaises(AssertionError): - self.make_pool([0], [], KVLayout.V41_FP4) + for main in (KVLayout.V41, KVLayout.V41_NEW): + for option, encoding in ( + (None, KVLayout.V41_FP4), + ("fp8", KVLayout.V41), + ): + expect = encoding.with_interleaved(main.is_interleaved) + with self.subTest(main=main, compressed=option): + pool = self.make_pool([0, 0, 2, 1, 1], [2, 3], main, option) + self.assert_kernel_requirements(pool.swa_kv_pool, main) + self.assertIs(pool.get_swa_key_layout(), main) + self.assertEqual(pool.get_swa_key_bytes_per_token(), 528) + for ratio in (1, 2): + layer_id = pool.sources_by_ratio[ratio][0] + self.assertIs(pool.get_extra_key_layout(layer_id), expect) + self.assertEqual( + pool.get_extra_key_bytes_per_token(layer_id), + expect.bytes_per_token, + ) + self.assertTrue(is_valid_kv_layout_pair(pool.kv_layout, expect)) + self.assert_kernel_requirements(pool.kv_pools[ratio], expect) + # A pool of an fp4 layout cannot be the main cache. + for fp4 in (KVLayout.V41_FP4, KVLayout.V41_FP4_NEW): + with self.assertRaises(AssertionError): + self.make_pool([0], [], fp4) def test_v41_pool_with_c4_c128(self): - pool = self.make_pool( - [0, 4, 128], - [], - KVLayout.V41, - c4_size=PAGE_SIZE, - c128_size=PAGE_SIZE, - c4_state_pool_size=16, - c128_state_pool_size=16, - ) - for ratio in (4, 128): - self.assertEqual(pool.kv_pools[ratio].page_size, PAGE_SIZE // ratio) - # The 2-token c128 page is the only production page that pads. - self.assertEqual(pool.kv_pools[128].bytes_per_page_padded, 1536) + # The 2-token c128 page is the only production page the block-split + # layout pads; the interleaved one never pads. + for main, c128_page_bytes in ((KVLayout.V41, 1536), (KVLayout.V41_NEW, 1056)): + with self.subTest(main=main): + pool = self.make_pool( + [0, 4, 128], + [], + main, + c4_size=PAGE_SIZE, + c128_size=PAGE_SIZE, + c4_state_pool_size=16, + c128_state_pool_size=16, + ) + for ratio in (4, 128): + self.assertEqual(pool.kv_pools[ratio].page_size, PAGE_SIZE // ratio) + self.assertIs(pool.kv_pools[ratio].kv_layout, main) + self.assertEqual( + pool.kv_pools[128].bytes_per_page_padded, c128_page_bytes + ) + + def test_pd_layout_names_the_arrangement(self): + """Prefill and decode servers must refuse to pair the two arrangements: + their pages have the same size, so a mismatch would read garbage.""" + names = { + main: self.make_pool([0, 0, 2, 1, 1], [2, 3], main).pd_kv_layout() + for main in (KVLayout.V41, KVLayout.V41_NEW) + } + self.assertNotEqual(names[KVLayout.V41], names[KVLayout.V41_NEW]) + self.assertIn("v41_fp4_new", names[KVLayout.V41_NEW]) def test_pd_entries_cover_the_c4_kv_page_count(self): """The PD transfer walks every registered entry with the c4 KV pool's