[DSV4.1] Add opt-in TRT-LLM sparse attention support (#41603)

This commit is contained in:
Yuwei An
2026-10-06 12:02:10 +08:00
committed by GitHub
parent c76faa1ff2
commit 620c76f25d
16 changed files with 684 additions and 70 deletions
@@ -191,8 +191,8 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
if (out_loc <= 0) return;
const auto kv_row = Paged::row(params.kvcache, out_loc);
if constexpr (kLayout == KVLayout::V41) {
// fp8 with one ue8m0 scale per 32 elements over the whole row, RoPE included.
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
// FP8 over the whole row, RoPE included; the layout selects the scaling.
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, data);
}
@@ -258,8 +258,8 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
const auto kv_row = Paged::row(params.kvcache, out_loc);
if constexpr (kLayout == KVLayout::V41) {
// fp8 with one ue8m0 scale per 32 elements over the whole row, RoPE included.
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
// FP8 over the whole row, RoPE included; the layout selects the scaling.
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
}
@@ -35,7 +35,7 @@ namespace sglang {
namespace deepseek_v4 {
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2 };
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 };
template <KVLayout kLayout>
struct KVLayoutTraits;
@@ -67,6 +67,15 @@ struct KVLayoutTraits<KVLayout::V41_FP4> {
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};
template <>
struct KVLayoutTraits<KVLayout::UNIFORM_FP8> {
static constexpr int64_t kDataBytes = 512;
static constexpr int64_t kScaleBytes = 0;
static constexpr int64_t kTileSize = 512;
static constexpr int64_t kPageAlign = 512;
static constexpr int64_t kBytesPerToken = kDataBytes;
};
/// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride.
template <KVLayout kLayout>
constexpr int64_t kv_page_bytes(int64_t page_size) {
@@ -206,6 +215,13 @@ SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, co
static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller");
if constexpr (kLayout == KVLayout::V41) {
store_row_fp8(data_row, scale_row, tx, v);
} else if constexpr (kLayout == KVLayout::UNIFORM_FP8) {
device::AlignedVector<fp8x2_e4m3_t, kVecSize / 2> out;
#pragma unroll
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
out[i] = fp8::pack_fp8(v[2 * i], v[2 * i + 1]);
}
out.store(data_row, tx);
} else {
store_row_fp4(data_row, scale_row, tx, v);
}
@@ -175,6 +175,14 @@ def fused_q_norm_rope(
freqs_cis: torch.Tensor,
positions: torch.Tensor,
) -> None:
# Reuse the BF16 kernel and preserve its rounding before FP8 conversion.
# Keep the temporary separate from q_input: callers may reuse the input.
fp8_output = q_output if q_output.dtype == torch.float8_e4m3fn else None
if fp8_output is not None:
assert q_input.dtype == torch.bfloat16
assert q_output.shape == q_input.shape
assert q_output.device == q_input.device
q_output = torch.empty_like(q_input)
freqs_real = torch.view_as_real(freqs_cis).flatten(-2)
head_dim = q_input.shape[-1]
rope_dim = freqs_real.shape[-1]
@@ -186,6 +194,8 @@ def fused_q_norm_rope(
q_input.dtype, head_dim, rope_dim, eps is not None
)
module.forward(q_input, q_output, freqs_real, positions, eps or 0.0)
if fp8_output is not None:
fp8_output.copy_(q_output)
def fused_q_indexer_rope_hadamard_quant(
@@ -20,19 +20,36 @@ class KVLayout(str, enum.Enum):
V41 = "v41"
# 512 e2m1 packed two per byte (even index low nibble), 32 e4m3 scales per 16 values.
V41_FP4 = "v41_fp4"
# TRT-LLM only: 512 e4m3 values with a fixed external unit scale.
UNIFORM_FP8 = "uniform_fp8"
@property
def data_bytes(self) -> int:
return {KVLayout.V4: 576, KVLayout.V41: 512, KVLayout.V41_FP4: 256}[self]
return {
KVLayout.V4: 576,
KVLayout.V41: 512,
KVLayout.V41_FP4: 256,
KVLayout.UNIFORM_FP8: 512,
}[self]
@property
def scale_bytes(self) -> int:
return {KVLayout.V4: 8, KVLayout.V41: 16, KVLayout.V41_FP4: 32}[self]
return {
KVLayout.V4: 8,
KVLayout.V41: 16,
KVLayout.V41_FP4: 32,
KVLayout.UNIFORM_FP8: 0,
}[self]
@property
def tile_size(self) -> int:
"""Values sharing one scale."""
return {KVLayout.V4: 64, KVLayout.V41: 32, KVLayout.V41_FP4: 16}[self]
return {
KVLayout.V4: 64,
KVLayout.V41: 32,
KVLayout.V41_FP4: 16,
KVLayout.UNIFORM_FP8: 512,
}[self]
@property
def bytes_per_token(self) -> int:
@@ -41,7 +58,12 @@ class KVLayout(str, enum.Enum):
@property
def page_align(self) -> int:
"""Unit the page stride is padded to: the reader's TMA row stride."""
return {KVLayout.V4: 576, KVLayout.V41: 512, KVLayout.V41_FP4: 256}[self]
return {
KVLayout.V4: 576,
KVLayout.V41: 512,
KVLayout.V41_FP4: 256,
KVLayout.UNIFORM_FP8: 512,
}[self]
@property
def is_fp4(self) -> bool:
@@ -0,0 +1,99 @@
"""Normalize/rotate KV and store directly into the TRT uniform-FP8 cache."""
import torch
import triton
import triton.language as tl
@triton.jit
def _kv_norm_rope_fp8_store(
X,
W,
F,
POS,
LOC,
CACHE,
EPS: tl.constexpr,
SX: tl.constexpr,
SF: tl.constexpr,
CACHE_ROWS: tl.constexpr,
):
row = tl.program_id(0)
base = row.to(tl.int64) * SX
d = tl.arange(0, 512)
x = tl.load(X + base + d).to(tl.float32)
w = tl.load(W + d).to(tl.float32)
inv = tl.rsqrt(tl.sum(x * x, 0) / 512 + EPS)
y = x * inv * w
# Match _fused_norm_rope_kernel's reduction and RoPE operation order.
pair = tl.arange(0, 32)
even = tl.load(X + base + 448 + pair * 2).to(tl.float32)
odd = tl.load(X + base + 449 + pair * 2).to(tl.float32)
we = tl.load(W + 448 + pair * 2).to(tl.float32)
wo = tl.load(W + 449 + pair * 2).to(tl.float32)
even = even * inv * we
odd = odd * inv * wo
pos = tl.load(POS + row).to(tl.int64)
c = tl.load(F + pos * SF + pair * 2).to(tl.float32)
s = tl.load(F + pos * SF + pair * 2 + 1).to(tl.float32)
re = even * c - odd * s
im = even * s + odd * c
slot = tl.load(LOC + row).to(tl.int64)
# Preserve PyTorch indexed assignment's negative-index interpretation.
slot = tl.where(slot < 0, slot + CACHE_ROWS, slot)
dst = CACHE + slot * 512
# The old path materializes BF16/FP16 before torch's FP8 conversion.
# Keep that rounding boundary without writing the intermediate to HBM.
y = y.to(X.dtype.element_ty).to(tl.float32)
re = re.to(X.dtype.element_ty).to(tl.float32)
im = im.to(X.dtype.element_ty).to(tl.float32)
tl.store(dst + d, y, d < 448)
tl.store(dst + 448 + pair * 2, re)
tl.store(dst + 449 + pair * 2, im)
def fused_k_norm_rope_uniform_fp8(
kv: torch.Tensor,
weight: torch.Tensor,
eps: float,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
loc: torch.Tensor,
cache: torch.Tensor,
) -> None:
"""Store norm+RoPE KV as unit-scale E4M3, without modifying ``kv``.
``loc`` addresses contiguous 512-byte rows in the uniform-FP8 pool.
Active rows must have valid, distinct destinations, as with index_put.
"""
assert kv.ndim == 2 and kv.shape[1] == 512 and kv.stride(1) == 1
assert kv.dtype in (torch.bfloat16, torch.float16, torch.float32)
assert weight.shape == (512,) and weight.is_contiguous()
assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous()
assert freqs_cis.ndim == 2 and freqs_cis.shape[1] == 32
assert positions.shape == loc.shape == (kv.shape[0],)
assert positions.dtype in (torch.int32, torch.int64)
assert loc.dtype in (torch.int32, torch.int64)
assert positions.is_contiguous() and loc.is_contiguous()
assert cache.dtype == torch.float8_e4m3fn and cache.is_contiguous()
assert cache.numel() % 512 == 0
assert all(
t.device == kv.device for t in (weight, freqs_cis, positions, loc, cache)
)
if kv.shape[0] == 0:
return
_kv_norm_rope_fp8_store[(kv.shape[0],)](
kv,
weight,
torch.view_as_real(freqs_cis),
positions,
loc,
cache,
EPS=eps,
SX=kv.stride(0),
SF=freqs_cis.stride(0) * 2,
CACHE_ROWS=cache.numel() // 512,
num_warps=4,
)
@@ -0,0 +1,69 @@
"""Refresh the layer-dependent tail of TRT-LLM's sparse attention tables."""
import torch
import triton
import triton.language as tl
@triton.jit
def _pack_sparse_tail(
INDICES,
LENGTHS,
TABLE,
TOTAL_LENGTHS,
INDEX_STRIDE: tl.constexpr,
TABLE_STRIDE: tl.constexpr,
LENGTH_STRIDE: tl.constexpr,
TOTAL_STRIDE: tl.constexpr,
WIDTH: tl.constexpr,
SWA_WIDTH: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
col = tl.arange(0, BLOCK)
values = tl.load(INDICES + row * INDEX_STRIDE + col, col < WIDTH, -1)
tl.store(TABLE + row * TABLE_STRIDE + SWA_WIDTH + col, values, col < WIDTH)
length = tl.load(LENGTHS + row * LENGTH_STRIDE)
tl.store(TOTAL_LENGTHS + row * TOTAL_STRIDE, SWA_WIDTH + length)
def pack_sparse_tail(
indices: torch.Tensor,
lengths: torch.Tensor,
table: torch.Tensor,
total_lengths: torch.Tensor,
swa_width: int = 128,
) -> None:
"""Copy compressed indices and add the fixed SWA width in one launch.
The caller owns the tile-padded buffers and their invariant SWA columns.
Only active rows are modified, preserving inert graph/tile padding.
Compressed entries start at column swa_width even for short sequences.
total_lengths describes the table span to scan, not the valid-token count:
adding only the actual SWA length would truncate the compressed tail.
Unused SWA slots retain their -1 sentinel and are ignored by attention.
"""
rows, width = indices.shape
assert table.shape == (rows, swa_width + width)
assert lengths.shape == total_lengths.shape == (rows,)
assert indices.stride(1) == table.stride(1) == 1
assert indices.dtype == table.dtype == total_lengths.dtype == torch.int32
assert lengths.dtype in (torch.int32, torch.int64)
assert indices.device == lengths.device == table.device == total_lengths.device
if rows == 0:
return
_pack_sparse_tail[(rows,)](
indices,
lengths,
table,
total_lengths,
indices.stride(0),
table.stride(0),
lengths.stride(0),
total_lengths.stride(0),
width,
swa_width,
triton.next_power_of_2(width),
num_warps=4,
)
@@ -16,6 +16,7 @@ from sglang.srt.arg_groups.overrides import (
run_post_process_pass,
)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of
from sglang.srt.utils.common import is_gfx95_supported, is_npu
@@ -217,9 +218,14 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
)
return
if (
cfg.dsv4_attn_backend == "trtllm"
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
):
raise ValueError(
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
)
if cfg.enable_encoder_swa_bounded_replay:
from sglang.srt.model_executor.cuda_graph_config import Backend
incompatible = (
(
"hardware other than CUDA or gfx950",
@@ -264,8 +270,14 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
),
("HiSparse", cfg.enable_hisparse),
("the unified KV layout", is_unified_kv_triton()),
# The trtllm-gen path has no uniform-FP8 pool for V4.1's ratio-1/2 layers.
("the trtllm DSv4 attention backend", cfg.dsv4_attn_backend == "trtllm"),
(
"TRT-LLM with SWA bounded replay",
cfg.dsv4_attn_backend == "trtllm"
and (
cfg.enable_encoder_swa_bounded_replay
or cfg.enable_decoder_swa_bounded_replay
),
),
("two-batch overlap", cfg.enable_two_batch_overlap),
("pipeline parallelism", cfg.pp_size > 1),
)
@@ -296,8 +308,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"block size and TP size."
)
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
prefill_graph = cfg.cuda_graph_config.prefill
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
# The captured low-ratio indexer scores a static context width; 16k
@@ -315,8 +325,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
)
if cfg.enable_decoder_swa_bounded_replay:
from sglang.srt.model_executor.cuda_graph_config import Backend
# Late layers see a per-request tail slice, not the captured prefill shape.
incompatible = (
(
@@ -516,11 +516,13 @@ class DSV4AttnMetadata:
raise ValueError(f"invalid {compress_ratio=}")
def init_trtllm_sparse_buffers(self) -> None:
"""Decode tables of 128 SWA columns then compressed KV, -1 for an invalid
index, lens counting all 128 SWA slots; only the c4 tail is per layer."""
"""Build tile-padded TRT-LLM tables and lengths for indexed attention
or SWA-only DSpark blocks; indexed tails are refreshed per layer."""
num_tokens = self.seq_lens_casual.shape[0]
assert self.swa_page_indices.shape == (num_tokens, SWA_WINDOW)
swa_width = self.swa_page_indices.shape[1]
assert self.swa_page_indices.shape[0] == num_tokens
assert swa_width >= SWA_WINDOW
# VarSeq reads rows to the 64-token tile boundary. Back every live view
# with an aligned parent whose extra rows contain inert values.
@@ -536,17 +538,40 @@ class DSV4AttnMetadata:
if n_pad != num_tokens:
self.seq_lens_casual = _tile_padded(1, self.seq_lens_casual)
self.swa_page_indices = _tile_padded(
-1, self.swa_page_indices, width=SWA_WINDOW
-1, self.swa_page_indices, width=swa_width
)
self.trtllm_swa_lens = _tile_padded(SWA_WINDOW)
if self.c4_sparse_page_indices is not None:
w4 = self.c4_sparse_page_indices.shape[-1]
self.trtllm_swa_lens = _tile_padded(
SWA_WINDOW,
self.swa_topk_lengths.clamp_min(SWA_WINDOW)
if swa_width > SWA_WINDOW
else None,
)
for ratio in self.low_ratios:
lengths = self.sparse_topk_lengths(ratio)
if lengths is not None:
# The FlashMLA metadata clamps empty compressed histories to
# one. TRT-LLM needs the true zero length: -1 loads zero KV,
# but does not by itself remove its softmax contribution.
torch.minimum(lengths, self.seq_lens_casual // ratio, out=lengths)
indexed_tables = [
table
for ratio in (1, 2, 4)
if ratio in self.present_ratios
if (table := self.sparse_page_indices(ratio)) is not None
]
if indexed_tables:
assert swa_width == SWA_WINDOW
# Ratio 1/2/4 all select index_topk entries. Reuse the per-layer
# table; its contents are overwritten before every attention call.
w4 = indexed_tables[0].shape[-1]
assert all(table.shape[-1] == w4 for table in indexed_tables)
assert w4 % 4 == 0, f"{w4=}"
# Unwritten c4 rows must remain inert until the per-layer fill.
# Unwritten indexed rows remain inert until the per-layer fill.
self.trtllm_c4_indices = _tile_padded(-1, width=SWA_WINDOW + w4)
self.trtllm_c4_indices[:, :SWA_WINDOW].copy_(self.swa_page_indices)
self.trtllm_c4_lens = _tile_padded(SWA_WINDOW)
if self.c128_page_indices is not None:
assert swa_width == SWA_WINDOW
w128 = self.c128_page_indices.shape[-1]
assert w128 % 4 == 0, f"{w128=}"
self.trtllm_c128_indices = _tile_padded(-1, width=SWA_WINDOW + w128)
@@ -2111,6 +2136,8 @@ class DeepseekV4AttnBackend(
)
metadata.core_attn_metadata.swa_page_indices = swa_page_indices
metadata.core_attn_metadata.swa_topk_lengths = swa_topk_lengths
if self.trtllm_attn:
metadata.core_attn_metadata.init_trtllm_sparse_buffers()
def _dspark_seq_lens_casual(
self, *, seq_lens: torch.Tensor, block_size: int
@@ -2921,7 +2948,11 @@ class DeepseekV4AttnBackend(
kv_cache = pool.get_extra_key_buffer(layer_id)
page_size = pool.get_extra_key_page_size(layer_id)
# The pool's page format: V4, or the V4.1 fp8 / fp4 layouts.
kv_layout = pool.get_extra_key_layout(layer_id)
kv_layout = (
KVLayout.UNIFORM_FP8
if pool.uniform_fp8
else pool.get_extra_key_layout(layer_id)
)
assert kv_cache is not None
if layer.compress_ratio == 1:
@@ -3352,7 +3383,10 @@ class DeepseekV4AttnBackend(
**_,
) -> torch.Tensor:
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
return q.new_empty(q.shape[0], q.shape[1], layer.v_head_dim)
return q.new_empty(
(q.shape[0], q.shape[1], layer.v_head_dim),
dtype=torch.bfloat16 if self.trtllm_attn else q.dtype,
)
assert k is v, "DeepseekV4 shares k and v"
swa_k = k
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Literal, Optional, Tuple
import torch
from sglang.kernels.ops.attention.dsv4.trtllm_metadata import pack_sparse_tail
from sglang.srt.environ import envs
from sglang.srt.layers.attention.deepseek_v4_backend import (
SWA_WINDOW,
@@ -18,6 +19,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4MultiStepBackend,
)
from sglang.srt.runtime_context import (
get_buffer,
get_exec,
get_parallel,
get_schedule,
@@ -25,6 +27,14 @@ from sglang.srt.runtime_context import (
max_prefill_buffer_tokens,
)
try:
import flashinfer.mla._core as _fi_core
from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4
except ImportError as exc:
_flashinfer_import_error = exc
else:
_flashinfer_import_error = None
if TYPE_CHECKING:
from sglang.srt.layers.attention.deepseek_v4_backend import DSV4AttnMetadata
from sglang.srt.layers.radix_attention import RadixAttention
@@ -38,8 +48,6 @@ _TRTLLM_GEN_WORKSPACE_SIZE_MB = 128
def _get_trtllm_workspace_buffer(device: torch.device) -> torch.Tensor:
from sglang.srt.runtime_context import get_buffer
return get_buffer(
"trtllm_dsv4_zero_workspace",
lambda: torch.zeros(
@@ -84,11 +92,13 @@ def _install_persistent_trtllm_semaphores(capacity_rows: int) -> None:
FlashInfer accepts a caller-owned buffer.
"""
global _trtllm_semaphore_installed, _trtllm_semaphore_rows
if _flashinfer_import_error is not None:
raise ImportError(
"--dsv4-attn-backend trtllm requires FlashInfer sparse MLA support"
) from _flashinfer_import_error
_trtllm_semaphore_rows = max(_trtllm_semaphore_rows, capacity_rows)
if _trtllm_semaphore_installed:
return
import flashinfer.mla._core as _fi_core
_orig = _fi_core._get_trtllm_gen_multi_ctas_kv_counter_buffer
# Allocate once outside graph capture. Stream ordering and the kernel's
# counter reset make one shared buffer safe across launches.
@@ -166,7 +176,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
*,
q: torch.Tensor,
layer: RadixAttention,
compress_ratio: Literal[0, 4, 128],
compress_ratio: Literal[0, 1, 2, 4, 128],
core_attn_metadata: DSV4AttnMetadata,
forward_batch: ForwardBatch,
attn_sink: torch.Tensor,
@@ -175,6 +185,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
extra_topk_lengths: Optional[torch.Tensor],
) -> torch.Tensor:
assert attn_sink is not None
if self.is_dsv41:
# FlashMLA pads TP query heads to 64. TRT-LLM accepts the native
# width; do not compute attention for the discarded padding.
q = q[:, : layer.tp_q_head_num, :]
attn_sink = attn_sink[: layer.tp_q_head_num]
if (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
@@ -221,7 +236,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
return (self.softmax_scale, 1.0)
def _trtllm_kv_cache_views(
self, layer_id: int, compress_ratio: Literal[0, 4, 128]
self, layer_id: int, compress_ratio: Literal[0, 1, 2, 4, 128]
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Return HND views of the uniform-FP8 SWA and compressed pools.
@@ -248,7 +263,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
*,
q: torch.Tensor,
layer: RadixAttention,
compress_ratio: Literal[0, 4, 128],
compress_ratio: Literal[0, 1, 2, 4, 128],
core_attn_metadata: DSV4AttnMetadata,
attn_sink: torch.Tensor,
swa_page_indices: torch.Tensor,
@@ -257,8 +272,6 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
) -> torch.Tensor:
"""Run sparse MLA decode with preallocated metadata tables."""
from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4
bs, num_heads, head_dim = q.shape
assert head_dim == 512
@@ -278,8 +291,10 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
extra_topk_lengths = extra_topk_lengths[:n_meta_rows]
bs = n_meta_rows
# Only the c4 tail and lens vary by layer; other table data is prebuilt.
assert swa_page_indices.shape == (bs, SWA_WINDOW)
# Indexed tails and lengths vary by layer; other table data is prebuilt.
swa_width = swa_page_indices.shape[1]
assert swa_page_indices.shape[0] == bs
assert swa_width == SWA_WINDOW or compress_ratio == 0
if compress_ratio == 0:
# Use the metadata view backed by 64-row-aligned storage because
# the VarSeq kernel reads table rows to the tile boundary.
@@ -302,22 +317,21 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
assert sparse_topk_lens.shape[0] > bs, f"{sparse_topk_lens.shape=}"
sparse_topk_lens = sparse_topk_lens[:bs]
if compress_ratio == 4:
if compress_ratio in (1, 2, 4):
assert extra_indices is not None and extra_topk_lengths is not None
width = extra_indices.shape[-1]
assert SWA_WINDOW + width == sparse_indices.shape[1], (
f"{width=} {sparse_indices.shape=}"
)
sparse_indices[:, SWA_WINDOW:].copy_(extra_indices)
# Lens include all 128 SWA slots; seq_lens controls their validity.
sparse_topk_lens.copy_(extra_topk_lengths)
sparse_topk_lens.add_(SWA_WINDOW)
pack_sparse_tail(
extra_indices, extra_topk_lengths, sparse_indices, sparse_topk_lens
)
swa_kv_cache, compressed_kv_cache = self._trtllm_kv_cache_views(
layer.layer_id, compress_ratio
)
# RoPE is already applied; the unit scale makes this a plain e4m3 cast.
# Q preparation already returns FP8 on V4.1; to() is a no-op in that path.
q_fp8 = q.to(torch.float8_e4m3fn).view(bs, 1, num_heads, 512)
bmm1_scale, bmm2_scale = self._get_trtllm_bmm_scales(layer)
@@ -326,6 +340,13 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
if seq_lens.shape[0] != bs:
assert seq_lens.shape[0] > bs, f"{seq_lens.shape=} {bs=}"
seq_lens = seq_lens[:bs]
if swa_width > SWA_WINDOW:
# DSpark attends noncausally to its context window plus the whole
# draft block. The tail still indexes SWA storage, not compressed
# KV. Use the complete block's visible length rather than each
# token's causal length, including for short requests.
compressed_kv_cache = swa_kv_cache
seq_lens = core_attn_metadata.swa_topk_lengths[:bs]
assert attn_sink.dtype == torch.float32
assert self.trtllm_workspace_buffer is not None
_check_trtllm_query_rows(bs)
@@ -342,6 +363,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
bmm2_scale=bmm2_scale,
sinks=attn_sink,
kv_layout="HND",
backend="trtllm-gen",
)
if out_pad_tail is not None:
out_pad_tail[:bs] = out.view(bs, num_heads, 512)
@@ -353,7 +375,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
*,
q: torch.Tensor,
layer: RadixAttention,
compress_ratio: Literal[0, 4, 128],
compress_ratio: Literal[0, 1, 2, 4, 128],
forward_batch: ForwardBatch,
attn_sink: torch.Tensor,
swa_page_indices: torch.Tensor,
@@ -366,8 +388,6 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
validity from it. This path runs eagerly.
"""
from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4
assert q.ndim == 3, f"{q.shape=}"
num_qo_padded, num_heads, head_dim = q.shape
assert head_dim == 512
@@ -418,11 +438,10 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
assert width % 4 == 0, f"{width=}"
table = _tile_padded_pf(-1, width=SWA_WINDOW + width)
table[:, :SWA_WINDOW].copy_(swa_indices)
table[:, SWA_WINDOW:].copy_(extra_indices[:sum_q])
assert extra_topk_lengths is not None
lens = _tile_padded_pf(
SWA_WINDOW,
extra_topk_lengths[:sum_q].to(torch.int32) + SWA_WINDOW,
lens = _tile_padded_pf(SWA_WINDOW)
pack_sparse_tail(
extra_indices[:sum_q], extra_topk_lengths[:sum_q], table, lens
)
core.trtllm_prefill_c128 = (table, lens)
sparse_indices, sparse_topk_lens = core.trtllm_prefill_c128
@@ -441,14 +460,15 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
sum_q,
SWA_WINDOW + width,
), f"{sparse_indices.shape=} {width=}"
sparse_indices[:, SWA_WINDOW:].copy_(extra_indices[:sum_q])
# Lens include 128 SWA slots; VarSeq metadata controls validity.
sparse_topk_lens = _tile_padded_pf(
SWA_WINDOW,
extra_topk_lengths[:sum_q].to(torch.int32) + SWA_WINDOW,
sparse_topk_lens = _tile_padded_pf(SWA_WINDOW)
pack_sparse_tail(
extra_indices[:sum_q],
extra_topk_lengths[:sum_q],
sparse_indices,
sparse_topk_lens,
)
# RoPE is already applied; the unit scale makes this a plain e4m3 cast.
# Retain a unit-scale cast for callers without fused FP8 Q preparation.
q_fp8 = q[:sum_q].to(torch.float8_e4m3fn)
swa_kv_cache, compressed_kv_cache = self._trtllm_kv_cache_views(
@@ -483,6 +503,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
bmm2_scale=bmm2_scale,
sinks=attn_sink,
kv_layout="HND",
backend="trtllm-gen",
cum_seq_lens_q=cum_seq_lens_q,
max_q_len=max_q_len,
)
@@ -20,6 +20,9 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import (
KVLayout,
is_valid_kv_layout_pair,
)
from sglang.kernels.ops.attention.dsv4.kv_norm_rope_store import (
fused_k_norm_rope_uniform_fp8,
)
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
@@ -165,6 +168,11 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
AITER attention."""
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
if get_exec().kernel.dsv4_attn_backend == "trtllm":
assert mode in ("auto", "v4") and option in ("auto", "fp8"), (
"trtllm uses uniform FP8 pools, not the packed V4.1 KV layouts"
)
return KVLayout.V4, None
if mode == "v4":
return KVLayout.V4, None if option == "auto" else option
assert mode in ("v41", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}"
@@ -2042,21 +2050,14 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
"""q ([B, H, head_dim]): rope its query heads in the same launch."""
if self.uniform_fp8:
assert q is None, "uniform FP8 store does not fuse query RoPE"
# Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again),
# then an e4m3 cast + scatter with per-tensor scale 1.0.
from sglang.kernels.ops.attention.deepseek_v4_rope import (
fused_norm_rope_inplace_triton,
)
fused_norm_rope_inplace_triton(
fused_k_norm_rope_uniform_fp8(
kv,
kv_weight,
eps,
freqs_cis,
positions=positions,
)
self.swa_kv_pool.set_key_buffer_fused(
self._swa_local_layer_id(layer_id), swa_loc, kv
positions,
swa_loc,
self.swa_kv_pool.get_key_buffer(self._swa_local_layer_id(layer_id)),
)
return
fused_k_norm_rope_flashmla(
+16 -1
View File
@@ -1339,6 +1339,9 @@ class MQALayer(MqaAttentionBase):
q, _ = self.wq_b(q)
q = q.view(-1, self.n_local_heads, self.head_dim)
if not self.q_head_norm:
if q_out is not None and q_out.dtype == torch.float8_e4m3fn:
fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions)
return q_out
# TODO: enable zero-copy path to skip the extra no-rope copy overhead
if (_is_cuda or _is_gfx95_supported) and q_out is not None:
fused_q_norm_rope(
@@ -2252,7 +2255,19 @@ class MQALayer(MqaAttentionBase):
tp_slice, q_padded, q_out, q_rope = slice(None), None, None, None
k_nope, k_rope = None, None
if unified_fp8_decode or unified_fp8_prefill:
if (
self.is_dsv41
and not self.q_head_norm
and get_token_to_kv_pool().uniform_fp8
and self.n_local_heads in (8, 16, 32, 64, 128)
):
kernel_num_heads = self.n_local_heads
q_out = torch.empty(
(x.shape[0], self.n_local_heads, self.head_dim),
dtype=torch.float8_e4m3fn,
device=x.device,
)
elif unified_fp8_decode or unified_fp8_prefill:
# width and dtype come off the pools themselves; the kernel reads Q
# with the kv row stride, so the two must not drift
kv_pool = get_token_to_kv_pool()
+14 -1
View File
@@ -191,6 +191,9 @@ class DSparkAttention(MqaAttentionBase):
q, _ = self.wq_b(q)
q = q.view(-1, self.n_local_heads, self.head_dim)
if not self.q_head_norm:
if q_out is not None and q_out.dtype == torch.float8_e4m3fn:
fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions)
return q_out
if self._use_fast_kernel and not _is_npu:
fused_rope_inplace(
q[..., -self.rope_head_dim :],
@@ -270,7 +273,17 @@ class DSparkAttention(MqaAttentionBase):
q_padded: Optional[torch.Tensor] = None
q_out: Optional[torch.Tensor] = None
if self.n_local_heads < _PAD_NUM_HEADS:
if (
pool.uniform_fp8
and not self.q_head_norm
and self.n_local_heads in (8, 16, 32, 64, 128)
):
q_out = torch.empty(
(hidden_states.shape[0], self.n_local_heads, self.head_dim),
dtype=torch.float8_e4m3fn,
device=hidden_states.device,
)
elif self.n_local_heads < _PAD_NUM_HEADS:
q_padded = hidden_states.new_empty(
hidden_states.shape[0], _PAD_NUM_HEADS, self.head_dim
)