[DSV4.1] Add opt-in TRT-LLM sparse attention support (#41603)
This commit is contained in:
@@ -191,8 +191,8 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
|
||||
if (out_loc <= 0) return;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
// fp8 with one ue8m0 scale per 32 elements over the whole row, RoPE included.
|
||||
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
|
||||
// FP8 over the whole row, RoPE included; the layout selects the scaling.
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, data);
|
||||
}
|
||||
|
||||
|
||||
@@ -258,8 +258,8 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
// fp8 with one ue8m0 scale per 32 elements over the whole row, RoPE included.
|
||||
if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
|
||||
// FP8 over the whole row, RoPE included; the layout selects the scaling.
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ namespace sglang {
|
||||
|
||||
namespace deepseek_v4 {
|
||||
|
||||
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2 };
|
||||
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 };
|
||||
|
||||
template <KVLayout kLayout>
|
||||
struct KVLayoutTraits;
|
||||
@@ -67,6 +67,15 @@ struct KVLayoutTraits<KVLayout::V41_FP4> {
|
||||
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct KVLayoutTraits<KVLayout::UNIFORM_FP8> {
|
||||
static constexpr int64_t kDataBytes = 512;
|
||||
static constexpr int64_t kScaleBytes = 0;
|
||||
static constexpr int64_t kTileSize = 512;
|
||||
static constexpr int64_t kPageAlign = 512;
|
||||
static constexpr int64_t kBytesPerToken = kDataBytes;
|
||||
};
|
||||
|
||||
/// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride.
|
||||
template <KVLayout kLayout>
|
||||
constexpr int64_t kv_page_bytes(int64_t page_size) {
|
||||
@@ -206,6 +215,13 @@ SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, co
|
||||
static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller");
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
store_row_fp8(data_row, scale_row, tx, v);
|
||||
} else if constexpr (kLayout == KVLayout::UNIFORM_FP8) {
|
||||
device::AlignedVector<fp8x2_e4m3_t, kVecSize / 2> out;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kVecSize / 2; ++i) {
|
||||
out[i] = fp8::pack_fp8(v[2 * i], v[2 * i + 1]);
|
||||
}
|
||||
out.store(data_row, tx);
|
||||
} else {
|
||||
store_row_fp4(data_row, scale_row, tx, v);
|
||||
}
|
||||
|
||||
@@ -175,6 +175,14 @@ def fused_q_norm_rope(
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
# Reuse the BF16 kernel and preserve its rounding before FP8 conversion.
|
||||
# Keep the temporary separate from q_input: callers may reuse the input.
|
||||
fp8_output = q_output if q_output.dtype == torch.float8_e4m3fn else None
|
||||
if fp8_output is not None:
|
||||
assert q_input.dtype == torch.bfloat16
|
||||
assert q_output.shape == q_input.shape
|
||||
assert q_output.device == q_input.device
|
||||
q_output = torch.empty_like(q_input)
|
||||
freqs_real = torch.view_as_real(freqs_cis).flatten(-2)
|
||||
head_dim = q_input.shape[-1]
|
||||
rope_dim = freqs_real.shape[-1]
|
||||
@@ -186,6 +194,8 @@ def fused_q_norm_rope(
|
||||
q_input.dtype, head_dim, rope_dim, eps is not None
|
||||
)
|
||||
module.forward(q_input, q_output, freqs_real, positions, eps or 0.0)
|
||||
if fp8_output is not None:
|
||||
fp8_output.copy_(q_output)
|
||||
|
||||
|
||||
def fused_q_indexer_rope_hadamard_quant(
|
||||
|
||||
@@ -20,19 +20,36 @@ class KVLayout(str, enum.Enum):
|
||||
V41 = "v41"
|
||||
# 512 e2m1 packed two per byte (even index low nibble), 32 e4m3 scales per 16 values.
|
||||
V41_FP4 = "v41_fp4"
|
||||
# TRT-LLM only: 512 e4m3 values with a fixed external unit scale.
|
||||
UNIFORM_FP8 = "uniform_fp8"
|
||||
|
||||
@property
|
||||
def data_bytes(self) -> int:
|
||||
return {KVLayout.V4: 576, KVLayout.V41: 512, KVLayout.V41_FP4: 256}[self]
|
||||
return {
|
||||
KVLayout.V4: 576,
|
||||
KVLayout.V41: 512,
|
||||
KVLayout.V41_FP4: 256,
|
||||
KVLayout.UNIFORM_FP8: 512,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
def scale_bytes(self) -> int:
|
||||
return {KVLayout.V4: 8, KVLayout.V41: 16, KVLayout.V41_FP4: 32}[self]
|
||||
return {
|
||||
KVLayout.V4: 8,
|
||||
KVLayout.V41: 16,
|
||||
KVLayout.V41_FP4: 32,
|
||||
KVLayout.UNIFORM_FP8: 0,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
def tile_size(self) -> int:
|
||||
"""Values sharing one scale."""
|
||||
return {KVLayout.V4: 64, KVLayout.V41: 32, KVLayout.V41_FP4: 16}[self]
|
||||
return {
|
||||
KVLayout.V4: 64,
|
||||
KVLayout.V41: 32,
|
||||
KVLayout.V41_FP4: 16,
|
||||
KVLayout.UNIFORM_FP8: 512,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
def bytes_per_token(self) -> int:
|
||||
@@ -41,7 +58,12 @@ class KVLayout(str, enum.Enum):
|
||||
@property
|
||||
def page_align(self) -> int:
|
||||
"""Unit the page stride is padded to: the reader's TMA row stride."""
|
||||
return {KVLayout.V4: 576, KVLayout.V41: 512, KVLayout.V41_FP4: 256}[self]
|
||||
return {
|
||||
KVLayout.V4: 576,
|
||||
KVLayout.V41: 512,
|
||||
KVLayout.V41_FP4: 256,
|
||||
KVLayout.UNIFORM_FP8: 512,
|
||||
}[self]
|
||||
|
||||
@property
|
||||
def is_fp4(self) -> bool:
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Normalize/rotate KV and store directly into the TRT uniform-FP8 cache."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kv_norm_rope_fp8_store(
|
||||
X,
|
||||
W,
|
||||
F,
|
||||
POS,
|
||||
LOC,
|
||||
CACHE,
|
||||
EPS: tl.constexpr,
|
||||
SX: tl.constexpr,
|
||||
SF: tl.constexpr,
|
||||
CACHE_ROWS: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
base = row.to(tl.int64) * SX
|
||||
d = tl.arange(0, 512)
|
||||
x = tl.load(X + base + d).to(tl.float32)
|
||||
w = tl.load(W + d).to(tl.float32)
|
||||
inv = tl.rsqrt(tl.sum(x * x, 0) / 512 + EPS)
|
||||
y = x * inv * w
|
||||
|
||||
# Match _fused_norm_rope_kernel's reduction and RoPE operation order.
|
||||
pair = tl.arange(0, 32)
|
||||
even = tl.load(X + base + 448 + pair * 2).to(tl.float32)
|
||||
odd = tl.load(X + base + 449 + pair * 2).to(tl.float32)
|
||||
we = tl.load(W + 448 + pair * 2).to(tl.float32)
|
||||
wo = tl.load(W + 449 + pair * 2).to(tl.float32)
|
||||
even = even * inv * we
|
||||
odd = odd * inv * wo
|
||||
pos = tl.load(POS + row).to(tl.int64)
|
||||
c = tl.load(F + pos * SF + pair * 2).to(tl.float32)
|
||||
s = tl.load(F + pos * SF + pair * 2 + 1).to(tl.float32)
|
||||
re = even * c - odd * s
|
||||
im = even * s + odd * c
|
||||
|
||||
slot = tl.load(LOC + row).to(tl.int64)
|
||||
# Preserve PyTorch indexed assignment's negative-index interpretation.
|
||||
slot = tl.where(slot < 0, slot + CACHE_ROWS, slot)
|
||||
dst = CACHE + slot * 512
|
||||
# The old path materializes BF16/FP16 before torch's FP8 conversion.
|
||||
# Keep that rounding boundary without writing the intermediate to HBM.
|
||||
y = y.to(X.dtype.element_ty).to(tl.float32)
|
||||
re = re.to(X.dtype.element_ty).to(tl.float32)
|
||||
im = im.to(X.dtype.element_ty).to(tl.float32)
|
||||
tl.store(dst + d, y, d < 448)
|
||||
tl.store(dst + 448 + pair * 2, re)
|
||||
tl.store(dst + 449 + pair * 2, im)
|
||||
|
||||
|
||||
def fused_k_norm_rope_uniform_fp8(
|
||||
kv: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
cache: torch.Tensor,
|
||||
) -> None:
|
||||
"""Store norm+RoPE KV as unit-scale E4M3, without modifying ``kv``.
|
||||
|
||||
``loc`` addresses contiguous 512-byte rows in the uniform-FP8 pool.
|
||||
Active rows must have valid, distinct destinations, as with index_put.
|
||||
"""
|
||||
assert kv.ndim == 2 and kv.shape[1] == 512 and kv.stride(1) == 1
|
||||
assert kv.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
assert weight.shape == (512,) and weight.is_contiguous()
|
||||
assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous()
|
||||
assert freqs_cis.ndim == 2 and freqs_cis.shape[1] == 32
|
||||
assert positions.shape == loc.shape == (kv.shape[0],)
|
||||
assert positions.dtype in (torch.int32, torch.int64)
|
||||
assert loc.dtype in (torch.int32, torch.int64)
|
||||
assert positions.is_contiguous() and loc.is_contiguous()
|
||||
assert cache.dtype == torch.float8_e4m3fn and cache.is_contiguous()
|
||||
assert cache.numel() % 512 == 0
|
||||
assert all(
|
||||
t.device == kv.device for t in (weight, freqs_cis, positions, loc, cache)
|
||||
)
|
||||
if kv.shape[0] == 0:
|
||||
return
|
||||
_kv_norm_rope_fp8_store[(kv.shape[0],)](
|
||||
kv,
|
||||
weight,
|
||||
torch.view_as_real(freqs_cis),
|
||||
positions,
|
||||
loc,
|
||||
cache,
|
||||
EPS=eps,
|
||||
SX=kv.stride(0),
|
||||
SF=freqs_cis.stride(0) * 2,
|
||||
CACHE_ROWS=cache.numel() // 512,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Refresh the layer-dependent tail of TRT-LLM's sparse attention tables."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_sparse_tail(
|
||||
INDICES,
|
||||
LENGTHS,
|
||||
TABLE,
|
||||
TOTAL_LENGTHS,
|
||||
INDEX_STRIDE: tl.constexpr,
|
||||
TABLE_STRIDE: tl.constexpr,
|
||||
LENGTH_STRIDE: tl.constexpr,
|
||||
TOTAL_STRIDE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
SWA_WIDTH: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
values = tl.load(INDICES + row * INDEX_STRIDE + col, col < WIDTH, -1)
|
||||
tl.store(TABLE + row * TABLE_STRIDE + SWA_WIDTH + col, values, col < WIDTH)
|
||||
length = tl.load(LENGTHS + row * LENGTH_STRIDE)
|
||||
tl.store(TOTAL_LENGTHS + row * TOTAL_STRIDE, SWA_WIDTH + length)
|
||||
|
||||
|
||||
def pack_sparse_tail(
|
||||
indices: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
table: torch.Tensor,
|
||||
total_lengths: torch.Tensor,
|
||||
swa_width: int = 128,
|
||||
) -> None:
|
||||
"""Copy compressed indices and add the fixed SWA width in one launch.
|
||||
|
||||
The caller owns the tile-padded buffers and their invariant SWA columns.
|
||||
Only active rows are modified, preserving inert graph/tile padding.
|
||||
|
||||
Compressed entries start at column swa_width even for short sequences.
|
||||
total_lengths describes the table span to scan, not the valid-token count:
|
||||
adding only the actual SWA length would truncate the compressed tail.
|
||||
Unused SWA slots retain their -1 sentinel and are ignored by attention.
|
||||
"""
|
||||
rows, width = indices.shape
|
||||
assert table.shape == (rows, swa_width + width)
|
||||
assert lengths.shape == total_lengths.shape == (rows,)
|
||||
assert indices.stride(1) == table.stride(1) == 1
|
||||
assert indices.dtype == table.dtype == total_lengths.dtype == torch.int32
|
||||
assert lengths.dtype in (torch.int32, torch.int64)
|
||||
assert indices.device == lengths.device == table.device == total_lengths.device
|
||||
if rows == 0:
|
||||
return
|
||||
_pack_sparse_tail[(rows,)](
|
||||
indices,
|
||||
lengths,
|
||||
table,
|
||||
total_lengths,
|
||||
indices.stride(0),
|
||||
table.stride(0),
|
||||
lengths.stride(0),
|
||||
total_lengths.stride(0),
|
||||
width,
|
||||
swa_width,
|
||||
triton.next_power_of_2(width),
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -16,6 +16,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
run_post_process_pass,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of
|
||||
from sglang.srt.utils.common import is_gfx95_supported, is_npu
|
||||
|
||||
@@ -217,9 +218,14 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
|
||||
)
|
||||
return
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
|
||||
)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
incompatible = (
|
||||
(
|
||||
"hardware other than CUDA or gfx950",
|
||||
@@ -264,8 +270,14 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
),
|
||||
("HiSparse", cfg.enable_hisparse),
|
||||
("the unified KV layout", is_unified_kv_triton()),
|
||||
# The trtllm-gen path has no uniform-FP8 pool for V4.1's ratio-1/2 layers.
|
||||
("the trtllm DSv4 attention backend", cfg.dsv4_attn_backend == "trtllm"),
|
||||
(
|
||||
"TRT-LLM with SWA bounded replay",
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and (
|
||||
cfg.enable_encoder_swa_bounded_replay
|
||||
or cfg.enable_decoder_swa_bounded_replay
|
||||
),
|
||||
),
|
||||
("two-batch overlap", cfg.enable_two_batch_overlap),
|
||||
("pipeline parallelism", cfg.pp_size > 1),
|
||||
)
|
||||
@@ -296,8 +308,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"block size and TP size."
|
||||
)
|
||||
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
prefill_graph = cfg.cuda_graph_config.prefill
|
||||
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
|
||||
# The captured low-ratio indexer scores a static context width; 16k
|
||||
@@ -315,8 +325,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
if cfg.enable_decoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
# Late layers see a per-request tail slice, not the captured prefill shape.
|
||||
incompatible = (
|
||||
(
|
||||
|
||||
@@ -516,11 +516,13 @@ class DSV4AttnMetadata:
|
||||
raise ValueError(f"invalid {compress_ratio=}")
|
||||
|
||||
def init_trtllm_sparse_buffers(self) -> None:
|
||||
"""Decode tables of 128 SWA columns then compressed KV, -1 for an invalid
|
||||
index, lens counting all 128 SWA slots; only the c4 tail is per layer."""
|
||||
"""Build tile-padded TRT-LLM tables and lengths for indexed attention
|
||||
or SWA-only DSpark blocks; indexed tails are refreshed per layer."""
|
||||
|
||||
num_tokens = self.seq_lens_casual.shape[0]
|
||||
assert self.swa_page_indices.shape == (num_tokens, SWA_WINDOW)
|
||||
swa_width = self.swa_page_indices.shape[1]
|
||||
assert self.swa_page_indices.shape[0] == num_tokens
|
||||
assert swa_width >= SWA_WINDOW
|
||||
|
||||
# VarSeq reads rows to the 64-token tile boundary. Back every live view
|
||||
# with an aligned parent whose extra rows contain inert values.
|
||||
@@ -536,17 +538,40 @@ class DSV4AttnMetadata:
|
||||
if n_pad != num_tokens:
|
||||
self.seq_lens_casual = _tile_padded(1, self.seq_lens_casual)
|
||||
self.swa_page_indices = _tile_padded(
|
||||
-1, self.swa_page_indices, width=SWA_WINDOW
|
||||
-1, self.swa_page_indices, width=swa_width
|
||||
)
|
||||
self.trtllm_swa_lens = _tile_padded(SWA_WINDOW)
|
||||
if self.c4_sparse_page_indices is not None:
|
||||
w4 = self.c4_sparse_page_indices.shape[-1]
|
||||
self.trtllm_swa_lens = _tile_padded(
|
||||
SWA_WINDOW,
|
||||
self.swa_topk_lengths.clamp_min(SWA_WINDOW)
|
||||
if swa_width > SWA_WINDOW
|
||||
else None,
|
||||
)
|
||||
for ratio in self.low_ratios:
|
||||
lengths = self.sparse_topk_lengths(ratio)
|
||||
if lengths is not None:
|
||||
# The FlashMLA metadata clamps empty compressed histories to
|
||||
# one. TRT-LLM needs the true zero length: -1 loads zero KV,
|
||||
# but does not by itself remove its softmax contribution.
|
||||
torch.minimum(lengths, self.seq_lens_casual // ratio, out=lengths)
|
||||
indexed_tables = [
|
||||
table
|
||||
for ratio in (1, 2, 4)
|
||||
if ratio in self.present_ratios
|
||||
if (table := self.sparse_page_indices(ratio)) is not None
|
||||
]
|
||||
if indexed_tables:
|
||||
assert swa_width == SWA_WINDOW
|
||||
# Ratio 1/2/4 all select index_topk entries. Reuse the per-layer
|
||||
# table; its contents are overwritten before every attention call.
|
||||
w4 = indexed_tables[0].shape[-1]
|
||||
assert all(table.shape[-1] == w4 for table in indexed_tables)
|
||||
assert w4 % 4 == 0, f"{w4=}"
|
||||
# Unwritten c4 rows must remain inert until the per-layer fill.
|
||||
# Unwritten indexed rows remain inert until the per-layer fill.
|
||||
self.trtllm_c4_indices = _tile_padded(-1, width=SWA_WINDOW + w4)
|
||||
self.trtllm_c4_indices[:, :SWA_WINDOW].copy_(self.swa_page_indices)
|
||||
self.trtllm_c4_lens = _tile_padded(SWA_WINDOW)
|
||||
if self.c128_page_indices is not None:
|
||||
assert swa_width == SWA_WINDOW
|
||||
w128 = self.c128_page_indices.shape[-1]
|
||||
assert w128 % 4 == 0, f"{w128=}"
|
||||
self.trtllm_c128_indices = _tile_padded(-1, width=SWA_WINDOW + w128)
|
||||
@@ -2111,6 +2136,8 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
metadata.core_attn_metadata.swa_page_indices = swa_page_indices
|
||||
metadata.core_attn_metadata.swa_topk_lengths = swa_topk_lengths
|
||||
if self.trtllm_attn:
|
||||
metadata.core_attn_metadata.init_trtllm_sparse_buffers()
|
||||
|
||||
def _dspark_seq_lens_casual(
|
||||
self, *, seq_lens: torch.Tensor, block_size: int
|
||||
@@ -2921,7 +2948,11 @@ class DeepseekV4AttnBackend(
|
||||
kv_cache = pool.get_extra_key_buffer(layer_id)
|
||||
page_size = pool.get_extra_key_page_size(layer_id)
|
||||
# The pool's page format: V4, or the V4.1 fp8 / fp4 layouts.
|
||||
kv_layout = pool.get_extra_key_layout(layer_id)
|
||||
kv_layout = (
|
||||
KVLayout.UNIFORM_FP8
|
||||
if pool.uniform_fp8
|
||||
else pool.get_extra_key_layout(layer_id)
|
||||
)
|
||||
assert kv_cache is not None
|
||||
|
||||
if layer.compress_ratio == 1:
|
||||
@@ -3352,7 +3383,10 @@ class DeepseekV4AttnBackend(
|
||||
**_,
|
||||
) -> torch.Tensor:
|
||||
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
|
||||
return q.new_empty(q.shape[0], q.shape[1], layer.v_head_dim)
|
||||
return q.new_empty(
|
||||
(q.shape[0], q.shape[1], layer.v_head_dim),
|
||||
dtype=torch.bfloat16 if self.trtllm_attn else q.dtype,
|
||||
)
|
||||
|
||||
assert k is v, "DeepseekV4 shares k and v"
|
||||
swa_k = k
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Literal, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.trtllm_metadata import pack_sparse_tail
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
SWA_WINDOW,
|
||||
@@ -18,6 +19,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
DeepseekV4MultiStepBackend,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_buffer,
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
@@ -25,6 +27,14 @@ from sglang.srt.runtime_context import (
|
||||
max_prefill_buffer_tokens,
|
||||
)
|
||||
|
||||
try:
|
||||
import flashinfer.mla._core as _fi_core
|
||||
from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4
|
||||
except ImportError as exc:
|
||||
_flashinfer_import_error = exc
|
||||
else:
|
||||
_flashinfer_import_error = None
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import DSV4AttnMetadata
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
@@ -38,8 +48,6 @@ _TRTLLM_GEN_WORKSPACE_SIZE_MB = 128
|
||||
|
||||
|
||||
def _get_trtllm_workspace_buffer(device: torch.device) -> torch.Tensor:
|
||||
from sglang.srt.runtime_context import get_buffer
|
||||
|
||||
return get_buffer(
|
||||
"trtllm_dsv4_zero_workspace",
|
||||
lambda: torch.zeros(
|
||||
@@ -84,11 +92,13 @@ def _install_persistent_trtllm_semaphores(capacity_rows: int) -> None:
|
||||
FlashInfer accepts a caller-owned buffer.
|
||||
"""
|
||||
global _trtllm_semaphore_installed, _trtllm_semaphore_rows
|
||||
if _flashinfer_import_error is not None:
|
||||
raise ImportError(
|
||||
"--dsv4-attn-backend trtllm requires FlashInfer sparse MLA support"
|
||||
) from _flashinfer_import_error
|
||||
_trtllm_semaphore_rows = max(_trtllm_semaphore_rows, capacity_rows)
|
||||
if _trtllm_semaphore_installed:
|
||||
return
|
||||
import flashinfer.mla._core as _fi_core
|
||||
|
||||
_orig = _fi_core._get_trtllm_gen_multi_ctas_kv_counter_buffer
|
||||
# Allocate once outside graph capture. Stream ordering and the kernel's
|
||||
# counter reset make one shared buffer safe across launches.
|
||||
@@ -166,7 +176,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
compress_ratio: Literal[0, 4, 128],
|
||||
compress_ratio: Literal[0, 1, 2, 4, 128],
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
forward_batch: ForwardBatch,
|
||||
attn_sink: torch.Tensor,
|
||||
@@ -175,6 +185,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
extra_topk_lengths: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
assert attn_sink is not None
|
||||
if self.is_dsv41:
|
||||
# FlashMLA pads TP query heads to 64. TRT-LLM accepts the native
|
||||
# width; do not compute attention for the discarded padding.
|
||||
q = q[:, : layer.tp_q_head_num, :]
|
||||
attn_sink = attn_sink[: layer.tp_q_head_num]
|
||||
if (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
@@ -221,7 +236,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
return (self.softmax_scale, 1.0)
|
||||
|
||||
def _trtllm_kv_cache_views(
|
||||
self, layer_id: int, compress_ratio: Literal[0, 4, 128]
|
||||
self, layer_id: int, compress_ratio: Literal[0, 1, 2, 4, 128]
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return HND views of the uniform-FP8 SWA and compressed pools.
|
||||
|
||||
@@ -248,7 +263,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
compress_ratio: Literal[0, 4, 128],
|
||||
compress_ratio: Literal[0, 1, 2, 4, 128],
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
attn_sink: torch.Tensor,
|
||||
swa_page_indices: torch.Tensor,
|
||||
@@ -257,8 +272,6 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
) -> torch.Tensor:
|
||||
"""Run sparse MLA decode with preallocated metadata tables."""
|
||||
|
||||
from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4
|
||||
|
||||
bs, num_heads, head_dim = q.shape
|
||||
assert head_dim == 512
|
||||
|
||||
@@ -278,8 +291,10 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
extra_topk_lengths = extra_topk_lengths[:n_meta_rows]
|
||||
bs = n_meta_rows
|
||||
|
||||
# Only the c4 tail and lens vary by layer; other table data is prebuilt.
|
||||
assert swa_page_indices.shape == (bs, SWA_WINDOW)
|
||||
# Indexed tails and lengths vary by layer; other table data is prebuilt.
|
||||
swa_width = swa_page_indices.shape[1]
|
||||
assert swa_page_indices.shape[0] == bs
|
||||
assert swa_width == SWA_WINDOW or compress_ratio == 0
|
||||
if compress_ratio == 0:
|
||||
# Use the metadata view backed by 64-row-aligned storage because
|
||||
# the VarSeq kernel reads table rows to the tile boundary.
|
||||
@@ -302,22 +317,21 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
assert sparse_topk_lens.shape[0] > bs, f"{sparse_topk_lens.shape=}"
|
||||
sparse_topk_lens = sparse_topk_lens[:bs]
|
||||
|
||||
if compress_ratio == 4:
|
||||
if compress_ratio in (1, 2, 4):
|
||||
assert extra_indices is not None and extra_topk_lengths is not None
|
||||
width = extra_indices.shape[-1]
|
||||
assert SWA_WINDOW + width == sparse_indices.shape[1], (
|
||||
f"{width=} {sparse_indices.shape=}"
|
||||
)
|
||||
sparse_indices[:, SWA_WINDOW:].copy_(extra_indices)
|
||||
# Lens include all 128 SWA slots; seq_lens controls their validity.
|
||||
sparse_topk_lens.copy_(extra_topk_lengths)
|
||||
sparse_topk_lens.add_(SWA_WINDOW)
|
||||
pack_sparse_tail(
|
||||
extra_indices, extra_topk_lengths, sparse_indices, sparse_topk_lens
|
||||
)
|
||||
|
||||
swa_kv_cache, compressed_kv_cache = self._trtllm_kv_cache_views(
|
||||
layer.layer_id, compress_ratio
|
||||
)
|
||||
|
||||
# RoPE is already applied; the unit scale makes this a plain e4m3 cast.
|
||||
# Q preparation already returns FP8 on V4.1; to() is a no-op in that path.
|
||||
q_fp8 = q.to(torch.float8_e4m3fn).view(bs, 1, num_heads, 512)
|
||||
|
||||
bmm1_scale, bmm2_scale = self._get_trtllm_bmm_scales(layer)
|
||||
@@ -326,6 +340,13 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
if seq_lens.shape[0] != bs:
|
||||
assert seq_lens.shape[0] > bs, f"{seq_lens.shape=} {bs=}"
|
||||
seq_lens = seq_lens[:bs]
|
||||
if swa_width > SWA_WINDOW:
|
||||
# DSpark attends noncausally to its context window plus the whole
|
||||
# draft block. The tail still indexes SWA storage, not compressed
|
||||
# KV. Use the complete block's visible length rather than each
|
||||
# token's causal length, including for short requests.
|
||||
compressed_kv_cache = swa_kv_cache
|
||||
seq_lens = core_attn_metadata.swa_topk_lengths[:bs]
|
||||
assert attn_sink.dtype == torch.float32
|
||||
assert self.trtllm_workspace_buffer is not None
|
||||
_check_trtllm_query_rows(bs)
|
||||
@@ -342,6 +363,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
bmm2_scale=bmm2_scale,
|
||||
sinks=attn_sink,
|
||||
kv_layout="HND",
|
||||
backend="trtllm-gen",
|
||||
)
|
||||
if out_pad_tail is not None:
|
||||
out_pad_tail[:bs] = out.view(bs, num_heads, 512)
|
||||
@@ -353,7 +375,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
compress_ratio: Literal[0, 4, 128],
|
||||
compress_ratio: Literal[0, 1, 2, 4, 128],
|
||||
forward_batch: ForwardBatch,
|
||||
attn_sink: torch.Tensor,
|
||||
swa_page_indices: torch.Tensor,
|
||||
@@ -366,8 +388,6 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
validity from it. This path runs eagerly.
|
||||
"""
|
||||
|
||||
from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4
|
||||
|
||||
assert q.ndim == 3, f"{q.shape=}"
|
||||
num_qo_padded, num_heads, head_dim = q.shape
|
||||
assert head_dim == 512
|
||||
@@ -418,11 +438,10 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
assert width % 4 == 0, f"{width=}"
|
||||
table = _tile_padded_pf(-1, width=SWA_WINDOW + width)
|
||||
table[:, :SWA_WINDOW].copy_(swa_indices)
|
||||
table[:, SWA_WINDOW:].copy_(extra_indices[:sum_q])
|
||||
assert extra_topk_lengths is not None
|
||||
lens = _tile_padded_pf(
|
||||
SWA_WINDOW,
|
||||
extra_topk_lengths[:sum_q].to(torch.int32) + SWA_WINDOW,
|
||||
lens = _tile_padded_pf(SWA_WINDOW)
|
||||
pack_sparse_tail(
|
||||
extra_indices[:sum_q], extra_topk_lengths[:sum_q], table, lens
|
||||
)
|
||||
core.trtllm_prefill_c128 = (table, lens)
|
||||
sparse_indices, sparse_topk_lens = core.trtllm_prefill_c128
|
||||
@@ -441,14 +460,15 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
sum_q,
|
||||
SWA_WINDOW + width,
|
||||
), f"{sparse_indices.shape=} {width=}"
|
||||
sparse_indices[:, SWA_WINDOW:].copy_(extra_indices[:sum_q])
|
||||
# Lens include 128 SWA slots; VarSeq metadata controls validity.
|
||||
sparse_topk_lens = _tile_padded_pf(
|
||||
SWA_WINDOW,
|
||||
extra_topk_lengths[:sum_q].to(torch.int32) + SWA_WINDOW,
|
||||
sparse_topk_lens = _tile_padded_pf(SWA_WINDOW)
|
||||
pack_sparse_tail(
|
||||
extra_indices[:sum_q],
|
||||
extra_topk_lengths[:sum_q],
|
||||
sparse_indices,
|
||||
sparse_topk_lens,
|
||||
)
|
||||
|
||||
# RoPE is already applied; the unit scale makes this a plain e4m3 cast.
|
||||
# Retain a unit-scale cast for callers without fused FP8 Q preparation.
|
||||
q_fp8 = q[:sum_q].to(torch.float8_e4m3fn)
|
||||
|
||||
swa_kv_cache, compressed_kv_cache = self._trtllm_kv_cache_views(
|
||||
@@ -483,6 +503,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend):
|
||||
bmm2_scale=bmm2_scale,
|
||||
sinks=attn_sink,
|
||||
kv_layout="HND",
|
||||
backend="trtllm-gen",
|
||||
cum_seq_lens_q=cum_seq_lens_q,
|
||||
max_q_len=max_q_len,
|
||||
)
|
||||
|
||||
@@ -20,6 +20,9 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import (
|
||||
KVLayout,
|
||||
is_valid_kv_layout_pair,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.kv_norm_rope_store import (
|
||||
fused_k_norm_rope_uniform_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
@@ -165,6 +168,11 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
AITER attention."""
|
||||
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
|
||||
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
|
||||
if get_exec().kernel.dsv4_attn_backend == "trtllm":
|
||||
assert mode in ("auto", "v4") and option in ("auto", "fp8"), (
|
||||
"trtllm uses uniform FP8 pools, not the packed V4.1 KV layouts"
|
||||
)
|
||||
return KVLayout.V4, None
|
||||
if mode == "v4":
|
||||
return KVLayout.V4, None if option == "auto" else option
|
||||
assert mode in ("v41", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}"
|
||||
@@ -2042,21 +2050,14 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
"""q ([B, H, head_dim]): rope its query heads in the same launch."""
|
||||
if self.uniform_fp8:
|
||||
assert q is None, "uniform FP8 store does not fuse query RoPE"
|
||||
# Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again),
|
||||
# then an e4m3 cast + scatter with per-tensor scale 1.0.
|
||||
from sglang.kernels.ops.attention.deepseek_v4_rope import (
|
||||
fused_norm_rope_inplace_triton,
|
||||
)
|
||||
|
||||
fused_norm_rope_inplace_triton(
|
||||
fused_k_norm_rope_uniform_fp8(
|
||||
kv,
|
||||
kv_weight,
|
||||
eps,
|
||||
freqs_cis,
|
||||
positions=positions,
|
||||
)
|
||||
self.swa_kv_pool.set_key_buffer_fused(
|
||||
self._swa_local_layer_id(layer_id), swa_loc, kv
|
||||
positions,
|
||||
swa_loc,
|
||||
self.swa_kv_pool.get_key_buffer(self._swa_local_layer_id(layer_id)),
|
||||
)
|
||||
return
|
||||
fused_k_norm_rope_flashmla(
|
||||
|
||||
@@ -1339,6 +1339,9 @@ class MQALayer(MqaAttentionBase):
|
||||
q, _ = self.wq_b(q)
|
||||
q = q.view(-1, self.n_local_heads, self.head_dim)
|
||||
if not self.q_head_norm:
|
||||
if q_out is not None and q_out.dtype == torch.float8_e4m3fn:
|
||||
fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions)
|
||||
return q_out
|
||||
# TODO: enable zero-copy path to skip the extra no-rope copy overhead
|
||||
if (_is_cuda or _is_gfx95_supported) and q_out is not None:
|
||||
fused_q_norm_rope(
|
||||
@@ -2252,7 +2255,19 @@ class MQALayer(MqaAttentionBase):
|
||||
|
||||
tp_slice, q_padded, q_out, q_rope = slice(None), None, None, None
|
||||
k_nope, k_rope = None, None
|
||||
if unified_fp8_decode or unified_fp8_prefill:
|
||||
if (
|
||||
self.is_dsv41
|
||||
and not self.q_head_norm
|
||||
and get_token_to_kv_pool().uniform_fp8
|
||||
and self.n_local_heads in (8, 16, 32, 64, 128)
|
||||
):
|
||||
kernel_num_heads = self.n_local_heads
|
||||
q_out = torch.empty(
|
||||
(x.shape[0], self.n_local_heads, self.head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=x.device,
|
||||
)
|
||||
elif unified_fp8_decode or unified_fp8_prefill:
|
||||
# width and dtype come off the pools themselves; the kernel reads Q
|
||||
# with the kv row stride, so the two must not drift
|
||||
kv_pool = get_token_to_kv_pool()
|
||||
|
||||
@@ -191,6 +191,9 @@ class DSparkAttention(MqaAttentionBase):
|
||||
q, _ = self.wq_b(q)
|
||||
q = q.view(-1, self.n_local_heads, self.head_dim)
|
||||
if not self.q_head_norm:
|
||||
if q_out is not None and q_out.dtype == torch.float8_e4m3fn:
|
||||
fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions)
|
||||
return q_out
|
||||
if self._use_fast_kernel and not _is_npu:
|
||||
fused_rope_inplace(
|
||||
q[..., -self.rope_head_dim :],
|
||||
@@ -270,7 +273,17 @@ class DSparkAttention(MqaAttentionBase):
|
||||
|
||||
q_padded: Optional[torch.Tensor] = None
|
||||
q_out: Optional[torch.Tensor] = None
|
||||
if self.n_local_heads < _PAD_NUM_HEADS:
|
||||
if (
|
||||
pool.uniform_fp8
|
||||
and not self.q_head_norm
|
||||
and self.n_local_heads in (8, 16, 32, 64, 128)
|
||||
):
|
||||
q_out = torch.empty(
|
||||
(hidden_states.shape[0], self.n_local_heads, self.head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
elif self.n_local_heads < _PAD_NUM_HEADS:
|
||||
q_padded = hidden_states.new_empty(
|
||||
hidden_states.shape[0], _PAD_NUM_HEADS, self.head_dim
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user