From 620c76f25db14f1599aa7c7ccb141a91e4944b63 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Mon, 5 Oct 2026 21:02:10 -0700 Subject: [PATCH] [DSV4.1] Add opt-in TRT-LLM sparse attention support (#41603) --- .../kernels/jit/csrc/deepseek_v4/c1.cuh | 4 +- .../kernels/jit/csrc/deepseek_v4/c2.cuh | 4 +- .../sgl_kernel/deepseek_v4/kv_layout.cuh | 18 ++- .../kernels/ops/attention/dsv4/elementwise.py | 10 ++ .../kernels/ops/attention/dsv4/kv_layout.py | 30 ++++- .../ops/attention/dsv4/kv_norm_rope_store.py | 99 +++++++++++++++++ .../ops/attention/dsv4/trtllm_metadata.py | 69 ++++++++++++ .../sglang/srt/arg_groups/deepseek_v4_hook.py | 24 ++-- .../layers/attention/deepseek_v4_backend.py | 54 +++++++-- .../attention/deepseek_v4_trtllm_backend.py | 81 +++++++++----- .../srt/mem_cache/deepseek_v4_memory_pool.py | 23 ++-- python/sglang/srt/models/deepseek_v4.py | 17 ++- .../sglang/srt/models/deepseek_v4_dspark.py | 15 ++- .../attention/test_dsv4_trtllm_prefill.py | 103 +++++++++++++++++ .../attention/test_dsv4_trtllm_q_metadata.py | 98 ++++++++++++++++ .../test_trtllm_kv_norm_rope_store.py | 105 ++++++++++++++++++ 16 files changed, 684 insertions(+), 70 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/dsv4/kv_norm_rope_store.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/trtllm_metadata.py create mode 100644 test/manual/kernels/attention/test_dsv4_trtllm_prefill.py create mode 100644 test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py create mode 100644 test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh index 1e22d39f7a..3e3511c955 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh @@ -191,8 +191,8 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel( if (out_loc <= 0) return; const auto kv_row = Paged::row(params.kvcache, out_loc); - if constexpr (kLayout == KVLayout::V41) { - // fp8 with one ue8m0 scale per 32 elements over the whole row, RoPE included. + if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) { + // FP8 over the whole row, RoPE included; the layout selects the scaling. return deepseek_v4::v41::store_row(kv_row.data, kv_row.scale, tx, data); } diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh index f9784dbd59..175fbb7627 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh @@ -258,8 +258,8 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel( // `raw_out_loc / ratio`; ratio 2 makes it a shift. const auto kv_row = Paged::row(params.kvcache, out_loc); - if constexpr (kLayout == KVLayout::V41) { - // fp8 with one ue8m0 scale per 32 elements over the whole row, RoPE included. + if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) { + // FP8 over the whole row, RoPE included; the layout selects the scaling. return deepseek_v4::v41::store_row(kv_row.data, kv_row.scale, tx, staged); } diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh index 456203bbe5..028766b9e7 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh @@ -35,7 +35,7 @@ namespace sglang { namespace deepseek_v4 { -enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2 }; +enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 }; template struct KVLayoutTraits; @@ -67,6 +67,15 @@ struct KVLayoutTraits { static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes; }; +template <> +struct KVLayoutTraits { + static constexpr int64_t kDataBytes = 512; + static constexpr int64_t kScaleBytes = 0; + static constexpr int64_t kTileSize = 512; + static constexpr int64_t kPageAlign = 512; + static constexpr int64_t kBytesPerToken = kDataBytes; +}; + /// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride. template constexpr int64_t kv_page_bytes(int64_t page_size) { @@ -206,6 +215,13 @@ SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, co static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller"); if constexpr (kLayout == KVLayout::V41) { store_row_fp8(data_row, scale_row, tx, v); + } else if constexpr (kLayout == KVLayout::UNIFORM_FP8) { + device::AlignedVector out; +#pragma unroll + for (uint32_t i = 0; i < kVecSize / 2; ++i) { + out[i] = fp8::pack_fp8(v[2 * i], v[2 * i + 1]); + } + out.store(data_row, tx); } else { store_row_fp4(data_row, scale_row, tx, v); } diff --git a/python/sglang/kernels/ops/attention/dsv4/elementwise.py b/python/sglang/kernels/ops/attention/dsv4/elementwise.py index a104b0bcc7..ceceb1860c 100644 --- a/python/sglang/kernels/ops/attention/dsv4/elementwise.py +++ b/python/sglang/kernels/ops/attention/dsv4/elementwise.py @@ -175,6 +175,14 @@ def fused_q_norm_rope( freqs_cis: torch.Tensor, positions: torch.Tensor, ) -> None: + # Reuse the BF16 kernel and preserve its rounding before FP8 conversion. + # Keep the temporary separate from q_input: callers may reuse the input. + fp8_output = q_output if q_output.dtype == torch.float8_e4m3fn else None + if fp8_output is not None: + assert q_input.dtype == torch.bfloat16 + assert q_output.shape == q_input.shape + assert q_output.device == q_input.device + q_output = torch.empty_like(q_input) freqs_real = torch.view_as_real(freqs_cis).flatten(-2) head_dim = q_input.shape[-1] rope_dim = freqs_real.shape[-1] @@ -186,6 +194,8 @@ def fused_q_norm_rope( q_input.dtype, head_dim, rope_dim, eps is not None ) module.forward(q_input, q_output, freqs_real, positions, eps or 0.0) + if fp8_output is not None: + fp8_output.copy_(q_output) def fused_q_indexer_rope_hadamard_quant( diff --git a/python/sglang/kernels/ops/attention/dsv4/kv_layout.py b/python/sglang/kernels/ops/attention/dsv4/kv_layout.py index bb51e43611..73a53ea412 100644 --- a/python/sglang/kernels/ops/attention/dsv4/kv_layout.py +++ b/python/sglang/kernels/ops/attention/dsv4/kv_layout.py @@ -20,19 +20,36 @@ class KVLayout(str, enum.Enum): V41 = "v41" # 512 e2m1 packed two per byte (even index low nibble), 32 e4m3 scales per 16 values. V41_FP4 = "v41_fp4" + # TRT-LLM only: 512 e4m3 values with a fixed external unit scale. + UNIFORM_FP8 = "uniform_fp8" @property def data_bytes(self) -> int: - return {KVLayout.V4: 576, KVLayout.V41: 512, KVLayout.V41_FP4: 256}[self] + return { + KVLayout.V4: 576, + KVLayout.V41: 512, + KVLayout.V41_FP4: 256, + KVLayout.UNIFORM_FP8: 512, + }[self] @property def scale_bytes(self) -> int: - return {KVLayout.V4: 8, KVLayout.V41: 16, KVLayout.V41_FP4: 32}[self] + return { + KVLayout.V4: 8, + KVLayout.V41: 16, + KVLayout.V41_FP4: 32, + KVLayout.UNIFORM_FP8: 0, + }[self] @property def tile_size(self) -> int: """Values sharing one scale.""" - return {KVLayout.V4: 64, KVLayout.V41: 32, KVLayout.V41_FP4: 16}[self] + return { + KVLayout.V4: 64, + KVLayout.V41: 32, + KVLayout.V41_FP4: 16, + KVLayout.UNIFORM_FP8: 512, + }[self] @property def bytes_per_token(self) -> int: @@ -41,7 +58,12 @@ class KVLayout(str, enum.Enum): @property def page_align(self) -> int: """Unit the page stride is padded to: the reader's TMA row stride.""" - return {KVLayout.V4: 576, KVLayout.V41: 512, KVLayout.V41_FP4: 256}[self] + return { + KVLayout.V4: 576, + KVLayout.V41: 512, + KVLayout.V41_FP4: 256, + KVLayout.UNIFORM_FP8: 512, + }[self] @property def is_fp4(self) -> bool: diff --git a/python/sglang/kernels/ops/attention/dsv4/kv_norm_rope_store.py b/python/sglang/kernels/ops/attention/dsv4/kv_norm_rope_store.py new file mode 100644 index 0000000000..a4a6023741 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/kv_norm_rope_store.py @@ -0,0 +1,99 @@ +"""Normalize/rotate KV and store directly into the TRT uniform-FP8 cache.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _kv_norm_rope_fp8_store( + X, + W, + F, + POS, + LOC, + CACHE, + EPS: tl.constexpr, + SX: tl.constexpr, + SF: tl.constexpr, + CACHE_ROWS: tl.constexpr, +): + row = tl.program_id(0) + base = row.to(tl.int64) * SX + d = tl.arange(0, 512) + x = tl.load(X + base + d).to(tl.float32) + w = tl.load(W + d).to(tl.float32) + inv = tl.rsqrt(tl.sum(x * x, 0) / 512 + EPS) + y = x * inv * w + + # Match _fused_norm_rope_kernel's reduction and RoPE operation order. + pair = tl.arange(0, 32) + even = tl.load(X + base + 448 + pair * 2).to(tl.float32) + odd = tl.load(X + base + 449 + pair * 2).to(tl.float32) + we = tl.load(W + 448 + pair * 2).to(tl.float32) + wo = tl.load(W + 449 + pair * 2).to(tl.float32) + even = even * inv * we + odd = odd * inv * wo + pos = tl.load(POS + row).to(tl.int64) + c = tl.load(F + pos * SF + pair * 2).to(tl.float32) + s = tl.load(F + pos * SF + pair * 2 + 1).to(tl.float32) + re = even * c - odd * s + im = even * s + odd * c + + slot = tl.load(LOC + row).to(tl.int64) + # Preserve PyTorch indexed assignment's negative-index interpretation. + slot = tl.where(slot < 0, slot + CACHE_ROWS, slot) + dst = CACHE + slot * 512 + # The old path materializes BF16/FP16 before torch's FP8 conversion. + # Keep that rounding boundary without writing the intermediate to HBM. + y = y.to(X.dtype.element_ty).to(tl.float32) + re = re.to(X.dtype.element_ty).to(tl.float32) + im = im.to(X.dtype.element_ty).to(tl.float32) + tl.store(dst + d, y, d < 448) + tl.store(dst + 448 + pair * 2, re) + tl.store(dst + 449 + pair * 2, im) + + +def fused_k_norm_rope_uniform_fp8( + kv: torch.Tensor, + weight: torch.Tensor, + eps: float, + freqs_cis: torch.Tensor, + positions: torch.Tensor, + loc: torch.Tensor, + cache: torch.Tensor, +) -> None: + """Store norm+RoPE KV as unit-scale E4M3, without modifying ``kv``. + + ``loc`` addresses contiguous 512-byte rows in the uniform-FP8 pool. + Active rows must have valid, distinct destinations, as with index_put. + """ + assert kv.ndim == 2 and kv.shape[1] == 512 and kv.stride(1) == 1 + assert kv.dtype in (torch.bfloat16, torch.float16, torch.float32) + assert weight.shape == (512,) and weight.is_contiguous() + assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous() + assert freqs_cis.ndim == 2 and freqs_cis.shape[1] == 32 + assert positions.shape == loc.shape == (kv.shape[0],) + assert positions.dtype in (torch.int32, torch.int64) + assert loc.dtype in (torch.int32, torch.int64) + assert positions.is_contiguous() and loc.is_contiguous() + assert cache.dtype == torch.float8_e4m3fn and cache.is_contiguous() + assert cache.numel() % 512 == 0 + assert all( + t.device == kv.device for t in (weight, freqs_cis, positions, loc, cache) + ) + if kv.shape[0] == 0: + return + _kv_norm_rope_fp8_store[(kv.shape[0],)]( + kv, + weight, + torch.view_as_real(freqs_cis), + positions, + loc, + cache, + EPS=eps, + SX=kv.stride(0), + SF=freqs_cis.stride(0) * 2, + CACHE_ROWS=cache.numel() // 512, + num_warps=4, + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/trtllm_metadata.py b/python/sglang/kernels/ops/attention/dsv4/trtllm_metadata.py new file mode 100644 index 0000000000..8c5b1ded48 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/trtllm_metadata.py @@ -0,0 +1,69 @@ +"""Refresh the layer-dependent tail of TRT-LLM's sparse attention tables.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _pack_sparse_tail( + INDICES, + LENGTHS, + TABLE, + TOTAL_LENGTHS, + INDEX_STRIDE: tl.constexpr, + TABLE_STRIDE: tl.constexpr, + LENGTH_STRIDE: tl.constexpr, + TOTAL_STRIDE: tl.constexpr, + WIDTH: tl.constexpr, + SWA_WIDTH: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + col = tl.arange(0, BLOCK) + values = tl.load(INDICES + row * INDEX_STRIDE + col, col < WIDTH, -1) + tl.store(TABLE + row * TABLE_STRIDE + SWA_WIDTH + col, values, col < WIDTH) + length = tl.load(LENGTHS + row * LENGTH_STRIDE) + tl.store(TOTAL_LENGTHS + row * TOTAL_STRIDE, SWA_WIDTH + length) + + +def pack_sparse_tail( + indices: torch.Tensor, + lengths: torch.Tensor, + table: torch.Tensor, + total_lengths: torch.Tensor, + swa_width: int = 128, +) -> None: + """Copy compressed indices and add the fixed SWA width in one launch. + + The caller owns the tile-padded buffers and their invariant SWA columns. + Only active rows are modified, preserving inert graph/tile padding. + + Compressed entries start at column swa_width even for short sequences. + total_lengths describes the table span to scan, not the valid-token count: + adding only the actual SWA length would truncate the compressed tail. + Unused SWA slots retain their -1 sentinel and are ignored by attention. + """ + rows, width = indices.shape + assert table.shape == (rows, swa_width + width) + assert lengths.shape == total_lengths.shape == (rows,) + assert indices.stride(1) == table.stride(1) == 1 + assert indices.dtype == table.dtype == total_lengths.dtype == torch.int32 + assert lengths.dtype in (torch.int32, torch.int64) + assert indices.device == lengths.device == table.device == total_lengths.device + if rows == 0: + return + _pack_sparse_tail[(rows,)]( + indices, + lengths, + table, + total_lengths, + indices.stride(0), + table.stride(0), + lengths.stride(0), + total_lengths.stride(0), + width, + swa_width, + triton.next_power_of_2(width), + num_warps=4, + ) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 09a1321d74..e9fc4bea65 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -16,6 +16,7 @@ from sglang.srt.arg_groups.overrides import ( run_post_process_pass, ) from sglang.srt.environ import envs +from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of from sglang.srt.utils.common import is_gfx95_supported, is_npu @@ -217,9 +218,14 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1" ) return + if ( + cfg.dsv4_attn_backend == "trtllm" + and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED + ): + raise ValueError( + "DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled" + ) if cfg.enable_encoder_swa_bounded_replay: - from sglang.srt.model_executor.cuda_graph_config import Backend - incompatible = ( ( "hardware other than CUDA or gfx950", @@ -264,8 +270,14 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: ), ("HiSparse", cfg.enable_hisparse), ("the unified KV layout", is_unified_kv_triton()), - # The trtllm-gen path has no uniform-FP8 pool for V4.1's ratio-1/2 layers. - ("the trtllm DSv4 attention backend", cfg.dsv4_attn_backend == "trtllm"), + ( + "TRT-LLM with SWA bounded replay", + cfg.dsv4_attn_backend == "trtllm" + and ( + cfg.enable_encoder_swa_bounded_replay + or cfg.enable_decoder_swa_bounded_replay + ), + ), ("two-batch overlap", cfg.enable_two_batch_overlap), ("pipeline parallelism", cfg.pp_size > 1), ) @@ -296,8 +308,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "block size and TP size." ) - from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase - prefill_graph = cfg.cuda_graph_config.prefill if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None: # The captured low-ratio indexer scores a static context width; 16k @@ -315,8 +325,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: ) if cfg.enable_decoder_swa_bounded_replay: - from sglang.srt.model_executor.cuda_graph_config import Backend - # Late layers see a per-request tail slice, not the captured prefill shape. incompatible = ( ( diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 4a29949afd..4420fdecd3 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -516,11 +516,13 @@ class DSV4AttnMetadata: raise ValueError(f"invalid {compress_ratio=}") def init_trtllm_sparse_buffers(self) -> None: - """Decode tables of 128 SWA columns then compressed KV, -1 for an invalid - index, lens counting all 128 SWA slots; only the c4 tail is per layer.""" + """Build tile-padded TRT-LLM tables and lengths for indexed attention + or SWA-only DSpark blocks; indexed tails are refreshed per layer.""" num_tokens = self.seq_lens_casual.shape[0] - assert self.swa_page_indices.shape == (num_tokens, SWA_WINDOW) + swa_width = self.swa_page_indices.shape[1] + assert self.swa_page_indices.shape[0] == num_tokens + assert swa_width >= SWA_WINDOW # VarSeq reads rows to the 64-token tile boundary. Back every live view # with an aligned parent whose extra rows contain inert values. @@ -536,17 +538,40 @@ class DSV4AttnMetadata: if n_pad != num_tokens: self.seq_lens_casual = _tile_padded(1, self.seq_lens_casual) self.swa_page_indices = _tile_padded( - -1, self.swa_page_indices, width=SWA_WINDOW + -1, self.swa_page_indices, width=swa_width ) - self.trtllm_swa_lens = _tile_padded(SWA_WINDOW) - if self.c4_sparse_page_indices is not None: - w4 = self.c4_sparse_page_indices.shape[-1] + self.trtllm_swa_lens = _tile_padded( + SWA_WINDOW, + self.swa_topk_lengths.clamp_min(SWA_WINDOW) + if swa_width > SWA_WINDOW + else None, + ) + for ratio in self.low_ratios: + lengths = self.sparse_topk_lengths(ratio) + if lengths is not None: + # The FlashMLA metadata clamps empty compressed histories to + # one. TRT-LLM needs the true zero length: -1 loads zero KV, + # but does not by itself remove its softmax contribution. + torch.minimum(lengths, self.seq_lens_casual // ratio, out=lengths) + indexed_tables = [ + table + for ratio in (1, 2, 4) + if ratio in self.present_ratios + if (table := self.sparse_page_indices(ratio)) is not None + ] + if indexed_tables: + assert swa_width == SWA_WINDOW + # Ratio 1/2/4 all select index_topk entries. Reuse the per-layer + # table; its contents are overwritten before every attention call. + w4 = indexed_tables[0].shape[-1] + assert all(table.shape[-1] == w4 for table in indexed_tables) assert w4 % 4 == 0, f"{w4=}" - # Unwritten c4 rows must remain inert until the per-layer fill. + # Unwritten indexed rows remain inert until the per-layer fill. self.trtllm_c4_indices = _tile_padded(-1, width=SWA_WINDOW + w4) self.trtllm_c4_indices[:, :SWA_WINDOW].copy_(self.swa_page_indices) self.trtllm_c4_lens = _tile_padded(SWA_WINDOW) if self.c128_page_indices is not None: + assert swa_width == SWA_WINDOW w128 = self.c128_page_indices.shape[-1] assert w128 % 4 == 0, f"{w128=}" self.trtllm_c128_indices = _tile_padded(-1, width=SWA_WINDOW + w128) @@ -2111,6 +2136,8 @@ class DeepseekV4AttnBackend( ) metadata.core_attn_metadata.swa_page_indices = swa_page_indices metadata.core_attn_metadata.swa_topk_lengths = swa_topk_lengths + if self.trtllm_attn: + metadata.core_attn_metadata.init_trtllm_sparse_buffers() def _dspark_seq_lens_casual( self, *, seq_lens: torch.Tensor, block_size: int @@ -2921,7 +2948,11 @@ class DeepseekV4AttnBackend( kv_cache = pool.get_extra_key_buffer(layer_id) page_size = pool.get_extra_key_page_size(layer_id) # The pool's page format: V4, or the V4.1 fp8 / fp4 layouts. - kv_layout = pool.get_extra_key_layout(layer_id) + kv_layout = ( + KVLayout.UNIFORM_FP8 + if pool.uniform_fp8 + else pool.get_extra_key_layout(layer_id) + ) assert kv_cache is not None if layer.compress_ratio == 1: @@ -3352,7 +3383,10 @@ class DeepseekV4AttnBackend( **_, ) -> torch.Tensor: if self.mtp_enabled and forward_batch.forward_mode.is_idle(): - return q.new_empty(q.shape[0], q.shape[1], layer.v_head_dim) + return q.new_empty( + (q.shape[0], q.shape[1], layer.v_head_dim), + dtype=torch.bfloat16 if self.trtllm_attn else q.dtype, + ) assert k is v, "DeepseekV4 shares k and v" swa_k = k diff --git a/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py index 634c6a63ed..b98d89159e 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Literal, Optional, Tuple import torch +from sglang.kernels.ops.attention.dsv4.trtllm_metadata import pack_sparse_tail from sglang.srt.environ import envs from sglang.srt.layers.attention.deepseek_v4_backend import ( SWA_WINDOW, @@ -18,6 +19,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4MultiStepBackend, ) from sglang.srt.runtime_context import ( + get_buffer, get_exec, get_parallel, get_schedule, @@ -25,6 +27,14 @@ from sglang.srt.runtime_context import ( max_prefill_buffer_tokens, ) +try: + import flashinfer.mla._core as _fi_core + from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4 +except ImportError as exc: + _flashinfer_import_error = exc +else: + _flashinfer_import_error = None + if TYPE_CHECKING: from sglang.srt.layers.attention.deepseek_v4_backend import DSV4AttnMetadata from sglang.srt.layers.radix_attention import RadixAttention @@ -38,8 +48,6 @@ _TRTLLM_GEN_WORKSPACE_SIZE_MB = 128 def _get_trtllm_workspace_buffer(device: torch.device) -> torch.Tensor: - from sglang.srt.runtime_context import get_buffer - return get_buffer( "trtllm_dsv4_zero_workspace", lambda: torch.zeros( @@ -84,11 +92,13 @@ def _install_persistent_trtllm_semaphores(capacity_rows: int) -> None: FlashInfer accepts a caller-owned buffer. """ global _trtllm_semaphore_installed, _trtllm_semaphore_rows + if _flashinfer_import_error is not None: + raise ImportError( + "--dsv4-attn-backend trtllm requires FlashInfer sparse MLA support" + ) from _flashinfer_import_error _trtllm_semaphore_rows = max(_trtllm_semaphore_rows, capacity_rows) if _trtllm_semaphore_installed: return - import flashinfer.mla._core as _fi_core - _orig = _fi_core._get_trtllm_gen_multi_ctas_kv_counter_buffer # Allocate once outside graph capture. Stream ordering and the kernel's # counter reset make one shared buffer safe across launches. @@ -166,7 +176,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): *, q: torch.Tensor, layer: RadixAttention, - compress_ratio: Literal[0, 4, 128], + compress_ratio: Literal[0, 1, 2, 4, 128], core_attn_metadata: DSV4AttnMetadata, forward_batch: ForwardBatch, attn_sink: torch.Tensor, @@ -175,6 +185,11 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): extra_topk_lengths: Optional[torch.Tensor], ) -> torch.Tensor: assert attn_sink is not None + if self.is_dsv41: + # FlashMLA pads TP query heads to 64. TRT-LLM accepts the native + # width; do not compute attention for the discarded padding. + q = q[:, : layer.tp_q_head_num, :] + attn_sink = attn_sink[: layer.tp_q_head_num] if ( forward_batch.forward_mode.is_decode_or_idle() or forward_batch.forward_mode.is_target_verify() @@ -221,7 +236,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): return (self.softmax_scale, 1.0) def _trtllm_kv_cache_views( - self, layer_id: int, compress_ratio: Literal[0, 4, 128] + self, layer_id: int, compress_ratio: Literal[0, 1, 2, 4, 128] ) -> Tuple[torch.Tensor, torch.Tensor]: """Return HND views of the uniform-FP8 SWA and compressed pools. @@ -248,7 +263,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): *, q: torch.Tensor, layer: RadixAttention, - compress_ratio: Literal[0, 4, 128], + compress_ratio: Literal[0, 1, 2, 4, 128], core_attn_metadata: DSV4AttnMetadata, attn_sink: torch.Tensor, swa_page_indices: torch.Tensor, @@ -257,8 +272,6 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): ) -> torch.Tensor: """Run sparse MLA decode with preallocated metadata tables.""" - from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4 - bs, num_heads, head_dim = q.shape assert head_dim == 512 @@ -278,8 +291,10 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): extra_topk_lengths = extra_topk_lengths[:n_meta_rows] bs = n_meta_rows - # Only the c4 tail and lens vary by layer; other table data is prebuilt. - assert swa_page_indices.shape == (bs, SWA_WINDOW) + # Indexed tails and lengths vary by layer; other table data is prebuilt. + swa_width = swa_page_indices.shape[1] + assert swa_page_indices.shape[0] == bs + assert swa_width == SWA_WINDOW or compress_ratio == 0 if compress_ratio == 0: # Use the metadata view backed by 64-row-aligned storage because # the VarSeq kernel reads table rows to the tile boundary. @@ -302,22 +317,21 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): assert sparse_topk_lens.shape[0] > bs, f"{sparse_topk_lens.shape=}" sparse_topk_lens = sparse_topk_lens[:bs] - if compress_ratio == 4: + if compress_ratio in (1, 2, 4): assert extra_indices is not None and extra_topk_lengths is not None width = extra_indices.shape[-1] assert SWA_WINDOW + width == sparse_indices.shape[1], ( f"{width=} {sparse_indices.shape=}" ) - sparse_indices[:, SWA_WINDOW:].copy_(extra_indices) - # Lens include all 128 SWA slots; seq_lens controls their validity. - sparse_topk_lens.copy_(extra_topk_lengths) - sparse_topk_lens.add_(SWA_WINDOW) + pack_sparse_tail( + extra_indices, extra_topk_lengths, sparse_indices, sparse_topk_lens + ) swa_kv_cache, compressed_kv_cache = self._trtllm_kv_cache_views( layer.layer_id, compress_ratio ) - # RoPE is already applied; the unit scale makes this a plain e4m3 cast. + # Q preparation already returns FP8 on V4.1; to() is a no-op in that path. q_fp8 = q.to(torch.float8_e4m3fn).view(bs, 1, num_heads, 512) bmm1_scale, bmm2_scale = self._get_trtllm_bmm_scales(layer) @@ -326,6 +340,13 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): if seq_lens.shape[0] != bs: assert seq_lens.shape[0] > bs, f"{seq_lens.shape=} {bs=}" seq_lens = seq_lens[:bs] + if swa_width > SWA_WINDOW: + # DSpark attends noncausally to its context window plus the whole + # draft block. The tail still indexes SWA storage, not compressed + # KV. Use the complete block's visible length rather than each + # token's causal length, including for short requests. + compressed_kv_cache = swa_kv_cache + seq_lens = core_attn_metadata.swa_topk_lengths[:bs] assert attn_sink.dtype == torch.float32 assert self.trtllm_workspace_buffer is not None _check_trtllm_query_rows(bs) @@ -342,6 +363,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): bmm2_scale=bmm2_scale, sinks=attn_sink, kv_layout="HND", + backend="trtllm-gen", ) if out_pad_tail is not None: out_pad_tail[:bs] = out.view(bs, num_heads, 512) @@ -353,7 +375,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): *, q: torch.Tensor, layer: RadixAttention, - compress_ratio: Literal[0, 4, 128], + compress_ratio: Literal[0, 1, 2, 4, 128], forward_batch: ForwardBatch, attn_sink: torch.Tensor, swa_page_indices: torch.Tensor, @@ -366,8 +388,6 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): validity from it. This path runs eagerly. """ - from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4 - assert q.ndim == 3, f"{q.shape=}" num_qo_padded, num_heads, head_dim = q.shape assert head_dim == 512 @@ -418,11 +438,10 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): assert width % 4 == 0, f"{width=}" table = _tile_padded_pf(-1, width=SWA_WINDOW + width) table[:, :SWA_WINDOW].copy_(swa_indices) - table[:, SWA_WINDOW:].copy_(extra_indices[:sum_q]) assert extra_topk_lengths is not None - lens = _tile_padded_pf( - SWA_WINDOW, - extra_topk_lengths[:sum_q].to(torch.int32) + SWA_WINDOW, + lens = _tile_padded_pf(SWA_WINDOW) + pack_sparse_tail( + extra_indices[:sum_q], extra_topk_lengths[:sum_q], table, lens ) core.trtllm_prefill_c128 = (table, lens) sparse_indices, sparse_topk_lens = core.trtllm_prefill_c128 @@ -441,14 +460,15 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): sum_q, SWA_WINDOW + width, ), f"{sparse_indices.shape=} {width=}" - sparse_indices[:, SWA_WINDOW:].copy_(extra_indices[:sum_q]) - # Lens include 128 SWA slots; VarSeq metadata controls validity. - sparse_topk_lens = _tile_padded_pf( - SWA_WINDOW, - extra_topk_lengths[:sum_q].to(torch.int32) + SWA_WINDOW, + sparse_topk_lens = _tile_padded_pf(SWA_WINDOW) + pack_sparse_tail( + extra_indices[:sum_q], + extra_topk_lengths[:sum_q], + sparse_indices, + sparse_topk_lens, ) - # RoPE is already applied; the unit scale makes this a plain e4m3 cast. + # Retain a unit-scale cast for callers without fused FP8 Q preparation. q_fp8 = q[:sum_q].to(torch.float8_e4m3fn) swa_kv_cache, compressed_kv_cache = self._trtllm_kv_cache_views( @@ -483,6 +503,7 @@ class DeepseekV4TrtllmAttnBackend(DeepseekV4AttnBackend): bmm2_scale=bmm2_scale, sinks=attn_sink, kv_layout="HND", + backend="trtllm-gen", cum_seq_lens_q=cum_seq_lens_q, max_q_len=max_q_len, ) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index cb0a66052b..f97e319802 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -20,6 +20,9 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import ( KVLayout, is_valid_kv_layout_pair, ) +from sglang.kernels.ops.attention.dsv4.kv_norm_rope_store import ( + fused_k_norm_rope_uniform_fp8, +) from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs @@ -165,6 +168,11 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: AITER attention.""" mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower() option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower() + if get_exec().kernel.dsv4_attn_backend == "trtllm": + assert mode in ("auto", "v4") and option in ("auto", "fp8"), ( + "trtllm uses uniform FP8 pools, not the packed V4.1 KV layouts" + ) + return KVLayout.V4, None if mode == "v4": return KVLayout.V4, None if option == "auto" else option assert mode in ("v41", "auto"), f"unknown SGLANG_DSV4_KV_LAYOUT={mode!r}" @@ -2042,21 +2050,14 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): """q ([B, H, head_dim]): rope its query heads in the same launch.""" if self.uniform_fp8: assert q is None, "uniform FP8 store does not fuse query RoPE" - # Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again), - # then an e4m3 cast + scatter with per-tensor scale 1.0. - from sglang.kernels.ops.attention.deepseek_v4_rope import ( - fused_norm_rope_inplace_triton, - ) - - fused_norm_rope_inplace_triton( + fused_k_norm_rope_uniform_fp8( kv, kv_weight, eps, freqs_cis, - positions=positions, - ) - self.swa_kv_pool.set_key_buffer_fused( - self._swa_local_layer_id(layer_id), swa_loc, kv + positions, + swa_loc, + self.swa_kv_pool.get_key_buffer(self._swa_local_layer_id(layer_id)), ) return fused_k_norm_rope_flashmla( diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index e0771b6c97..270909408b 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1339,6 +1339,9 @@ class MQALayer(MqaAttentionBase): q, _ = self.wq_b(q) q = q.view(-1, self.n_local_heads, self.head_dim) if not self.q_head_norm: + if q_out is not None and q_out.dtype == torch.float8_e4m3fn: + fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions) + return q_out # TODO: enable zero-copy path to skip the extra no-rope copy overhead if (_is_cuda or _is_gfx95_supported) and q_out is not None: fused_q_norm_rope( @@ -2252,7 +2255,19 @@ class MQALayer(MqaAttentionBase): tp_slice, q_padded, q_out, q_rope = slice(None), None, None, None k_nope, k_rope = None, None - if unified_fp8_decode or unified_fp8_prefill: + if ( + self.is_dsv41 + and not self.q_head_norm + and get_token_to_kv_pool().uniform_fp8 + and self.n_local_heads in (8, 16, 32, 64, 128) + ): + kernel_num_heads = self.n_local_heads + q_out = torch.empty( + (x.shape[0], self.n_local_heads, self.head_dim), + dtype=torch.float8_e4m3fn, + device=x.device, + ) + elif unified_fp8_decode or unified_fp8_prefill: # width and dtype come off the pools themselves; the kernel reads Q # with the kv row stride, so the two must not drift kv_pool = get_token_to_kv_pool() diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index ffff25666b..f2f3ccbc25 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -191,6 +191,9 @@ class DSparkAttention(MqaAttentionBase): q, _ = self.wq_b(q) q = q.view(-1, self.n_local_heads, self.head_dim) if not self.q_head_norm: + if q_out is not None and q_out.dtype == torch.float8_e4m3fn: + fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions) + return q_out if self._use_fast_kernel and not _is_npu: fused_rope_inplace( q[..., -self.rope_head_dim :], @@ -270,7 +273,17 @@ class DSparkAttention(MqaAttentionBase): q_padded: Optional[torch.Tensor] = None q_out: Optional[torch.Tensor] = None - if self.n_local_heads < _PAD_NUM_HEADS: + if ( + pool.uniform_fp8 + and not self.q_head_norm + and self.n_local_heads in (8, 16, 32, 64, 128) + ): + q_out = torch.empty( + (hidden_states.shape[0], self.n_local_heads, self.head_dim), + dtype=torch.float8_e4m3fn, + device=hidden_states.device, + ) + elif self.n_local_heads < _PAD_NUM_HEADS: q_padded = hidden_states.new_empty( hidden_states.shape[0], _PAD_NUM_HEADS, self.head_dim ) diff --git a/test/manual/kernels/attention/test_dsv4_trtllm_prefill.py b/test/manual/kernels/attention/test_dsv4_trtllm_prefill.py new file mode 100644 index 0000000000..a3b3ab5137 --- /dev/null +++ b/test/manual/kernels/attention/test_dsv4_trtllm_prefill.py @@ -0,0 +1,103 @@ +"""Blackwell regression for DSv4 TRT-LLM long-query dispatch. + +Run directly on SM100/SM103. Compare FP8 attention with a float32 reference +across the short-query boundary, then verify the TP4 prefill tile at 8K. +""" + +import unittest + +import torch +from flashinfer.mla import trtllm_batch_decode_sparse_mla_dsv4 + +from sglang.srt.layers.attention.deepseek_v4_trtllm_backend import ( + _install_persistent_trtllm_semaphores, +) + + +@unittest.skipUnless( + torch.cuda.is_available() + and torch.cuda.get_device_capability() in ((10, 0), (10, 3)), + "requires SM100/SM103", +) +class TestTrtllmPrefill(unittest.TestCase): + def test_prefill_reference_and_dispatch(self): + torch.manual_seed(41603) + _install_persistent_trtllm_semaphores(8192) + device = "cuda" + dtype = torch.float8_e4m3fn + heads, dim, topk = 16, 512, 2176 + workspace = torch.zeros(128 * 1024 * 1024, dtype=torch.uint8, device=device) + swa = torch.randn(64, 1, 64, dim, device=device).to(dtype) + compressed = torch.randn(64, 1, 64, dim, device=device).to(dtype) + sink = torch.randn(heads, device=device) + for rows in (1, 16, 17, 8192): + with self.subTest(rows=rows): + q = torch.randn(rows, heads, dim, device=device).to(dtype) + indices = torch.randint( + 0, 4096, (rows, topk), dtype=torch.int32, device=device + ) + lens = torch.randint( + 128, topk + 1, (rows,), dtype=torch.int32, device=device + ) + cu = torch.tensor([0, rows], dtype=torch.int32, device=device) + # A cached prefix ensures that all 128 SWA slots are valid. + seq_lens = torch.tensor([rows + 128], dtype=torch.int32, device=device) + + def run(): + return trtllm_batch_decode_sparse_mla_dsv4( + query=q, + swa_kv_cache=swa, + workspace_buffer=workspace, + sparse_indices=indices, + compressed_kv_cache=compressed, + sparse_topk_lens=lens, + seq_lens=seq_lens, + bmm1_scale=dim**-0.5, + bmm2_scale=1.0, + sinks=sink, + kv_layout="HND", + backend="trtllm-gen", + cum_seq_lens_q=cu, + max_q_len=rows, + ) + + out = run() + torch.cuda.synchronize() + self.assertTrue(torch.isfinite(out).all().item()) + samples = sorted(set([0, rows // 2, rows - 1])) + for row in samples: + length = lens[row].item() + keys = torch.cat( + ( + swa.float().reshape(-1, dim)[indices[row, :128].long()], + compressed.float().reshape(-1, dim)[ + indices[row, 128:length].long() + ], + ) + ) + logits = q[row].float() @ keys.T * dim**-0.5 + probs = torch.cat((logits, sink[:, None]), dim=-1).softmax(-1) + expected = probs[:, :-1] @ keys + # Allow FP8 attention rounding relative to the FP32 reference. + torch.testing.assert_close( + out[row].float(), expected, atol=0.02, rtol=0.03 + ) + if rows == 8192: + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA] + ) as prof: + run() + torch.cuda.synchronize() + names = [ + event.name + for event in prof.events() + if "fmhaSm100" in event.name + ] + self.assertTrue(names, "No TRT-LLM attention kernel recorded") + self.assertTrue( + all("VarSeqQ16Kv128" in name for name in names), names + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py b/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py new file mode 100644 index 0000000000..1cf9bcd715 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py @@ -0,0 +1,98 @@ +import sys + +import pytest +import torch + +from sglang.kernels.ops.attention.dsv4.elementwise import fused_q_norm_rope +from sglang.kernels.ops.attention.dsv4.trtllm_metadata import pack_sparse_tail +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@pytest.mark.parametrize("rows,heads", [(6, 16), (9, 128), (4097, 16)]) +@pytest.mark.parametrize("position_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("eps", [None, 1e-6]) +def test_fp8_rope_output_padding_and_graph(rows, heads, position_dtype, eps): + torch.manual_seed(911) + q = torch.randn(rows, heads + 1, 512, device="cuda", dtype=torch.bfloat16)[ + :, :heads + ] + storage = torch.full( + (rows, heads + 1, 512), 7.0, device="cuda", dtype=torch.float8_e4m3fn + ) + output = storage[:, :heads] + freqs = torch.polar( + torch.ones(8192, 32, device="cuda"), torch.randn(8192, 32, device="cuda") + ) + positions = torch.randint(8192, (rows,), device="cuda", dtype=position_dtype) + expected = torch.empty_like(q) + fused_q_norm_rope(q, output, eps, freqs, positions) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + fused_q_norm_rope(q, output, eps, freqs, positions) + for _ in range(3): + q.normal_() + original = q.clone() + positions.random_(0, 8192) + graph.replay() + fused_q_norm_rope(q, expected, eps, freqs, positions) + torch.testing.assert_close( + output.float(), expected.to(output.dtype).float(), rtol=0, atol=0 + ) + torch.testing.assert_close(q, original, rtol=0, atol=0) + assert (storage[:, heads:].float() == 7).all() + + +def test_fp8_rope_128_heads_int64_offsets(): + # Both Q and output cross 2**31 elements. Check the rows on either side + # without allocating another full-sized reference tensor. + rows, heads = 32769, 128 + required = rows * heads * 512 * 5 + if torch.cuda.mem_get_info()[0] < required + (1 << 30): + pytest.skip("Requires about 11 GiB of free GPU memory") + q = torch.ones(rows, heads, 512, device="cuda", dtype=torch.bfloat16) + output = torch.empty(q.shape, device="cuda", dtype=torch.float8_e4m3fn) + freqs = torch.ones(1, 32, device="cuda", dtype=torch.complex64) + positions = torch.zeros(rows, device="cuda", dtype=torch.int32) + fused_q_norm_rope(q, output, None, freqs, positions) + torch.testing.assert_close(output[-2:].float(), q[-2:].float(), rtol=0, atol=0) + + +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +def test_sparse_tail_short_sequence_and_padding(dtype): + # A short sequence has one valid SWA entry but still reserves 128 columns. + table = torch.full((64, 132), -1, device="cuda", dtype=torch.int32) + table[0, 0] = 42 + indices = torch.tensor([[100, 101, -1, -1]], device="cuda", dtype=torch.int32) + lengths = torch.tensor([2], device="cuda", dtype=dtype) + totals = torch.zeros(64, device="cuda", dtype=torch.int32) + pack_sparse_tail(indices, lengths, table[:1], totals[:1]) + assert totals[0].item() == 130 + assert table[0, 0].item() == 42 + assert (table[0, 1:128] == -1).all() + torch.testing.assert_close(table[:1, 128:], indices) + assert (table[1:] == -1).all() + assert (totals[1:] == 0).all() + + +def test_sparse_tail_rejects_float_lengths(): + indices = torch.empty((0, 4), dtype=torch.int32) + table = torch.empty((0, 132), dtype=torch.int32) + with pytest.raises(AssertionError): + pack_sparse_tail( + indices, torch.empty(0), table, torch.empty(0, dtype=torch.int32) + ) + + +@pytest.mark.parametrize("different_device", range(4)) +def test_sparse_tail_rejects_mixed_devices(different_device): + shapes = [(0, 4), (0,), (0, 132), (0,)] + tensors = [torch.empty(shape, dtype=torch.int32) for shape in shapes] + tensors[different_device] = tensors[different_device].to("meta") + with pytest.raises(AssertionError): + pack_sparse_tail(*tensors) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py b/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py new file mode 100644 index 0000000000..6204405fda --- /dev/null +++ b/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py @@ -0,0 +1,105 @@ +"""Byte-level parity with the existing norm/RoPE -> cast -> index_put path.""" + +import sys + +import pytest +import torch + +from sglang.kernels.ops.attention.deepseek_v4_rope import fused_norm_rope_inplace_triton +from sglang.kernels.ops.attention.dsv4.kv_norm_rope_store import ( + fused_k_norm_rope_uniform_fp8, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def inputs(n, dtype=torch.bfloat16, strided=False, index_dtype=torch.int64): + torch.manual_seed(42) + width = 1792 if strided else 512 + x = torch.randn(n, width, device="cuda", dtype=dtype)[:, -512:] + weight = torch.randn(512, device="cuda", dtype=dtype) * 0.5 + 1 + angles = torch.randn(1024, 32, device="cuda", dtype=torch.float32) + freq = torch.polar(torch.ones_like(angles), angles) + pos = torch.randint(0, 1024, (n,), device="cuda", dtype=index_dtype) + loc = torch.randperm(n + 17, device="cuda")[:n].to(index_dtype).contiguous() + cache = torch.zeros(n + 17, 512, device="cuda", dtype=torch.float8_e4m3fn) + return x, weight, freq, pos, loc, cache + + +def reference(x, weight, freq, pos, loc, cache, eps=1e-6): + y = x.clone() + fused_norm_rope_inplace_triton(y, weight, eps, freq, positions=pos) + cache.view(torch.uint8)[loc.long()] = y.to(torch.float8_e4m3fn).view(torch.uint8) + + +def check(x, weight, freq, pos, loc, cache, eps=1e-6): + original = x.clone() + expected = cache.clone() + reference(x, weight, freq, pos, loc, expected, eps) + fused_k_norm_rope_uniform_fp8(x, weight, eps, freq, pos, loc, cache) + torch.cuda.synchronize() + assert torch.equal(cache.view(torch.uint8), expected.view(torch.uint8)) + assert torch.equal(x, original) + + +@pytest.mark.parametrize("n", [0, 1, 5, 6, 63, 64, 65, 315, 378, 384, 1024, 16384]) +@pytest.mark.parametrize( + "strided,index_dtype", [(False, torch.int64), (True, torch.int32)] +) +def test_parity(n, strided, index_dtype): + check(*inputs(n, strided=strided, index_dtype=index_dtype)) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +def test_other_input_dtypes(dtype): + check(*inputs(65, dtype=dtype, strided=True)) + + +def test_zero_and_negative_slots(): + x, weight, freq, pos, loc, cache = inputs(65) + x.zero_() + loc[:10] -= cache.shape[0] + check(x, weight, freq, pos, loc, cache) + + +def test_fp8_overflow(): + x, weight, freq, pos, loc, cache = inputs(65) + weight.mul_(400) + check(x, weight, freq, pos, loc, cache) + + +def test_graph_replay_updates(): + x, weight, freq, pos, loc, cache = inputs(65, strided=True, index_dtype=torch.int32) + fused_k_norm_rope_uniform_fp8(x, weight, 1e-6, freq, pos, loc, cache) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + fused_k_norm_rope_uniform_fp8(x, weight, 1e-6, freq, pos, loc, cache) + for seed in [2, 3]: + torch.manual_seed(seed) + x.copy_(torch.randn_like(x)) + pos.copy_(torch.randint(0, 1024, pos.shape, device="cuda", dtype=pos.dtype)) + loc.copy_(torch.randperm(cache.shape[0], device="cuda")[: x.shape[0]]) + cache.zero_() + expected = cache.clone() + reference(x, weight, freq, pos, loc, expected) + g.replay() + torch.cuda.synchronize() + assert torch.equal(cache.view(torch.uint8), expected.view(torch.uint8)) + + +def test_large_cache_offset(): + x, weight, freq, pos, _, _ = inputs(2) + # Sparse writes cross the signed 32-bit byte-offset boundary. + rows = (1 << 22) + 2 + cache = torch.empty(rows, 512, device="cuda", dtype=torch.float8_e4m3fn) + loc = torch.tensor([0, rows - 1], device="cuda", dtype=torch.int64) + expected = torch.zeros(2, 512, device="cuda", dtype=torch.float8_e4m3fn) + reference(x, weight, freq, pos, torch.arange(2, device="cuda"), expected) + fused_k_norm_rope_uniform_fp8(x, weight, 1e-6, freq, pos, loc, cache) + torch.cuda.synchronize() + assert torch.equal(cache.view(torch.uint8)[loc], expected.view(torch.uint8)) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))