diff --git a/python/sglang/kernels/ops/attention/dsv4/opus_sparse_prefill_hip.py b/python/sglang/kernels/ops/attention/dsv4/opus_sparse_prefill_hip.py new file mode 100644 index 0000000000..8fd672cd56 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/opus_sparse_prefill_hip.py @@ -0,0 +1,101 @@ +"""DeepSeek-V4 sparse prefill attention on gfx950 through aiter's OPUS kernel. + +OPUS attends over two bf16 row sources, each addressed by a CSR index list. The +caller dequantizes rows into bf16 workspaces and builds padded per-query lists +(``SparsePrefillChunkCache``); this module turns those lists into CSR and runs +the kernel. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +# (int32 indices, int32 indptr [T + 1]) +Csr = Tuple[torch.Tensor, torch.Tensor] + + +@triton.jit +def _combined_to_csr_kernel( + indices_ptr, + indices_stride, + lens_ptr, + indptr_ptr, + out_ptr, + row_start, + row_end, + BLOCK: tl.constexpr, +): + # OPUS reads every CSR entry, so -1 holes inside the scanned prefix and + # entries outside [row_start, row_end) are dropped here. + row = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, BLOCK) + length = tl.load(lens_ptr + row) + idx = tl.load( + indices_ptr + row * indices_stride + cols, mask=cols < length, other=-1 + ) + keep = (idx >= row_start) & (idx < row_end) + dst = tl.cumsum(keep.to(tl.int32), axis=0) - 1 + start = tl.load(indptr_ptr + row) + tl.store(out_ptr + start + dst, idx - row_start, mask=keep) + + +def combined_to_csr( + indices: torch.Tensor, lens: torch.Tensor, row_start: int, row_end: int +) -> Csr: + """int32 [T, W] padded index lists with per-row scanned lengths [T] -> the CSR + (T * W capacity) of their entries in [row_start, row_end), rebased to row_start.""" + num_rows, width = indices.shape + device = indices.device + cols = torch.arange(width, device=device, dtype=torch.int32) + keep = ( + (cols[None, :] < lens[:, None]) & (indices >= row_start) & (indices < row_end) + ) + indptr = torch.zeros(num_rows + 1, dtype=torch.int32, device=device) + torch.cumsum(keep.sum(dim=1, dtype=torch.int32), dim=0, out=indptr[1:]) + csr = torch.empty(max(num_rows * width, 1), dtype=torch.int32, device=device) + if num_rows > 0: + _combined_to_csr_kernel[(num_rows,)]( + indices, + indices.stride(0), + lens, + indptr, + csr, + row_start, + row_end, + BLOCK=triton.next_power_of_2(width), + ) + return csr, indptr + + +def opus_sparse_prefill( + q: torch.Tensor, + kv: torch.Tensor, + csr: Csr, + attn_sink: torch.Tensor, + softmax_scale: float, + extend_kv: Optional[torch.Tensor] = None, + extend_csr: Optional[Csr] = None, +) -> torch.Tensor: + """Attention of q [T, H, 512] bf16 over the bf16 rows of kv [R, 512] named by + csr, then those of extend_kv named by extend_csr, with a per-head sink.""" + from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus + + indices, indptr = csr + if extend_kv is None: + # every query reads an empty row of the unused extend source + extend_kv = kv[:1] + extend_csr = (indices[:1], torch.zeros_like(indptr)) + extend_indices, extend_indptr = extend_csr + return pa_sparse_prefill_opus( + q.contiguous(), + kv, + indices, + indptr, + extend_kv, + extend_indices, + extend_indptr, + attn_sink, + softmax_scale, + ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 5ac07bcd5e..d4dc410de8 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1579,6 +1579,9 @@ class Envs: SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) + # gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16 + # dequant of the chunk's SWA and compressed history, instead of the decode kernel. + SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL = EnvBool(False) # DSpark draft block on the HIP radix backend: build the attention metadata inside the # draft CUDA graph from the raw inputs instead of eagerly before every replay. SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 4c4e88a5ba..5a3835b969 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -44,10 +44,16 @@ from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( SOFTMAX_SCALE as SWAPAB_SOFTMAX_SCALE, ) +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import dequantize_k_cache_paged from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( init_compression_metadata as _init_compression_metadata_triton, ) +from sglang.kernels.ops.attention.dsv4.opus_sparse_prefill_hip import ( + Csr, + combined_to_csr, + opus_sparse_prefill, +) from sglang.kernels.ops.attention.dsv4.swapab_gluon_hip import swapab_attention from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( BuildCausalSwaPageIndices, @@ -81,12 +87,18 @@ from sglang.srt.layers.attention.dsv4.metadata import ( copy_metadata, maybe_copy_inplace, ) +from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import ( + SparsePrefillChunkCache, + SparsePrefillWorkspace, +) from sglang.srt.layers.attention.hip_flash_mla import ( + _apply_inverse_rope, flash_mla_with_kvcache_entrypoint, hip_attn_kv_splits, resolve_hip_flashmla_backend, ) from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask +from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.dsv41_request_window import window_layout from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode @@ -97,7 +109,7 @@ from sglang.srt.runtime_context import ( ) from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout -from sglang.srt.utils import ceil_align +from sglang.srt.utils import ceil_align, is_gfx95_supported if TYPE_CHECKING: from sgl_kernel.flash_mla import FlashMLASchedMeta @@ -130,6 +142,34 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T: return F.pad(x, pad=(0, target_size - curr_size), mode="constant", value=-1) +# OPUS prefill applies from this many query rows; below it the decode kernel splits KV +# and the dequant is not amortized. +_OPUS_PREFILL_MIN_TOKENS = 1024 +# The OPUS workspace holds every request's whole compressed history, dequantized once +# per kv_source layer. Past this many history rows per query row that dequant costs +# more than OPUS saves (gfx950, uniformly random top-k): a ratio-2 source serves fewer +# layers than the ratio-1 one. +_OPUS_PREFILL_MAX_ROWS_PER_TOKEN = {1: 256, 2: 128} +# One ratio's bf16 workspace stays within 1 GiB. +_OPUS_PREFILL_MAX_WORKSPACE_ROWS = 1 << 20 + + +@dataclass +class _OpusPrefillState: + """OPUS prefill inputs of one forward. A V4.1 ratio-1/2 layer reads the + compressed KV of its kv_source layer and the top-k of its index_source layer, + so each is built once and reused by the layers that follow its source.""" + + cache: SparsePrefillChunkCache + # the SWA rows every layer lays out alike + swa_csr: Csr + # ratio -> (data_ptr of the compressed cache, its bf16 rows) + compressed_kv: Dict[int, Tuple[int, torch.Tensor]] = field(default_factory=dict) + # ratio -> CSR of the top-k into compressed_kv[ratio]; dropped when the indexer + # rewrites that ratio's top-k + topk_csr: Dict[int, Csr] = field(default_factory=dict) + + def _fold_lengths_into_index_lists( indices: torch.Tensor, lengths: Optional[torch.Tensor], @@ -625,11 +665,17 @@ class DSV4AttnMetadata: self.c0_flashmla_metadata = _create_flashmla_metadata() self.c4_flashmla_metadata = _create_flashmla_metadata() self.c128_flashmla_metadata = _create_flashmla_metadata() + # -1 filled like the page indices: identity rows write only their reachable columns, + # and the OPUS prefill reads the raw indices without the top-k lengths if 1 in self.low_ratios: - self.c1_sparse_raw_indices = torch.empty_like(self.c1_sparse_page_indices) + self.c1_sparse_raw_indices = torch.full_like( + self.c1_sparse_page_indices, -1 + ) self.c1_flashmla_metadata = _create_flashmla_metadata() if 2 in self.low_ratios: - self.c2_sparse_raw_indices = torch.empty_like(self.c2_sparse_page_indices) + self.c2_sparse_raw_indices = torch.full_like( + self.c2_sparse_page_indices, -1 + ) self.c2_flashmla_metadata = _create_flashmla_metadata() @@ -973,6 +1019,12 @@ class DeepseekV4HipRadixBackend( DSV4RawVerifyMetadata, DSV4RawDecodeMetadata, ] = None + self.opus_prefill_workspace = SparsePrefillWorkspace(self.device) + self.opus_compressed_workspaces = { + ratio: SparsePrefillWorkspace(self.device) for ratio in (1, 2) + } + # Reset with the metadata: its inputs belong to one prefill forward. + self._opus_prefill_state: Optional[_OpusPrefillState] = None def _move_to_device(self, x: List[int]) -> torch.Tensor: pin_tensor = torch.tensor(x, dtype=torch.int32, pin_memory=True) @@ -1996,6 +2048,7 @@ class DeepseekV4HipRadixBackend( if self.mtp_enabled and forward_batch.forward_mode.is_idle(): return + self._opus_prefill_state = None self.forward_metadata = self._build_forward_metadata(forward_batch) self.init_forward_metadata_in_graph(forward_batch) self._refresh_fp4_prefill_workspace(forward_batch) @@ -2210,6 +2263,7 @@ class DeepseekV4HipRadixBackend( def init_forward_metadata_for_breakable_cuda_graph_capture( self, forward_batch: ForwardBatch ) -> DSV4Metadata: + self._opus_prefill_state = None if not self.low_ratios: self.forward_metadata = self._build_forward_metadata( forward_batch, @@ -2253,6 +2307,7 @@ class DeepseekV4HipRadixBackend( *, static_forward_batch: Optional[ForwardBatch] = None, ) -> None: + self._opus_prefill_state = None if not self.low_ratios: replay_batch = ( static_forward_batch @@ -2821,6 +2876,169 @@ class DeepseekV4HipRadixBackend( cache_k=swa_k, ) + def _opus_prefill_applies( + self, q: torch.Tensor, forward_batch: ForwardBatch, compress_ratio: int + ) -> bool: + if not envs.SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL.get(): + return False + if ( + compress_ratio not in (0, 1, 2) + or self.is_draft_worker + or not forward_batch.forward_mode.is_extend_without_speculative() + or q.ndim != 3 + or q.shape[0] < _OPUS_PREFILL_MIN_TOKENS + or q.shape[-1] != 512 + or q.dtype != torch.bfloat16 + or not is_gfx95_supported() + or self.token_to_kv_pool.request_window is not None + or self.forward_metadata.late_layer_tail is not None + or is_cp_active(forward_batch) + or forward_batch.seq_lens_cpu is None + or forward_batch.extend_seq_lens_cpu is None + ): + return False + if compress_ratio == 0: + return True + seq_lens = forward_batch.seq_lens_cpu + history_rows = len(seq_lens) * max(int(seq_lens.max()) // compress_ratio, 1) + return history_rows <= min( + _OPUS_PREFILL_MAX_ROWS_PER_TOKEN[compress_ratio] * q.shape[0], + _OPUS_PREFILL_MAX_WORKSPACE_ROWS, + ) + + def _build_opus_prefill_cache( + self, + forward_batch: ForwardBatch, + core_attn_metadata: DSV4AttnMetadata, + num_tokens: int, + ) -> SparsePrefillChunkCache: + seq_lens_cpu = forward_batch.seq_lens_cpu.tolist() + total_swa = sum( + min(int(seq_len), int(extend_len) + SWA_WINDOW - 1) + for seq_len, extend_len in zip( + seq_lens_cpu, forward_batch.extend_seq_lens_cpu, strict=True + ) + ) + extend_seq_lens = forward_batch.extend_seq_lens.to(torch.int32) + query_pos = core_attn_metadata.seq_lens_casual[:num_tokens] - 1 + assert query_pos.shape[0] == num_tokens, (query_pos.shape, num_tokens) + return SparsePrefillChunkCache.build( + seq_lens=forward_batch.seq_lens.to(torch.int32), + extend_seq_lens=extend_seq_lens, + query_lens=extend_seq_lens, + query_pos=query_pos.to(torch.int32), + req_pool_indices=forward_batch.req_pool_indices.to(torch.int32), + req_to_token=self.req_to_token, + full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping, + swa_window_size=SWA_WINDOW, + swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size, + num_qo_tokens=num_tokens, + max_seq_len=max(seq_lens_cpu), + total_swa=total_swa, + ) + + def _forward_prefill_opus( + self, + q: torch.Tensor, + layer_id: int, + compress_ratio: int, + forward_batch: ForwardBatch, + core_attn_metadata: DSV4AttnMetadata, + attn_sink: torch.Tensor, + inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]], + ) -> torch.Tensor: + """Sparse prefill through OPUS over the layer's SWA rows and, for ratios 1 / 2, + the compressed history its top-k selects.""" + pool = self.token_to_kv_pool + num_tokens = q.shape[0] + state = self._opus_prefill_state + if state is None or state.cache.num_qo_tokens != num_tokens: + cache = self._build_opus_prefill_cache( + forward_batch, core_attn_metadata, num_tokens + ) + swa_csr = combined_to_csr( + cache.c0_combined_indices, + cache.c0_combined_lens, + 0, + cache.swa_token_ids.shape[0], + ) + state = self._opus_prefill_state = _OpusPrefillState(cache, swa_csr) + cache = state.cache + num_swa_rows = cache.swa_token_ids.shape[0] + swa_rows = self.opus_prefill_workspace.get(num_swa_rows) + dequantize_k_cache_paged( + pool.get_swa_key_buffer_radix(layer_id), + cache.swa_token_ids, + page_size=cache.swa_page_size, + out=swa_rows, + layout=pool.get_swa_key_layout(), + ) + swa_kv = swa_rows.view(num_swa_rows, -1) + if compress_ratio == 0: + out = opus_sparse_prefill( + q, swa_kv, state.swa_csr, attn_sink, self.softmax_scale + ) + else: + compressed_kv, topk_csr = self._opus_compressed_inputs( + state, layer_id, compress_ratio, core_attn_metadata + ) + out = opus_sparse_prefill( + q, + compressed_kv, + topk_csr, + attn_sink, + self.softmax_scale, + extend_kv=swa_kv, + extend_csr=state.swa_csr, + ) + if inv_rope is not None: + _apply_inverse_rope(out, inv_rope) + return out + + def _opus_compressed_inputs( + self, + state: _OpusPrefillState, + layer_id: int, + compress_ratio: int, + core_attn_metadata: DSV4AttnMetadata, + ) -> Tuple[torch.Tensor, Csr]: + """The bf16 compressed history of the layer's kv_source and the CSR of the + ratio's current top-k into it.""" + pool = self.token_to_kv_pool + cache = state.cache + c_cache = pool.get_extra_key_buffer(layer_id) + entry = state.compressed_kv.get(compress_ratio) + if entry is None or entry[0] != c_cache.data_ptr(): + c_page_size = pool.get_extra_key_page_size(layer_id) + gather = cache.ensure_compressed( + compress_ratio, core_attn_metadata.page_table, c_page_size + ) + num_rows = gather.flat_token_ids.shape[0] + rows = self.opus_compressed_workspaces[compress_ratio].get(num_rows) + dequantize_k_cache_paged( + c_cache, + gather.flat_token_ids, + page_size=c_page_size, + out=rows, + layout=pool.get_extra_key_layout(layer_id), + ) + entry = (c_cache.data_ptr(), rows.view(num_rows, -1)) + state.compressed_kv[compress_ratio] = entry + compressed_kv = entry[1] + topk_csr = state.topk_csr.get(compress_ratio) + if topk_csr is None: + raw_indices = core_attn_metadata.sparse_raw_indices(compress_ratio) + indices, lens = cache.combine_compressed( + compress_ratio, raw_indices[: cache.num_qo_tokens] + ) + topk_csr = combined_to_csr(indices, lens, 0, compressed_kv.shape[0]) + state.topk_csr[compress_ratio] = topk_csr + return compressed_kv, topk_csr + + def _drop_opus_topk_csr(self, compress_ratio: int) -> None: + if self._opus_prefill_state is not None: + self._opus_prefill_state.topk_csr.pop(compress_ratio, None) + def _forward_attention( self, q: torch.Tensor, @@ -2875,6 +3093,16 @@ class DeepseekV4HipRadixBackend( if isinstance(core_attn_metadata, DSV4AttnMetadata): if save_kv_cache: self.store_cache(layer_id, swa_k, forward_batch) + if self._opus_prefill_applies(q, forward_batch, compress_ratio): + return self._forward_prefill_opus( + q, + layer_id, + compress_ratio, + forward_batch, + core_attn_metadata, + attn_sink, + inv_rope, + ) swa_k_cache = token_to_kv_pool.get_swa_key_buffer_radix(layer_id) extra_k_cache, extra_indices, extra_topk_lengths = None, None, None @@ -3029,6 +3257,7 @@ class DeepseekV4HipRadixBackend( self.forward_metadata.core_metadata.drop_folded_sparse_indices( layer.compress_ratio ) + self._drop_opus_topk_csr(layer.compress_ratio) if ( forward_batch.forward_mode.is_decode() or forward_batch.forward_mode.is_target_verify() diff --git a/test/registered/attention/unittests/dsv4/test_dsv41_opus_sparse_prefill_hip.py b/test/registered/attention/unittests/dsv4/test_dsv41_opus_sparse_prefill_hip.py new file mode 100644 index 0000000000..f1a3d51055 --- /dev/null +++ b/test/registered/attention/unittests/dsv4/test_dsv41_opus_sparse_prefill_hip.py @@ -0,0 +1,161 @@ +"""The DSV4.1 OPUS sparse prefill path on gfx950: the -1 padded lists to CSR conversion, the +two-source OPUS attention against an fp32 reference, and the -1 filled raw top-k buffers it reads.""" + +import unittest + +import torch + +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=20, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +D = 512 +SCALE = D**-0.5 +# about 3x the measured bf16 error of the kernel (~3e-3 of the output's max) +TOL = 1e-2 + + +def _opus_available() -> bool: + if not (is_hip() and is_gfx95_supported()): + return False + try: + from aiter.ops.pa_sparse_prefill_opus import ( # noqa: F401 + pa_sparse_prefill_opus, + ) + except ImportError: + return False + return True + + +def _padded_lists(rows, width, num_src_rows, gen, *, max_len=None): + """-1 padded lists [rows, width] with per-row scanned lengths: valid entries, -1 holes inside + the scanned prefix, and stale entries past it.""" + device = "cuda" + max_len = width if max_len is None else max_len + lens = torch.randint(0, max_len + 1, (rows,), generator=gen, device=device) + idx = torch.randint(0, num_src_rows, (rows, width), generator=gen, device=device) + holes = torch.rand(rows, width, generator=gen, device=device) < 0.2 + idx = torch.where(holes, -1, idx) + return idx.to(torch.int32), lens.to(torch.int32) + + +def _csr_reference(indices, lens, row_start, row_end): + out, indptr = [], [0] + for row, n in zip(indices.tolist(), lens.tolist()): + kept = [i - row_start for i in row[:n] if row_start <= i < row_end] + out.extend(kept) + indptr.append(indptr[-1] + len(kept)) + return out, indptr + + +def _attention_reference(q, sources, sink): + """fp32 attention with sink: every query row over the union of its rows in each source.""" + T, H, _ = q.shape + out = torch.zeros(T, H, D, dtype=torch.float32, device=q.device) + for t in range(T): + keys = [] + for kv, (indices, indptr) in sources: + rows = indices[indptr[t] : indptr[t + 1]].long() + keys.append(kv[rows].float()) + k = torch.cat(keys) + s = q[t].float() @ k.T * SCALE + logits = torch.cat([s, sink[:, None].float()], dim=1) + p = torch.softmax(logits, dim=1)[:, :-1] + out[t] = p @ k + return out + + +@unittest.skipUnless(_opus_available(), "gfx950 with aiter's OPUS sparse prefill") +class TestCombinedToCsr(CustomTestCase): + def test_matches_reference(self): + from sglang.kernels.ops.attention.dsv4.opus_sparse_prefill_hip import ( + combined_to_csr, + ) + + gen = torch.Generator(device="cuda").manual_seed(0) + for rows, width, row_start, row_end in [(37, 128, 0, 500), (64, 640, 100, 400)]: + indices, lens = _padded_lists(rows, width, 600, gen) + csr, indptr = combined_to_csr(indices, lens, row_start, row_end) + ref, ref_indptr = _csr_reference(indices, lens, row_start, row_end) + self.assertEqual(indptr.tolist(), ref_indptr) + self.assertEqual(csr[: ref_indptr[-1]].tolist(), ref) + + +@unittest.skipUnless(_opus_available(), "gfx950 with aiter's OPUS sparse prefill") +class TestOpusSparsePrefill(CustomTestCase): + def _case(self, heads, two_sources, gen): + from sglang.kernels.ops.attention.dsv4.opus_sparse_prefill_hip import ( + combined_to_csr, + opus_sparse_prefill, + ) + + T, n_c, n_s = 48, 700, 300 + q = (torch.randn(T, heads, D, generator=gen, device="cuda") * 0.5).to( + torch.bfloat16 + ) + sink = torch.randn(heads, generator=gen, device="cuda") * 0.5 + swa_kv = (torch.randn(n_s, D, generator=gen, device="cuda") * 0.5).to( + torch.bfloat16 + ) + swa_idx, swa_lens = _padded_lists(T, 128, n_s, gen) + # every query row sees at least one key, as in prefill (its own SWA row) + swa_idx[:, 0] = torch.arange(T, device="cuda", dtype=torch.int32) % n_s + swa_lens = swa_lens.clamp(min=1) + swa_csr = combined_to_csr(swa_idx, swa_lens, 0, n_s) + if not two_sources: + out = opus_sparse_prefill(q, swa_kv, swa_csr, sink, SCALE) + ref = _attention_reference(q, [(swa_kv, swa_csr)], sink) + else: + c_kv = (torch.randn(n_c, D, generator=gen, device="cuda") * 0.5).to( + torch.bfloat16 + ) + c_idx, c_lens = _padded_lists(T, 512, n_c, gen) + # a row with no compressed history, like a request's first tokens + c_lens[0] = 0 + c_csr = combined_to_csr(c_idx, c_lens, 0, n_c) + out = opus_sparse_prefill( + q, c_kv, c_csr, sink, SCALE, extend_kv=swa_kv, extend_csr=swa_csr + ) + ref = _attention_reference(q, [(c_kv, c_csr), (swa_kv, swa_csr)], sink) + err = (out.float() - ref).abs().max().item() / ref.abs().max().item() + self.assertLess(err, TOL, (heads, two_sources, err)) + + def test_matches_fp32_reference(self): + gen = torch.Generator(device="cuda").manual_seed(0) + for heads in (32, 16): + for two_sources in (True, False): + self._case(heads, two_sources, gen) + + +@unittest.skipUnless(is_hip() and is_gfx95_supported(), "gfx950") +class TestRawTopkBuffersAreMinusOneFilled(CustomTestCase): + def test_fresh_buffers_hold_no_stale_rows(self): + from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( + DSV4AttnMetadata, + ) + + rows, topk = 1024, 512 + # hand the caching allocator back blocks of valid-looking row indices + junk = torch.full((rows * topk * 4,), 7, dtype=torch.int32, device="cuda") + del junk + + meta = DSV4AttnMetadata.__new__(DSV4AttnMetadata) + meta.index_topk = topk + meta.low_ratios = (1, 2) + lengths = torch.full((rows,), 3, dtype=torch.int32, device="cuda") + meta.c4_topk_lengths_clamp1 = lengths + meta.c4_topk_lengths_raw = lengths + meta.c1_topk_lengths_clamp1 = lengths + meta.c2_topk_lengths_clamp1 = lengths + meta.init_flashmla_related(is_prefill=True) + + # identity rows write only their reachable columns; OPUS reads the rest as rows + for ratio in (1, 2): + raw = meta.sparse_raw_indices(ratio) + self.assertTrue(bool((raw == -1).all()), ratio) + + +if __name__ == "__main__": + unittest.main()