From 476fe86a12535173ea0eee1e849f80e350cc6ffd Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Mon, 19 Oct 2026 22:25:04 -0700 Subject: [PATCH] [Perf] deepseek-v4.1: Score a request's verify rows as one request in the full-scan indexer (#45251) Co-authored-by: Claude Fable 5.1 --- .../layers/attention/deepseek_v4_backend.py | 15 ++++- .../srt/layers/attention/dsv4/metadata.py | 23 ++++++-- .../attention/dsv4/v41_indexer/full_topk.py | 14 +++-- .../attention/dsv4/v41_indexer/scoring.py | 15 +++++ .../dsv4/v41_indexer/sparse_table.py | 14 +++-- .../unit/layers/test_dsv4_nonpaged_indexer.py | 59 +++++++++++++++++++ 6 files changed, 121 insertions(+), 19 deletions(-) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 0053836cbe..8453d41b1d 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1413,6 +1413,7 @@ class DeepseekV4AttnBackend( *, compress_ratio: int = 4, use_prefill_cuda_graph: bool = False, + rows_per_request: int = 1, ): page_table = core_attn_metadata.page_table index_page_size = 0 @@ -1450,6 +1451,7 @@ class DeepseekV4AttnBackend( ), use_prefill_cuda_graph=use_prefill_cuda_graph, compress_ratio=compress_ratio, + rows_per_request=rows_per_request, ) def init_forward_metadata_decode( @@ -2242,16 +2244,25 @@ class DeepseekV4AttnBackend( if c128_compress_metadata is None and self.has_c128: c128_compress_metadata = create(compress_ratio=128) low = core_attn_metadata.low_ratios + rows_per_request = 1 if is_ragged else num_draft_tokens return DSV4Metadata( core_attn_metadata, indexer_metadata, c1_indexer_metadata=( - self.init_forward_metadata_indexer(core_attn_metadata, compress_ratio=1) + self.init_forward_metadata_indexer( + core_attn_metadata, + compress_ratio=1, + rows_per_request=rows_per_request, + ) if 1 in low else None ), c2_indexer_metadata=( - self.init_forward_metadata_indexer(core_attn_metadata, compress_ratio=2) + self.init_forward_metadata_indexer( + core_attn_metadata, + compress_ratio=2, + rows_per_request=rows_per_request, + ) if 2 in low else None ), diff --git a/python/sglang/srt/layers/attention/dsv4/metadata.py b/python/sglang/srt/layers/attention/dsv4/metadata.py index bd850d38b3..af39d75f4a 100644 --- a/python/sglang/srt/layers/attention/dsv4/metadata.py +++ b/python/sglang/srt/layers/attention/dsv4/metadata.py @@ -188,6 +188,7 @@ class PagedIndexerMetadata: # Rows per logits chunk for the prefill CUDA graph low-ratio indexer; 0 plans # all rows at once. row_chunk: int = 0 + rows_per_request: int = 1 # A list when the dynamic-budget forward is row-chunked: one schedule per chunk. deep_gemm_metadata: Any = field(init=False, repr=False) topk_metadata: torch.Tensor = field(init=False, repr=False) @@ -211,9 +212,13 @@ class PagedIndexerMetadata: else: import deep_gemm - use_jit_indexer = not self.force_deep_gemm_metadata and ( - envs.SGLANG_OPT_USE_JIT_INDEXER_METADATA.get() - or self.compressed_seq_lens.numel() > _LARGE_INDEXER_QUERY_THRESHOLD + use_jit_indexer = ( + not self.force_deep_gemm_metadata + and self.rows_per_request == 1 + and ( + envs.SGLANG_OPT_USE_JIT_INDEXER_METADATA.get() + or self.compressed_seq_lens.numel() > _LARGE_INDEXER_QUERY_THRESHOLD + ) ) if use_jit_indexer: from sglang.kernels.ops.attention.dsv4 import ( @@ -226,6 +231,11 @@ class PagedIndexerMetadata: if compressed_seq_lens.dim() == 1: compressed_seq_lens = compressed_seq_lens.unsqueeze(-1) num_rows = compressed_seq_lens.shape[0] + group = self.rows_per_request + if group > 1: + assert compressed_seq_lens.shape[1] == 1 and num_rows % group == 0, ( + f"{num_rows} query rows do not group by {group}" + ) self.mqa_logits_budget_bytes = self._mqa_logits_budget(num_rows=num_rows) self.rows_per_chunk = plan_indexer_row_chunks( num_rows=num_rows, @@ -233,6 +243,8 @@ class PagedIndexerMetadata: budget_bytes=self.mqa_logits_budget_bytes, sm120_row_cap=_SM120_INDEXER_M_CHUNK if _IS_SM120 else None, ) + if self.rows_per_chunk is not None and group > 1: + self.rows_per_chunk = max(group, self.rows_per_chunk // group * group) if self.rows_per_chunk is not None: logger.debug( "DSV4 indexer chunks %d query rows x %d compressed cols into " @@ -257,7 +269,7 @@ class PagedIndexerMetadata: # Chunk metadata is shared by all indexer layers in this forward. self.deep_gemm_metadata = [ get_paged_mqa_logits_metadata( - compressed_seq_lens[rows], + compressed_seq_lens[rows].view(-1, group), self.compressed_page_size, deep_gemm.get_num_sms(), ) @@ -267,7 +279,7 @@ class PagedIndexerMetadata: ] else: self.deep_gemm_metadata = get_paged_mqa_logits_metadata( - compressed_seq_lens, + compressed_seq_lens.view(-1, group), self.compressed_page_size, deep_gemm.get_num_sms(), ) @@ -377,6 +389,7 @@ class PagedIndexerMetadata: "force_deep_gemm_metadata", "use_prefill_cuda_graph", "use_topk_v2", + "rows_per_request", ], copy_fields=copy_fields, assign_fields=assign_fields, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py index 883cf05757..04e19803b4 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py @@ -99,12 +99,13 @@ class FullTopKIndexer: topk_plans = metadata.topk_metadata_chunks assert not metadata.use_topk_v2 or topk_plans is not None for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()): + q, lens, table = data.paged_logits_args(metadata, rows) logits = deep_gemm_fp4_paged_mqa_logits( - (data.q_fp4[rows], data.q_sf[rows]), + q, data.k_cache, data.weights[rows], - metadata.compressed_seq_lens[rows], - metadata.page_table[rows], + lens, + table, plan, metadata.max_compressed_seq_len, ) @@ -119,12 +120,13 @@ class FullTopKIndexer: ), ) return + q, lens, table = data.paged_logits_args(metadata) logits = deep_gemm_fp4_paged_mqa_logits( - (data.q_fp4, data.q_sf), + q, data.k_cache, data.weights, - metadata.compressed_seq_lens, - metadata.page_table, + lens, + table, metadata.deep_gemm_metadata, metadata.max_compressed_seq_len, ) diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index a6ddca3f92..9b65fc25c3 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -41,6 +41,21 @@ class DeepGEMMDecodeData(msgspec.Struct, frozen=True): weights: torch.Tensor # [rows, heads] bf16/fp32 head weights k_cache: torch.Tensor # [pages, page_size, 1, 68] uint8, the layer's index-K pool + def paged_logits_args(self, metadata, rows=slice(None)): + """``(q, lengths, page_table)`` for ``deep_gemm_fp4_paged_mqa_logits`` over + ``rows``, ``metadata.rows_per_request`` consecutive rows per request.""" + n = metadata.rows_per_request + q_fp4, q_sf = self.q_fp4[rows], self.q_sf[rows] + lens = metadata.compressed_seq_lens[rows] + table = metadata.page_table[rows] + if n > 1: + heads = q_fp4.shape[2] + q_fp4 = q_fp4.view(-1, n, heads, 64) + q_sf = q_sf.view(-1, n, heads) + lens = lens.view(-1, n) + table = table[::n] + return (q_fp4, q_sf), lens, table + class DeepGEMMPrefillData(msgspec.Struct, frozen=True): """The rows of one request are consecutive; a selection is a flattened-K diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py index cbdca3e415..382da22b22 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py @@ -267,12 +267,13 @@ class SparseTableBackend: seq_lens = metadata.compressed_seq_lens.reshape(-1) if isinstance(metadata.deep_gemm_metadata, list): return self._publish_decode_chunked(inputs, data, seq_lens) + q, lens, table = data.paged_logits_args(metadata) logits = deep_gemm_fp4_paged_mqa_logits( - (data.q_fp4, data.q_sf), + q, data.k_cache, data.weights, - metadata.compressed_seq_lens, - metadata.page_table, + lens, + table, metadata.deep_gemm_metadata, metadata.max_compressed_seq_len, ) @@ -344,12 +345,13 @@ class SparseTableBackend: topk_plans = metadata.topk_metadata_chunks assert not metadata.use_topk_v2 or topk_plans is not None for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()): + q, lens, table = data.paged_logits_args(metadata, rows) logits = deep_gemm_fp4_paged_mqa_logits( - (data.q_fp4[rows], data.q_sf[rows]), + q, data.k_cache, data.weights[rows], - metadata.compressed_seq_lens[rows], - metadata.page_table[rows], + lens, + table, plan, metadata.max_compressed_seq_len, ) diff --git a/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py b/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py index f8f21b4938..9b9e7f277f 100644 --- a/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py +++ b/test/registered/unit/layers/test_dsv4_nonpaged_indexer.py @@ -984,6 +984,7 @@ class TestChunkedPagedDecode(CustomTestCase): compressed_page_size=64, topk_metadata_chunks=self.topk_plans, use_topk_v2=True, + rows_per_request=1, row_chunks=lambda: list(zip(self.chunks, self.plans)), ) inputs = DecodeInputs( @@ -1095,6 +1096,64 @@ class TestChunkedPagedDecode(CustomTestCase): self._assert_chunked_calls(deep_gemm, topk) +class TestGroupedVerifyRows(CustomTestCase): + """Static verify rows reach the paged logits kernel as one request per block.""" + + def _args(self, rows_per_request, rows=slice(None)): + from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import ( + DeepGEMMDecodeData, + ) + + requests, block, heads, pages = 3, 6, 2, 4 + num_rows = requests * block + lens = torch.arange(num_rows, dtype=torch.int32) + 100 + table = torch.arange(requests * pages, dtype=torch.int32).view(requests, pages) + metadata = SimpleNamespace( + compressed_seq_lens=lens, + page_table=table.repeat_interleave(block, dim=0), + rows_per_request=rows_per_request, + ) + data = DeepGEMMDecodeData( + q_fp4=torch.arange(num_rows * heads * 64, dtype=torch.int8).view( + num_rows, 1, heads, 64 + ), + q_sf=torch.arange(num_rows * heads, dtype=torch.int32).view( + num_rows, 1, heads + ), + weights=torch.zeros((num_rows, heads), dtype=torch.float32), + k_cache=torch.zeros((1, 64, 1, 68), dtype=torch.uint8), + ) + return data, metadata, table, data.paged_logits_args(metadata, rows) + + def test_one_row_per_request_keeps_the_per_row_layout(self): + data, metadata, _table, ((q, sf), lens, table) = self._args(1) + self.assertTrue(torch.equal(q, data.q_fp4)) + self.assertTrue(torch.equal(sf, data.q_sf)) + self.assertTrue(torch.equal(lens, metadata.compressed_seq_lens)) + self.assertTrue(torch.equal(table, metadata.page_table)) + + def test_block_rows_become_one_request(self): + data, metadata, per_request_table, ((q, sf), lens, table) = self._args(6) + self.assertEqual(tuple(q.shape), (3, 6, 2, 64)) + self.assertEqual(tuple(sf.shape), (3, 6, 2)) + self.assertTrue(torch.equal(q.reshape(-1, 1, 2, 64), data.q_fp4)) + self.assertTrue(torch.equal(lens, metadata.compressed_seq_lens.view(3, 6))) + self.assertEqual(tuple(table.shape), (3, 4)) + self.assertTrue(torch.equal(table, per_request_table)) + self.assertEqual(table.stride(1), 1) + + def test_row_chunk_groups_its_own_rows(self): + data, metadata, per_request_table, ((q, sf), lens, table) = self._args( + 6, slice(6, 18) + ) + self.assertEqual(tuple(q.shape), (2, 6, 2, 64)) + self.assertTrue(torch.equal(q.reshape(-1, 1, 2, 64), data.q_fp4[6:18])) + self.assertTrue( + torch.equal(lens, metadata.compressed_seq_lens[6:18].view(2, 6)) + ) + self.assertTrue(torch.equal(table, per_request_table[1:])) + + class TestCandidateIndexerGating(CustomTestCase): def test_candidate_indexer_gating(self): from sglang.srt.layers.attention.dsv4 import v41_indexer