[Perf] deepseek-v4.1: Score a request's verify rows as one request in the full-scan indexer (#45251)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-10-19 22:25:04 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 35c873cff4
commit 476fe86a12
6 changed files with 121 additions and 19 deletions
@@ -1413,6 +1413,7 @@ class DeepseekV4AttnBackend(
*,
compress_ratio: int = 4,
use_prefill_cuda_graph: bool = False,
rows_per_request: int = 1,
):
page_table = core_attn_metadata.page_table
index_page_size = 0
@@ -1450,6 +1451,7 @@ class DeepseekV4AttnBackend(
),
use_prefill_cuda_graph=use_prefill_cuda_graph,
compress_ratio=compress_ratio,
rows_per_request=rows_per_request,
)
def init_forward_metadata_decode(
@@ -2242,16 +2244,25 @@ class DeepseekV4AttnBackend(
if c128_compress_metadata is None and self.has_c128:
c128_compress_metadata = create(compress_ratio=128)
low = core_attn_metadata.low_ratios
rows_per_request = 1 if is_ragged else num_draft_tokens
return DSV4Metadata(
core_attn_metadata,
indexer_metadata,
c1_indexer_metadata=(
self.init_forward_metadata_indexer(core_attn_metadata, compress_ratio=1)
self.init_forward_metadata_indexer(
core_attn_metadata,
compress_ratio=1,
rows_per_request=rows_per_request,
)
if 1 in low
else None
),
c2_indexer_metadata=(
self.init_forward_metadata_indexer(core_attn_metadata, compress_ratio=2)
self.init_forward_metadata_indexer(
core_attn_metadata,
compress_ratio=2,
rows_per_request=rows_per_request,
)
if 2 in low
else None
),
@@ -188,6 +188,7 @@ class PagedIndexerMetadata:
# Rows per logits chunk for the prefill CUDA graph low-ratio indexer; 0 plans
# all rows at once.
row_chunk: int = 0
rows_per_request: int = 1
# A list when the dynamic-budget forward is row-chunked: one schedule per chunk.
deep_gemm_metadata: Any = field(init=False, repr=False)
topk_metadata: torch.Tensor = field(init=False, repr=False)
@@ -211,9 +212,13 @@ class PagedIndexerMetadata:
else:
import deep_gemm
use_jit_indexer = not self.force_deep_gemm_metadata and (
envs.SGLANG_OPT_USE_JIT_INDEXER_METADATA.get()
or self.compressed_seq_lens.numel() > _LARGE_INDEXER_QUERY_THRESHOLD
use_jit_indexer = (
not self.force_deep_gemm_metadata
and self.rows_per_request == 1
and (
envs.SGLANG_OPT_USE_JIT_INDEXER_METADATA.get()
or self.compressed_seq_lens.numel() > _LARGE_INDEXER_QUERY_THRESHOLD
)
)
if use_jit_indexer:
from sglang.kernels.ops.attention.dsv4 import (
@@ -226,6 +231,11 @@ class PagedIndexerMetadata:
if compressed_seq_lens.dim() == 1:
compressed_seq_lens = compressed_seq_lens.unsqueeze(-1)
num_rows = compressed_seq_lens.shape[0]
group = self.rows_per_request
if group > 1:
assert compressed_seq_lens.shape[1] == 1 and num_rows % group == 0, (
f"{num_rows} query rows do not group by {group}"
)
self.mqa_logits_budget_bytes = self._mqa_logits_budget(num_rows=num_rows)
self.rows_per_chunk = plan_indexer_row_chunks(
num_rows=num_rows,
@@ -233,6 +243,8 @@ class PagedIndexerMetadata:
budget_bytes=self.mqa_logits_budget_bytes,
sm120_row_cap=_SM120_INDEXER_M_CHUNK if _IS_SM120 else None,
)
if self.rows_per_chunk is not None and group > 1:
self.rows_per_chunk = max(group, self.rows_per_chunk // group * group)
if self.rows_per_chunk is not None:
logger.debug(
"DSV4 indexer chunks %d query rows x %d compressed cols into "
@@ -257,7 +269,7 @@ class PagedIndexerMetadata:
# Chunk metadata is shared by all indexer layers in this forward.
self.deep_gemm_metadata = [
get_paged_mqa_logits_metadata(
compressed_seq_lens[rows],
compressed_seq_lens[rows].view(-1, group),
self.compressed_page_size,
deep_gemm.get_num_sms(),
)
@@ -267,7 +279,7 @@ class PagedIndexerMetadata:
]
else:
self.deep_gemm_metadata = get_paged_mqa_logits_metadata(
compressed_seq_lens,
compressed_seq_lens.view(-1, group),
self.compressed_page_size,
deep_gemm.get_num_sms(),
)
@@ -377,6 +389,7 @@ class PagedIndexerMetadata:
"force_deep_gemm_metadata",
"use_prefill_cuda_graph",
"use_topk_v2",
"rows_per_request",
],
copy_fields=copy_fields,
assign_fields=assign_fields,
@@ -99,12 +99,13 @@ class FullTopKIndexer:
topk_plans = metadata.topk_metadata_chunks
assert not metadata.use_topk_v2 or topk_plans is not None
for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()):
q, lens, table = data.paged_logits_args(metadata, rows)
logits = deep_gemm_fp4_paged_mqa_logits(
(data.q_fp4[rows], data.q_sf[rows]),
q,
data.k_cache,
data.weights[rows],
metadata.compressed_seq_lens[rows],
metadata.page_table[rows],
lens,
table,
plan,
metadata.max_compressed_seq_len,
)
@@ -119,12 +120,13 @@ class FullTopKIndexer:
),
)
return
q, lens, table = data.paged_logits_args(metadata)
logits = deep_gemm_fp4_paged_mqa_logits(
(data.q_fp4, data.q_sf),
q,
data.k_cache,
data.weights,
metadata.compressed_seq_lens,
metadata.page_table,
lens,
table,
metadata.deep_gemm_metadata,
metadata.max_compressed_seq_len,
)
@@ -41,6 +41,21 @@ class DeepGEMMDecodeData(msgspec.Struct, frozen=True):
weights: torch.Tensor # [rows, heads] bf16/fp32 head weights
k_cache: torch.Tensor # [pages, page_size, 1, 68] uint8, the layer's index-K pool
def paged_logits_args(self, metadata, rows=slice(None)):
"""``(q, lengths, page_table)`` for ``deep_gemm_fp4_paged_mqa_logits`` over
``rows``, ``metadata.rows_per_request`` consecutive rows per request."""
n = metadata.rows_per_request
q_fp4, q_sf = self.q_fp4[rows], self.q_sf[rows]
lens = metadata.compressed_seq_lens[rows]
table = metadata.page_table[rows]
if n > 1:
heads = q_fp4.shape[2]
q_fp4 = q_fp4.view(-1, n, heads, 64)
q_sf = q_sf.view(-1, n, heads)
lens = lens.view(-1, n)
table = table[::n]
return (q_fp4, q_sf), lens, table
class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
"""The rows of one request are consecutive; a selection is a flattened-K
@@ -267,12 +267,13 @@ class SparseTableBackend:
seq_lens = metadata.compressed_seq_lens.reshape(-1)
if isinstance(metadata.deep_gemm_metadata, list):
return self._publish_decode_chunked(inputs, data, seq_lens)
q, lens, table = data.paged_logits_args(metadata)
logits = deep_gemm_fp4_paged_mqa_logits(
(data.q_fp4, data.q_sf),
q,
data.k_cache,
data.weights,
metadata.compressed_seq_lens,
metadata.page_table,
lens,
table,
metadata.deep_gemm_metadata,
metadata.max_compressed_seq_len,
)
@@ -344,12 +345,13 @@ class SparseTableBackend:
topk_plans = metadata.topk_metadata_chunks
assert not metadata.use_topk_v2 or topk_plans is not None
for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()):
q, lens, table = data.paged_logits_args(metadata, rows)
logits = deep_gemm_fp4_paged_mqa_logits(
(data.q_fp4[rows], data.q_sf[rows]),
q,
data.k_cache,
data.weights[rows],
metadata.compressed_seq_lens[rows],
metadata.page_table[rows],
lens,
table,
plan,
metadata.max_compressed_seq_len,
)