[Perf] deepseek-v4.1: Score a request's verify rows as one request in the full-scan indexer (#45251)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
35c873cff4
commit
476fe86a12
@@ -1413,6 +1413,7 @@ class DeepseekV4AttnBackend(
|
||||
*,
|
||||
compress_ratio: int = 4,
|
||||
use_prefill_cuda_graph: bool = False,
|
||||
rows_per_request: int = 1,
|
||||
):
|
||||
page_table = core_attn_metadata.page_table
|
||||
index_page_size = 0
|
||||
@@ -1450,6 +1451,7 @@ class DeepseekV4AttnBackend(
|
||||
),
|
||||
use_prefill_cuda_graph=use_prefill_cuda_graph,
|
||||
compress_ratio=compress_ratio,
|
||||
rows_per_request=rows_per_request,
|
||||
)
|
||||
|
||||
def init_forward_metadata_decode(
|
||||
@@ -2242,16 +2244,25 @@ class DeepseekV4AttnBackend(
|
||||
if c128_compress_metadata is None and self.has_c128:
|
||||
c128_compress_metadata = create(compress_ratio=128)
|
||||
low = core_attn_metadata.low_ratios
|
||||
rows_per_request = 1 if is_ragged else num_draft_tokens
|
||||
return DSV4Metadata(
|
||||
core_attn_metadata,
|
||||
indexer_metadata,
|
||||
c1_indexer_metadata=(
|
||||
self.init_forward_metadata_indexer(core_attn_metadata, compress_ratio=1)
|
||||
self.init_forward_metadata_indexer(
|
||||
core_attn_metadata,
|
||||
compress_ratio=1,
|
||||
rows_per_request=rows_per_request,
|
||||
)
|
||||
if 1 in low
|
||||
else None
|
||||
),
|
||||
c2_indexer_metadata=(
|
||||
self.init_forward_metadata_indexer(core_attn_metadata, compress_ratio=2)
|
||||
self.init_forward_metadata_indexer(
|
||||
core_attn_metadata,
|
||||
compress_ratio=2,
|
||||
rows_per_request=rows_per_request,
|
||||
)
|
||||
if 2 in low
|
||||
else None
|
||||
),
|
||||
|
||||
@@ -188,6 +188,7 @@ class PagedIndexerMetadata:
|
||||
# Rows per logits chunk for the prefill CUDA graph low-ratio indexer; 0 plans
|
||||
# all rows at once.
|
||||
row_chunk: int = 0
|
||||
rows_per_request: int = 1
|
||||
# A list when the dynamic-budget forward is row-chunked: one schedule per chunk.
|
||||
deep_gemm_metadata: Any = field(init=False, repr=False)
|
||||
topk_metadata: torch.Tensor = field(init=False, repr=False)
|
||||
@@ -211,9 +212,13 @@ class PagedIndexerMetadata:
|
||||
else:
|
||||
import deep_gemm
|
||||
|
||||
use_jit_indexer = not self.force_deep_gemm_metadata and (
|
||||
envs.SGLANG_OPT_USE_JIT_INDEXER_METADATA.get()
|
||||
or self.compressed_seq_lens.numel() > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
use_jit_indexer = (
|
||||
not self.force_deep_gemm_metadata
|
||||
and self.rows_per_request == 1
|
||||
and (
|
||||
envs.SGLANG_OPT_USE_JIT_INDEXER_METADATA.get()
|
||||
or self.compressed_seq_lens.numel() > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
)
|
||||
)
|
||||
if use_jit_indexer:
|
||||
from sglang.kernels.ops.attention.dsv4 import (
|
||||
@@ -226,6 +231,11 @@ class PagedIndexerMetadata:
|
||||
if compressed_seq_lens.dim() == 1:
|
||||
compressed_seq_lens = compressed_seq_lens.unsqueeze(-1)
|
||||
num_rows = compressed_seq_lens.shape[0]
|
||||
group = self.rows_per_request
|
||||
if group > 1:
|
||||
assert compressed_seq_lens.shape[1] == 1 and num_rows % group == 0, (
|
||||
f"{num_rows} query rows do not group by {group}"
|
||||
)
|
||||
self.mqa_logits_budget_bytes = self._mqa_logits_budget(num_rows=num_rows)
|
||||
self.rows_per_chunk = plan_indexer_row_chunks(
|
||||
num_rows=num_rows,
|
||||
@@ -233,6 +243,8 @@ class PagedIndexerMetadata:
|
||||
budget_bytes=self.mqa_logits_budget_bytes,
|
||||
sm120_row_cap=_SM120_INDEXER_M_CHUNK if _IS_SM120 else None,
|
||||
)
|
||||
if self.rows_per_chunk is not None and group > 1:
|
||||
self.rows_per_chunk = max(group, self.rows_per_chunk // group * group)
|
||||
if self.rows_per_chunk is not None:
|
||||
logger.debug(
|
||||
"DSV4 indexer chunks %d query rows x %d compressed cols into "
|
||||
@@ -257,7 +269,7 @@ class PagedIndexerMetadata:
|
||||
# Chunk metadata is shared by all indexer layers in this forward.
|
||||
self.deep_gemm_metadata = [
|
||||
get_paged_mqa_logits_metadata(
|
||||
compressed_seq_lens[rows],
|
||||
compressed_seq_lens[rows].view(-1, group),
|
||||
self.compressed_page_size,
|
||||
deep_gemm.get_num_sms(),
|
||||
)
|
||||
@@ -267,7 +279,7 @@ class PagedIndexerMetadata:
|
||||
]
|
||||
else:
|
||||
self.deep_gemm_metadata = get_paged_mqa_logits_metadata(
|
||||
compressed_seq_lens,
|
||||
compressed_seq_lens.view(-1, group),
|
||||
self.compressed_page_size,
|
||||
deep_gemm.get_num_sms(),
|
||||
)
|
||||
@@ -377,6 +389,7 @@ class PagedIndexerMetadata:
|
||||
"force_deep_gemm_metadata",
|
||||
"use_prefill_cuda_graph",
|
||||
"use_topk_v2",
|
||||
"rows_per_request",
|
||||
],
|
||||
copy_fields=copy_fields,
|
||||
assign_fields=assign_fields,
|
||||
|
||||
@@ -99,12 +99,13 @@ class FullTopKIndexer:
|
||||
topk_plans = metadata.topk_metadata_chunks
|
||||
assert not metadata.use_topk_v2 or topk_plans is not None
|
||||
for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()):
|
||||
q, lens, table = data.paged_logits_args(metadata, rows)
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(data.q_fp4[rows], data.q_sf[rows]),
|
||||
q,
|
||||
data.k_cache,
|
||||
data.weights[rows],
|
||||
metadata.compressed_seq_lens[rows],
|
||||
metadata.page_table[rows],
|
||||
lens,
|
||||
table,
|
||||
plan,
|
||||
metadata.max_compressed_seq_len,
|
||||
)
|
||||
@@ -119,12 +120,13 @@ class FullTopKIndexer:
|
||||
),
|
||||
)
|
||||
return
|
||||
q, lens, table = data.paged_logits_args(metadata)
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(data.q_fp4, data.q_sf),
|
||||
q,
|
||||
data.k_cache,
|
||||
data.weights,
|
||||
metadata.compressed_seq_lens,
|
||||
metadata.page_table,
|
||||
lens,
|
||||
table,
|
||||
metadata.deep_gemm_metadata,
|
||||
metadata.max_compressed_seq_len,
|
||||
)
|
||||
|
||||
@@ -41,6 +41,21 @@ class DeepGEMMDecodeData(msgspec.Struct, frozen=True):
|
||||
weights: torch.Tensor # [rows, heads] bf16/fp32 head weights
|
||||
k_cache: torch.Tensor # [pages, page_size, 1, 68] uint8, the layer's index-K pool
|
||||
|
||||
def paged_logits_args(self, metadata, rows=slice(None)):
|
||||
"""``(q, lengths, page_table)`` for ``deep_gemm_fp4_paged_mqa_logits`` over
|
||||
``rows``, ``metadata.rows_per_request`` consecutive rows per request."""
|
||||
n = metadata.rows_per_request
|
||||
q_fp4, q_sf = self.q_fp4[rows], self.q_sf[rows]
|
||||
lens = metadata.compressed_seq_lens[rows]
|
||||
table = metadata.page_table[rows]
|
||||
if n > 1:
|
||||
heads = q_fp4.shape[2]
|
||||
q_fp4 = q_fp4.view(-1, n, heads, 64)
|
||||
q_sf = q_sf.view(-1, n, heads)
|
||||
lens = lens.view(-1, n)
|
||||
table = table[::n]
|
||||
return (q_fp4, q_sf), lens, table
|
||||
|
||||
|
||||
class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
|
||||
"""The rows of one request are consecutive; a selection is a flattened-K
|
||||
|
||||
@@ -267,12 +267,13 @@ class SparseTableBackend:
|
||||
seq_lens = metadata.compressed_seq_lens.reshape(-1)
|
||||
if isinstance(metadata.deep_gemm_metadata, list):
|
||||
return self._publish_decode_chunked(inputs, data, seq_lens)
|
||||
q, lens, table = data.paged_logits_args(metadata)
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(data.q_fp4, data.q_sf),
|
||||
q,
|
||||
data.k_cache,
|
||||
data.weights,
|
||||
metadata.compressed_seq_lens,
|
||||
metadata.page_table,
|
||||
lens,
|
||||
table,
|
||||
metadata.deep_gemm_metadata,
|
||||
metadata.max_compressed_seq_len,
|
||||
)
|
||||
@@ -344,12 +345,13 @@ class SparseTableBackend:
|
||||
topk_plans = metadata.topk_metadata_chunks
|
||||
assert not metadata.use_topk_v2 or topk_plans is not None
|
||||
for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()):
|
||||
q, lens, table = data.paged_logits_args(metadata, rows)
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(data.q_fp4[rows], data.q_sf[rows]),
|
||||
q,
|
||||
data.k_cache,
|
||||
data.weights[rows],
|
||||
metadata.compressed_seq_lens[rows],
|
||||
metadata.page_table[rows],
|
||||
lens,
|
||||
table,
|
||||
plan,
|
||||
metadata.max_compressed_seq_len,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user