[Perf] dsv41: Use the fused ratio-2 compressor in the prefill graph (#46230)

This commit is contained in:
Liangsheng Yin
2026-10-23 21:33:42 -07:00
committed by GitHub
parent 54df642fe6
commit 7d996300b9
@@ -63,6 +63,7 @@ from sglang.srt.layers.attention.dsv4.compressor_v2 import (
)
from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
_rope_fq4,
fused_low_ratio_compress_supported,
token_req_indices,
)
from sglang.srt.layers.attention.dsv4.indexer import C4IndexerBackendMixin
@@ -303,10 +304,13 @@ def _low_ratio_source_projections(
return
if run_compressor and layer.compressor is not None:
kv, score = layer.compressor.project(x[:real])
put("kv", kv)
if score is not None:
put("score", score)
if layer.compressor.use_fused_gate:
put("kv_score", layer.compressor.project_fused(x[:real]))
else:
kv, score = layer.compressor.project(x[:real])
put("kv", kv)
if score is not None:
put("score", score)
if run_indexer and layer.indexer is not None:
# Only the captured consumers read these; a full top-k layer projects
# once inside its eager break (_index_q_and_weights), so skip them there.
@@ -1053,6 +1057,9 @@ class DSV4Metadata:
# Shared by all low-ratio source layers; graph replay refreshes them live.
low_ratio_req_indices: Optional[torch.Tensor] = None
low_ratio_pos_i64: Optional[torch.Tensor] = None
# The prefill graph's int32 [2, request slots]: each request's first row and row
# count, for the captured ratio-2 compressor. Slots past the batch hold no rows.
low_ratio_extend_offsets: Optional[torch.Tensor] = None
# Each low ratio's index-K slots (scoring.prefill_k_slots), gathered once by the
# extend's metadata builder; the late-layer tail's metadata reads the same dict.
low_ratio_k_slots: Optional[Dict[int, torch.Tensor]] = None
@@ -1106,6 +1113,10 @@ class DSV4Metadata:
maybe_copy_inplace(
self.low_ratio_pos_i64, src=static_metadata.low_ratio_pos_i64
)
maybe_copy_inplace(
self.low_ratio_extend_offsets,
src=static_metadata.low_ratio_extend_offsets,
)
self.low_ratio_k_slots = static_metadata.low_ratio_k_slots
self.low_ratio_index_plans = static_metadata.low_ratio_index_plans
maybe_copy_inplace(
@@ -1687,6 +1698,17 @@ class DeepseekV4AttnBackend(
metadata.low_ratio_pos_i64 = core_attn_metadata.positions_casual.to(
torch.int64
)
# One slot per request the bucket can hold. DP padding extends
# extend_seq_lens past the requests; those rows belong to no slot.
num_reqs = extend_start_loc.shape[0]
offsets = torch.zeros(
(2, min(self.req_to_token.shape[0], padded_num_tokens)),
dtype=torch.int32,
device=extend_seq_lens.device,
)
offsets[0, :num_reqs] = extend_start_loc
offsets[1, :num_reqs] = extend_seq_lens[:num_reqs]
metadata.low_ratio_extend_offsets = offsets
return metadata
@property
@@ -3231,13 +3253,14 @@ class DeepseekV4AttnBackend(
bufs = {
"q": zeros(num_tokens, heads, dim, dtype=torch.bfloat16),
"w": zeros(num_tokens, heads, dtype=torch.bfloat16),
"kv": zeros(
num_tokens,
latent,
dtype=torch.bfloat16 if ratio == 1 else torch.float32,
),
}
if ratio == 2:
if ratio == 1:
bufs["kv"] = zeros(num_tokens, latent, dtype=torch.bfloat16)
elif fused_low_ratio_compress_supported():
# The fused compressor reads | kv | score | rows.
bufs["kv_score"] = zeros(num_tokens, 2 * latent, dtype=torch.float32)
else:
bufs["kv"] = zeros(num_tokens, latent, dtype=torch.float32)
bufs["score"] = zeros(num_tokens, latent, dtype=torch.float32)
# Keep the previous allocations alive: captured graphs still hold them.
sets[ratio] = [bufs] + (have or [])
@@ -3442,9 +3465,19 @@ class DeepseekV4AttnBackend(
layer, x, q_lora, pos, bufs, run_compressor, captured_index
)
if run_compressor and layer.compressor is not None:
self._low_ratio_compress_torch(
layer, x, req, pos, projected=(bufs["kv"], bufs.get("score"))
)
if layer.compressor.use_fused_gate:
self._low_ratio_compress_fused(
layer,
x,
req,
pos,
projected=bufs["kv_score"],
extend_offsets=meta.low_ratio_extend_offsets.unbind(),
)
else:
self._low_ratio_compress_torch(
layer, x, req, pos, projected=(bufs["kv"], bufs.get("score"))
)
if run_indexer and layer.indexer is not None:
if captured_index:
# Captured: sparse scores of the published blocks only, so
@@ -3611,10 +3644,11 @@ class DeepseekV4AttnBackend(
)
def _low_ratio_compress_fused(
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None, projected=None
) -> None:
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
(first row, row count) of each request."""
(first row, row count) of each request. ``projected`` is the compressor's
projection of ``x`` when the caller already has it."""
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
index_k_norm_rope_pack_store,
)
@@ -3643,7 +3677,7 @@ class DeepseekV4AttnBackend(
if layer.compress_ratio == 1:
# At ratio 1, c1_out_loc equals the int64 raw_out_loc supplied by the scheduler.
latent = c1_decode_norm_rope_store(
compressor.wkv(x),
compressor.wkv(x) if projected is None else projected,
compressor.norm.weight.data,
pos,
core.raw_out_loc,
@@ -3659,7 +3693,7 @@ class DeepseekV4AttnBackend(
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
state = pool.get_attention_compress_states(layer_id)
args = (
compressor.project_fused(x),
compressor.project_fused(x) if projected is None else projected,
state.kv_score_buffer.kv_score,
compressor.norm.weight.data,
pos,