[Perf] deepseek-v4.1: Skip wasted layer 20 work on earlier prompt tokens under decoder bounded replay (#42901)
This commit is contained in:
@@ -259,7 +259,9 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
flashmla_metadata.num_splits = num_splits
|
||||
|
||||
|
||||
def _low_ratio_source_projections(layer, x, q_lora, positions, bufs):
|
||||
def _low_ratio_source_projections(
|
||||
layer, x, q_lora, positions, bufs, run_compressor, run_indexer
|
||||
):
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
|
||||
get_tc_piecewise_forward_context,
|
||||
)
|
||||
@@ -284,12 +286,12 @@ def _low_ratio_source_projections(layer, x, q_lora, positions, bufs):
|
||||
buf.zero_()
|
||||
return
|
||||
|
||||
if layer.compressor is not None:
|
||||
if run_compressor and layer.compressor is not None:
|
||||
kv, score = layer.compressor.project(x[:real])
|
||||
put("kv", kv)
|
||||
if score is not None:
|
||||
put("score", score)
|
||||
if layer.indexer is not None:
|
||||
if run_indexer and layer.indexer is not None:
|
||||
indexer = layer.indexer
|
||||
put("q", indexer.queries(q_lora[:real], layer.freqs_cis[positions[:real]]))
|
||||
put("w", indexer.head_weights(x[:real]))
|
||||
@@ -908,8 +910,8 @@ class DSV4AttnMetadata:
|
||||
|
||||
|
||||
class LateLayerTail(msgspec.Struct, frozen=True):
|
||||
"""Rows the layers after the last kv_source layer run over under decoder SWA
|
||||
bounded replay: the last tail tokens of each request in the extend."""
|
||||
"""Rows the layers from late_layer_start on run over under decoder SWA bounded
|
||||
replay: the last tail tokens of each request in the extend."""
|
||||
|
||||
token_indices: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
@@ -1238,6 +1240,8 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
# The model switches onto this metadata in enter_late_layer_tail.
|
||||
self.tail_forward_metadata: Optional[DSV4Metadata] = None
|
||||
# Set by write_tail_source_kv until exit_late_layer_tail.
|
||||
self.tail_source_kv_layer_id: Optional[int] = None
|
||||
# Prefill CUDA graph tail block (bounded replay inside the graph): row
|
||||
# capacity, persistent index/location buffers, held metadata per block size.
|
||||
self._bcg_tail_capacity: Optional[int] = None
|
||||
@@ -1940,6 +1944,22 @@ class DeepseekV4AttnBackend(
|
||||
pad_rows=pad_rows,
|
||||
)
|
||||
|
||||
def write_tail_source_kv(
|
||||
self, *, layer, x, positions, forward_batch: ForwardBatch
|
||||
) -> None:
|
||||
"""Write every row's compressed KV for the source layer the tail starts at,
|
||||
before enter_late_layer_tail; the layer's own pass inside the tail then
|
||||
skips its compressor."""
|
||||
self.forward_low_ratio_sources(
|
||||
layer=layer,
|
||||
x=x,
|
||||
q_lora=None,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
run_indexer=False,
|
||||
)
|
||||
self.tail_source_kv_layer_id = layer.layer_id
|
||||
|
||||
def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple:
|
||||
"""Switch the late layers onto the tail; the return value goes back to
|
||||
exit_late_layer_tail."""
|
||||
@@ -1969,31 +1989,7 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
forward_batch.global_num_token_non_padded = bound
|
||||
forward_batch.num_token_non_padded = bound
|
||||
# The layers before the switch published top-k into the full metadata's
|
||||
# buffers; carry the tail rows into the tail metadata's (padding stays -1).
|
||||
full_core = saved[0].core_attn_metadata
|
||||
tail_core = tail_metadata.core_attn_metadata
|
||||
for ratio in tail_core.low_ratios:
|
||||
for full_buf, tail_buf in (
|
||||
(
|
||||
full_core.sparse_page_indices(ratio),
|
||||
tail_core.sparse_page_indices(ratio),
|
||||
),
|
||||
(
|
||||
full_core.sparse_topk_lengths(ratio),
|
||||
tail_core.sparse_topk_lengths(ratio),
|
||||
),
|
||||
(
|
||||
full_core.sparse_raw_indices(ratio),
|
||||
tail_core.sparse_raw_indices(ratio),
|
||||
),
|
||||
):
|
||||
if full_buf is None or tail_buf is None:
|
||||
continue
|
||||
rows = tail.real_rows(full_buf)
|
||||
if tail.real_rows_count is not None:
|
||||
rows = rows[: tail.real_rows_count]
|
||||
tail_buf[: rows.shape[0]].copy_(rows)
|
||||
self.forward_metadata = tail_metadata
|
||||
if self.token_to_kv_pool.request_window is not None:
|
||||
self.token_to_kv_pool.request_window.activate(
|
||||
@@ -2017,6 +2013,7 @@ class DeepseekV4AttnBackend(
|
||||
forward_batch.global_num_token_non_padded_cpu = non_padded
|
||||
forward_batch.global_num_token_non_padded = non_padded_gpu
|
||||
forward_batch.num_token_non_padded = local_non_padded
|
||||
self.tail_source_kv_layer_id = None
|
||||
|
||||
# The prefill CUDA graph runs the tail switch as eager breaks: these two run
|
||||
# at capture and again at every replay (see deepseek_v4.bcg_enter/exit).
|
||||
@@ -3108,6 +3105,9 @@ class DeepseekV4AttnBackend(
|
||||
return
|
||||
if forward_batch.encoder_swa_replay:
|
||||
run_compressor = False
|
||||
if layer.layer_id == self.tail_source_kv_layer_id:
|
||||
run_compressor = False
|
||||
meta = self.forward_metadata
|
||||
if dsa_use_prefill_cp(forward_batch) and forward_batch.forward_mode.is_extend():
|
||||
self._forward_low_ratio_sources_cp(
|
||||
layer=layer,
|
||||
@@ -3119,7 +3119,6 @@ class DeepseekV4AttnBackend(
|
||||
run_indexer=run_indexer,
|
||||
)
|
||||
return
|
||||
meta = self.forward_metadata
|
||||
hoisted_req = getattr(meta, "low_ratio_req_indices", None)
|
||||
hoisted_pos = getattr(meta, "low_ratio_pos_i64", None)
|
||||
if (
|
||||
@@ -3138,7 +3137,9 @@ class DeepseekV4AttnBackend(
|
||||
and self._low_ratio_in_prefill_graph()
|
||||
):
|
||||
bufs = self._source_projection_buffers(x.shape[0], layer.compress_ratio)
|
||||
_bcg_low_ratio_source_projections(layer, x, q_lora, pos, bufs)
|
||||
_bcg_low_ratio_source_projections(
|
||||
layer, x, q_lora, pos, bufs, run_compressor, run_indexer
|
||||
)
|
||||
if run_compressor and layer.compressor is not None:
|
||||
self._low_ratio_compress_torch(
|
||||
layer, x, req, pos, projected=(bufs["kv"], bufs.get("score"))
|
||||
@@ -3680,15 +3681,6 @@ class DeepseekV4AttnBackend(
|
||||
if published is None:
|
||||
return
|
||||
self.forward_metadata.candidate_metadata = published
|
||||
tail_metadata = self.tail_forward_metadata
|
||||
if tail_metadata is None or tail_metadata is self.forward_metadata:
|
||||
return
|
||||
tail = tail_metadata.late_layer_tail
|
||||
tail_metadata.candidate_metadata = published.tail(
|
||||
tail.local_lens_cpu
|
||||
if tail.cp_metadata is not None
|
||||
else tail.extend_seq_lens_cpu
|
||||
)
|
||||
|
||||
def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
"""Idle always re-translates at store time: its metadata may be stale, and
|
||||
|
||||
@@ -973,6 +973,9 @@ class DeepseekV4HipRadixBackend(
|
||||
)
|
||||
# the model switches onto it after the last kv_source layer (enter_late_layer_tail)
|
||||
self.tail_forward_metadata: Optional[DSV4Metadata] = None
|
||||
# Read by the shared forward_low_ratio_sources; the ROCm tail starts after
|
||||
# the source layer, so nothing sets it.
|
||||
self.tail_source_kv_layer_id: Optional[int] = None
|
||||
self.encoder_replay = False
|
||||
self.topk = get_spec().speculative_eagle_topk or 0
|
||||
assert self.topk in [0, 1], "MTP Topk > 1 not supported for DeepSeek V4"
|
||||
|
||||
@@ -3652,16 +3652,38 @@ class DeepseekV4Model(nn.Module):
|
||||
|
||||
self.dspark_layers_to_capture: Optional[List[int]] = None
|
||||
|
||||
# Decoder SWA bounded replay: layers past the last kv_source layer run over
|
||||
# each request's last SWA_WINDOW extend tokens only.
|
||||
# Decoder SWA bounded replay: layers from late_layer_start on run over each
|
||||
# request's last SWA_WINDOW extend tokens only.
|
||||
self.late_layer_start: Optional[int] = None
|
||||
# The tail starts at the last kv_source layer, whose compressed KV is first
|
||||
# written for every row (DeepseekV4AttnBackend.write_tail_source_kv).
|
||||
self.tail_writes_source_kv = False
|
||||
if get_exec().features.enable_decoder_swa_bounded_replay:
|
||||
assert config.kv_source_layer_ids, (
|
||||
"decoder SWA bounded replay needs kv_source_layer_ids"
|
||||
)
|
||||
self.late_layer_start = max(config.kv_source_layer_ids) + 1
|
||||
last_source = max(config.kv_source_layer_ids)
|
||||
# ROCm starts the tail after the source layer: the gfx950 fused boundary
|
||||
# runs a whole layer in one call, and the ROCm backend carries the source
|
||||
# layer's top-k and candidates into the tail itself.
|
||||
self.tail_writes_source_kv = not _is_hip
|
||||
self.late_layer_start = (
|
||||
last_source if self.tail_writes_source_kv else last_source + 1
|
||||
)
|
||||
if self.tail_writes_source_kv:
|
||||
assert self.layers[last_source].engram is None, (
|
||||
"the last kv_source layer must not carry an Engram"
|
||||
)
|
||||
# The tail layers read the top-k and candidates the source layer
|
||||
# publishes inside the tail; no earlier layer carries them over.
|
||||
assert (
|
||||
last_source in config.index_source_layer_ids
|
||||
and config.candidate_source_layer_id in (-1, last_source)
|
||||
), (
|
||||
"the last kv_source layer must also be the index and candidate source"
|
||||
)
|
||||
late_ratios = set(
|
||||
config.compress_ratios[self.late_layer_start : config.num_hidden_layers]
|
||||
config.compress_ratios[last_source + 1 : config.num_hidden_layers]
|
||||
)
|
||||
assert late_ratios <= {
|
||||
0,
|
||||
@@ -3719,7 +3741,7 @@ class DeepseekV4Model(nn.Module):
|
||||
def _check_late_layer_tail_readers(
|
||||
self, forward_batch: ForwardBatch, tail: LateLayerTail
|
||||
) -> None:
|
||||
# Rows outside the tail are never computed past the last kv_source layer;
|
||||
# Rows outside the tail are never computed from late_layer_start on;
|
||||
# late_layer_whole_extend gives the requests that read them a whole tail.
|
||||
extend_lens = forward_batch.extend_seq_lens_cpu
|
||||
if (
|
||||
@@ -3840,6 +3862,14 @@ class DeepseekV4Model(nn.Module):
|
||||
hidden_states, forward_batch, next=next_boundary
|
||||
)
|
||||
if tail is not None and i == self.late_layer_start:
|
||||
if self.tail_writes_source_kv:
|
||||
source = self.layers[i]
|
||||
attn_backend.write_tail_source_kv(
|
||||
layer=source.self_attn,
|
||||
x=mhc.combine(source.attn_hc, state),
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
# Decode reaches back at most SWA_WINDOW positions.
|
||||
if tail_in_graph:
|
||||
# Eager break: re-run at replay so the backend swaps onto the
|
||||
|
||||
Reference in New Issue
Block a user