[Fix] [NVIDIA] deepseek-v4.1: Fix the breakable prefill CUDA graph in the late-layer tail on SM90 (#45407)
Co-authored-by: JohnZhao01 <325926370+JohnZhao01@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
JohnZhao01
Claude Opus 5.5
hnyls2002
parent
a95ae5f1c5
commit
26396bf566
@@ -1950,15 +1950,16 @@ class DeepseekV4AttnBackend(
|
||||
) -> None:
|
||||
"""Write every row's compressed KV for the source layer the tail starts at,
|
||||
before enter_late_layer_tail; the layer's own pass inside the tail then
|
||||
skips its compressor."""
|
||||
self.forward_low_ratio_sources(
|
||||
layer=layer,
|
||||
x=x,
|
||||
q_lora=None,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
run_indexer=False,
|
||||
)
|
||||
skips its compressor, also on a rank with no rows to write."""
|
||||
if x.shape[0] > 0:
|
||||
self.forward_low_ratio_sources(
|
||||
layer=layer,
|
||||
x=x,
|
||||
q_lora=None,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
run_indexer=False,
|
||||
)
|
||||
self.tail_source_kv_layer_id = layer.layer_id
|
||||
|
||||
def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple:
|
||||
@@ -3211,23 +3212,18 @@ class DeepseekV4AttnBackend(
|
||||
return
|
||||
hoisted_req = getattr(meta, "low_ratio_req_indices", None)
|
||||
hoisted_pos = getattr(meta, "low_ratio_pos_i64", None)
|
||||
if hoisted_pos is not None and hoisted_pos.shape[0] < positions.shape[0]:
|
||||
# DP attention pads an eager extend past the rows extend_seq_lens cover.
|
||||
num_real = hoisted_pos.shape[0]
|
||||
x, positions = x[:num_real], positions[:num_real]
|
||||
if q_lora is not None:
|
||||
q_lora = q_lora[:num_real]
|
||||
if (
|
||||
hoisted_req is not None
|
||||
and hoisted_pos is not None
|
||||
and hoisted_pos.shape[0] == positions.shape[0]
|
||||
):
|
||||
# Bucket-sized under the prefill graph; an eager break sees the
|
||||
# live rows only and falls through.
|
||||
req, pos = hoisted_req, hoisted_pos
|
||||
else:
|
||||
if hoisted_pos is None:
|
||||
req = token_req_indices(forward_batch, num_tokens=positions.shape[0])
|
||||
pos = positions
|
||||
else:
|
||||
# The hoisted rows list the live rows first. DP attention pads an eager
|
||||
# extend past them; an eager break in the late-layer tail sees fewer, and
|
||||
# there extend_seq_lens still spans the whole extend.
|
||||
n = min(hoisted_pos.shape[0], positions.shape[0])
|
||||
x, positions = x[:n], positions[:n]
|
||||
if q_lora is not None:
|
||||
q_lora = q_lora[:n]
|
||||
req, pos = hoisted_req[:n], hoisted_pos[:n]
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and self._low_ratio_in_prefill_graph()
|
||||
|
||||
@@ -849,6 +849,34 @@ bcg_deepseek_v4_low_ratio_sources = eager_on_graph(
|
||||
)
|
||||
|
||||
|
||||
def _low_ratio_sources_break(attn_backend, forward_batch) -> bool:
|
||||
"""Whether the ratio-1/2 sources run as an eager break: inside a breakable graph
|
||||
without the in-graph low-ratio path, or in a decoder tail graph."""
|
||||
return (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and is_in_breakable_cuda_graph()
|
||||
and (
|
||||
not getattr(attn_backend, "low_ratio_prefill_graph", False)
|
||||
or in_replay_tail_graph()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def deepseek_v4_write_tail_source_kv(layer, x, positions) -> None:
|
||||
# Like the sources above: the compressor syncs with the host.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
real_num_tokens = forward_batch.global_num_token_non_padded_cpu
|
||||
get_attn_backend().write_tail_source_kv(
|
||||
layer=layer,
|
||||
x=x[:real_num_tokens],
|
||||
positions=positions[:real_num_tokens],
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
|
||||
bcg_deepseek_v4_write_tail_source_kv = eager_on_graph(deepseek_v4_write_tail_source_kv)
|
||||
|
||||
|
||||
def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
# The hasher reads per-request rows, so it cannot run inside the CUDA graph.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
@@ -2425,14 +2453,7 @@ class MQALayer(MqaAttentionBase):
|
||||
if self.compress_ratio in (1, 2) and (
|
||||
self.compressor is not None or self.indexer is not None
|
||||
):
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and is_in_breakable_cuda_graph()
|
||||
and (
|
||||
not getattr(attn_backend, "low_ratio_prefill_graph", False)
|
||||
or in_replay_tail_graph()
|
||||
)
|
||||
):
|
||||
if _low_ratio_sources_break(attn_backend, forward_batch):
|
||||
bcg_deepseek_v4_low_ratio_sources(self, x, q_lora, positions)
|
||||
else:
|
||||
attn_backend.forward_low_ratio_sources(
|
||||
@@ -3975,12 +3996,17 @@ class DeepseekV4Model(nn.Module):
|
||||
source_x = fused_boundary_attn_input(source, state)
|
||||
else:
|
||||
source_x = mhc.combine(source.attn_hc, state)
|
||||
attn_backend.write_tail_source_kv(
|
||||
layer=source.self_attn,
|
||||
x=source_x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if _low_ratio_sources_break(attn_backend, forward_batch):
|
||||
bcg_deepseek_v4_write_tail_source_kv(
|
||||
source.self_attn, source_x, positions
|
||||
)
|
||||
else:
|
||||
attn_backend.write_tail_source_kv(
|
||||
layer=source.self_attn,
|
||||
x=source_x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
# Decode reaches back at most SWA_WINDOW positions.
|
||||
if tail_in_graph:
|
||||
# Eager break: re-run at replay so the backend swaps onto the
|
||||
|
||||
Reference in New Issue
Block a user