[Fix] deepseek-v4.1: Keep the padded query buffer inside decoder tail graphs (#45301)

This commit is contained in:
Liangsheng Yin
2026-10-19 22:41:53 -07:00
committed by GitHub
parent 476fe86a12
commit 472f0d540b
3 changed files with 38 additions and 17 deletions
@@ -1046,6 +1046,8 @@ class DSV4Metadata:
# Per-step scratch for TP-padded query heads, zeroed by the first user.
# Later layers overwrite real heads and preserve the zero padding.
q_pad_buffer: Optional[torch.Tensor] = None
# Set once a CUDA graph captures q_pad_buffer; it may not grow after that.
q_pad_captured: bool = False
# What the candidate-source layer published for the index-source layers after
# it, in the implementation's own type; never copied from the host.
@@ -691,6 +691,8 @@ class DSV4Metadata:
low_ratio_pos_i64: Optional[torch.Tensor] = None
# Per-step scratch for the TP-padded query heads (models/deepseek_v4.py).
q_pad_buffer: Optional[torch.Tensor] = None
# Set once a CUDA graph captures q_pad_buffer; it may not grow after that.
q_pad_captured: bool = False
# low-ratio FlyDSL indexer workspaces by ratio; address-pinned like the c4 ones, rebuilt in place
fp4_low_ratio_decode_workspaces: Dict[
int, Union[FP4DecodeWorkspace, FP4RowgroupDecodeWorkspace]
@@ -746,7 +748,10 @@ class DSV4Metadata:
)
# allocated inside a captured segment (graph-pool storage), so a refresh must keep it
BREAKABLE_CUDA_GRAPH_KEEP_FIELDS: ClassVar[Tuple[str, ...]] = ("q_pad_buffer",)
BREAKABLE_CUDA_GRAPH_KEEP_FIELDS: ClassVar[Tuple[str, ...]] = (
"q_pad_buffer",
"q_pad_captured",
)
def refresh_for_breakable_cuda_graph_replay_(self, other: DSV4Metadata) -> None:
self.core_attn_metadata.refresh_for_breakable_cuda_graph_replay_(
+30 -16
View File
@@ -1107,6 +1107,34 @@ class MqaAttentionBase(nn.Module):
# Other FlashMLA implementations retain their existing padded shape.
return 64 if self.n_local_heads <= 64 else self.n_heads
def _zero_padded_q(self, x, kernel_num_heads, attn_backend) -> torch.Tensor:
# One zeroed buffer per metadata, grown to the most rows it has served; each
# call takes a row slice. A graph that keeps the metadata keeps the buffer,
# so the buffer must not grow once a graph has captured it.
meta = getattr(attn_backend, "forward_metadata", None)
buf = getattr(meta, "q_pad_buffer", None)
if (
buf is None
or buf.shape[0] < x.shape[0]
or buf.shape[1:] != (kernel_num_heads, self.head_dim)
or buf.dtype != x.dtype
):
assert not getattr(meta, "q_pad_captured", False), (
"the padded query buffer grew after a CUDA graph captured it"
)
buf = x.new_zeros(x.shape[0], kernel_num_heads, self.head_dim)
if meta is not None:
try:
meta.q_pad_buffer = buf
except (AttributeError, TypeError):
pass
if meta is not None and torch.cuda.is_current_stream_capturing():
try:
meta.q_pad_captured = True
except (AttributeError, TypeError):
pass
return buf[: x.shape[0]]
def _wo_a_quant_freqs(
self, forward_batch: ForwardBatch, q_rope: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
@@ -2555,22 +2583,8 @@ class MQALayer(MqaAttentionBase):
# Backends without an exact-head specialization retain the existing
# padded shape. attn_sink is sliced to this rank and padded to match.
if self.is_dsv41:
# V4.1 kernels read all padded heads, so the padding must be
# zero. The buffer is reused per layer; no consumer may keep it.
want = (x.shape[0], kernel_num_heads, self.head_dim)
meta = getattr(attn_backend, "forward_metadata", None)
q_padded = getattr(meta, "q_pad_buffer", None)
if (
q_padded is None
or tuple(q_padded.shape) != want
or q_padded.dtype != x.dtype
):
q_padded = x.new_zeros(*want)
if meta is not None:
try:
meta.q_pad_buffer = q_padded
except (AttributeError, TypeError):
pass
# V4.1 kernels read all padded heads, so the padding must be zero.
q_padded = self._zero_padded_q(x, kernel_num_heads, attn_backend)
elif _is_gfx942_supported:
# Uninitialized padded TP heads inject NaN into attention on gfx942
# (fnuz), so zero-init there; other archs tolerate new_empty and skip