[Fix] dsv41: Bound MoE routing to live SWA graph-tail rows (#44347)

This commit is contained in:
Khoa Pham
2026-10-14 21:14:28 -07:00
committed by GitHub
parent 08f15e76bb
commit d4daf52369
@@ -1201,11 +1201,7 @@ class DeepseekV4AttnBackend(
self._bcg_tail_capacity: Optional[int] = None
self._bcg_tail_buffers: Optional[tuple] = None
self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {}
# Prefill CUDA graph tail block (bounded replay inside the graph): row
# capacity, persistent index/location buffers, held metadata per block size.
self._bcg_tail_capacity: Optional[int] = None
self._bcg_tail_buffers: Optional[tuple] = None
self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {}
self._bcg_tail_num_token_non_padded: Optional[torch.Tensor] = None
self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner)
self.dsv4_prefill_backend = getattr(kernel, "dsv4_prefill_backend", "auto")
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
@@ -1879,13 +1875,24 @@ class DeepseekV4AttnBackend(
get_local_dp_buffer_len(),
)
tail = tail_metadata.late_layer_tail
# Inside the prefill CUDA graph the tail block is padded to a fixed capacity;
# the eager breaks (attention, low-ratio sources) clip their inputs to the
# batch's non-padded count, so point it at the real tail rows while the
# late layers run.
saved = saved + (forward_batch.global_num_token_non_padded_cpu,)
saved = saved + (
forward_batch.global_num_token_non_padded_cpu,
forward_batch.global_num_token_non_padded,
forward_batch.num_token_non_padded,
)
if tail.real_rows_count is not None:
forward_batch.global_num_token_non_padded_cpu = tail.real_rows_count
non_padded = forward_batch.global_num_token_non_padded
if non_padded is not None:
if self._bcg_tail_num_token_non_padded is None:
self._bcg_tail_num_token_non_padded = torch.empty_like(non_padded)
bound = torch.clamp(
non_padded,
max=tail.real_rows_count,
out=self._bcg_tail_num_token_non_padded,
)
forward_batch.global_num_token_non_padded = bound
forward_batch.num_token_non_padded = bound
# The layers before the switch published top-k into the full metadata's
# buffers; carry the tail rows into the tail metadata's (padding stays -1).
full_core = saved[0].core_attn_metadata
@@ -1927,9 +1934,13 @@ class DeepseekV4AttnBackend(
forward_batch.attn_cp_metadata,
local_dp_buffer_len,
non_padded,
non_padded_gpu,
local_non_padded,
) = saved
set_local_dp_buffer_len(local_dp_buffer_len)
forward_batch.global_num_token_non_padded_cpu = non_padded
forward_batch.global_num_token_non_padded = non_padded_gpu
forward_batch.num_token_non_padded = local_non_padded
# The prefill CUDA graph runs the tail switch as eager breaks: these two run
# at capture and again at every replay (see deepseek_v4.bcg_enter/exit).