[Fix] dsv41: Bound MoE routing to live SWA graph-tail rows (#44347)
This commit is contained in:
@@ -1201,11 +1201,7 @@ class DeepseekV4AttnBackend(
|
||||
self._bcg_tail_capacity: Optional[int] = None
|
||||
self._bcg_tail_buffers: Optional[tuple] = None
|
||||
self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {}
|
||||
# Prefill CUDA graph tail block (bounded replay inside the graph): row
|
||||
# capacity, persistent index/location buffers, held metadata per block size.
|
||||
self._bcg_tail_capacity: Optional[int] = None
|
||||
self._bcg_tail_buffers: Optional[tuple] = None
|
||||
self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {}
|
||||
self._bcg_tail_num_token_non_padded: Optional[torch.Tensor] = None
|
||||
self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner)
|
||||
self.dsv4_prefill_backend = getattr(kernel, "dsv4_prefill_backend", "auto")
|
||||
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
|
||||
@@ -1879,13 +1875,24 @@ class DeepseekV4AttnBackend(
|
||||
get_local_dp_buffer_len(),
|
||||
)
|
||||
tail = tail_metadata.late_layer_tail
|
||||
# Inside the prefill CUDA graph the tail block is padded to a fixed capacity;
|
||||
# the eager breaks (attention, low-ratio sources) clip their inputs to the
|
||||
# batch's non-padded count, so point it at the real tail rows while the
|
||||
# late layers run.
|
||||
saved = saved + (forward_batch.global_num_token_non_padded_cpu,)
|
||||
saved = saved + (
|
||||
forward_batch.global_num_token_non_padded_cpu,
|
||||
forward_batch.global_num_token_non_padded,
|
||||
forward_batch.num_token_non_padded,
|
||||
)
|
||||
if tail.real_rows_count is not None:
|
||||
forward_batch.global_num_token_non_padded_cpu = tail.real_rows_count
|
||||
non_padded = forward_batch.global_num_token_non_padded
|
||||
if non_padded is not None:
|
||||
if self._bcg_tail_num_token_non_padded is None:
|
||||
self._bcg_tail_num_token_non_padded = torch.empty_like(non_padded)
|
||||
bound = torch.clamp(
|
||||
non_padded,
|
||||
max=tail.real_rows_count,
|
||||
out=self._bcg_tail_num_token_non_padded,
|
||||
)
|
||||
forward_batch.global_num_token_non_padded = bound
|
||||
forward_batch.num_token_non_padded = bound
|
||||
# The layers before the switch published top-k into the full metadata's
|
||||
# buffers; carry the tail rows into the tail metadata's (padding stays -1).
|
||||
full_core = saved[0].core_attn_metadata
|
||||
@@ -1927,9 +1934,13 @@ class DeepseekV4AttnBackend(
|
||||
forward_batch.attn_cp_metadata,
|
||||
local_dp_buffer_len,
|
||||
non_padded,
|
||||
non_padded_gpu,
|
||||
local_non_padded,
|
||||
) = saved
|
||||
set_local_dp_buffer_len(local_dp_buffer_len)
|
||||
forward_batch.global_num_token_non_padded_cpu = non_padded
|
||||
forward_batch.global_num_token_non_padded = non_padded_gpu
|
||||
forward_batch.num_token_non_padded = local_non_padded
|
||||
|
||||
# The prefill CUDA graph runs the tail switch as eager breaks: these two run
|
||||
# at capture and again at every replay (see deepseek_v4.bcg_enter/exit).
|
||||
|
||||
Reference in New Issue
Block a user