diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index ecb84751d1..a608ea3935 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1201,11 +1201,7 @@ class DeepseekV4AttnBackend( self._bcg_tail_capacity: Optional[int] = None self._bcg_tail_buffers: Optional[tuple] = None self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {} - # Prefill CUDA graph tail block (bounded replay inside the graph): row - # capacity, persistent index/location buffers, held metadata per block size. - self._bcg_tail_capacity: Optional[int] = None - self._bcg_tail_buffers: Optional[tuple] = None - self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {} + self._bcg_tail_num_token_non_padded: Optional[torch.Tensor] = None self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner) self.dsv4_prefill_backend = getattr(kernel, "dsv4_prefill_backend", "auto") if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend): @@ -1879,13 +1875,24 @@ class DeepseekV4AttnBackend( get_local_dp_buffer_len(), ) tail = tail_metadata.late_layer_tail - # Inside the prefill CUDA graph the tail block is padded to a fixed capacity; - # the eager breaks (attention, low-ratio sources) clip their inputs to the - # batch's non-padded count, so point it at the real tail rows while the - # late layers run. - saved = saved + (forward_batch.global_num_token_non_padded_cpu,) + saved = saved + ( + forward_batch.global_num_token_non_padded_cpu, + forward_batch.global_num_token_non_padded, + forward_batch.num_token_non_padded, + ) if tail.real_rows_count is not None: forward_batch.global_num_token_non_padded_cpu = tail.real_rows_count + non_padded = forward_batch.global_num_token_non_padded + if non_padded is not None: + if self._bcg_tail_num_token_non_padded is None: + self._bcg_tail_num_token_non_padded = torch.empty_like(non_padded) + bound = torch.clamp( + non_padded, + max=tail.real_rows_count, + out=self._bcg_tail_num_token_non_padded, + ) + forward_batch.global_num_token_non_padded = bound + forward_batch.num_token_non_padded = bound # The layers before the switch published top-k into the full metadata's # buffers; carry the tail rows into the tail metadata's (padding stays -1). full_core = saved[0].core_attn_metadata @@ -1927,9 +1934,13 @@ class DeepseekV4AttnBackend( forward_batch.attn_cp_metadata, local_dp_buffer_len, non_padded, + non_padded_gpu, + local_non_padded, ) = saved set_local_dp_buffer_len(local_dp_buffer_len) forward_batch.global_num_token_non_padded_cpu = non_padded + forward_batch.global_num_token_non_padded = non_padded_gpu + forward_batch.num_token_non_padded = local_non_padded # The prefill CUDA graph runs the tail switch as eager breaks: these two run # at capture and again at every replay (see deepseek_v4.bcg_enter/exit).