[Bugfix] Fix full prefill CUDA graph padding and EAGLE capture (#35588)

This commit is contained in:
Aurick Qiao
2026-08-30 21:30:24 -07:00
committed by GitHub
parent 2ea6d17eab
commit 9a9e167179
7 changed files with 171 additions and 42 deletions
@@ -815,7 +815,7 @@ def build_prefill_registry(
source: Optional[Any] = None,
) -> CudaGraphBufferRegistry:
"""Registry mirroring the **token-axis** FB-shared buffers for the
piecewise / breakable (prefill) cuda-graph runners.
piecewise / breakable / full (prefill) cuda-graph runners.
``register_input_embeds`` (default ``True``) registers the multimodal
``input_embeds`` slot; the eager extend path passes ``False`` so it is
@@ -910,13 +910,18 @@ def build_prefill_registry(
# blank real tokens whenever raw < bucket. Recompute the local
# count against the padded bucket from the batch's un-adjusted
# global count, mirroring the decode registry's post_fill.
if require_gathered_buffer and not enable_prefill_cp:
buf.fill_(
compute_local_num_token_non_padded_cpu(
global_num_token_non_padded=fb.num_token_non_padded_cpu,
num_tokens_per_dp=ctx.padded_num_tokens,
if require_gathered_buffer:
if not enable_prefill_cp:
buf.fill_(
compute_local_num_token_non_padded_cpu(
global_num_token_non_padded=fb.num_token_non_padded_cpu,
num_tokens_per_dp=ctx.padded_num_tokens,
)
)
)
else:
# Non-gathered FullCG still needs the live boundary rather
# than a stale/absent ForwardBatch tensor.
buf.fill_(ctx.raw_num_tokens)
slots.append(
GraphSlot(
@@ -322,14 +322,14 @@ def capture_prefill_graph(
# Skip prefill CG for EAGLE target on tc_piecewise when the fixed server
# capture ceiling is below FULL. EAGLE target prefill requests FULL, so a
# NULL or LAST graph is dead; capturing it can perturb FP4/TRTLLM-MoE
# state and corrupt decode replay (see #28386 and #28870). BCG captures
# FULL for EAGLE target in PrefillCudaGraphRunner.__init__, so it does not
# need this skip.
# state and corrupt decode replay (see #28386 and #28870). BCG and FullCG
# capture FULL for EAGLE targets in PrefillCudaGraphRunner.__init__, so
# they do not need this skip.
if (
model_runner.spec_algorithm.is_eagle()
and not model_runner.is_draft_worker
and get_server_return_hidden_states_mode() < CaptureHiddenMode.FULL
and not check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
and check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
):
logger.info(
"Disable prefill CUDA graph for EAGLE target on tc_piecewise "
@@ -296,16 +296,20 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.capture_forward_mode = ForwardMode.EXTEND
# Hidden-state capture mode cases:
# - Breakable EAGLE draft: LAST.
# - Breakable EAGLE target: FULL.
# - EAGLE target: FULL.
# - Return-hidden-states or DFLASH: FULL.
# - Otherwise: NULL.
is_breakable_eagle = (
is_eagle = model_runner.spec_algorithm.is_eagle()
is_breakable_eagle_draft = (
self.prefill_backend_name == Backend.BREAKABLE
and model_runner.spec_algorithm.is_eagle()
and is_eagle
and model_runner.is_draft_worker
)
if is_breakable_eagle and model_runner.is_draft_worker:
if is_breakable_eagle_draft:
self.capture_hidden_mode = CaptureHiddenMode.LAST
elif is_breakable_eagle or model_runner.spec_algorithm.is_dflash_family():
elif (is_eagle and not model_runner.is_draft_worker) or (
model_runner.spec_algorithm.is_dflash_family()
):
self.capture_hidden_mode = CaptureHiddenMode.FULL
else:
self.capture_hidden_mode = self.return_hidden_states_mode
@@ -350,7 +354,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
hidden_size=input_embeds_hidden_size,
embed_dtype=self.model_runner.dtype,
enable_mamba_track=self.mamba_track_enabled,
enable_num_token_non_padded=enable_num_token_non_padded(),
# FullCG always pads to a capture bucket. Models that mask padded
# hidden rows need the live boundary even without expert parallelism.
enable_num_token_non_padded=(
enable_num_token_non_padded()
or self.prefill_backend_name == Backend.FULL
),
require_gathered_buffer=require_gathered_buffer(),
enable_prefill_cp=(
is_dsa_enable_prefill_cp() or is_mla_prefill_cp_enabled()