[Bugfix] Fix full prefill CUDA graph padding and EAGLE capture (#35588)
This commit is contained in:
@@ -815,7 +815,7 @@ def build_prefill_registry(
|
||||
source: Optional[Any] = None,
|
||||
) -> CudaGraphBufferRegistry:
|
||||
"""Registry mirroring the **token-axis** FB-shared buffers for the
|
||||
piecewise / breakable (prefill) cuda-graph runners.
|
||||
piecewise / breakable / full (prefill) cuda-graph runners.
|
||||
|
||||
``register_input_embeds`` (default ``True``) registers the multimodal
|
||||
``input_embeds`` slot; the eager extend path passes ``False`` so it is
|
||||
@@ -910,13 +910,18 @@ def build_prefill_registry(
|
||||
# blank real tokens whenever raw < bucket. Recompute the local
|
||||
# count against the padded bucket from the batch's un-adjusted
|
||||
# global count, mirroring the decode registry's post_fill.
|
||||
if require_gathered_buffer and not enable_prefill_cp:
|
||||
buf.fill_(
|
||||
compute_local_num_token_non_padded_cpu(
|
||||
global_num_token_non_padded=fb.num_token_non_padded_cpu,
|
||||
num_tokens_per_dp=ctx.padded_num_tokens,
|
||||
if require_gathered_buffer:
|
||||
if not enable_prefill_cp:
|
||||
buf.fill_(
|
||||
compute_local_num_token_non_padded_cpu(
|
||||
global_num_token_non_padded=fb.num_token_non_padded_cpu,
|
||||
num_tokens_per_dp=ctx.padded_num_tokens,
|
||||
)
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Non-gathered FullCG still needs the live boundary rather
|
||||
# than a stale/absent ForwardBatch tensor.
|
||||
buf.fill_(ctx.raw_num_tokens)
|
||||
|
||||
slots.append(
|
||||
GraphSlot(
|
||||
|
||||
@@ -322,14 +322,14 @@ def capture_prefill_graph(
|
||||
# Skip prefill CG for EAGLE target on tc_piecewise when the fixed server
|
||||
# capture ceiling is below FULL. EAGLE target prefill requests FULL, so a
|
||||
# NULL or LAST graph is dead; capturing it can perturb FP4/TRTLLM-MoE
|
||||
# state and corrupt decode replay (see #28386 and #28870). BCG captures
|
||||
# FULL for EAGLE target in PrefillCudaGraphRunner.__init__, so it does not
|
||||
# need this skip.
|
||||
# state and corrupt decode replay (see #28386 and #28870). BCG and FullCG
|
||||
# capture FULL for EAGLE targets in PrefillCudaGraphRunner.__init__, so
|
||||
# they do not need this skip.
|
||||
if (
|
||||
model_runner.spec_algorithm.is_eagle()
|
||||
and not model_runner.is_draft_worker
|
||||
and get_server_return_hidden_states_mode() < CaptureHiddenMode.FULL
|
||||
and not check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
|
||||
and check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
):
|
||||
logger.info(
|
||||
"Disable prefill CUDA graph for EAGLE target on tc_piecewise "
|
||||
|
||||
@@ -296,16 +296,20 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.capture_forward_mode = ForwardMode.EXTEND
|
||||
# Hidden-state capture mode cases:
|
||||
# - Breakable EAGLE draft: LAST.
|
||||
# - Breakable EAGLE target: FULL.
|
||||
# - EAGLE target: FULL.
|
||||
# - Return-hidden-states or DFLASH: FULL.
|
||||
# - Otherwise: NULL.
|
||||
is_breakable_eagle = (
|
||||
is_eagle = model_runner.spec_algorithm.is_eagle()
|
||||
is_breakable_eagle_draft = (
|
||||
self.prefill_backend_name == Backend.BREAKABLE
|
||||
and model_runner.spec_algorithm.is_eagle()
|
||||
and is_eagle
|
||||
and model_runner.is_draft_worker
|
||||
)
|
||||
if is_breakable_eagle and model_runner.is_draft_worker:
|
||||
if is_breakable_eagle_draft:
|
||||
self.capture_hidden_mode = CaptureHiddenMode.LAST
|
||||
elif is_breakable_eagle or model_runner.spec_algorithm.is_dflash_family():
|
||||
elif (is_eagle and not model_runner.is_draft_worker) or (
|
||||
model_runner.spec_algorithm.is_dflash_family()
|
||||
):
|
||||
self.capture_hidden_mode = CaptureHiddenMode.FULL
|
||||
else:
|
||||
self.capture_hidden_mode = self.return_hidden_states_mode
|
||||
@@ -350,7 +354,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
hidden_size=input_embeds_hidden_size,
|
||||
embed_dtype=self.model_runner.dtype,
|
||||
enable_mamba_track=self.mamba_track_enabled,
|
||||
enable_num_token_non_padded=enable_num_token_non_padded(),
|
||||
# FullCG always pads to a capture bucket. Models that mask padded
|
||||
# hidden rows need the live boundary even without expert parallelism.
|
||||
enable_num_token_non_padded=(
|
||||
enable_num_token_non_padded()
|
||||
or self.prefill_backend_name == Backend.FULL
|
||||
),
|
||||
require_gathered_buffer=require_gathered_buffer(),
|
||||
enable_prefill_cp=(
|
||||
is_dsa_enable_prefill_cp() or is_mla_prefill_cp_enabled()
|
||||
|
||||
Reference in New Issue
Block a user