fix(spec): enable Qwen3.5 EAGLE3 capture and streaming overlap (#41154)

Co-authored-by: Yuhao Wang <59914901+jin-chuan@users.noreply.github.com>
Co-authored-by: Liangsheng Yin <95566987+hnyls2002@users.noreply.github.com>
This commit is contained in:
Doğaç Eldenk
2026-09-30 14:41:46 -07:00
committed by GitHub
co-authored by Yuhao Wang Liangsheng Yin
parent a9c97c9f69
commit f050ab194a
6 changed files with 90 additions and 19 deletions
@@ -837,16 +837,17 @@ def build_prefill_registry(
enable_prefill_cp: bool = False,
# Per-bucket attn-TP sharded (SP) predicate; defaults to replicated.
attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False,
register_input_embeds: bool = True,
register_input_embeds: Optional[bool] = None,
share_pool: bool = True,
source: Optional[Any] = None,
) -> CudaGraphBufferRegistry:
"""Registry mirroring the **token-axis** FB-shared buffers for the
piecewise / breakable / full (prefill) cuda-graph runners.
``register_input_embeds`` (default ``True``) registers the multimodal
``input_embeds`` slot; the eager extend path passes ``False`` so it is
carried from the batch (a read input) rather than written in-graph.
``register_input_embeds`` defaults to ``is_multimodal``. The EAGLE3 draft
passes True because the draft model itself is text-only. The eager extend
path passes ``False`` so embeddings are carried from the batch rather
than written in-graph.
Padding policies match the inline copy/zero in
``PiecewiseCudaGraphRunner.load_batch``: ``input_ids`` / ``positions``
@@ -899,6 +900,9 @@ def build_prefill_registry(
padding_policy=PaddingPolicy.ZERO,
),
]
if register_input_embeds is None:
register_input_embeds = is_multimodal
if is_multimodal:
slots.append(
GraphSlot(
@@ -910,17 +914,17 @@ def build_prefill_registry(
slice_fn=lambda buf, n: buf[:, :n],
)
)
if register_input_embeds:
slots.append(
GraphSlot(
"input_embeds",
lambda _bs2, mt: (mt, hidden_size),
embed_dtype,
axis="tokens",
padding_policy=PaddingPolicy.ZERO,
copy_from_fb=False,
)
if register_input_embeds:
slots.append(
GraphSlot(
"input_embeds",
lambda _bs2, mt: (mt, hidden_size),
embed_dtype,
axis="tokens",
padding_policy=PaddingPolicy.ZERO,
copy_from_fb=False,
)
)
if enable_mamba_track:
slots.append(GraphSlot("mamba_track_indices", _bs, torch.int64, axis="bs"))
slots.append(GraphSlot("mamba_track_mask", _bs, torch.bool, axis="bs"))
@@ -285,6 +285,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
buffer population, attention metadata init, and output slicing.
"""
_use_draft_input_embeds = False
_backend_can_run_prefill_cuda_graph = None
def __init__(self, model_runner: ModelRunner):
@@ -354,6 +355,14 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
and is_eagle
and model_runner.is_draft_worker
)
# A multimodal target prepares the EAGLE3 draft inputs as a merged
# embedding sequence, so capture that path instead of baking
# embed_tokens(input_ids). EAGLE(MTP) drafts are excluded since
# they handle embeds in their own forward and reject an explicit
# input_embeds argument.
self._use_draft_input_embeds = (
is_breakable_eagle_draft and model_runner.spec_algorithm.is_eagle3()
)
if is_breakable_eagle_draft:
self.capture_hidden_mode = CaptureHiddenMode.LAST
elif (is_eagle and not model_runner.is_draft_worker) or (
@@ -381,6 +390,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
hidden_size=input_embeds_hidden_size,
dtype=self.model_runner.dtype,
enable_mamba_track=self.mamba_track_enabled,
enable_input_embeds=(self.is_multimodal or self._use_draft_input_embeds),
pp_size=self.pp_size,
is_first_pp_rank=self.model_runner.pp_group.is_first_rank,
hc_hidden_size=getattr(
@@ -415,6 +425,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
require_gathered_buffer=require_gathered_buffer(),
enable_prefill_cp=(is_dsa_enable_prefill_cp() or is_mla_cp_enabled()),
attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded,
register_input_embeds=(self.is_multimodal or self._use_draft_input_embeds),
source=self.buffers,
)
@@ -1730,6 +1741,29 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
pp_proxy_tensors=kwargs.get("pp_proxy_tensors"),
)
if self._use_draft_input_embeds:
# Keep one address-stable embedding input for the captured draft
# body. Multimodal requests use the target-composed embeds with
# the tail-token row replaced, and text requests use an
# embed_tokens lookup.
draft_input_embeds = forward_batch.mm_input_embeds
if draft_input_embeds is not None:
# Replace each request's last row with the embedding of its
# sampled tail token, mirroring the native EAGLE drafts.
last_indices = (
forward_batch.extend_start_loc + forward_batch.extend_seq_lens - 1
).long()
draft_input_embeds[last_indices] = self.layer_model.embed_tokens(
forward_batch.input_ids[last_indices]
)
else:
draft_input_embeds = self.layer_model.embed_tokens(
forward_batch.input_ids
)
self.buffer_registry.get_slot("input_embeds").buffer[:num_tokens].copy_(
draft_input_embeds
)
registry = self.buffer_registry
def _slot(name):
@@ -2010,6 +2044,16 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# MTP consumes the target model's live multimodal embeddings in its
# eager wrapper before the captured transformer body is replayed.
tail_batch.mm_input_embeds = forward_batch.mm_input_embeds
model_kwargs = kwargs
if self._use_draft_input_embeds:
# Draft forwards that accept `input_embeds` (all current EAGLE
# draft classes) route it into the draft transformer, selecting
# the same explicit-embedding branch as capture instead of
# replaying the captured token lookup.
model_kwargs = {
**kwargs,
"input_embeds": static_forward_batch.input_embeds,
}
try:
with self._prefill_forward_context(
static_forward_batch,
@@ -2020,7 +2064,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
tail_batch.input_ids,
tail_batch.positions,
tail_batch,
**kwargs,
**model_kwargs,
)
finally:
self.layer_model.forward = original_layer_forward
@@ -292,6 +292,7 @@ class PrefillInputBuffers(ForwardInputBuffers):
hidden_size: int,
dtype: torch.dtype,
enable_mamba_track: bool,
enable_input_embeds: Optional[bool] = None,
pp_size: int = 1,
is_first_pp_rank: bool = False,
hc_hidden_size: Optional[int] = None,
@@ -318,11 +319,17 @@ class PrefillInputBuffers(ForwardInputBuffers):
)
positions = torch.zeros((max_num_tokens,), dtype=torch.int64)
if is_multimodal:
if enable_input_embeds is None:
enable_input_embeds = is_multimodal
if enable_input_embeds:
input_embeds = torch.zeros((max_num_tokens, hidden_size), dtype=dtype)
mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64)
else:
input_embeds = None
if is_multimodal:
mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64)
else:
mrope_positions = None
pp_proxy_tensors = (
+5
View File
@@ -1771,6 +1771,11 @@ class Qwen3_5ForCausalLM(nn.Module):
for layer_id in self.layers_to_capture:
setattr(self.layers[layer_id], "_is_layer_to_capture", True)
def set_eagle3_layers_to_capture(self, layers_to_capture: list[int]):
self.layers_to_capture = layers_to_capture
for layer_id in self.layers_to_capture:
setattr(self.layers[layer_id], "_is_layer_to_capture", True)
@property
def start_layer(self) -> int:
return self._start_layer
+9 -2
View File
@@ -1763,17 +1763,24 @@ class Qwen3VLForConditionalGeneration(nn.Module):
return self.model.embed_tokens.weight, self.lm_head.weight
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
if not self.pp_group.is_last_rank:
return
self.capture_aux_hidden_states = True
self.model.capture_aux_hidden_states = True
if layer_ids is None:
num_layers = self.config.num_hidden_layers
self.model.layers_to_capture = [
layers_to_capture = [
2,
num_layers // 2,
num_layers - 3,
] # Specific layers for EAGLE3 support
else:
self.model.layers_to_capture = [val + 1 for val in layer_ids]
layers_to_capture = [val + 1 for val in layer_ids]
if hasattr(self.model, "set_eagle3_layers_to_capture"):
self.model.set_eagle3_layers_to_capture(layers_to_capture)
else:
self.model.layers_to_capture = layers_to_capture
def _require_vision(model) -> None:
@@ -1051,6 +1051,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
next_token_ids = batch_result.next_token_ids.to(torch.int64)
# Prepare for draft extend in a separate stream
if self.plan_stream:
self.plan_stream.wait_stream(
torch.get_device_module(self.device).current_stream()
)
with self.plan_stream_ctx:
forward_batch = prepare_for_draft_extend(
draft_extend_input,