fix(spec): enable Qwen3.5 EAGLE3 capture and streaming overlap (#41154)
Co-authored-by: Yuhao Wang <59914901+jin-chuan@users.noreply.github.com> Co-authored-by: Liangsheng Yin <95566987+hnyls2002@users.noreply.github.com>
This commit is contained in:
co-authored by
Yuhao Wang
Liangsheng Yin
parent
a9c97c9f69
commit
f050ab194a
@@ -837,16 +837,17 @@ def build_prefill_registry(
|
||||
enable_prefill_cp: bool = False,
|
||||
# Per-bucket attn-TP sharded (SP) predicate; defaults to replicated.
|
||||
attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False,
|
||||
register_input_embeds: bool = True,
|
||||
register_input_embeds: Optional[bool] = None,
|
||||
share_pool: bool = True,
|
||||
source: Optional[Any] = None,
|
||||
) -> CudaGraphBufferRegistry:
|
||||
"""Registry mirroring the **token-axis** FB-shared buffers for the
|
||||
piecewise / breakable / full (prefill) cuda-graph runners.
|
||||
|
||||
``register_input_embeds`` (default ``True``) registers the multimodal
|
||||
``input_embeds`` slot; the eager extend path passes ``False`` so it is
|
||||
carried from the batch (a read input) rather than written in-graph.
|
||||
``register_input_embeds`` defaults to ``is_multimodal``. The EAGLE3 draft
|
||||
passes True because the draft model itself is text-only. The eager extend
|
||||
path passes ``False`` so embeddings are carried from the batch rather
|
||||
than written in-graph.
|
||||
|
||||
Padding policies match the inline copy/zero in
|
||||
``PiecewiseCudaGraphRunner.load_batch``: ``input_ids`` / ``positions``
|
||||
@@ -899,6 +900,9 @@ def build_prefill_registry(
|
||||
padding_policy=PaddingPolicy.ZERO,
|
||||
),
|
||||
]
|
||||
if register_input_embeds is None:
|
||||
register_input_embeds = is_multimodal
|
||||
|
||||
if is_multimodal:
|
||||
slots.append(
|
||||
GraphSlot(
|
||||
@@ -910,17 +914,17 @@ def build_prefill_registry(
|
||||
slice_fn=lambda buf, n: buf[:, :n],
|
||||
)
|
||||
)
|
||||
if register_input_embeds:
|
||||
slots.append(
|
||||
GraphSlot(
|
||||
"input_embeds",
|
||||
lambda _bs2, mt: (mt, hidden_size),
|
||||
embed_dtype,
|
||||
axis="tokens",
|
||||
padding_policy=PaddingPolicy.ZERO,
|
||||
copy_from_fb=False,
|
||||
)
|
||||
if register_input_embeds:
|
||||
slots.append(
|
||||
GraphSlot(
|
||||
"input_embeds",
|
||||
lambda _bs2, mt: (mt, hidden_size),
|
||||
embed_dtype,
|
||||
axis="tokens",
|
||||
padding_policy=PaddingPolicy.ZERO,
|
||||
copy_from_fb=False,
|
||||
)
|
||||
)
|
||||
if enable_mamba_track:
|
||||
slots.append(GraphSlot("mamba_track_indices", _bs, torch.int64, axis="bs"))
|
||||
slots.append(GraphSlot("mamba_track_mask", _bs, torch.bool, axis="bs"))
|
||||
|
||||
@@ -285,6 +285,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
buffer population, attention metadata init, and output slicing.
|
||||
"""
|
||||
|
||||
_use_draft_input_embeds = False
|
||||
_backend_can_run_prefill_cuda_graph = None
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
@@ -354,6 +355,14 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
and is_eagle
|
||||
and model_runner.is_draft_worker
|
||||
)
|
||||
# A multimodal target prepares the EAGLE3 draft inputs as a merged
|
||||
# embedding sequence, so capture that path instead of baking
|
||||
# embed_tokens(input_ids). EAGLE(MTP) drafts are excluded since
|
||||
# they handle embeds in their own forward and reject an explicit
|
||||
# input_embeds argument.
|
||||
self._use_draft_input_embeds = (
|
||||
is_breakable_eagle_draft and model_runner.spec_algorithm.is_eagle3()
|
||||
)
|
||||
if is_breakable_eagle_draft:
|
||||
self.capture_hidden_mode = CaptureHiddenMode.LAST
|
||||
elif (is_eagle and not model_runner.is_draft_worker) or (
|
||||
@@ -381,6 +390,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
hidden_size=input_embeds_hidden_size,
|
||||
dtype=self.model_runner.dtype,
|
||||
enable_mamba_track=self.mamba_track_enabled,
|
||||
enable_input_embeds=(self.is_multimodal or self._use_draft_input_embeds),
|
||||
pp_size=self.pp_size,
|
||||
is_first_pp_rank=self.model_runner.pp_group.is_first_rank,
|
||||
hc_hidden_size=getattr(
|
||||
@@ -415,6 +425,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
require_gathered_buffer=require_gathered_buffer(),
|
||||
enable_prefill_cp=(is_dsa_enable_prefill_cp() or is_mla_cp_enabled()),
|
||||
attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded,
|
||||
register_input_embeds=(self.is_multimodal or self._use_draft_input_embeds),
|
||||
source=self.buffers,
|
||||
)
|
||||
|
||||
@@ -1730,6 +1741,29 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
pp_proxy_tensors=kwargs.get("pp_proxy_tensors"),
|
||||
)
|
||||
|
||||
if self._use_draft_input_embeds:
|
||||
# Keep one address-stable embedding input for the captured draft
|
||||
# body. Multimodal requests use the target-composed embeds with
|
||||
# the tail-token row replaced, and text requests use an
|
||||
# embed_tokens lookup.
|
||||
draft_input_embeds = forward_batch.mm_input_embeds
|
||||
if draft_input_embeds is not None:
|
||||
# Replace each request's last row with the embedding of its
|
||||
# sampled tail token, mirroring the native EAGLE drafts.
|
||||
last_indices = (
|
||||
forward_batch.extend_start_loc + forward_batch.extend_seq_lens - 1
|
||||
).long()
|
||||
draft_input_embeds[last_indices] = self.layer_model.embed_tokens(
|
||||
forward_batch.input_ids[last_indices]
|
||||
)
|
||||
else:
|
||||
draft_input_embeds = self.layer_model.embed_tokens(
|
||||
forward_batch.input_ids
|
||||
)
|
||||
self.buffer_registry.get_slot("input_embeds").buffer[:num_tokens].copy_(
|
||||
draft_input_embeds
|
||||
)
|
||||
|
||||
registry = self.buffer_registry
|
||||
|
||||
def _slot(name):
|
||||
@@ -2010,6 +2044,16 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# MTP consumes the target model's live multimodal embeddings in its
|
||||
# eager wrapper before the captured transformer body is replayed.
|
||||
tail_batch.mm_input_embeds = forward_batch.mm_input_embeds
|
||||
model_kwargs = kwargs
|
||||
if self._use_draft_input_embeds:
|
||||
# Draft forwards that accept `input_embeds` (all current EAGLE
|
||||
# draft classes) route it into the draft transformer, selecting
|
||||
# the same explicit-embedding branch as capture instead of
|
||||
# replaying the captured token lookup.
|
||||
model_kwargs = {
|
||||
**kwargs,
|
||||
"input_embeds": static_forward_batch.input_embeds,
|
||||
}
|
||||
try:
|
||||
with self._prefill_forward_context(
|
||||
static_forward_batch,
|
||||
@@ -2020,7 +2064,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
tail_batch.input_ids,
|
||||
tail_batch.positions,
|
||||
tail_batch,
|
||||
**kwargs,
|
||||
**model_kwargs,
|
||||
)
|
||||
finally:
|
||||
self.layer_model.forward = original_layer_forward
|
||||
|
||||
@@ -292,6 +292,7 @@ class PrefillInputBuffers(ForwardInputBuffers):
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_mamba_track: bool,
|
||||
enable_input_embeds: Optional[bool] = None,
|
||||
pp_size: int = 1,
|
||||
is_first_pp_rank: bool = False,
|
||||
hc_hidden_size: Optional[int] = None,
|
||||
@@ -318,11 +319,17 @@ class PrefillInputBuffers(ForwardInputBuffers):
|
||||
)
|
||||
positions = torch.zeros((max_num_tokens,), dtype=torch.int64)
|
||||
|
||||
if is_multimodal:
|
||||
if enable_input_embeds is None:
|
||||
enable_input_embeds = is_multimodal
|
||||
|
||||
if enable_input_embeds:
|
||||
input_embeds = torch.zeros((max_num_tokens, hidden_size), dtype=dtype)
|
||||
mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64)
|
||||
else:
|
||||
input_embeds = None
|
||||
|
||||
if is_multimodal:
|
||||
mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64)
|
||||
else:
|
||||
mrope_positions = None
|
||||
|
||||
pp_proxy_tensors = (
|
||||
|
||||
@@ -1771,6 +1771,11 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
for layer_id in self.layers_to_capture:
|
||||
setattr(self.layers[layer_id], "_is_layer_to_capture", True)
|
||||
|
||||
def set_eagle3_layers_to_capture(self, layers_to_capture: list[int]):
|
||||
self.layers_to_capture = layers_to_capture
|
||||
for layer_id in self.layers_to_capture:
|
||||
setattr(self.layers[layer_id], "_is_layer_to_capture", True)
|
||||
|
||||
@property
|
||||
def start_layer(self) -> int:
|
||||
return self._start_layer
|
||||
|
||||
@@ -1763,17 +1763,24 @@ class Qwen3VLForConditionalGeneration(nn.Module):
|
||||
return self.model.embed_tokens.weight, self.lm_head.weight
|
||||
|
||||
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
|
||||
if not self.pp_group.is_last_rank:
|
||||
return
|
||||
self.capture_aux_hidden_states = True
|
||||
self.model.capture_aux_hidden_states = True
|
||||
if layer_ids is None:
|
||||
num_layers = self.config.num_hidden_layers
|
||||
self.model.layers_to_capture = [
|
||||
layers_to_capture = [
|
||||
2,
|
||||
num_layers // 2,
|
||||
num_layers - 3,
|
||||
] # Specific layers for EAGLE3 support
|
||||
else:
|
||||
self.model.layers_to_capture = [val + 1 for val in layer_ids]
|
||||
layers_to_capture = [val + 1 for val in layer_ids]
|
||||
|
||||
if hasattr(self.model, "set_eagle3_layers_to_capture"):
|
||||
self.model.set_eagle3_layers_to_capture(layers_to_capture)
|
||||
else:
|
||||
self.model.layers_to_capture = layers_to_capture
|
||||
|
||||
|
||||
def _require_vision(model) -> None:
|
||||
|
||||
@@ -1051,6 +1051,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
next_token_ids = batch_result.next_token_ids.to(torch.int64)
|
||||
|
||||
# Prepare for draft extend in a separate stream
|
||||
if self.plan_stream:
|
||||
self.plan_stream.wait_stream(
|
||||
torch.get_device_module(self.device).current_stream()
|
||||
)
|
||||
with self.plan_stream_ctx:
|
||||
forward_batch = prepare_for_draft_extend(
|
||||
draft_extend_input,
|
||||
|
||||
Reference in New Issue
Block a user