From f050ab194ab5b1c6d792be06925658826c56bbd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fa=C3=A7=20Eldenk?= Date: Wed, 30 Sep 2026 14:41:46 -0700 Subject: [PATCH] fix(spec): enable Qwen3.5 EAGLE3 capture and streaming overlap (#41154) Co-authored-by: Yuhao Wang <59914901+jin-chuan@users.noreply.github.com> Co-authored-by: Liangsheng Yin <95566987+hnyls2002@users.noreply.github.com> --- .../cuda_graph_buffer_registry.py | 32 +++++++------ .../runner/prefill_cuda_graph_runner.py | 46 ++++++++++++++++++- .../model_executor/runner_utils/buffers.py | 11 ++++- python/sglang/srt/models/qwen3_5.py | 5 ++ python/sglang/srt/models/qwen3_vl.py | 11 ++++- .../sglang/srt/speculative/eagle_worker_v2.py | 4 ++ 6 files changed, 90 insertions(+), 19 deletions(-) diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index a461e6e625..385f099090 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -837,16 +837,17 @@ def build_prefill_registry( enable_prefill_cp: bool = False, # Per-bucket attn-TP sharded (SP) predicate; defaults to replicated. attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False, - register_input_embeds: bool = True, + register_input_embeds: Optional[bool] = None, share_pool: bool = True, source: Optional[Any] = None, ) -> CudaGraphBufferRegistry: """Registry mirroring the **token-axis** FB-shared buffers for the piecewise / breakable / full (prefill) cuda-graph runners. - ``register_input_embeds`` (default ``True``) registers the multimodal - ``input_embeds`` slot; the eager extend path passes ``False`` so it is - carried from the batch (a read input) rather than written in-graph. + ``register_input_embeds`` defaults to ``is_multimodal``. The EAGLE3 draft + passes True because the draft model itself is text-only. The eager extend + path passes ``False`` so embeddings are carried from the batch rather + than written in-graph. Padding policies match the inline copy/zero in ``PiecewiseCudaGraphRunner.load_batch``: ``input_ids`` / ``positions`` @@ -899,6 +900,9 @@ def build_prefill_registry( padding_policy=PaddingPolicy.ZERO, ), ] + if register_input_embeds is None: + register_input_embeds = is_multimodal + if is_multimodal: slots.append( GraphSlot( @@ -910,17 +914,17 @@ def build_prefill_registry( slice_fn=lambda buf, n: buf[:, :n], ) ) - if register_input_embeds: - slots.append( - GraphSlot( - "input_embeds", - lambda _bs2, mt: (mt, hidden_size), - embed_dtype, - axis="tokens", - padding_policy=PaddingPolicy.ZERO, - copy_from_fb=False, - ) + if register_input_embeds: + slots.append( + GraphSlot( + "input_embeds", + lambda _bs2, mt: (mt, hidden_size), + embed_dtype, + axis="tokens", + padding_policy=PaddingPolicy.ZERO, + copy_from_fb=False, ) + ) if enable_mamba_track: slots.append(GraphSlot("mamba_track_indices", _bs, torch.int64, axis="bs")) slots.append(GraphSlot("mamba_track_mask", _bs, torch.bool, axis="bs")) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index d0678daa8c..b9fced80a5 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -285,6 +285,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): buffer population, attention metadata init, and output slicing. """ + _use_draft_input_embeds = False _backend_can_run_prefill_cuda_graph = None def __init__(self, model_runner: ModelRunner): @@ -354,6 +355,14 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): and is_eagle and model_runner.is_draft_worker ) + # A multimodal target prepares the EAGLE3 draft inputs as a merged + # embedding sequence, so capture that path instead of baking + # embed_tokens(input_ids). EAGLE(MTP) drafts are excluded since + # they handle embeds in their own forward and reject an explicit + # input_embeds argument. + self._use_draft_input_embeds = ( + is_breakable_eagle_draft and model_runner.spec_algorithm.is_eagle3() + ) if is_breakable_eagle_draft: self.capture_hidden_mode = CaptureHiddenMode.LAST elif (is_eagle and not model_runner.is_draft_worker) or ( @@ -381,6 +390,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): hidden_size=input_embeds_hidden_size, dtype=self.model_runner.dtype, enable_mamba_track=self.mamba_track_enabled, + enable_input_embeds=(self.is_multimodal or self._use_draft_input_embeds), pp_size=self.pp_size, is_first_pp_rank=self.model_runner.pp_group.is_first_rank, hc_hidden_size=getattr( @@ -415,6 +425,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): require_gathered_buffer=require_gathered_buffer(), enable_prefill_cp=(is_dsa_enable_prefill_cp() or is_mla_cp_enabled()), attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded, + register_input_embeds=(self.is_multimodal or self._use_draft_input_embeds), source=self.buffers, ) @@ -1730,6 +1741,29 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): pp_proxy_tensors=kwargs.get("pp_proxy_tensors"), ) + if self._use_draft_input_embeds: + # Keep one address-stable embedding input for the captured draft + # body. Multimodal requests use the target-composed embeds with + # the tail-token row replaced, and text requests use an + # embed_tokens lookup. + draft_input_embeds = forward_batch.mm_input_embeds + if draft_input_embeds is not None: + # Replace each request's last row with the embedding of its + # sampled tail token, mirroring the native EAGLE drafts. + last_indices = ( + forward_batch.extend_start_loc + forward_batch.extend_seq_lens - 1 + ).long() + draft_input_embeds[last_indices] = self.layer_model.embed_tokens( + forward_batch.input_ids[last_indices] + ) + else: + draft_input_embeds = self.layer_model.embed_tokens( + forward_batch.input_ids + ) + self.buffer_registry.get_slot("input_embeds").buffer[:num_tokens].copy_( + draft_input_embeds + ) + registry = self.buffer_registry def _slot(name): @@ -2010,6 +2044,16 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # MTP consumes the target model's live multimodal embeddings in its # eager wrapper before the captured transformer body is replayed. tail_batch.mm_input_embeds = forward_batch.mm_input_embeds + model_kwargs = kwargs + if self._use_draft_input_embeds: + # Draft forwards that accept `input_embeds` (all current EAGLE + # draft classes) route it into the draft transformer, selecting + # the same explicit-embedding branch as capture instead of + # replaying the captured token lookup. + model_kwargs = { + **kwargs, + "input_embeds": static_forward_batch.input_embeds, + } try: with self._prefill_forward_context( static_forward_batch, @@ -2020,7 +2064,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): tail_batch.input_ids, tail_batch.positions, tail_batch, - **kwargs, + **model_kwargs, ) finally: self.layer_model.forward = original_layer_forward diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index e0d173d515..d9157905af 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -292,6 +292,7 @@ class PrefillInputBuffers(ForwardInputBuffers): hidden_size: int, dtype: torch.dtype, enable_mamba_track: bool, + enable_input_embeds: Optional[bool] = None, pp_size: int = 1, is_first_pp_rank: bool = False, hc_hidden_size: Optional[int] = None, @@ -318,11 +319,17 @@ class PrefillInputBuffers(ForwardInputBuffers): ) positions = torch.zeros((max_num_tokens,), dtype=torch.int64) - if is_multimodal: + if enable_input_embeds is None: + enable_input_embeds = is_multimodal + + if enable_input_embeds: input_embeds = torch.zeros((max_num_tokens, hidden_size), dtype=dtype) - mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64) else: input_embeds = None + + if is_multimodal: + mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64) + else: mrope_positions = None pp_proxy_tensors = ( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 6523472fd2..7b96231593 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1771,6 +1771,11 @@ class Qwen3_5ForCausalLM(nn.Module): for layer_id in self.layers_to_capture: setattr(self.layers[layer_id], "_is_layer_to_capture", True) + def set_eagle3_layers_to_capture(self, layers_to_capture: list[int]): + self.layers_to_capture = layers_to_capture + for layer_id in self.layers_to_capture: + setattr(self.layers[layer_id], "_is_layer_to_capture", True) + @property def start_layer(self) -> int: return self._start_layer diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index f392eeefdf..9c447ec7fa 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1763,17 +1763,24 @@ class Qwen3VLForConditionalGeneration(nn.Module): return self.model.embed_tokens.weight, self.lm_head.weight def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None): + if not self.pp_group.is_last_rank: + return self.capture_aux_hidden_states = True self.model.capture_aux_hidden_states = True if layer_ids is None: num_layers = self.config.num_hidden_layers - self.model.layers_to_capture = [ + layers_to_capture = [ 2, num_layers // 2, num_layers - 3, ] # Specific layers for EAGLE3 support else: - self.model.layers_to_capture = [val + 1 for val in layer_ids] + layers_to_capture = [val + 1 for val in layer_ids] + + if hasattr(self.model, "set_eagle3_layers_to_capture"): + self.model.set_eagle3_layers_to_capture(layers_to_capture) + else: + self.model.layers_to_capture = layers_to_capture def _require_vision(model) -> None: diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 8da5d34fc1..738e98112e 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -1051,6 +1051,10 @@ class EagleDraftWorker(EagleDraftWorkerBase): next_token_ids = batch_result.next_token_ids.to(torch.int64) # Prepare for draft extend in a separate stream + if self.plan_stream: + self.plan_stream.wait_stream( + torch.get_device_module(self.device).current_stream() + ) with self.plan_stream_ctx: forward_batch = prepare_for_draft_extend( draft_extend_input,