diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index 2da921adf..e8f6f5bfa 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -152,12 +152,14 @@ class EAGLEDraftExtendCudaGraphRunner: dtype=self.model_runner.dtype, ) else: + # Use target config: hidden_states carries target output. + target_cfg = self.eagle_worker.target_worker.model_runner.model_config hidden_states = torch.zeros( ( self.max_num_token, - self.model_runner.model_config.spec_hidden_size, + target_cfg.spec_hidden_size, ), - dtype=self.model_runner.dtype, + dtype=target_cfg.dtype, ) self.seq_len_fill_value = ( self.model_runner.attn_backend.get_cuda_graph_seq_len_fill_value() diff --git a/python/sglang/srt/speculative/eagle_worker.py b/python/sglang/srt/speculative/eagle_worker.py index 7b86462af..c6fd83636 100644 --- a/python/sglang/srt/speculative/eagle_worker.py +++ b/python/sglang/srt/speculative/eagle_worker.py @@ -1130,16 +1130,17 @@ class EAGLEWorker(TpModelWorker): # All reqs finished this verify; swap to an idle ExtendInput. batch = batch.copy() batch.prepare_for_idle() + target_cfg = self.target_worker.model_runner.model_config hidden_size = ( - self.model_config.hidden_size * 3 + target_cfg.hidden_size * 3 if self.speculative_algorithm.is_eagle3() and self.eagle_use_aux_hidden_state - else self.model_config.spec_hidden_size + else target_cfg.spec_hidden_size ) draft_extend_input = EagleDraftExtendInput.create_idle_input( device=self.device, hidden_size=hidden_size, - dtype=self.model_config.dtype, + dtype=target_cfg.dtype, capture_hidden_mode=CaptureHiddenMode.LAST, ) batch.spec_info = draft_extend_input