minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: STANDALONE spec-decode hidden-size mismatch crash (#24217)

Browse Source 
Co-authored-by: root <root@sharp-heart-runs-fin-03.datacrunch.io>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Sleep_AllDay
2026-05-10 17:48:00 -07:00
committed by GitHub
co-authored by root github-actions[bot] kpham-sgl Claude Opus 4.7 hnyls2002 Liangsheng Yin
parent 8acb0270fd
commit aa7a9af12f
2 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py
+4 -2
View File
@@ -152,12 +152,14 @@ class EAGLEDraftExtendCudaGraphRunner:
dtype=self.model_runner.dtype, dtype=self.model_runner.dtype,
) )
else: else:
# Use target config: hidden_states carries target output.
target_cfg = self.eagle_worker.target_worker.model_runner.model_config
hidden_states = torch.zeros( hidden_states = torch.zeros(
( (
self.max_num_token, self.max_num_token,
self.model_runner.model_config.spec_hidden_size, target_cfg.spec_hidden_size,
), ),
dtype=self.model_runner.dtype, dtype=target_cfg.dtype,
) )
self.seq_len_fill_value = ( self.seq_len_fill_value = (
self.model_runner.attn_backend.get_cuda_graph_seq_len_fill_value() self.model_runner.attn_backend.get_cuda_graph_seq_len_fill_value()
python/sglang/srt/speculative/eagle_worker.py
+4 -3
View File
@@ -1130,16 +1130,17 @@ class EAGLEWorker(TpModelWorker):
# All reqs finished this verify; swap to an idle ExtendInput. # All reqs finished this verify; swap to an idle ExtendInput.
batch = batch.copy() batch = batch.copy()
batch.prepare_for_idle() batch.prepare_for_idle()
target_cfg = self.target_worker.model_runner.model_config
hidden_size = ( hidden_size = (
self.model_config.hidden_size * 3 target_cfg.hidden_size * 3
if self.speculative_algorithm.is_eagle3() if self.speculative_algorithm.is_eagle3()
and self.eagle_use_aux_hidden_state and self.eagle_use_aux_hidden_state
else self.model_config.spec_hidden_size else target_cfg.spec_hidden_size
) )
draft_extend_input = EagleDraftExtendInput.create_idle_input( draft_extend_input = EagleDraftExtendInput.create_idle_input(
device=self.device, device=self.device,
hidden_size=hidden_size, hidden_size=hidden_size,
dtype=self.model_config.dtype, dtype=target_cfg.dtype,
capture_hidden_mode=CaptureHiddenMode.LAST, capture_hidden_mode=CaptureHiddenMode.LAST,
) )
batch.spec_info = draft_extend_input batch.spec_info = draft_extend_input