[Bug][Spec] fix startup crash and reduce CUDA graph memory usage for speculative adaptive (#35275)

Co-authored-by: alphabetc1 <2508695655@qq.com>
Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
GoldPancake
2026-08-27 00:46:13 -07:00
committed by GitHub
co-authored by alphabetc1 Shuwen Wang Claude Opus 5
parent 1a91c232ea
commit b8a6adadfe
7 changed files with 110 additions and 10 deletions
@@ -0,0 +1,51 @@
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
class _FakeModelRunner:
max_decode_logits_rows = ModelRunner.max_decode_logits_rows
def __init__(self, *, initial_width: int, cuda_graph_bs: list[int]):
self.initial_width = initial_width
self.cuda_graph_bs = cuda_graph_bs
def decode_num_tokens_per_req(self, *, num_draft_tokens=None):
return self.initial_width if num_draft_tokens is None else num_draft_tokens
def _alignment_8_capture_bs(runner, width):
return ([bs for bs in runner.cuda_graph_bs if bs * width % 8 == 0], [])
class TestModelRunnerDecodeRows(unittest.TestCase):
def test_adaptive_sizing_covers_a_wider_candidate_width(self):
"""The shared logits buffer is sized for the widest adaptive candidate
width: bs 12 at width 6 needs 72 rows."""
runner = _FakeModelRunner(initial_width=4, cuda_graph_bs=[4, 8, 12])
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
f.write('{"1":{"candidate_steps":[3,5]}}')
f.flush()
spec = SimpleNamespace(
speculative_adaptive=True, speculative_adaptive_config=f.name
)
with patch(
"sglang.srt.model_executor.model_runner.get_spec", return_value=spec
), patch(
"sglang.srt.model_executor.model_runner.max_speculative_num_draft_tokens",
return_value=6,
), patch(
"sglang.srt.model_executor.model_runner.get_batch_sizes_to_capture",
side_effect=_alignment_8_capture_bs,
):
self.assertEqual(runner.max_decode_logits_rows(), 72)
if __name__ == "__main__":
unittest.main()