Files
sglang/test/registered/unit/spec/test_adaptive_spec_params.py
T
+2 97c6978369 GLM-5.3-Flash support (#36507)
Co-authored-by: zRzRzRzRzRzRzR <Yuxuan.Zhang2@liverpool.ac.uk>
Co-authored-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com>
Co-authored-by: zanes-ops <zanes@nvidia.com>
Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
Co-authored-by: Jian Chen <jianchen0311@gmail.com>
Co-authored-by: zijiexia <37504505+zijiexia@users.noreply.github.com>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Ehsan Akhgari <ehsan.akhgari@gmail.com>
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
Co-authored-by: BBuf <1182563586@qq.com>
Co-authored-by: Raiden Makoto <81530826+Raiden-Makoto@users.noreply.github.com>
2026-09-06 02:27:59 -07:00

426 lines
16 KiB
Python

import json
import tempfile
import unittest
from sglang.srt.speculative.adaptive_spec_params import (
AdaptiveSpeculativeParams,
AdaptiveStepSlot,
resolve_candidate_steps_from_config,
)
from sglang.test.ci.ci_register import register_cpu_ci, register_xpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
register_xpu_ci(est_time=10, suite="stage-a-test-1-gpu-xpu")
class TestAdaptiveStepSlot(CustomTestCase):
def _make_params_from_config(self, initial_steps: int, config: dict):
return AdaptiveStepSlot(initial_steps=initial_steps, cfg=config)
def test_initial_steps_snaps_to_middle_when_missing(self):
params = self._make_params_from_config(2, {"candidate_steps": [1, 3, 7]})
self.assertEqual(params.candidate_steps, [1, 3, 7])
self.assertEqual(params.current_steps, 3)
self.assertEqual(params.ema_accept_len, 2.0)
def test_update_respects_warmup_and_interval(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 1,
"update_interval": 2,
},
)
self.assertFalse(params.update([0, 0]))
self.assertEqual(params.current_steps, 3)
self.assertFalse(params.update([0, 0]))
self.assertEqual(params.current_steps, 3)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 1)
def test_empty_batches_do_not_consume_warmup_or_shift_steps(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 1,
"update_interval": 1,
},
)
self.assertFalse(params.update([]))
self.assertEqual(params.current_steps, 3)
self.assertEqual(params.ema_accept_len, 2.0)
self.assertFalse(params.update([0, 0]))
self.assertEqual(params.current_steps, 3)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 1)
def test_update_scales_up_across_candidates(self):
params = self._make_params_from_config(
1,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"up_hysteresis": 0.0,
},
)
self.assertTrue(params.update([1, 1]))
self.assertEqual(params.current_steps, 3)
self.assertTrue(params.update([3, 3]))
self.assertEqual(params.current_steps, 7)
def test_update_can_scale_down_across_candidates_in_one_recompute(self):
params = self._make_params_from_config(
7,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
},
)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 1)
def test_exact_rise_threshold_does_not_upshift(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"up_hysteresis": 0.0,
},
)
self.assertFalse(params.update([2, 3]))
self.assertEqual(params.current_steps, 3)
self.assertEqual(params.ema_accept_len, 2.5)
self.assertTrue(params.update([3, 3]))
self.assertEqual(params.current_steps, 7)
def test_exact_drop_threshold_does_downshift(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"down_hysteresis": 0.0,
"up_hysteresis": 0.5,
},
)
self.assertTrue(params.update([0, 1]))
self.assertEqual(params.current_steps, 1)
self.assertEqual(params.ema_accept_len, 0.5)
def test_hysteresis_can_prevent_premature_upshift(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"up_hysteresis": 0.75,
},
)
self.assertFalse(params.update([3, 3]))
self.assertEqual(params.current_steps, 3)
self.assertTrue(params.update([4, 4]))
self.assertEqual(params.current_steps, 7)
def test_down_hysteresis_can_prevent_premature_downshift(self):
params = self._make_params_from_config(
7,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"down_hysteresis": -0.75,
},
)
self.assertFalse(params.update([2, 2]))
self.assertEqual(params.current_steps, 7)
self.assertTrue(params.update([1, 1]))
self.assertEqual(params.current_steps, 3)
def test_multi_batch_sequence_can_ramp_up_then_back_down(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 0.5,
"warmup_batches": 0,
"update_interval": 1,
"up_hysteresis": 0.0,
"down_hysteresis": 0.0,
},
)
self.assertTrue(params.update([4, 4]))
self.assertEqual(params.current_steps, 7)
self.assertEqual(params.ema_accept_len, 3.0)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 3)
self.assertEqual(params.ema_accept_len, 1.5)
self.assertFalse(params.update([0, 0]))
self.assertEqual(params.current_steps, 3)
self.assertEqual(params.ema_accept_len, 0.75)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 1)
self.assertEqual(params.ema_accept_len, 0.375)
def test_zero_step_mixed_slot_drops_probes_and_rechecks(self):
params = self._make_params_from_config(
3,
{
"candidate_steps": [0, 3],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"down_hysteresis": 0.0,
},
)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 0)
self.assertEqual(params.ema_accept_len, 0.0)
self.assertTrue(params.update([3, 3]))
self.assertEqual(params.current_steps, 3)
self.assertEqual(params.ema_accept_len, 0.0)
self.assertTrue(params.update([0, 0]))
self.assertEqual(params.current_steps, 0)
self.assertEqual(params.ema_accept_len, 0.0)
def test_ceiling_coeff_caps_steps(self):
params = self._make_params_from_config(
7,
{
"candidate_steps": [1, 3, 7],
"ema_alpha": 1.0,
"warmup_batches": 0,
"update_interval": 1,
"ceiling_coeff": 1.0,
},
)
# Force low ema to trigger ceiling
params.ema_accept_len = 1.0
self.assertTrue(params.update([1, 1]))
# ceiling = ceil(1.0 * 1.0) = 1, target capped to 1
self.assertEqual(params.current_steps, 1)
def test_ceiling_disabled_by_default(self):
params = self._make_params_from_config(3, {"candidate_steps": [1, 3, 7]})
self.assertEqual(params.ceiling_coeff, 0)
class TestAdaptiveSpeculativeParams(CustomTestCase):
def test_default_config_loads(self):
params = AdaptiveSpeculativeParams(initial_steps=3)
self.assertEqual(params._bs_list, [1, 8, 32, 64])
self.assertEqual(params._slots[1].candidate_steps, [1, 3, 5, 7])
self.assertEqual(params._slots[8].candidate_steps, [0, 1, 3])
self.assertEqual(params._slots[32].candidate_steps, [0, 1])
self.assertEqual(params._slots[64].candidate_steps, [0])
def test_config_file(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump(
{
"1": {"candidate_steps": [1, 5], "up_hysteresis": 0.3},
"32": {"candidate_steps": [1, 2]},
},
f,
)
f.flush()
params = AdaptiveSpeculativeParams(initial_steps=5, cfg_path=f.name)
self.assertEqual(params._bs_list, [1, 32])
# Slots are built straight from the config; the launch flag never pollutes
# them. initial_steps just selects the smallest slot's starting step.
self.assertEqual(params._slots[1].candidate_steps, [1, 5])
self.assertEqual(params._slots[1].current_steps, 5)
self.assertEqual(params._slots[1].up_hysteresis, 0.3)
self.assertEqual(params._slots[32].candidate_steps, [1, 2])
def test_launch_flag_not_injected_into_slots(self):
# initial_steps lives only in a larger slot. It must NOT be merged into
# any other slot's candidates: slots come straight from the config.
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump(
{
"1": {"candidate_steps": [1, 5]},
"8": {"candidate_steps": [1, 3, 7]},
},
f,
)
f.flush()
params = AdaptiveSpeculativeParams(initial_steps=7, cfg_path=f.name)
self.assertEqual(params._slots[1].candidate_steps, [1, 5])
self.assertEqual(params._slots[8].candidate_steps, [1, 3, 7])
# The slot that does not own initial_steps starts at its own median.
self.assertEqual(params._slots[1].current_steps, 5)
self.assertEqual(params._slots[8].current_steps, 7)
def test_invalid_config_raises(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump({"not_a_bs": "bad"}, f)
f.flush()
with self.assertRaises(ValueError):
AdaptiveSpeculativeParams(initial_steps=3, cfg_path=f.name)
def test_invalid_steps_raises(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump({"1": {"candidate_steps": "bad"}}, f)
f.flush()
with self.assertRaises(ValueError):
AdaptiveSpeculativeParams(initial_steps=3, cfg_path=f.name)
def test_empty_steps_raises(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump({"1": {"candidate_steps": []}}, f)
f.flush()
with self.assertRaises(ValueError):
AdaptiveSpeculativeParams(initial_steps=3, cfg_path=f.name)
def test_global_hysteresis_inherited(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump(
{
"up_hysteresis": 0.5,
"1": {"candidate_steps": [1, 3]},
},
f,
)
f.flush()
params = AdaptiveSpeculativeParams(initial_steps=3, cfg_path=f.name)
self.assertEqual(params._slots[1].up_hysteresis, 0.5)
def test_entry_hysteresis_overrides_global(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump(
{
"up_hysteresis": 0.5,
"1": {"candidate_steps": [1, 3], "up_hysteresis": 0.1},
},
f,
)
f.flush()
params = AdaptiveSpeculativeParams(initial_steps=3, cfg_path=f.name)
self.assertEqual(params._slots[1].up_hysteresis, 0.1)
class TestBatchSizeRouting(CustomTestCase):
"""BS-aware routing: batch size selects the slot, CUDA-graph BS pads first."""
def _params(self):
# Slots: bs=1 -> [1,3,5,7], bs=8 -> [0,1,3], bs=32 -> [0,1].
return AdaptiveSpeculativeParams(initial_steps=3)
def test_routes_to_floor_slot_without_cuda_graph(self):
params = self._params()
# A batch maps to the largest slot BS <= batch (floor), capped at the top slot.
self.assertEqual(params._route(1).candidate_steps, [1, 3, 5, 7])
self.assertEqual(params._route(7).candidate_steps, [1, 3, 5, 7])
self.assertEqual(params._route(8).candidate_steps, [0, 1, 3])
self.assertEqual(params._route(31).candidate_steps, [0, 1, 3])
self.assertEqual(params._route(32).candidate_steps, [0, 1])
self.assertEqual(params._route(1000).candidate_steps, [0])
def test_cuda_graph_bs_pads_batch_up_before_routing(self):
params = self._params()
params.set_cuda_graph_bs([4, 8, 16, 32])
# bs=5 pads up to the captured graph BS 8 -> slot bs=8.
self.assertEqual(params._route(5).candidate_steps, [0, 1, 3])
# bs=17 pads up to 32 -> slot bs=32.
self.assertEqual(params._route(17).candidate_steps, [0, 1])
# A batch larger than every captured BS keeps its own value -> top slot.
self.assertEqual(params._route(100).candidate_steps, [0])
def test_cuda_graph_bs_for_step_prunes_unreachable_graphs(self):
params = self._params()
params.set_cuda_graph_bs([4, 8, 16, 32])
# step=1 is reachable from every slot.
self.assertEqual(params.cuda_graph_bs_for_step(1), [4, 8, 16, 32])
# step=3 lives in the bs=1 and bs=8 slots: graphs 4,8,16 floor into them.
self.assertEqual(params.cuda_graph_bs_for_step(3), [4, 8, 16])
# step=5 lives only in the bs=1 slot: only graph BS 4 floors into it.
self.assertEqual(params.cuda_graph_bs_for_step(5), [4])
# step=7 lives only in the bs=1 slot: only graph BS 4 floors into it.
self.assertEqual(params.cuda_graph_bs_for_step(7), [4])
def test_cuda_graph_bs_for_step_returns_none_when_disabled(self):
params = self._params()
self.assertIsNone(params.cuda_graph_bs_for_step(7))
params.set_cuda_graph_bs(None)
self.assertIsNone(params.cuda_graph_bs_for_step(7))
def test_observe_verify_feeds_the_routed_slot(self):
params = self._params()
# Drive the bs=1 slot up with perfect acceptance; the bs=32 slot is
# untouched and stays at its single candidate step.
for _ in range(40):
params.on_verify_complete([7, 7, 7], batch_size=1)
self.assertGreater(params.get_steps_for_batch(1), 1)
self.assertEqual(params.get_steps_for_batch(32), 1)
class TestResolveCandidateSteps(CustomTestCase):
def test_default_config(self):
steps = resolve_candidate_steps_from_config()
self.assertEqual(steps, [0, 1, 3, 5, 7])
def test_config_file(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump({"1": {"candidate_steps": [2, 4]}}, f)
f.flush()
steps = resolve_candidate_steps_from_config(cfg_path=f.name)
self.assertEqual(steps, [2, 4])
def test_unions_and_dedups_across_slots(self):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
json.dump(
{
"1": {"candidate_steps": [1, 5]},
"8": {"candidate_steps": [3, 5, 7]},
},
f,
)
f.flush()
steps = resolve_candidate_steps_from_config(cfg_path=f.name)
self.assertEqual(steps, [1, 3, 5, 7])
if __name__ == "__main__":
unittest.main()