diff --git a/python/sglang/srt/managers/scheduler_components/output_streamer.py b/python/sglang/srt/managers/scheduler_components/output_streamer.py index 752d456b9b..2b4e0d5b04 100644 --- a/python/sglang/srt/managers/scheduler_components/output_streamer.py +++ b/python/sglang/srt/managers/scheduler_components/output_streamer.py @@ -429,11 +429,16 @@ class _GenerationStreamAccumulator: req.sampling_params.stream_interval or self.default_stream_interval ) - # origin stream_interval logic + # Send every step at the default interval of 1. Otherwise send + # the first token at once, then whenever at least + # stream_interval tokens are unsent: a step can append several + # tokens (speculative decoding, dLLM), so it can pass a multiple + # of the interval without landing on it. + num_unsent = len(req.output_ids) - req.send_token_offset should_output = ( - len(req.output_ids) % stream_interval == 1 - if stream_interval > 1 - else len(req.output_ids) % stream_interval == 0 + stream_interval == 1 + or num_unsent >= stream_interval + or (req.send_token_offset == 0 and num_unsent > 0) ) if should_output: diff --git a/test/registered/unit/managers/test_output_streamer_customized_info.py b/test/registered/unit/managers/test_output_streamer_customized_info.py index a426f434a5..ec76d63694 100644 --- a/test/registered/unit/managers/test_output_streamer_customized_info.py +++ b/test/registered/unit/managers/test_output_streamer_customized_info.py @@ -214,6 +214,7 @@ class TestOutputStreamerCustomizedInfo(unittest.TestCase): quiet = _FakeReq("quiet", [10, 11]) quiet.stream = True quiet.sampling_params.stream_interval = 2 + quiet.send_token_offset = 1 terminal = _FakeReq("terminal", [20], finished=True) streamer._stream_output_generation([quiet, terminal], False) @@ -251,6 +252,7 @@ class TestOutputStreamerCustomizedInfo(unittest.TestCase): quiet = _FakeReq("quiet", [10, 11]) quiet.stream = True quiet.sampling_params.stream_interval = 2 + quiet.send_token_offset = 1 streamer._stream_output_generation([terminal, quiet], False) @@ -392,6 +394,33 @@ class TestOutputStreamerWeightVersions(unittest.TestCase): self.assertIsNone(payload.weight_versions) +class TestOutputStreamerStreamInterval(unittest.TestCase): + def test_sends_once_interval_tokens_are_unsent(self): + """With stream_interval=4, the first token goes out at once, then a chunk + whenever 4 or more tokens are unsent, however many each step appends.""" + cases = { + # One token per step keeps the cadence of sending at 1, 5, 9, ... + "single_token_steps": ([1] * 9, (), [1, 0, 0, 0, 4, 0, 0, 0, 4]), + # Speculative steps pass multiples of 4 without landing on them. + "multi_token_steps": ([1, 3, 3, 3, 3], (), [1, 0, 6, 0, 6]), + # A chunk held for a possible stop string goes out on the next step. + "held_for_stop_str": ([1] * 6, (4,), [1, 0, 0, 0, 0, 5]), + } + for name, (steps, held_steps, expected) in cases.items(): + with self.subTest(name): + req = _FakeReq("r0", []) + req.stream = True + req.sampling_params.stream_interval = 4 + sent = [] + for step, num_tokens in enumerate(steps): + req.check_match_stop_str_prefix = lambda: step in held_steps + req.output_ids.extend(range(num_tokens)) + accumulator = _accumulator() + accumulator.accept(req=req) + sent.append(sum(map(len, accumulator.output_ids))) + self.assertEqual(sent, expected) + + _IPC_ROUND_TRIPS = { "pickle": lambda payload: pickle.loads( pickle.dumps(payload, protocol=pickle.HIGHEST_PROTOCOL)