Stream a chunk once stream_interval tokens are unsent (#42643)

This commit is contained in:
shodoco
2026-10-08 10:12:37 -07:00
committed by GitHub
parent 344b8a863b
commit e2fbc6a550
2 changed files with 38 additions and 4 deletions
@@ -429,11 +429,16 @@ class _GenerationStreamAccumulator:
req.sampling_params.stream_interval or self.default_stream_interval
)
# origin stream_interval logic
# Send every step at the default interval of 1. Otherwise send
# the first token at once, then whenever at least
# stream_interval tokens are unsent: a step can append several
# tokens (speculative decoding, dLLM), so it can pass a multiple
# of the interval without landing on it.
num_unsent = len(req.output_ids) - req.send_token_offset
should_output = (
len(req.output_ids) % stream_interval == 1
if stream_interval > 1
else len(req.output_ids) % stream_interval == 0
stream_interval == 1
or num_unsent >= stream_interval
or (req.send_token_offset == 0 and num_unsent > 0)
)
if should_output: