From 0b77284587313705c137637534becce0b8653535 Mon Sep 17 00:00:00 2001 From: Byron Hsu Date: Wed, 22 Apr 2026 13:05:40 -0700 Subject: [PATCH] [minor] Make DEFAULT_FORCE_STREAM_INTERVAL configurable via SGLANG_FORCE_STREAM_INTERVAL (#23215) --- python/sglang/srt/environ.py | 6 ++++++ .../sglang/srt/managers/scheduler_output_processor_mixin.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 709ef340c..c587b6ab3 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -266,6 +266,12 @@ class Envs: SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL = EnvInt(120) SGLANG_SWA_EVICTION_INTERVAL_MULTIPLIER = EnvFloat(1.0) + # For non-streaming requests, the scheduler still flushes intermediate + # output batches to the tokenizer manager every N decoded tokens so that + # `first_token_time`/TTFT can be recorded. Lower this (e.g. to 1) to get + # an accurate TTFT for benchmarking; the upstream default of 50 trades + # off some TTFT-metric accuracy for less IPC overhead. + SGLANG_FORCE_STREAM_INTERVAL = EnvInt(50) # Test: pd-disaggregation SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake") diff --git a/python/sglang/srt/managers/scheduler_output_processor_mixin.py b/python/sglang/srt/managers/scheduler_output_processor_mixin.py index ee3a463b0..c3e6bba9f 100644 --- a/python/sglang/srt/managers/scheduler_output_processor_mixin.py +++ b/python/sglang/srt/managers/scheduler_output_processor_mixin.py @@ -33,7 +33,9 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -DEFAULT_FORCE_STREAM_INTERVAL = 50 +# How often (in decoded tokens) the scheduler force-flushes an intermediate +# output batch for non-streaming requests. +DEFAULT_FORCE_STREAM_INTERVAL = envs.SGLANG_FORCE_STREAM_INTERVAL.get() class SchedulerOutputProcessorMixin: