minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: set first_token_time before computing decode_throughput for single-batch completions (#19984)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-03-06 16:11:41 -08:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 5c8e28698c
commit 7a12255b6e
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/tokenizer_manager.py
+5
View File
@@ -1612,6 +1612,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
state.finished = recv_obj.finished_reasons[i] is not None state.finished = recv_obj.finished_reasons[i] is not None
if state.finished: if state.finished:
# Ensure first_token_time is set before computing decode_throughput.
# Without this, requests that finish on their first output batch
# would have first_token_time=0.0, producing bogus throughput.
if state.time_stats.first_token_time == 0.0:
state.time_stats.set_first_token_time()
state.time_stats.trace_ctx.trace_set_root_attrs( state.time_stats.trace_ctx.trace_set_root_attrs(
self.convert_to_span_attrs(state, recv_obj, i) self.convert_to_span_attrs(state, recv_obj, i)
) )