minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix potential eos bug on decode instance when PD is enabled (#12206)

Browse Source 
Signed-off-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Shangming Cai
2025-10-28 01:29:02 -07:00
committed by GitHub
parent a1f2dc90e4
commit 41a113356a
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/disaggregation/decode.py
+6 -2
View File
@@ -773,8 +773,12 @@ class DecodeTransferQueue:
indices_to_remove.add(i) indices_to_remove.add(i)
decode_req.req.time_stats.wait_queue_entry_time = time.perf_counter() decode_req.req.time_stats.wait_queue_entry_time = time.perf_counter()
# special handling for sampling_params.max_new_tokens == 1 # special handling for corner cases
if decode_req.req.sampling_params.max_new_tokens == 1: should_finish = (
decode_req.req.sampling_params.max_new_tokens == 1
or output_id in decode_req.req.eos_token_ids
)
if should_finish:
# finish immediately # finish immediately
decode_req.req.time_stats.forward_entry_time = ( decode_req.req.time_stats.forward_entry_time = (
decode_req.req.time_stats.completion_time decode_req.req.time_stats.completion_time