minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add num_prealloc_ready_tokens to decode load snapshot (#38065)

Browse Source
This commit is contained in:
cctry
2026-09-04 15:05:57 -07:00
committed by GitHub
parent 320bdd1ee2
commit 0eff0f7460
2 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/load_snapshot.py
+1
View File
@@ -202,6 +202,7 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it # num_total_tokens minus tokens still awaiting a KV transfer (equal to it
# outside disaggregated decode). # outside disaggregated decode).
num_active_tokens: int = 0 num_active_tokens: int = 0
num_prealloc_ready_tokens: int = 0
max_total_num_tokens: int = 0 max_total_num_tokens: int = 0
max_running_requests: int = 0 max_running_requests: int = 0
token_usage: float = 0.0 token_usage: float = 0.0
python/sglang/srt/managers/scheduler_components/load_inquirer.py
+7 -3
View File
@@ -173,6 +173,7 @@ class SchedulerLoadInquirer:
prefill_bootstrap = prefill_inflight = 0 prefill_bootstrap = prefill_inflight = 0
decode_prealloc = decode_transfer = decode_retracted = 0 decode_prealloc = decode_transfer = decode_retracted = 0
decode_prealloc_ready = 0 decode_prealloc_ready = 0
num_prealloc_ready_tokens = 0
if self.disaggregation_mode == DisaggregationMode.PREFILL: if self.disaggregation_mode == DisaggregationMode.PREFILL:
mode_str = "prefill" mode_str = "prefill"
prefill_bootstrap = len(self.get_disagg_prefill_bootstrap_queue().queue) prefill_bootstrap = len(self.get_disagg_prefill_bootstrap_queue().queue)
@@ -184,11 +185,13 @@ class SchedulerLoadInquirer:
decode_retracted = len( decode_retracted = len(
self.get_disagg_decode_prealloc_queue().retracted_queue self.get_disagg_decode_prealloc_queue().retracted_queue
) )
decode_prealloc_ready = sum( ready_reqs = [
1 decode_req.req
for decode_req in self.get_disagg_decode_prealloc_queue().queue for decode_req in self.get_disagg_decode_prealloc_queue().queue
if decode_req.waiting_for_input if decode_req.waiting_for_input
) ]
decode_prealloc_ready = len(ready_reqs)
num_prealloc_ready_tokens = sum(req.seqlen for req in ready_reqs)
disaggregation = DisaggregationMetrics( disaggregation = DisaggregationMetrics(
mode=mode_str, mode=mode_str,
prefill_bootstrap_queue_reqs=prefill_bootstrap, prefill_bootstrap_queue_reqs=prefill_bootstrap,
@@ -220,6 +223,7 @@ class SchedulerLoadInquirer:
num_used_tokens=num_used_tokens, num_used_tokens=num_used_tokens,
num_total_tokens=num_total_tokens, num_total_tokens=num_total_tokens,
num_active_tokens=num_active_tokens, num_active_tokens=num_active_tokens,
num_prealloc_ready_tokens=num_prealloc_ready_tokens,
max_total_num_tokens=self.max_total_num_tokens, max_total_num_tokens=self.max_total_num_tokens,
max_running_requests=self.max_running_requests, max_running_requests=self.max_running_requests,
token_usage=round(kv_token_usage, 4), token_usage=round(kv_token_usage, 4),