diff --git a/python/sglang/srt/managers/load_snapshot.py b/python/sglang/srt/managers/load_snapshot.py index 950ef4f18..6b147c98c 100644 --- a/python/sglang/srt/managers/load_snapshot.py +++ b/python/sglang/srt/managers/load_snapshot.py @@ -171,6 +171,7 @@ class QueueMetrics(msgspec.Struct, array_like=True): grammar: int paused: int retracted: int + prealloc_ready: int # LoadSnapshot's nested sub-struct fields; every other struct field is a flat diff --git a/python/sglang/srt/managers/scheduler_components/load_inquirer.py b/python/sglang/srt/managers/scheduler_components/load_inquirer.py index f26a98e7e..409ca12e3 100644 --- a/python/sglang/srt/managers/scheduler_components/load_inquirer.py +++ b/python/sglang/srt/managers/scheduler_components/load_inquirer.py @@ -166,6 +166,7 @@ class SchedulerLoadInquirer: mode_str = "null" prefill_bootstrap = prefill_inflight = 0 decode_prealloc = decode_transfer = decode_retracted = 0 + decode_prealloc_ready = 0 if self.disaggregation_mode == DisaggregationMode.PREFILL: mode_str = "prefill" prefill_bootstrap = len(self.get_disagg_prefill_bootstrap_queue().queue) @@ -177,6 +178,11 @@ class SchedulerLoadInquirer: decode_retracted = len( self.get_disagg_decode_prealloc_queue().retracted_queue ) + decode_prealloc_ready = sum( + 1 + for decode_req in self.get_disagg_decode_prealloc_queue().queue + if decode_req.waiting_for_input + ) disaggregation = DisaggregationMetrics( mode=mode_str, prefill_bootstrap_queue_reqs=prefill_bootstrap, @@ -193,6 +199,7 @@ class SchedulerLoadInquirer: grammar=stats.num_grammar_queue_reqs, paused=stats.num_paused_reqs, retracted=stats.num_retracted_reqs, + prealloc_ready=decode_prealloc_ready, ) totals = self.get_decode_moment_totals() diff --git a/test/registered/unit/entrypoints/test_v1_loads_aggregate.py b/test/registered/unit/entrypoints/test_v1_loads_aggregate.py index 4aea88daf..66f059e88 100644 --- a/test/registered/unit/entrypoints/test_v1_loads_aggregate.py +++ b/test/registered/unit/entrypoints/test_v1_loads_aggregate.py @@ -171,7 +171,9 @@ class TestGetLoads(CustomTestCase): disaggregation=DisaggregationMetrics( mode="decode", decode_transfer_queue_reqs=4 ), - queues=QueueMetrics(waiting=2, grammar=1, paused=0, retracted=3), + queues=QueueMetrics( + waiting=2, grammar=1, paused=0, retracted=3, prealloc_ready=1 + ), ) )