From 53267ee18f2ed9b414a0ae563176dd1c8f849641 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sat, 7 Nov 2026 09:26:46 +0800 Subject: [PATCH] observability: time loop-other inflight poll and release/insert Adds loop-detail phases outside run_batch/process_result/get_next: * inflight_poll: process_disagg_prefill_inflight_queue() (overlap loop) * inflight_release: release_kv_cache(checkpoint=True) + tree_cache.finish() inside it, so the checkpoint/insert cost is separated from the poll * launch_sample: launch_batch_sample_if_needed() The get_next side already publishes match/adder/hicache/bootstrap/mlp_sync and a pass residual (sglang.other_ms) on scheduler.prefill.pass. --- python/sglang/srt/disaggregation/prefill.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index ddafd46e18..bdfd812db6 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1052,11 +1052,16 @@ class SchedulerDisaggregationPrefillMixin: # When the server is idle, do self-check and re-init some states self.on_idle() - self.process_disagg_prefill_inflight_queue() + # `loop other` (outside run_batch/process_result/get_next) hosts the + # inflight poll + release_kv_cache(checkpoint/insert) and the sample + # launch. Timed as its own phases so the loop-other residual shrinks. + with batch_telemetry.detail_phase("inflight_poll"): + self.process_disagg_prefill_inflight_queue() # Run sample of the current batch # It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed. - self.launch_batch_sample_if_needed(batch_result, batch) + with batch_telemetry.detail_phase("launch_sample"): + self.launch_batch_sample_if_needed(batch_result, batch) # Update last_batch and scheduler status self._sched_idled = batch is None @@ -1457,10 +1462,20 @@ class SchedulerDisaggregationPrefillMixin: if not isinstance(req.finished_reason, FINISH_ABORT): req.finished_reason = FINISH_LENGTH(length=0) # unlock the tree + _release_t0 = ( + time.monotonic_ns() + if batch_telemetry.loop_detail_enabled() + else None + ) release_kv_cache(req, self.tree_cache, checkpoint=True) self.tree_cache.finish( req.cache_request_handle, CacheRequestOutcome.SUCCESS ) + if _release_t0 is not None: + # Nested in inflight_poll: checkpoint + radix/hicache insert. + batch_telemetry.detail_add( + "inflight_release", _release_t0, time.monotonic_ns() + ) # Consume the join summary before clear() releases the room. _join_stats = None if batch_telemetry.mooncake_detail_enabled():