observability: time loop-other inflight poll and release/insert

Adds loop-detail phases outside run_batch/process_result/get_next:

* inflight_poll: process_disagg_prefill_inflight_queue() (overlap loop)
* inflight_release: release_kv_cache(checkpoint=True) + tree_cache.finish()
  inside it, so the checkpoint/insert cost is separated from the poll
* launch_sample: launch_batch_sample_if_needed()

The get_next side already publishes match/adder/hicache/bootstrap/mlp_sync
and a pass residual (sglang.other_ms) on scheduler.prefill.pass.
This commit is contained in:
2026-11-07 09:26:46 +08:00
parent aa53670fcd
commit 53267ee18f
+17 -2
View File
@@ -1052,11 +1052,16 @@ class SchedulerDisaggregationPrefillMixin:
# When the server is idle, do self-check and re-init some states
self.on_idle()
self.process_disagg_prefill_inflight_queue()
# `loop other` (outside run_batch/process_result/get_next) hosts the
# inflight poll + release_kv_cache(checkpoint/insert) and the sample
# launch. Timed as its own phases so the loop-other residual shrinks.
with batch_telemetry.detail_phase("inflight_poll"):
self.process_disagg_prefill_inflight_queue()
# Run sample of the current batch
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
self.launch_batch_sample_if_needed(batch_result, batch)
with batch_telemetry.detail_phase("launch_sample"):
self.launch_batch_sample_if_needed(batch_result, batch)
# Update last_batch and scheduler status
self._sched_idled = batch is None
@@ -1457,10 +1462,20 @@ class SchedulerDisaggregationPrefillMixin:
if not isinstance(req.finished_reason, FINISH_ABORT):
req.finished_reason = FINISH_LENGTH(length=0)
# unlock the tree
_release_t0 = (
time.monotonic_ns()
if batch_telemetry.loop_detail_enabled()
else None
)
release_kv_cache(req, self.tree_cache, checkpoint=True)
self.tree_cache.finish(
req.cache_request_handle, CacheRequestOutcome.SUCCESS
)
if _release_t0 is not None:
# Nested in inflight_poll: checkpoint + radix/hicache insert.
batch_telemetry.detail_add(
"inflight_release", _release_t0, time.monotonic_ns()
)
# Consume the join summary before clear() releases the room.
_join_stats = None
if batch_telemetry.mooncake_detail_enabled():