observability: time loop-other inflight poll and release/insert
Adds loop-detail phases outside run_batch/process_result/get_next: * inflight_poll: process_disagg_prefill_inflight_queue() (overlap loop) * inflight_release: release_kv_cache(checkpoint=True) + tree_cache.finish() inside it, so the checkpoint/insert cost is separated from the poll * launch_sample: launch_batch_sample_if_needed() The get_next side already publishes match/adder/hicache/bootstrap/mlp_sync and a pass residual (sglang.other_ms) on scheduler.prefill.pass.
This commit is contained in:
@@ -1052,11 +1052,16 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
# `loop other` (outside run_batch/process_result/get_next) hosts the
|
||||
# inflight poll + release_kv_cache(checkpoint/insert) and the sample
|
||||
# launch. Timed as its own phases so the loop-other residual shrinks.
|
||||
with batch_telemetry.detail_phase("inflight_poll"):
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
self.launch_batch_sample_if_needed(batch_result, batch)
|
||||
with batch_telemetry.detail_phase("launch_sample"):
|
||||
self.launch_batch_sample_if_needed(batch_result, batch)
|
||||
|
||||
# Update last_batch and scheduler status
|
||||
self._sched_idled = batch is None
|
||||
@@ -1457,10 +1462,20 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if not isinstance(req.finished_reason, FINISH_ABORT):
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
# unlock the tree
|
||||
_release_t0 = (
|
||||
time.monotonic_ns()
|
||||
if batch_telemetry.loop_detail_enabled()
|
||||
else None
|
||||
)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=True)
|
||||
self.tree_cache.finish(
|
||||
req.cache_request_handle, CacheRequestOutcome.SUCCESS
|
||||
)
|
||||
if _release_t0 is not None:
|
||||
# Nested in inflight_poll: checkpoint + radix/hicache insert.
|
||||
batch_telemetry.detail_add(
|
||||
"inflight_release", _release_t0, time.monotonic_ns()
|
||||
)
|
||||
# Consume the join summary before clear() releases the room.
|
||||
_join_stats = None
|
||||
if batch_telemetry.mooncake_detail_enabled():
|
||||
|
||||
Reference in New Issue
Block a user