feat(metrics): expose deferred decode KV release metrics (#41128)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
FREETRUMP
2026-10-04 04:52:21 +08:00
committed by GitHub
co-authored by Shangming Cai Claude Fable 5
parent 749a89abd6
commit 8bf780dda5
6 changed files with 379 additions and 40 deletions
+47 -11
View File
@@ -2407,8 +2407,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
envs.SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE_TIMEOUT.get()
)
# Aborted-mid-transfer requests whose KV pages/slot are held until drained
# or timed out. Entries: (decode_req, deadline, metadata_idx, required_acks).
self._deferred_releases: List[Tuple[DecodeRequest, float, int, int]] = []
# or timed out. Entries:
# (decode_req, start_time, deadline, metadata_idx, required_acks).
self._deferred_releases: List[Tuple[DecodeRequest, float, float, int, int]] = []
def add(self, decode_req: DecodeRequest) -> None:
self.queue.append(decode_req)
@@ -2799,12 +2800,19 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
return transferred_reqs
def _defer_release(self, decode_req: DecodeRequest) -> None:
deadline = time.monotonic() + self.deferred_kv_release_timeout
start_time = time.monotonic()
deadline = start_time + self.deferred_kv_release_timeout
# Require an ack from every notified prefill rank (dummy-proof). Snapshot
# now -- the receiver may be cleared by resolve time.
required_acks = len(decode_req.kv_receiver.bootstrap_infos)
self._deferred_releases.append(
(decode_req, deadline, decode_req.metadata_buffer_index, required_acks)
(
decode_req,
start_time,
deadline,
decode_req.metadata_buffer_index,
required_acks,
)
)
def _do_release(self, decode_req: DecodeRequest, idx: int) -> None:
@@ -2822,6 +2830,18 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
def has_pending_deferred_releases(self) -> bool:
return bool(self._deferred_releases)
def num_pending_deferred_releases(self) -> int:
return len(self._deferred_releases)
def _observe_deferred_kv_release(
self, duration_seconds: float, outcome: str
) -> None:
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.observe_decode_deferred_kv_release(
duration_seconds=duration_seconds,
outcome=outcome,
)
def resolve_deferred_releases(self) -> None:
"""Release drained transfers; device destinations may also time out."""
if not self._deferred_releases:
@@ -2831,7 +2851,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
req.kv_receiver.kv_mgr.is_abort_release_safe(
req.req.bootstrap_room, acks
)
for req, _, _, acks in self._deferred_releases
for req, _, _, _, acks in self._deferred_releases
if self.enable_host_receive and req.host_staged
],
dtype=torch.int32,
@@ -2845,36 +2865,52 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
now = time.monotonic()
still_held = []
to_release = []
for decode_req, deadline, idx, required_acks in self._deferred_releases:
for (
decode_req,
start_time,
deadline,
idx,
required_acks,
) in self._deferred_releases:
room = decode_req.req.bootstrap_room
if self.enable_host_receive and decode_req.host_staged:
if next(host_ready):
to_release.append((decode_req, idx, room, True))
to_release.append((decode_req, start_time, idx, room, True))
else:
# A timeout cannot prove that a remote write has stopped.
still_held.append((decode_req, deadline, idx, required_acks))
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
)
continue
kv_mgr = decode_req.kv_receiver.kv_mgr
drained = kv_mgr.is_abort_release_safe(room, required_acks)
if not drained and now < deadline:
still_held.append((decode_req, deadline, idx, required_acks))
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
)
else:
to_release.append((decode_req, idx, room, drained))
to_release.append((decode_req, start_time, idx, room, drained))
# Commit the survivors before releasing so a _do_release exception can't
# leave a released entry in the list (double-free / None receiver on retry).
self._deferred_releases = still_held
for decode_req, idx, room, drained in to_release:
for decode_req, start_time, idx, room, drained in to_release:
if not drained:
logger.warning(
f"Deferred KV release for room {room} timed out after "
f"{self.deferred_kv_release_timeout}s without a full drain "
f"ack from prefill; releasing anyway."
)
outcome = "drained" if drained else "timeout"
try:
self._do_release(decode_req, idx)
except Exception:
# Isolate a failed release so the rest still run; entry already dropped.
logger.exception(f"Deferred KV release failed for room {room}")
outcome = "error"
self._observe_deferred_kv_release(
duration_seconds=max(0.0, now - start_time),
outcome=outcome,
)
def release_memory_occupation(self):
"""Clean up in-flight transfers before releasing GPU memory."""
@@ -1392,6 +1392,9 @@ class SchedulerMetricsReporter:
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
transfer_queue, priority_enabled
)
self.stats.num_decode_deferred_kv_release_reqs = (
self.scheduler.disagg_decode_transfer_queue.num_pending_deferred_releases()
)
host_reqs = (
[req for req in transfer_queue if req.host_staged]
if get_disagg().disaggregation_decode_host_receive_threshold > 0
@@ -138,6 +138,7 @@ class SchedulerStats:
num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_deferred_kv_release_reqs: int = 0
kv_transfer_speed_gb_s: float = 0.0
kv_transfer_latency_ms: float = 0.0
pending_prealloc_token_usage: float = 0.0
@@ -533,6 +534,26 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
documentation="Total requests admitted to receive prefill KV in host memory.",
labelnames=labels.keys(),
)
self.num_decode_deferred_kv_release_reqs = Gauge(
name="sglang:num_decode_deferred_kv_release_reqs",
documentation=(
"The number of failed decode requests whose KV allocations are "
"held pending transfer drain acknowledgement or timeout."
),
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.decode_deferred_kv_release_seconds = Histogram(
name="sglang:decode_deferred_kv_release_seconds",
documentation="Histogram of time spent holding deferred decode KV releases.",
labelnames=labels.keys(),
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60),
)
self.decode_deferred_kv_release_total = Counter(
name="sglang:decode_deferred_kv_release_total",
documentation="Total deferred decode KV release resolution attempts.",
labelnames=list(labels.keys()) + ["outcome"],
)
self.kv_transfer_speed_gb_s = Histogram(
name="sglang:kv_transfer_speed_gb_s",
documentation="Histogram of KV cache transfer speed in GB/s.",
@@ -1208,6 +1229,16 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
def increment_decode_host_receive_reqs(self) -> None:
self.num_decode_host_receive_reqs.labels(**self.labels).inc(1)
def observe_decode_deferred_kv_release(
self, duration_seconds: float, outcome: str
) -> None:
if outcome not in ("drained", "timeout", "error"):
raise ValueError(f"Invalid deferred KV release outcome: {outcome}")
self._log_histogram(self.decode_deferred_kv_release_seconds, duration_seconds)
self.decode_deferred_kv_release_total.labels(
**self.labels, outcome=outcome
).inc(1)
def increment_prefill_retries(self, count: int) -> None:
if count > 0:
self.num_prefill_retries_total.labels(**self.labels).inc(count)
@@ -1442,6 +1473,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
self.num_decode_host_receive_queue_reqs,
stats.num_decode_host_receive_queue_reqs,
)
self._log_gauge(
self.num_decode_deferred_kv_release_reqs,
stats.num_decode_deferred_kv_release_reqs,
)
self._log_gauge(
self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage
)