feat(metrics): expose deferred decode KV release metrics (#41128)
Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Shangming Cai
Claude Fable 5
parent
749a89abd6
commit
8bf780dda5
@@ -2407,8 +2407,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
envs.SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE_TIMEOUT.get()
|
||||
)
|
||||
# Aborted-mid-transfer requests whose KV pages/slot are held until drained
|
||||
# or timed out. Entries: (decode_req, deadline, metadata_idx, required_acks).
|
||||
self._deferred_releases: List[Tuple[DecodeRequest, float, int, int]] = []
|
||||
# or timed out. Entries:
|
||||
# (decode_req, start_time, deadline, metadata_idx, required_acks).
|
||||
self._deferred_releases: List[Tuple[DecodeRequest, float, float, int, int]] = []
|
||||
|
||||
def add(self, decode_req: DecodeRequest) -> None:
|
||||
self.queue.append(decode_req)
|
||||
@@ -2799,12 +2800,19 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
return transferred_reqs
|
||||
|
||||
def _defer_release(self, decode_req: DecodeRequest) -> None:
|
||||
deadline = time.monotonic() + self.deferred_kv_release_timeout
|
||||
start_time = time.monotonic()
|
||||
deadline = start_time + self.deferred_kv_release_timeout
|
||||
# Require an ack from every notified prefill rank (dummy-proof). Snapshot
|
||||
# now -- the receiver may be cleared by resolve time.
|
||||
required_acks = len(decode_req.kv_receiver.bootstrap_infos)
|
||||
self._deferred_releases.append(
|
||||
(decode_req, deadline, decode_req.metadata_buffer_index, required_acks)
|
||||
(
|
||||
decode_req,
|
||||
start_time,
|
||||
deadline,
|
||||
decode_req.metadata_buffer_index,
|
||||
required_acks,
|
||||
)
|
||||
)
|
||||
|
||||
def _do_release(self, decode_req: DecodeRequest, idx: int) -> None:
|
||||
@@ -2822,6 +2830,18 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
def has_pending_deferred_releases(self) -> bool:
|
||||
return bool(self._deferred_releases)
|
||||
|
||||
def num_pending_deferred_releases(self) -> int:
|
||||
return len(self._deferred_releases)
|
||||
|
||||
def _observe_deferred_kv_release(
|
||||
self, duration_seconds: float, outcome: str
|
||||
) -> None:
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.observe_decode_deferred_kv_release(
|
||||
duration_seconds=duration_seconds,
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def resolve_deferred_releases(self) -> None:
|
||||
"""Release drained transfers; device destinations may also time out."""
|
||||
if not self._deferred_releases:
|
||||
@@ -2831,7 +2851,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
req.req.bootstrap_room, acks
|
||||
)
|
||||
for req, _, _, acks in self._deferred_releases
|
||||
for req, _, _, _, acks in self._deferred_releases
|
||||
if self.enable_host_receive and req.host_staged
|
||||
],
|
||||
dtype=torch.int32,
|
||||
@@ -2845,36 +2865,52 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
now = time.monotonic()
|
||||
still_held = []
|
||||
to_release = []
|
||||
for decode_req, deadline, idx, required_acks in self._deferred_releases:
|
||||
for (
|
||||
decode_req,
|
||||
start_time,
|
||||
deadline,
|
||||
idx,
|
||||
required_acks,
|
||||
) in self._deferred_releases:
|
||||
room = decode_req.req.bootstrap_room
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if next(host_ready):
|
||||
to_release.append((decode_req, idx, room, True))
|
||||
to_release.append((decode_req, start_time, idx, room, True))
|
||||
else:
|
||||
# A timeout cannot prove that a remote write has stopped.
|
||||
still_held.append((decode_req, deadline, idx, required_acks))
|
||||
still_held.append(
|
||||
(decode_req, start_time, deadline, idx, required_acks)
|
||||
)
|
||||
continue
|
||||
kv_mgr = decode_req.kv_receiver.kv_mgr
|
||||
drained = kv_mgr.is_abort_release_safe(room, required_acks)
|
||||
if not drained and now < deadline:
|
||||
still_held.append((decode_req, deadline, idx, required_acks))
|
||||
still_held.append(
|
||||
(decode_req, start_time, deadline, idx, required_acks)
|
||||
)
|
||||
else:
|
||||
to_release.append((decode_req, idx, room, drained))
|
||||
to_release.append((decode_req, start_time, idx, room, drained))
|
||||
# Commit the survivors before releasing so a _do_release exception can't
|
||||
# leave a released entry in the list (double-free / None receiver on retry).
|
||||
self._deferred_releases = still_held
|
||||
for decode_req, idx, room, drained in to_release:
|
||||
for decode_req, start_time, idx, room, drained in to_release:
|
||||
if not drained:
|
||||
logger.warning(
|
||||
f"Deferred KV release for room {room} timed out after "
|
||||
f"{self.deferred_kv_release_timeout}s without a full drain "
|
||||
f"ack from prefill; releasing anyway."
|
||||
)
|
||||
outcome = "drained" if drained else "timeout"
|
||||
try:
|
||||
self._do_release(decode_req, idx)
|
||||
except Exception:
|
||||
# Isolate a failed release so the rest still run; entry already dropped.
|
||||
logger.exception(f"Deferred KV release failed for room {room}")
|
||||
outcome = "error"
|
||||
self._observe_deferred_kv_release(
|
||||
duration_seconds=max(0.0, now - start_time),
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def release_memory_occupation(self):
|
||||
"""Clean up in-flight transfers before releasing GPU memory."""
|
||||
|
||||
@@ -1392,6 +1392,9 @@ class SchedulerMetricsReporter:
|
||||
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
|
||||
transfer_queue, priority_enabled
|
||||
)
|
||||
self.stats.num_decode_deferred_kv_release_reqs = (
|
||||
self.scheduler.disagg_decode_transfer_queue.num_pending_deferred_releases()
|
||||
)
|
||||
host_reqs = (
|
||||
[req for req in transfer_queue if req.host_staged]
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0
|
||||
|
||||
@@ -138,6 +138,7 @@ class SchedulerStats:
|
||||
num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_deferred_kv_release_reqs: int = 0
|
||||
kv_transfer_speed_gb_s: float = 0.0
|
||||
kv_transfer_latency_ms: float = 0.0
|
||||
pending_prealloc_token_usage: float = 0.0
|
||||
@@ -533,6 +534,26 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
documentation="Total requests admitted to receive prefill KV in host memory.",
|
||||
labelnames=labels.keys(),
|
||||
)
|
||||
self.num_decode_deferred_kv_release_reqs = Gauge(
|
||||
name="sglang:num_decode_deferred_kv_release_reqs",
|
||||
documentation=(
|
||||
"The number of failed decode requests whose KV allocations are "
|
||||
"held pending transfer drain acknowledgement or timeout."
|
||||
),
|
||||
labelnames=labels.keys(),
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self.decode_deferred_kv_release_seconds = Histogram(
|
||||
name="sglang:decode_deferred_kv_release_seconds",
|
||||
documentation="Histogram of time spent holding deferred decode KV releases.",
|
||||
labelnames=labels.keys(),
|
||||
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60),
|
||||
)
|
||||
self.decode_deferred_kv_release_total = Counter(
|
||||
name="sglang:decode_deferred_kv_release_total",
|
||||
documentation="Total deferred decode KV release resolution attempts.",
|
||||
labelnames=list(labels.keys()) + ["outcome"],
|
||||
)
|
||||
self.kv_transfer_speed_gb_s = Histogram(
|
||||
name="sglang:kv_transfer_speed_gb_s",
|
||||
documentation="Histogram of KV cache transfer speed in GB/s.",
|
||||
@@ -1208,6 +1229,16 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
def increment_decode_host_receive_reqs(self) -> None:
|
||||
self.num_decode_host_receive_reqs.labels(**self.labels).inc(1)
|
||||
|
||||
def observe_decode_deferred_kv_release(
|
||||
self, duration_seconds: float, outcome: str
|
||||
) -> None:
|
||||
if outcome not in ("drained", "timeout", "error"):
|
||||
raise ValueError(f"Invalid deferred KV release outcome: {outcome}")
|
||||
self._log_histogram(self.decode_deferred_kv_release_seconds, duration_seconds)
|
||||
self.decode_deferred_kv_release_total.labels(
|
||||
**self.labels, outcome=outcome
|
||||
).inc(1)
|
||||
|
||||
def increment_prefill_retries(self, count: int) -> None:
|
||||
if count > 0:
|
||||
self.num_prefill_retries_total.labels(**self.labels).inc(count)
|
||||
@@ -1442,6 +1473,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
self.num_decode_host_receive_queue_reqs,
|
||||
stats.num_decode_host_receive_queue_reqs,
|
||||
)
|
||||
self._log_gauge(
|
||||
self.num_decode_deferred_kv_release_reqs,
|
||||
stats.num_decode_deferred_kv_release_reqs,
|
||||
)
|
||||
self._log_gauge(
|
||||
self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user