feat(metrics): expose deferred decode KV release metrics (#41128)
Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Shangming Cai
Claude Fable 5
parent
749a89abd6
commit
8bf780dda5
@@ -2407,8 +2407,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
envs.SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE_TIMEOUT.get()
|
||||
)
|
||||
# Aborted-mid-transfer requests whose KV pages/slot are held until drained
|
||||
# or timed out. Entries: (decode_req, deadline, metadata_idx, required_acks).
|
||||
self._deferred_releases: List[Tuple[DecodeRequest, float, int, int]] = []
|
||||
# or timed out. Entries:
|
||||
# (decode_req, start_time, deadline, metadata_idx, required_acks).
|
||||
self._deferred_releases: List[Tuple[DecodeRequest, float, float, int, int]] = []
|
||||
|
||||
def add(self, decode_req: DecodeRequest) -> None:
|
||||
self.queue.append(decode_req)
|
||||
@@ -2799,12 +2800,19 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
return transferred_reqs
|
||||
|
||||
def _defer_release(self, decode_req: DecodeRequest) -> None:
|
||||
deadline = time.monotonic() + self.deferred_kv_release_timeout
|
||||
start_time = time.monotonic()
|
||||
deadline = start_time + self.deferred_kv_release_timeout
|
||||
# Require an ack from every notified prefill rank (dummy-proof). Snapshot
|
||||
# now -- the receiver may be cleared by resolve time.
|
||||
required_acks = len(decode_req.kv_receiver.bootstrap_infos)
|
||||
self._deferred_releases.append(
|
||||
(decode_req, deadline, decode_req.metadata_buffer_index, required_acks)
|
||||
(
|
||||
decode_req,
|
||||
start_time,
|
||||
deadline,
|
||||
decode_req.metadata_buffer_index,
|
||||
required_acks,
|
||||
)
|
||||
)
|
||||
|
||||
def _do_release(self, decode_req: DecodeRequest, idx: int) -> None:
|
||||
@@ -2822,6 +2830,18 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
def has_pending_deferred_releases(self) -> bool:
|
||||
return bool(self._deferred_releases)
|
||||
|
||||
def num_pending_deferred_releases(self) -> int:
|
||||
return len(self._deferred_releases)
|
||||
|
||||
def _observe_deferred_kv_release(
|
||||
self, duration_seconds: float, outcome: str
|
||||
) -> None:
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.observe_decode_deferred_kv_release(
|
||||
duration_seconds=duration_seconds,
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def resolve_deferred_releases(self) -> None:
|
||||
"""Release drained transfers; device destinations may also time out."""
|
||||
if not self._deferred_releases:
|
||||
@@ -2831,7 +2851,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
req.req.bootstrap_room, acks
|
||||
)
|
||||
for req, _, _, acks in self._deferred_releases
|
||||
for req, _, _, _, acks in self._deferred_releases
|
||||
if self.enable_host_receive and req.host_staged
|
||||
],
|
||||
dtype=torch.int32,
|
||||
@@ -2845,36 +2865,52 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
now = time.monotonic()
|
||||
still_held = []
|
||||
to_release = []
|
||||
for decode_req, deadline, idx, required_acks in self._deferred_releases:
|
||||
for (
|
||||
decode_req,
|
||||
start_time,
|
||||
deadline,
|
||||
idx,
|
||||
required_acks,
|
||||
) in self._deferred_releases:
|
||||
room = decode_req.req.bootstrap_room
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if next(host_ready):
|
||||
to_release.append((decode_req, idx, room, True))
|
||||
to_release.append((decode_req, start_time, idx, room, True))
|
||||
else:
|
||||
# A timeout cannot prove that a remote write has stopped.
|
||||
still_held.append((decode_req, deadline, idx, required_acks))
|
||||
still_held.append(
|
||||
(decode_req, start_time, deadline, idx, required_acks)
|
||||
)
|
||||
continue
|
||||
kv_mgr = decode_req.kv_receiver.kv_mgr
|
||||
drained = kv_mgr.is_abort_release_safe(room, required_acks)
|
||||
if not drained and now < deadline:
|
||||
still_held.append((decode_req, deadline, idx, required_acks))
|
||||
still_held.append(
|
||||
(decode_req, start_time, deadline, idx, required_acks)
|
||||
)
|
||||
else:
|
||||
to_release.append((decode_req, idx, room, drained))
|
||||
to_release.append((decode_req, start_time, idx, room, drained))
|
||||
# Commit the survivors before releasing so a _do_release exception can't
|
||||
# leave a released entry in the list (double-free / None receiver on retry).
|
||||
self._deferred_releases = still_held
|
||||
for decode_req, idx, room, drained in to_release:
|
||||
for decode_req, start_time, idx, room, drained in to_release:
|
||||
if not drained:
|
||||
logger.warning(
|
||||
f"Deferred KV release for room {room} timed out after "
|
||||
f"{self.deferred_kv_release_timeout}s without a full drain "
|
||||
f"ack from prefill; releasing anyway."
|
||||
)
|
||||
outcome = "drained" if drained else "timeout"
|
||||
try:
|
||||
self._do_release(decode_req, idx)
|
||||
except Exception:
|
||||
# Isolate a failed release so the rest still run; entry already dropped.
|
||||
logger.exception(f"Deferred KV release failed for room {room}")
|
||||
outcome = "error"
|
||||
self._observe_deferred_kv_release(
|
||||
duration_seconds=max(0.0, now - start_time),
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def release_memory_occupation(self):
|
||||
"""Clean up in-flight transfers before releasing GPU memory."""
|
||||
|
||||
@@ -1392,6 +1392,9 @@ class SchedulerMetricsReporter:
|
||||
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
|
||||
transfer_queue, priority_enabled
|
||||
)
|
||||
self.stats.num_decode_deferred_kv_release_reqs = (
|
||||
self.scheduler.disagg_decode_transfer_queue.num_pending_deferred_releases()
|
||||
)
|
||||
host_reqs = (
|
||||
[req for req in transfer_queue if req.host_staged]
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0
|
||||
|
||||
@@ -138,6 +138,7 @@ class SchedulerStats:
|
||||
num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_deferred_kv_release_reqs: int = 0
|
||||
kv_transfer_speed_gb_s: float = 0.0
|
||||
kv_transfer_latency_ms: float = 0.0
|
||||
pending_prealloc_token_usage: float = 0.0
|
||||
@@ -533,6 +534,26 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
documentation="Total requests admitted to receive prefill KV in host memory.",
|
||||
labelnames=labels.keys(),
|
||||
)
|
||||
self.num_decode_deferred_kv_release_reqs = Gauge(
|
||||
name="sglang:num_decode_deferred_kv_release_reqs",
|
||||
documentation=(
|
||||
"The number of failed decode requests whose KV allocations are "
|
||||
"held pending transfer drain acknowledgement or timeout."
|
||||
),
|
||||
labelnames=labels.keys(),
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self.decode_deferred_kv_release_seconds = Histogram(
|
||||
name="sglang:decode_deferred_kv_release_seconds",
|
||||
documentation="Histogram of time spent holding deferred decode KV releases.",
|
||||
labelnames=labels.keys(),
|
||||
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60),
|
||||
)
|
||||
self.decode_deferred_kv_release_total = Counter(
|
||||
name="sglang:decode_deferred_kv_release_total",
|
||||
documentation="Total deferred decode KV release resolution attempts.",
|
||||
labelnames=list(labels.keys()) + ["outcome"],
|
||||
)
|
||||
self.kv_transfer_speed_gb_s = Histogram(
|
||||
name="sglang:kv_transfer_speed_gb_s",
|
||||
documentation="Histogram of KV cache transfer speed in GB/s.",
|
||||
@@ -1208,6 +1229,16 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
def increment_decode_host_receive_reqs(self) -> None:
|
||||
self.num_decode_host_receive_reqs.labels(**self.labels).inc(1)
|
||||
|
||||
def observe_decode_deferred_kv_release(
|
||||
self, duration_seconds: float, outcome: str
|
||||
) -> None:
|
||||
if outcome not in ("drained", "timeout", "error"):
|
||||
raise ValueError(f"Invalid deferred KV release outcome: {outcome}")
|
||||
self._log_histogram(self.decode_deferred_kv_release_seconds, duration_seconds)
|
||||
self.decode_deferred_kv_release_total.labels(
|
||||
**self.labels, outcome=outcome
|
||||
).inc(1)
|
||||
|
||||
def increment_prefill_retries(self, count: int) -> None:
|
||||
if count > 0:
|
||||
self.num_prefill_retries_total.labels(**self.labels).inc(count)
|
||||
@@ -1442,6 +1473,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
self.num_decode_host_receive_queue_reqs,
|
||||
stats.num_decode_host_receive_queue_reqs,
|
||||
)
|
||||
self._log_gauge(
|
||||
self.num_decode_deferred_kv_release_reqs,
|
||||
stats.num_decode_deferred_kv_release_reqs,
|
||||
)
|
||||
self._log_gauge(
|
||||
self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ import threading
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from typing import NamedTuple
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
from sglang.srt.disaggregation import decode as decode_mod
|
||||
from sglang.srt.disaggregation.base.conn import BaseKVManager, KVPoll
|
||||
@@ -642,15 +642,22 @@ class _FakeIdxAllocator:
|
||||
self.freed.append(idx)
|
||||
|
||||
|
||||
def _make_queue(timeout=30.0):
|
||||
def _make_queue(timeout=30.0, enable_metrics=False):
|
||||
q = DecodeTransferQueue.__new__(DecodeTransferQueue)
|
||||
q._deferred_releases = []
|
||||
q.deferred_kv_release_timeout = timeout
|
||||
q.enable_host_receive = False
|
||||
q.enable_staging = False
|
||||
q.staging_handler = None
|
||||
q.tree_cache = object()
|
||||
q.metadata_buffers = SimpleNamespace(bootstrap_room={})
|
||||
q.req_to_metadata_buffer_idx_allocator = _FakeIdxAllocator()
|
||||
q.scheduler = SimpleNamespace(
|
||||
metrics_reporter=SimpleNamespace(enable_metrics=enable_metrics),
|
||||
metrics_collector=SimpleNamespace(
|
||||
observe_decode_deferred_kv_release=MagicMock()
|
||||
),
|
||||
)
|
||||
return q
|
||||
|
||||
|
||||
@@ -713,63 +720,96 @@ class TestResolveDeferredReleases(CustomTestCase):
|
||||
def test_holds_until_drained_then_releases(self):
|
||||
mgr = _make_manager()
|
||||
room, idx = 200, 7
|
||||
q = _make_queue()
|
||||
q = _make_queue(enable_metrics=True)
|
||||
dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=2)
|
||||
# In production the receiver arms the room just before it sends the
|
||||
# ABORT (_send_abort_notification), before the scheduler defers here.
|
||||
generation = mgr.register_deferred_abort_room(room)
|
||||
q._defer_release(dreq)
|
||||
|
||||
with patch.object(decode_mod, "release_kv_cache") as rel:
|
||||
# Not yet acked -> held, not released.
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_not_called()
|
||||
self.assertEqual(len(q._deferred_releases), 1)
|
||||
with patch.object(
|
||||
decode_mod.time, "monotonic", side_effect=[10.0, 11.0, 12.0, 13.0]
|
||||
):
|
||||
q._defer_release(dreq)
|
||||
with patch.object(decode_mod, "release_kv_cache") as rel:
|
||||
# Not yet acked -> held, not released.
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_not_called()
|
||||
self.assertEqual(len(q._deferred_releases), 1)
|
||||
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called()
|
||||
|
||||
# One of two ranks acked -> still held.
|
||||
mgr.note_abort_ack(room, 0, generation)
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_not_called()
|
||||
self.assertEqual(len(q._deferred_releases), 1)
|
||||
# One of two ranks acked -> still held.
|
||||
mgr.note_abort_ack(room, 0, generation)
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_not_called()
|
||||
self.assertEqual(len(q._deferred_releases), 1)
|
||||
|
||||
# Both ranks acked -> released exactly once.
|
||||
mgr.note_abort_ack(room, 1, generation)
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
|
||||
# Both ranks acked -> released exactly once.
|
||||
mgr.note_abort_ack(room, 1, generation)
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
|
||||
|
||||
# Held state fully cleaned up.
|
||||
self.assertEqual(q._deferred_releases, [])
|
||||
self.assertEqual(q.num_pending_deferred_releases(), 0)
|
||||
self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx])
|
||||
self.assertEqual(q.metadata_buffers.bootstrap_room[idx], 0)
|
||||
self.assertNotIn(room, mgr._deferred_abort_ack_tracker)
|
||||
self.assertIsNone(dreq.kv_receiver)
|
||||
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_called_once_with(
|
||||
duration_seconds=3.0,
|
||||
outcome="drained",
|
||||
)
|
||||
|
||||
def test_releases_on_timeout_without_ack(self):
|
||||
mgr = _make_manager()
|
||||
room, idx = 300, 3
|
||||
q = _make_queue(timeout=30.0)
|
||||
q = _make_queue(timeout=30.0, enable_metrics=True)
|
||||
dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=1)
|
||||
# Force an already-expired deadline (no ack will ever arrive).
|
||||
q._deferred_releases.append((dreq, float("-inf"), idx, 1))
|
||||
q._deferred_releases.append((dreq, 10.0, float("-inf"), idx, 1))
|
||||
|
||||
with patch.object(decode_mod, "release_kv_cache") as rel:
|
||||
with (
|
||||
patch.object(decode_mod.time, "monotonic", return_value=42.0),
|
||||
patch.object(decode_mod, "release_kv_cache") as rel,
|
||||
):
|
||||
q.resolve_deferred_releases()
|
||||
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
|
||||
|
||||
self.assertEqual(q._deferred_releases, [])
|
||||
self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx])
|
||||
self.assertIsNone(dreq.kv_receiver)
|
||||
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_called_once_with(
|
||||
duration_seconds=32.0,
|
||||
outcome="timeout",
|
||||
)
|
||||
|
||||
def test_metrics_disabled_does_not_observe_release(self):
|
||||
mgr = _make_manager()
|
||||
room, idx = 301, 4
|
||||
q = _make_queue(enable_metrics=False)
|
||||
dreq = _make_decode_req(room, idx, mgr)
|
||||
q._deferred_releases.append((dreq, 10.0, float("-inf"), idx, 1))
|
||||
|
||||
with (
|
||||
patch.object(decode_mod.time, "monotonic", return_value=20.0),
|
||||
patch.object(decode_mod, "release_kv_cache") as rel,
|
||||
):
|
||||
q.resolve_deferred_releases()
|
||||
|
||||
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
|
||||
self.assertEqual(q.num_pending_deferred_releases(), 0)
|
||||
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called()
|
||||
|
||||
def test_failed_release_is_isolated_and_not_retried(self):
|
||||
# A raising _do_release must drop the entry (no double-free on retry) and
|
||||
# not brick resolve for the remaining entries or subsequent calls.
|
||||
mgr = _make_manager()
|
||||
q = _make_queue()
|
||||
q = _make_queue(enable_metrics=True)
|
||||
good = _make_decode_req(700, 1, mgr)
|
||||
bad = _make_decode_req(701, 2, mgr)
|
||||
# Both already past deadline -> both selected for release.
|
||||
q._deferred_releases.append((bad, float("-inf"), 2, 1))
|
||||
q._deferred_releases.append((good, float("-inf"), 1, 1))
|
||||
q._deferred_releases.append((bad, 10.0, float("-inf"), 2, 1))
|
||||
q._deferred_releases.append((good, 10.0, float("-inf"), 1, 1))
|
||||
|
||||
calls = []
|
||||
|
||||
@@ -778,24 +818,36 @@ class TestResolveDeferredReleases(CustomTestCase):
|
||||
if req is bad.req:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
with patch.object(decode_mod, "release_kv_cache", side_effect=fake_release):
|
||||
with (
|
||||
patch.object(decode_mod.time, "monotonic", return_value=40.0),
|
||||
patch.object(decode_mod, "release_kv_cache", side_effect=fake_release),
|
||||
):
|
||||
q.resolve_deferred_releases() # must not raise
|
||||
# The good one still released despite the bad one throwing.
|
||||
self.assertIn(good.req, calls)
|
||||
# Nothing left held, and a second call is a clean no-op (no retry).
|
||||
self.assertEqual(q._deferred_releases, [])
|
||||
q.resolve_deferred_releases()
|
||||
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_has_calls(
|
||||
[
|
||||
call(duration_seconds=30.0, outcome="error"),
|
||||
call(duration_seconds=30.0, outcome="timeout"),
|
||||
]
|
||||
)
|
||||
|
||||
def test_defer_release_records_deadline_and_idx(self):
|
||||
mgr = _make_manager()
|
||||
q = _make_queue(timeout=12.5)
|
||||
dreq = _make_decode_req(room=400, idx=9, mgr=mgr)
|
||||
q._defer_release(dreq)
|
||||
with patch.object(decode_mod.time, "monotonic", return_value=5.0):
|
||||
q._defer_release(dreq)
|
||||
self.assertEqual(len(q._deferred_releases), 1)
|
||||
held_req, deadline, held_idx, required = q._deferred_releases[0]
|
||||
held_req, start_time, deadline, held_idx, required = q._deferred_releases[0]
|
||||
self.assertIs(held_req, dreq)
|
||||
self.assertEqual(start_time, 5.0)
|
||||
self.assertEqual(deadline, 17.5)
|
||||
self.assertEqual(held_idx, 9)
|
||||
self.assertIsInstance(deadline, float)
|
||||
self.assertEqual(required, 1)
|
||||
|
||||
|
||||
def _make_pop_queue(mgr):
|
||||
|
||||
@@ -394,7 +394,8 @@ class TestIdleMetrics(CustomTestCase):
|
||||
for priority in (1, 2)
|
||||
]
|
||||
self.scheduler.disagg_decode_transfer_queue = types.SimpleNamespace(
|
||||
queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)]
|
||||
queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)],
|
||||
num_pending_deferred_releases=lambda: 0,
|
||||
)
|
||||
for _ in host_reqs:
|
||||
collector.increment_decode_host_receive_reqs()
|
||||
|
||||
@@ -21,9 +21,15 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import prometheus_client
|
||||
|
||||
from sglang.srt.disaggregation.decode import DecodeTransferQueue
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.managers.scheduler_components.metrics_reporter import (
|
||||
SchedulerMetricsReporter,
|
||||
)
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
STAT_LOGGER_ROLE_EXPERT_DISPATCH,
|
||||
STAT_LOGGER_ROLE_RADIX_CACHE,
|
||||
@@ -32,6 +38,7 @@ from sglang.srt.observability.metrics_collector import (
|
||||
STAT_LOGGER_ROLE_TOKENIZER,
|
||||
RadixCacheMetricsCollector,
|
||||
SchedulerMetricsCollector,
|
||||
SchedulerStats,
|
||||
StorageMetricsCollector,
|
||||
TokenizerMetricsCollector,
|
||||
radix_cache_metric_labels,
|
||||
@@ -225,6 +232,211 @@ class TestHiCacheMetrics(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestDeferredKVReleaseMetrics(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.labels = {
|
||||
"model_name": "test",
|
||||
"engine_type": "decode",
|
||||
"tp_rank": 0,
|
||||
"pp_rank": 0,
|
||||
"moe_ep_rank": 0,
|
||||
}
|
||||
self.collector = SchedulerMetricsCollector.__new__(SchedulerMetricsCollector)
|
||||
self.collector.labels = self.labels
|
||||
self.collector.decode_deferred_kv_release_seconds = _RecordingMetric(
|
||||
name="sglang:decode_deferred_kv_release_seconds",
|
||||
labelnames=self.labels.keys(),
|
||||
)
|
||||
self.collector.decode_deferred_kv_release_total = _RecordingMetric(
|
||||
name="sglang:decode_deferred_kv_release_total",
|
||||
labelnames=list(self.labels.keys()) + ["outcome"],
|
||||
)
|
||||
self.collector.num_decode_deferred_kv_release_reqs = _RecordingMetric(
|
||||
name="sglang:num_decode_deferred_kv_release_reqs",
|
||||
labelnames=self.labels.keys(),
|
||||
)
|
||||
|
||||
def test_records_duration_and_bounded_outcome(self):
|
||||
self.collector.observe_decode_deferred_kv_release(
|
||||
duration_seconds=0.25,
|
||||
outcome="drained",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
self.collector.decode_deferred_kv_release_seconds.observations,
|
||||
[(self.labels, 0.25)],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.collector.decode_deferred_kv_release_total.increments,
|
||||
[({**self.labels, "outcome": "drained"}, 1)],
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "Invalid deferred KV release outcome"):
|
||||
self.collector.observe_decode_deferred_kv_release(
|
||||
duration_seconds=0.5,
|
||||
outcome="room-123",
|
||||
)
|
||||
|
||||
def test_logs_current_deferred_release_count(self):
|
||||
queue = DecodeTransferQueue.__new__(DecodeTransferQueue)
|
||||
queue.queue = []
|
||||
queue._deferred_releases = [object(), object(), object()]
|
||||
reporter = SchedulerMetricsReporter.__new__(SchedulerMetricsReporter)
|
||||
reporter.scheduler = SimpleNamespace(
|
||||
disaggregation_mode=DisaggregationMode.DECODE,
|
||||
disagg_decode_transfer_queue=queue,
|
||||
disagg_decode_prealloc_queue=SimpleNamespace(queue=[]),
|
||||
running_batch=SimpleNamespace(reqs=[]),
|
||||
waiting_queue=[],
|
||||
grammar_manager=[],
|
||||
enable_priority_scheduling=False,
|
||||
pool_stats_observer=SimpleNamespace(
|
||||
get_pool_stats=lambda: SimpleNamespace(
|
||||
update_scheduler_stats=lambda stats: None,
|
||||
),
|
||||
streaming_session_count=lambda: 0,
|
||||
session_held_tokens=lambda: 0,
|
||||
),
|
||||
)
|
||||
reporter.stats = SchedulerStats()
|
||||
reporter.current_scheduler_metrics_enabled = True
|
||||
collector = MagicMock()
|
||||
collector.last_log_time = 0
|
||||
collector.num_decode_deferred_kv_release_reqs = (
|
||||
self.collector.num_decode_deferred_kv_release_reqs
|
||||
)
|
||||
collector.labels = self.labels
|
||||
collector.log_stats.side_effect = lambda stats: (
|
||||
SchedulerMetricsCollector.log_stats(collector, stats)
|
||||
)
|
||||
collector._log_gauge.side_effect = lambda gauge, data: (
|
||||
SchedulerMetricsCollector._log_gauge(collector, gauge, data)
|
||||
)
|
||||
reporter.metrics_collector = collector
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER",
|
||||
False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg",
|
||||
return_value=SimpleNamespace(
|
||||
disaggregation_decode_host_receive_threshold=0
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter",
|
||||
side_effect=[31.0, 31.0, 62.0, 62.0],
|
||||
),
|
||||
):
|
||||
reporter._maybe_log_idle_metrics()
|
||||
self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 3)
|
||||
queue._deferred_releases.clear()
|
||||
reporter._maybe_log_idle_metrics()
|
||||
self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 0)
|
||||
|
||||
self.assertEqual(
|
||||
self.collector.num_decode_deferred_kv_release_reqs.sets,
|
||||
[(self.labels, 3), (self.labels, 0)],
|
||||
)
|
||||
|
||||
def test_decode_report_logs_current_deferred_release_count(self):
|
||||
# Regression: report_decode_stats reads the queue in its message section
|
||||
# before the metrics block runs, so a reset inside the metrics block
|
||||
# would zero the gauge on every busy-path report.
|
||||
queue = DecodeTransferQueue.__new__(DecodeTransferQueue)
|
||||
queue.queue = []
|
||||
queue._deferred_releases = [object(), object()]
|
||||
batch = SimpleNamespace(
|
||||
reqs=[],
|
||||
seq_lens_cpu=None,
|
||||
forward_iter=1,
|
||||
dp_cooperation_info=None,
|
||||
)
|
||||
reporter = SchedulerMetricsReporter.__new__(SchedulerMetricsReporter)
|
||||
reporter.scheduler = SimpleNamespace(
|
||||
disaggregation_mode=DisaggregationMode.DECODE,
|
||||
disagg_decode_transfer_queue=queue,
|
||||
disagg_decode_prealloc_queue=SimpleNamespace(
|
||||
queue=[], retracted_queue=[], num_tokens_pre_allocated=0
|
||||
),
|
||||
running_batch=batch,
|
||||
waiting_queue=[],
|
||||
grammar_manager=[],
|
||||
enable_priority_scheduling=False,
|
||||
max_total_num_tokens=1,
|
||||
spec_algorithm=SimpleNamespace(is_none=lambda: True),
|
||||
pool_stats_observer=SimpleNamespace(
|
||||
get_pool_stats=lambda: SimpleNamespace(
|
||||
get_decode_usage_msg_parts=lambda: [],
|
||||
update_scheduler_stats=lambda stats: None,
|
||||
),
|
||||
streaming_session_count=lambda: 0,
|
||||
session_held_tokens=lambda: 0,
|
||||
),
|
||||
kv_events_publisher=SimpleNamespace(
|
||||
emit_kv_metrics=lambda: None, publish_kv_events=lambda: None
|
||||
),
|
||||
)
|
||||
reporter.stats = SchedulerStats()
|
||||
reporter.current_scheduler_metrics_enabled = True
|
||||
reporter.is_stats_logging_rank = False
|
||||
reporter.enable_mfu_metrics = False
|
||||
reporter.scheduler_status_logger = None
|
||||
reporter.forward_ct_decode = 0
|
||||
reporter.decode_log_interval = 1
|
||||
reporter.last_decode_stats_tic = 0.0
|
||||
reporter.num_generated_tokens = 0
|
||||
reporter.num_retracted_reqs = 0
|
||||
reporter.num_paused_reqs = 0
|
||||
reporter.fwd_occupancy = 0.0
|
||||
reporter._graph_backend_label = "cuda graph"
|
||||
reporter._calculate_utilization = lambda: None
|
||||
reporter._update_lora_metrics = lambda: None
|
||||
reporter._log_hicache_stats = lambda: None
|
||||
collector = MagicMock()
|
||||
collector.num_decode_deferred_kv_release_reqs = (
|
||||
self.collector.num_decode_deferred_kv_release_reqs
|
||||
)
|
||||
collector.labels = self.labels
|
||||
collector.log_stats.side_effect = lambda stats: (
|
||||
SchedulerMetricsCollector.log_stats(collector, stats)
|
||||
)
|
||||
collector._log_gauge.side_effect = lambda gauge, data: (
|
||||
SchedulerMetricsCollector._log_gauge(collector, gauge, data)
|
||||
)
|
||||
reporter.metrics_collector = collector
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER",
|
||||
False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.RECORD_STEP_TIME",
|
||||
False,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg",
|
||||
return_value=SimpleNamespace(
|
||||
disaggregation_decode_host_receive_threshold=0,
|
||||
language_only=False,
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter",
|
||||
return_value=10.0,
|
||||
),
|
||||
):
|
||||
reporter.report_decode_stats(can_run_cuda_graph=False, running_batch=batch)
|
||||
|
||||
self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 2)
|
||||
self.assertEqual(
|
||||
self.collector.num_decode_deferred_kv_release_reqs.sets,
|
||||
[(self.labels, 2)],
|
||||
)
|
||||
|
||||
|
||||
class TestRequestTimePerOutputToken(CustomTestCase):
|
||||
def _collector(self, labels):
|
||||
with get_context().override_server_args(
|
||||
|
||||
Reference in New Issue
Block a user