diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 536d4d9b57..309513062b 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -2407,8 +2407,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): envs.SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE_TIMEOUT.get() ) # Aborted-mid-transfer requests whose KV pages/slot are held until drained - # or timed out. Entries: (decode_req, deadline, metadata_idx, required_acks). - self._deferred_releases: List[Tuple[DecodeRequest, float, int, int]] = [] + # or timed out. Entries: + # (decode_req, start_time, deadline, metadata_idx, required_acks). + self._deferred_releases: List[Tuple[DecodeRequest, float, float, int, int]] = [] def add(self, decode_req: DecodeRequest) -> None: self.queue.append(decode_req) @@ -2799,12 +2800,19 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): return transferred_reqs def _defer_release(self, decode_req: DecodeRequest) -> None: - deadline = time.monotonic() + self.deferred_kv_release_timeout + start_time = time.monotonic() + deadline = start_time + self.deferred_kv_release_timeout # Require an ack from every notified prefill rank (dummy-proof). Snapshot # now -- the receiver may be cleared by resolve time. required_acks = len(decode_req.kv_receiver.bootstrap_infos) self._deferred_releases.append( - (decode_req, deadline, decode_req.metadata_buffer_index, required_acks) + ( + decode_req, + start_time, + deadline, + decode_req.metadata_buffer_index, + required_acks, + ) ) def _do_release(self, decode_req: DecodeRequest, idx: int) -> None: @@ -2822,6 +2830,18 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): def has_pending_deferred_releases(self) -> bool: return bool(self._deferred_releases) + def num_pending_deferred_releases(self) -> int: + return len(self._deferred_releases) + + def _observe_deferred_kv_release( + self, duration_seconds: float, outcome: str + ) -> None: + if self.scheduler.metrics_reporter.enable_metrics: + self.scheduler.metrics_collector.observe_decode_deferred_kv_release( + duration_seconds=duration_seconds, + outcome=outcome, + ) + def resolve_deferred_releases(self) -> None: """Release drained transfers; device destinations may also time out.""" if not self._deferred_releases: @@ -2831,7 +2851,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): req.kv_receiver.kv_mgr.is_abort_release_safe( req.req.bootstrap_room, acks ) - for req, _, _, acks in self._deferred_releases + for req, _, _, _, acks in self._deferred_releases if self.enable_host_receive and req.host_staged ], dtype=torch.int32, @@ -2845,36 +2865,52 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): now = time.monotonic() still_held = [] to_release = [] - for decode_req, deadline, idx, required_acks in self._deferred_releases: + for ( + decode_req, + start_time, + deadline, + idx, + required_acks, + ) in self._deferred_releases: room = decode_req.req.bootstrap_room if self.enable_host_receive and decode_req.host_staged: if next(host_ready): - to_release.append((decode_req, idx, room, True)) + to_release.append((decode_req, start_time, idx, room, True)) else: # A timeout cannot prove that a remote write has stopped. - still_held.append((decode_req, deadline, idx, required_acks)) + still_held.append( + (decode_req, start_time, deadline, idx, required_acks) + ) continue kv_mgr = decode_req.kv_receiver.kv_mgr drained = kv_mgr.is_abort_release_safe(room, required_acks) if not drained and now < deadline: - still_held.append((decode_req, deadline, idx, required_acks)) + still_held.append( + (decode_req, start_time, deadline, idx, required_acks) + ) else: - to_release.append((decode_req, idx, room, drained)) + to_release.append((decode_req, start_time, idx, room, drained)) # Commit the survivors before releasing so a _do_release exception can't # leave a released entry in the list (double-free / None receiver on retry). self._deferred_releases = still_held - for decode_req, idx, room, drained in to_release: + for decode_req, start_time, idx, room, drained in to_release: if not drained: logger.warning( f"Deferred KV release for room {room} timed out after " f"{self.deferred_kv_release_timeout}s without a full drain " f"ack from prefill; releasing anyway." ) + outcome = "drained" if drained else "timeout" try: self._do_release(decode_req, idx) except Exception: # Isolate a failed release so the rest still run; entry already dropped. logger.exception(f"Deferred KV release failed for room {room}") + outcome = "error" + self._observe_deferred_kv_release( + duration_seconds=max(0.0, now - start_time), + outcome=outcome, + ) def release_memory_occupation(self): """Clean up in-flight transfers before releasing GPU memory.""" diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index fb1b8b3197..d4a142ff02 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -1392,6 +1392,9 @@ class SchedulerMetricsReporter: self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs( transfer_queue, priority_enabled ) + self.stats.num_decode_deferred_kv_release_reqs = ( + self.scheduler.disagg_decode_transfer_queue.num_pending_deferred_releases() + ) host_reqs = ( [req for req in transfer_queue if req.host_staged] if get_disagg().disaggregation_decode_host_receive_threshold > 0 diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 0650f65600..270877c9e3 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -138,6 +138,7 @@ class SchedulerStats: num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount) num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount) num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount) + num_decode_deferred_kv_release_reqs: int = 0 kv_transfer_speed_gb_s: float = 0.0 kv_transfer_latency_ms: float = 0.0 pending_prealloc_token_usage: float = 0.0 @@ -533,6 +534,26 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): documentation="Total requests admitted to receive prefill KV in host memory.", labelnames=labels.keys(), ) + self.num_decode_deferred_kv_release_reqs = Gauge( + name="sglang:num_decode_deferred_kv_release_reqs", + documentation=( + "The number of failed decode requests whose KV allocations are " + "held pending transfer drain acknowledgement or timeout." + ), + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) + self.decode_deferred_kv_release_seconds = Histogram( + name="sglang:decode_deferred_kv_release_seconds", + documentation="Histogram of time spent holding deferred decode KV releases.", + labelnames=labels.keys(), + buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60), + ) + self.decode_deferred_kv_release_total = Counter( + name="sglang:decode_deferred_kv_release_total", + documentation="Total deferred decode KV release resolution attempts.", + labelnames=list(labels.keys()) + ["outcome"], + ) self.kv_transfer_speed_gb_s = Histogram( name="sglang:kv_transfer_speed_gb_s", documentation="Histogram of KV cache transfer speed in GB/s.", @@ -1208,6 +1229,16 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): def increment_decode_host_receive_reqs(self) -> None: self.num_decode_host_receive_reqs.labels(**self.labels).inc(1) + def observe_decode_deferred_kv_release( + self, duration_seconds: float, outcome: str + ) -> None: + if outcome not in ("drained", "timeout", "error"): + raise ValueError(f"Invalid deferred KV release outcome: {outcome}") + self._log_histogram(self.decode_deferred_kv_release_seconds, duration_seconds) + self.decode_deferred_kv_release_total.labels( + **self.labels, outcome=outcome + ).inc(1) + def increment_prefill_retries(self, count: int) -> None: if count > 0: self.num_prefill_retries_total.labels(**self.labels).inc(count) @@ -1442,6 +1473,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): self.num_decode_host_receive_queue_reqs, stats.num_decode_host_receive_queue_reqs, ) + self._log_gauge( + self.num_decode_deferred_kv_release_reqs, + stats.num_decode_deferred_kv_release_reqs, + ) self._log_gauge( self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage ) diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 21759941c2..48bb6ac02e 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -12,7 +12,7 @@ import threading import unittest from types import SimpleNamespace from typing import NamedTuple -from unittest.mock import patch +from unittest.mock import MagicMock, call, patch from sglang.srt.disaggregation import decode as decode_mod from sglang.srt.disaggregation.base.conn import BaseKVManager, KVPoll @@ -642,15 +642,22 @@ class _FakeIdxAllocator: self.freed.append(idx) -def _make_queue(timeout=30.0): +def _make_queue(timeout=30.0, enable_metrics=False): q = DecodeTransferQueue.__new__(DecodeTransferQueue) q._deferred_releases = [] q.deferred_kv_release_timeout = timeout + q.enable_host_receive = False q.enable_staging = False q.staging_handler = None q.tree_cache = object() q.metadata_buffers = SimpleNamespace(bootstrap_room={}) q.req_to_metadata_buffer_idx_allocator = _FakeIdxAllocator() + q.scheduler = SimpleNamespace( + metrics_reporter=SimpleNamespace(enable_metrics=enable_metrics), + metrics_collector=SimpleNamespace( + observe_decode_deferred_kv_release=MagicMock() + ), + ) return q @@ -713,63 +720,96 @@ class TestResolveDeferredReleases(CustomTestCase): def test_holds_until_drained_then_releases(self): mgr = _make_manager() room, idx = 200, 7 - q = _make_queue() + q = _make_queue(enable_metrics=True) dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=2) # In production the receiver arms the room just before it sends the # ABORT (_send_abort_notification), before the scheduler defers here. generation = mgr.register_deferred_abort_room(room) - q._defer_release(dreq) - with patch.object(decode_mod, "release_kv_cache") as rel: - # Not yet acked -> held, not released. - q.resolve_deferred_releases() - rel.assert_not_called() - self.assertEqual(len(q._deferred_releases), 1) + with patch.object( + decode_mod.time, "monotonic", side_effect=[10.0, 11.0, 12.0, 13.0] + ): + q._defer_release(dreq) + with patch.object(decode_mod, "release_kv_cache") as rel: + # Not yet acked -> held, not released. + q.resolve_deferred_releases() + rel.assert_not_called() + self.assertEqual(len(q._deferred_releases), 1) + q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called() - # One of two ranks acked -> still held. - mgr.note_abort_ack(room, 0, generation) - q.resolve_deferred_releases() - rel.assert_not_called() - self.assertEqual(len(q._deferred_releases), 1) + # One of two ranks acked -> still held. + mgr.note_abort_ack(room, 0, generation) + q.resolve_deferred_releases() + rel.assert_not_called() + self.assertEqual(len(q._deferred_releases), 1) - # Both ranks acked -> released exactly once. - mgr.note_abort_ack(room, 1, generation) - q.resolve_deferred_releases() - rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) + # Both ranks acked -> released exactly once. + mgr.note_abort_ack(room, 1, generation) + q.resolve_deferred_releases() + rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) # Held state fully cleaned up. self.assertEqual(q._deferred_releases, []) + self.assertEqual(q.num_pending_deferred_releases(), 0) self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx]) self.assertEqual(q.metadata_buffers.bootstrap_room[idx], 0) self.assertNotIn(room, mgr._deferred_abort_ack_tracker) self.assertIsNone(dreq.kv_receiver) + q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_called_once_with( + duration_seconds=3.0, + outcome="drained", + ) def test_releases_on_timeout_without_ack(self): mgr = _make_manager() room, idx = 300, 3 - q = _make_queue(timeout=30.0) + q = _make_queue(timeout=30.0, enable_metrics=True) dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=1) # Force an already-expired deadline (no ack will ever arrive). - q._deferred_releases.append((dreq, float("-inf"), idx, 1)) + q._deferred_releases.append((dreq, 10.0, float("-inf"), idx, 1)) - with patch.object(decode_mod, "release_kv_cache") as rel: + with ( + patch.object(decode_mod.time, "monotonic", return_value=42.0), + patch.object(decode_mod, "release_kv_cache") as rel, + ): q.resolve_deferred_releases() rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) self.assertEqual(q._deferred_releases, []) self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx]) self.assertIsNone(dreq.kv_receiver) + q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_called_once_with( + duration_seconds=32.0, + outcome="timeout", + ) + + def test_metrics_disabled_does_not_observe_release(self): + mgr = _make_manager() + room, idx = 301, 4 + q = _make_queue(enable_metrics=False) + dreq = _make_decode_req(room, idx, mgr) + q._deferred_releases.append((dreq, 10.0, float("-inf"), idx, 1)) + + with ( + patch.object(decode_mod.time, "monotonic", return_value=20.0), + patch.object(decode_mod, "release_kv_cache") as rel, + ): + q.resolve_deferred_releases() + + rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) + self.assertEqual(q.num_pending_deferred_releases(), 0) + q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called() def test_failed_release_is_isolated_and_not_retried(self): # A raising _do_release must drop the entry (no double-free on retry) and # not brick resolve for the remaining entries or subsequent calls. mgr = _make_manager() - q = _make_queue() + q = _make_queue(enable_metrics=True) good = _make_decode_req(700, 1, mgr) bad = _make_decode_req(701, 2, mgr) # Both already past deadline -> both selected for release. - q._deferred_releases.append((bad, float("-inf"), 2, 1)) - q._deferred_releases.append((good, float("-inf"), 1, 1)) + q._deferred_releases.append((bad, 10.0, float("-inf"), 2, 1)) + q._deferred_releases.append((good, 10.0, float("-inf"), 1, 1)) calls = [] @@ -778,24 +818,36 @@ class TestResolveDeferredReleases(CustomTestCase): if req is bad.req: raise RuntimeError("boom") - with patch.object(decode_mod, "release_kv_cache", side_effect=fake_release): + with ( + patch.object(decode_mod.time, "monotonic", return_value=40.0), + patch.object(decode_mod, "release_kv_cache", side_effect=fake_release), + ): q.resolve_deferred_releases() # must not raise # The good one still released despite the bad one throwing. self.assertIn(good.req, calls) # Nothing left held, and a second call is a clean no-op (no retry). self.assertEqual(q._deferred_releases, []) q.resolve_deferred_releases() + q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_has_calls( + [ + call(duration_seconds=30.0, outcome="error"), + call(duration_seconds=30.0, outcome="timeout"), + ] + ) def test_defer_release_records_deadline_and_idx(self): mgr = _make_manager() q = _make_queue(timeout=12.5) dreq = _make_decode_req(room=400, idx=9, mgr=mgr) - q._defer_release(dreq) + with patch.object(decode_mod.time, "monotonic", return_value=5.0): + q._defer_release(dreq) self.assertEqual(len(q._deferred_releases), 1) - held_req, deadline, held_idx, required = q._deferred_releases[0] + held_req, start_time, deadline, held_idx, required = q._deferred_releases[0] self.assertIs(held_req, dreq) + self.assertEqual(start_time, 5.0) + self.assertEqual(deadline, 17.5) self.assertEqual(held_idx, 9) - self.assertIsInstance(deadline, float) + self.assertEqual(required, 1) def _make_pop_queue(mgr): diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index 28c83b7471..096fff58a0 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -394,7 +394,8 @@ class TestIdleMetrics(CustomTestCase): for priority in (1, 2) ] self.scheduler.disagg_decode_transfer_queue = types.SimpleNamespace( - queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)] + queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)], + num_pending_deferred_releases=lambda: 0, ) for _ in host_reqs: collector.increment_decode_host_receive_reqs() diff --git a/test/registered/unit/observability/test_stat_loggers_di.py b/test/registered/unit/observability/test_stat_loggers_di.py index 512e303152..bf658711ed 100644 --- a/test/registered/unit/observability/test_stat_loggers_di.py +++ b/test/registered/unit/observability/test_stat_loggers_di.py @@ -21,9 +21,15 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest from types import SimpleNamespace +from unittest.mock import MagicMock, patch import prometheus_client +from sglang.srt.disaggregation.decode import DecodeTransferQueue +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.managers.scheduler_components.metrics_reporter import ( + SchedulerMetricsReporter, +) from sglang.srt.observability.metrics_collector import ( STAT_LOGGER_ROLE_EXPERT_DISPATCH, STAT_LOGGER_ROLE_RADIX_CACHE, @@ -32,6 +38,7 @@ from sglang.srt.observability.metrics_collector import ( STAT_LOGGER_ROLE_TOKENIZER, RadixCacheMetricsCollector, SchedulerMetricsCollector, + SchedulerStats, StorageMetricsCollector, TokenizerMetricsCollector, radix_cache_metric_labels, @@ -225,6 +232,211 @@ class TestHiCacheMetrics(unittest.TestCase): ) +class TestDeferredKVReleaseMetrics(CustomTestCase): + def setUp(self): + self.labels = { + "model_name": "test", + "engine_type": "decode", + "tp_rank": 0, + "pp_rank": 0, + "moe_ep_rank": 0, + } + self.collector = SchedulerMetricsCollector.__new__(SchedulerMetricsCollector) + self.collector.labels = self.labels + self.collector.decode_deferred_kv_release_seconds = _RecordingMetric( + name="sglang:decode_deferred_kv_release_seconds", + labelnames=self.labels.keys(), + ) + self.collector.decode_deferred_kv_release_total = _RecordingMetric( + name="sglang:decode_deferred_kv_release_total", + labelnames=list(self.labels.keys()) + ["outcome"], + ) + self.collector.num_decode_deferred_kv_release_reqs = _RecordingMetric( + name="sglang:num_decode_deferred_kv_release_reqs", + labelnames=self.labels.keys(), + ) + + def test_records_duration_and_bounded_outcome(self): + self.collector.observe_decode_deferred_kv_release( + duration_seconds=0.25, + outcome="drained", + ) + + self.assertEqual( + self.collector.decode_deferred_kv_release_seconds.observations, + [(self.labels, 0.25)], + ) + self.assertEqual( + self.collector.decode_deferred_kv_release_total.increments, + [({**self.labels, "outcome": "drained"}, 1)], + ) + with self.assertRaisesRegex(ValueError, "Invalid deferred KV release outcome"): + self.collector.observe_decode_deferred_kv_release( + duration_seconds=0.5, + outcome="room-123", + ) + + def test_logs_current_deferred_release_count(self): + queue = DecodeTransferQueue.__new__(DecodeTransferQueue) + queue.queue = [] + queue._deferred_releases = [object(), object(), object()] + reporter = SchedulerMetricsReporter.__new__(SchedulerMetricsReporter) + reporter.scheduler = SimpleNamespace( + disaggregation_mode=DisaggregationMode.DECODE, + disagg_decode_transfer_queue=queue, + disagg_decode_prealloc_queue=SimpleNamespace(queue=[]), + running_batch=SimpleNamespace(reqs=[]), + waiting_queue=[], + grammar_manager=[], + enable_priority_scheduling=False, + pool_stats_observer=SimpleNamespace( + get_pool_stats=lambda: SimpleNamespace( + update_scheduler_stats=lambda stats: None, + ), + streaming_session_count=lambda: 0, + session_held_tokens=lambda: 0, + ), + ) + reporter.stats = SchedulerStats() + reporter.current_scheduler_metrics_enabled = True + collector = MagicMock() + collector.last_log_time = 0 + collector.num_decode_deferred_kv_release_reqs = ( + self.collector.num_decode_deferred_kv_release_reqs + ) + collector.labels = self.labels + collector.log_stats.side_effect = lambda stats: ( + SchedulerMetricsCollector.log_stats(collector, stats) + ) + collector._log_gauge.side_effect = lambda gauge, data: ( + SchedulerMetricsCollector._log_gauge(collector, gauge, data) + ) + reporter.metrics_collector = collector + + with ( + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER", + False, + ), + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg", + return_value=SimpleNamespace( + disaggregation_decode_host_receive_threshold=0 + ), + ), + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter", + side_effect=[31.0, 31.0, 62.0, 62.0], + ), + ): + reporter._maybe_log_idle_metrics() + self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 3) + queue._deferred_releases.clear() + reporter._maybe_log_idle_metrics() + self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 0) + + self.assertEqual( + self.collector.num_decode_deferred_kv_release_reqs.sets, + [(self.labels, 3), (self.labels, 0)], + ) + + def test_decode_report_logs_current_deferred_release_count(self): + # Regression: report_decode_stats reads the queue in its message section + # before the metrics block runs, so a reset inside the metrics block + # would zero the gauge on every busy-path report. + queue = DecodeTransferQueue.__new__(DecodeTransferQueue) + queue.queue = [] + queue._deferred_releases = [object(), object()] + batch = SimpleNamespace( + reqs=[], + seq_lens_cpu=None, + forward_iter=1, + dp_cooperation_info=None, + ) + reporter = SchedulerMetricsReporter.__new__(SchedulerMetricsReporter) + reporter.scheduler = SimpleNamespace( + disaggregation_mode=DisaggregationMode.DECODE, + disagg_decode_transfer_queue=queue, + disagg_decode_prealloc_queue=SimpleNamespace( + queue=[], retracted_queue=[], num_tokens_pre_allocated=0 + ), + running_batch=batch, + waiting_queue=[], + grammar_manager=[], + enable_priority_scheduling=False, + max_total_num_tokens=1, + spec_algorithm=SimpleNamespace(is_none=lambda: True), + pool_stats_observer=SimpleNamespace( + get_pool_stats=lambda: SimpleNamespace( + get_decode_usage_msg_parts=lambda: [], + update_scheduler_stats=lambda stats: None, + ), + streaming_session_count=lambda: 0, + session_held_tokens=lambda: 0, + ), + kv_events_publisher=SimpleNamespace( + emit_kv_metrics=lambda: None, publish_kv_events=lambda: None + ), + ) + reporter.stats = SchedulerStats() + reporter.current_scheduler_metrics_enabled = True + reporter.is_stats_logging_rank = False + reporter.enable_mfu_metrics = False + reporter.scheduler_status_logger = None + reporter.forward_ct_decode = 0 + reporter.decode_log_interval = 1 + reporter.last_decode_stats_tic = 0.0 + reporter.num_generated_tokens = 0 + reporter.num_retracted_reqs = 0 + reporter.num_paused_reqs = 0 + reporter.fwd_occupancy = 0.0 + reporter._graph_backend_label = "cuda graph" + reporter._calculate_utilization = lambda: None + reporter._update_lora_metrics = lambda: None + reporter._log_hicache_stats = lambda: None + collector = MagicMock() + collector.num_decode_deferred_kv_release_reqs = ( + self.collector.num_decode_deferred_kv_release_reqs + ) + collector.labels = self.labels + collector.log_stats.side_effect = lambda stats: ( + SchedulerMetricsCollector.log_stats(collector, stats) + ) + collector._log_gauge.side_effect = lambda gauge, data: ( + SchedulerMetricsCollector._log_gauge(collector, gauge, data) + ) + reporter.metrics_collector = collector + + with ( + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER", + False, + ), + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.RECORD_STEP_TIME", + False, + ), + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg", + return_value=SimpleNamespace( + disaggregation_decode_host_receive_threshold=0, + language_only=False, + ), + ), + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter", + return_value=10.0, + ), + ): + reporter.report_decode_stats(can_run_cuda_graph=False, running_batch=batch) + + self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 2) + self.assertEqual( + self.collector.num_decode_deferred_kv_release_reqs.sets, + [(self.labels, 2)], + ) + + class TestRequestTimePerOutputToken(CustomTestCase): def _collector(self, labels): with get_context().override_server_args(