feat(metrics): expose deferred decode KV release metrics (#41128)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
FREETRUMP
2026-10-04 04:52:21 +08:00
committed by GitHub
co-authored by Shangming Cai Claude Fable 5
parent 749a89abd6
commit 8bf780dda5
6 changed files with 379 additions and 40 deletions
+47 -11
View File
@@ -2407,8 +2407,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
envs.SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE_TIMEOUT.get()
)
# Aborted-mid-transfer requests whose KV pages/slot are held until drained
# or timed out. Entries: (decode_req, deadline, metadata_idx, required_acks).
self._deferred_releases: List[Tuple[DecodeRequest, float, int, int]] = []
# or timed out. Entries:
# (decode_req, start_time, deadline, metadata_idx, required_acks).
self._deferred_releases: List[Tuple[DecodeRequest, float, float, int, int]] = []
def add(self, decode_req: DecodeRequest) -> None:
self.queue.append(decode_req)
@@ -2799,12 +2800,19 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
return transferred_reqs
def _defer_release(self, decode_req: DecodeRequest) -> None:
deadline = time.monotonic() + self.deferred_kv_release_timeout
start_time = time.monotonic()
deadline = start_time + self.deferred_kv_release_timeout
# Require an ack from every notified prefill rank (dummy-proof). Snapshot
# now -- the receiver may be cleared by resolve time.
required_acks = len(decode_req.kv_receiver.bootstrap_infos)
self._deferred_releases.append(
(decode_req, deadline, decode_req.metadata_buffer_index, required_acks)
(
decode_req,
start_time,
deadline,
decode_req.metadata_buffer_index,
required_acks,
)
)
def _do_release(self, decode_req: DecodeRequest, idx: int) -> None:
@@ -2822,6 +2830,18 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
def has_pending_deferred_releases(self) -> bool:
return bool(self._deferred_releases)
def num_pending_deferred_releases(self) -> int:
return len(self._deferred_releases)
def _observe_deferred_kv_release(
self, duration_seconds: float, outcome: str
) -> None:
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.observe_decode_deferred_kv_release(
duration_seconds=duration_seconds,
outcome=outcome,
)
def resolve_deferred_releases(self) -> None:
"""Release drained transfers; device destinations may also time out."""
if not self._deferred_releases:
@@ -2831,7 +2851,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
req.kv_receiver.kv_mgr.is_abort_release_safe(
req.req.bootstrap_room, acks
)
for req, _, _, acks in self._deferred_releases
for req, _, _, _, acks in self._deferred_releases
if self.enable_host_receive and req.host_staged
],
dtype=torch.int32,
@@ -2845,36 +2865,52 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
now = time.monotonic()
still_held = []
to_release = []
for decode_req, deadline, idx, required_acks in self._deferred_releases:
for (
decode_req,
start_time,
deadline,
idx,
required_acks,
) in self._deferred_releases:
room = decode_req.req.bootstrap_room
if self.enable_host_receive and decode_req.host_staged:
if next(host_ready):
to_release.append((decode_req, idx, room, True))
to_release.append((decode_req, start_time, idx, room, True))
else:
# A timeout cannot prove that a remote write has stopped.
still_held.append((decode_req, deadline, idx, required_acks))
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
)
continue
kv_mgr = decode_req.kv_receiver.kv_mgr
drained = kv_mgr.is_abort_release_safe(room, required_acks)
if not drained and now < deadline:
still_held.append((decode_req, deadline, idx, required_acks))
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
)
else:
to_release.append((decode_req, idx, room, drained))
to_release.append((decode_req, start_time, idx, room, drained))
# Commit the survivors before releasing so a _do_release exception can't
# leave a released entry in the list (double-free / None receiver on retry).
self._deferred_releases = still_held
for decode_req, idx, room, drained in to_release:
for decode_req, start_time, idx, room, drained in to_release:
if not drained:
logger.warning(
f"Deferred KV release for room {room} timed out after "
f"{self.deferred_kv_release_timeout}s without a full drain "
f"ack from prefill; releasing anyway."
)
outcome = "drained" if drained else "timeout"
try:
self._do_release(decode_req, idx)
except Exception:
# Isolate a failed release so the rest still run; entry already dropped.
logger.exception(f"Deferred KV release failed for room {room}")
outcome = "error"
self._observe_deferred_kv_release(
duration_seconds=max(0.0, now - start_time),
outcome=outcome,
)
def release_memory_occupation(self):
"""Clean up in-flight transfers before releasing GPU memory."""
@@ -1392,6 +1392,9 @@ class SchedulerMetricsReporter:
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
transfer_queue, priority_enabled
)
self.stats.num_decode_deferred_kv_release_reqs = (
self.scheduler.disagg_decode_transfer_queue.num_pending_deferred_releases()
)
host_reqs = (
[req for req in transfer_queue if req.host_staged]
if get_disagg().disaggregation_decode_host_receive_threshold > 0
@@ -138,6 +138,7 @@ class SchedulerStats:
num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_deferred_kv_release_reqs: int = 0
kv_transfer_speed_gb_s: float = 0.0
kv_transfer_latency_ms: float = 0.0
pending_prealloc_token_usage: float = 0.0
@@ -533,6 +534,26 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
documentation="Total requests admitted to receive prefill KV in host memory.",
labelnames=labels.keys(),
)
self.num_decode_deferred_kv_release_reqs = Gauge(
name="sglang:num_decode_deferred_kv_release_reqs",
documentation=(
"The number of failed decode requests whose KV allocations are "
"held pending transfer drain acknowledgement or timeout."
),
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.decode_deferred_kv_release_seconds = Histogram(
name="sglang:decode_deferred_kv_release_seconds",
documentation="Histogram of time spent holding deferred decode KV releases.",
labelnames=labels.keys(),
buckets=(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60),
)
self.decode_deferred_kv_release_total = Counter(
name="sglang:decode_deferred_kv_release_total",
documentation="Total deferred decode KV release resolution attempts.",
labelnames=list(labels.keys()) + ["outcome"],
)
self.kv_transfer_speed_gb_s = Histogram(
name="sglang:kv_transfer_speed_gb_s",
documentation="Histogram of KV cache transfer speed in GB/s.",
@@ -1208,6 +1229,16 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
def increment_decode_host_receive_reqs(self) -> None:
self.num_decode_host_receive_reqs.labels(**self.labels).inc(1)
def observe_decode_deferred_kv_release(
self, duration_seconds: float, outcome: str
) -> None:
if outcome not in ("drained", "timeout", "error"):
raise ValueError(f"Invalid deferred KV release outcome: {outcome}")
self._log_histogram(self.decode_deferred_kv_release_seconds, duration_seconds)
self.decode_deferred_kv_release_total.labels(
**self.labels, outcome=outcome
).inc(1)
def increment_prefill_retries(self, count: int) -> None:
if count > 0:
self.num_prefill_retries_total.labels(**self.labels).inc(count)
@@ -1442,6 +1473,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
self.num_decode_host_receive_queue_reqs,
stats.num_decode_host_receive_queue_reqs,
)
self._log_gauge(
self.num_decode_deferred_kv_release_reqs,
stats.num_decode_deferred_kv_release_reqs,
)
self._log_gauge(
self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage
)
@@ -12,7 +12,7 @@ import threading
import unittest
from types import SimpleNamespace
from typing import NamedTuple
from unittest.mock import patch
from unittest.mock import MagicMock, call, patch
from sglang.srt.disaggregation import decode as decode_mod
from sglang.srt.disaggregation.base.conn import BaseKVManager, KVPoll
@@ -642,15 +642,22 @@ class _FakeIdxAllocator:
self.freed.append(idx)
def _make_queue(timeout=30.0):
def _make_queue(timeout=30.0, enable_metrics=False):
q = DecodeTransferQueue.__new__(DecodeTransferQueue)
q._deferred_releases = []
q.deferred_kv_release_timeout = timeout
q.enable_host_receive = False
q.enable_staging = False
q.staging_handler = None
q.tree_cache = object()
q.metadata_buffers = SimpleNamespace(bootstrap_room={})
q.req_to_metadata_buffer_idx_allocator = _FakeIdxAllocator()
q.scheduler = SimpleNamespace(
metrics_reporter=SimpleNamespace(enable_metrics=enable_metrics),
metrics_collector=SimpleNamespace(
observe_decode_deferred_kv_release=MagicMock()
),
)
return q
@@ -713,63 +720,96 @@ class TestResolveDeferredReleases(CustomTestCase):
def test_holds_until_drained_then_releases(self):
mgr = _make_manager()
room, idx = 200, 7
q = _make_queue()
q = _make_queue(enable_metrics=True)
dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=2)
# In production the receiver arms the room just before it sends the
# ABORT (_send_abort_notification), before the scheduler defers here.
generation = mgr.register_deferred_abort_room(room)
q._defer_release(dreq)
with patch.object(decode_mod, "release_kv_cache") as rel:
# Not yet acked -> held, not released.
q.resolve_deferred_releases()
rel.assert_not_called()
self.assertEqual(len(q._deferred_releases), 1)
with patch.object(
decode_mod.time, "monotonic", side_effect=[10.0, 11.0, 12.0, 13.0]
):
q._defer_release(dreq)
with patch.object(decode_mod, "release_kv_cache") as rel:
# Not yet acked -> held, not released.
q.resolve_deferred_releases()
rel.assert_not_called()
self.assertEqual(len(q._deferred_releases), 1)
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called()
# One of two ranks acked -> still held.
mgr.note_abort_ack(room, 0, generation)
q.resolve_deferred_releases()
rel.assert_not_called()
self.assertEqual(len(q._deferred_releases), 1)
# One of two ranks acked -> still held.
mgr.note_abort_ack(room, 0, generation)
q.resolve_deferred_releases()
rel.assert_not_called()
self.assertEqual(len(q._deferred_releases), 1)
# Both ranks acked -> released exactly once.
mgr.note_abort_ack(room, 1, generation)
q.resolve_deferred_releases()
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
# Both ranks acked -> released exactly once.
mgr.note_abort_ack(room, 1, generation)
q.resolve_deferred_releases()
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
# Held state fully cleaned up.
self.assertEqual(q._deferred_releases, [])
self.assertEqual(q.num_pending_deferred_releases(), 0)
self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx])
self.assertEqual(q.metadata_buffers.bootstrap_room[idx], 0)
self.assertNotIn(room, mgr._deferred_abort_ack_tracker)
self.assertIsNone(dreq.kv_receiver)
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_called_once_with(
duration_seconds=3.0,
outcome="drained",
)
def test_releases_on_timeout_without_ack(self):
mgr = _make_manager()
room, idx = 300, 3
q = _make_queue(timeout=30.0)
q = _make_queue(timeout=30.0, enable_metrics=True)
dreq = _make_decode_req(room, idx, mgr, n_prefill_ranks=1)
# Force an already-expired deadline (no ack will ever arrive).
q._deferred_releases.append((dreq, float("-inf"), idx, 1))
q._deferred_releases.append((dreq, 10.0, float("-inf"), idx, 1))
with patch.object(decode_mod, "release_kv_cache") as rel:
with (
patch.object(decode_mod.time, "monotonic", return_value=42.0),
patch.object(decode_mod, "release_kv_cache") as rel,
):
q.resolve_deferred_releases()
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
self.assertEqual(q._deferred_releases, [])
self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx])
self.assertIsNone(dreq.kv_receiver)
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_called_once_with(
duration_seconds=32.0,
outcome="timeout",
)
def test_metrics_disabled_does_not_observe_release(self):
mgr = _make_manager()
room, idx = 301, 4
q = _make_queue(enable_metrics=False)
dreq = _make_decode_req(room, idx, mgr)
q._deferred_releases.append((dreq, 10.0, float("-inf"), idx, 1))
with (
patch.object(decode_mod.time, "monotonic", return_value=20.0),
patch.object(decode_mod, "release_kv_cache") as rel,
):
q.resolve_deferred_releases()
rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False)
self.assertEqual(q.num_pending_deferred_releases(), 0)
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called()
def test_failed_release_is_isolated_and_not_retried(self):
# A raising _do_release must drop the entry (no double-free on retry) and
# not brick resolve for the remaining entries or subsequent calls.
mgr = _make_manager()
q = _make_queue()
q = _make_queue(enable_metrics=True)
good = _make_decode_req(700, 1, mgr)
bad = _make_decode_req(701, 2, mgr)
# Both already past deadline -> both selected for release.
q._deferred_releases.append((bad, float("-inf"), 2, 1))
q._deferred_releases.append((good, float("-inf"), 1, 1))
q._deferred_releases.append((bad, 10.0, float("-inf"), 2, 1))
q._deferred_releases.append((good, 10.0, float("-inf"), 1, 1))
calls = []
@@ -778,24 +818,36 @@ class TestResolveDeferredReleases(CustomTestCase):
if req is bad.req:
raise RuntimeError("boom")
with patch.object(decode_mod, "release_kv_cache", side_effect=fake_release):
with (
patch.object(decode_mod.time, "monotonic", return_value=40.0),
patch.object(decode_mod, "release_kv_cache", side_effect=fake_release),
):
q.resolve_deferred_releases() # must not raise
# The good one still released despite the bad one throwing.
self.assertIn(good.req, calls)
# Nothing left held, and a second call is a clean no-op (no retry).
self.assertEqual(q._deferred_releases, [])
q.resolve_deferred_releases()
q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_has_calls(
[
call(duration_seconds=30.0, outcome="error"),
call(duration_seconds=30.0, outcome="timeout"),
]
)
def test_defer_release_records_deadline_and_idx(self):
mgr = _make_manager()
q = _make_queue(timeout=12.5)
dreq = _make_decode_req(room=400, idx=9, mgr=mgr)
q._defer_release(dreq)
with patch.object(decode_mod.time, "monotonic", return_value=5.0):
q._defer_release(dreq)
self.assertEqual(len(q._deferred_releases), 1)
held_req, deadline, held_idx, required = q._deferred_releases[0]
held_req, start_time, deadline, held_idx, required = q._deferred_releases[0]
self.assertIs(held_req, dreq)
self.assertEqual(start_time, 5.0)
self.assertEqual(deadline, 17.5)
self.assertEqual(held_idx, 9)
self.assertIsInstance(deadline, float)
self.assertEqual(required, 1)
def _make_pop_queue(mgr):
@@ -394,7 +394,8 @@ class TestIdleMetrics(CustomTestCase):
for priority in (1, 2)
]
self.scheduler.disagg_decode_transfer_queue = types.SimpleNamespace(
queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)]
queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)],
num_pending_deferred_releases=lambda: 0,
)
for _ in host_reqs:
collector.increment_decode_host_receive_reqs()
@@ -21,9 +21,15 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import prometheus_client
from sglang.srt.disaggregation.decode import DecodeTransferQueue
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.scheduler_components.metrics_reporter import (
SchedulerMetricsReporter,
)
from sglang.srt.observability.metrics_collector import (
STAT_LOGGER_ROLE_EXPERT_DISPATCH,
STAT_LOGGER_ROLE_RADIX_CACHE,
@@ -32,6 +38,7 @@ from sglang.srt.observability.metrics_collector import (
STAT_LOGGER_ROLE_TOKENIZER,
RadixCacheMetricsCollector,
SchedulerMetricsCollector,
SchedulerStats,
StorageMetricsCollector,
TokenizerMetricsCollector,
radix_cache_metric_labels,
@@ -225,6 +232,211 @@ class TestHiCacheMetrics(unittest.TestCase):
)
class TestDeferredKVReleaseMetrics(CustomTestCase):
def setUp(self):
self.labels = {
"model_name": "test",
"engine_type": "decode",
"tp_rank": 0,
"pp_rank": 0,
"moe_ep_rank": 0,
}
self.collector = SchedulerMetricsCollector.__new__(SchedulerMetricsCollector)
self.collector.labels = self.labels
self.collector.decode_deferred_kv_release_seconds = _RecordingMetric(
name="sglang:decode_deferred_kv_release_seconds",
labelnames=self.labels.keys(),
)
self.collector.decode_deferred_kv_release_total = _RecordingMetric(
name="sglang:decode_deferred_kv_release_total",
labelnames=list(self.labels.keys()) + ["outcome"],
)
self.collector.num_decode_deferred_kv_release_reqs = _RecordingMetric(
name="sglang:num_decode_deferred_kv_release_reqs",
labelnames=self.labels.keys(),
)
def test_records_duration_and_bounded_outcome(self):
self.collector.observe_decode_deferred_kv_release(
duration_seconds=0.25,
outcome="drained",
)
self.assertEqual(
self.collector.decode_deferred_kv_release_seconds.observations,
[(self.labels, 0.25)],
)
self.assertEqual(
self.collector.decode_deferred_kv_release_total.increments,
[({**self.labels, "outcome": "drained"}, 1)],
)
with self.assertRaisesRegex(ValueError, "Invalid deferred KV release outcome"):
self.collector.observe_decode_deferred_kv_release(
duration_seconds=0.5,
outcome="room-123",
)
def test_logs_current_deferred_release_count(self):
queue = DecodeTransferQueue.__new__(DecodeTransferQueue)
queue.queue = []
queue._deferred_releases = [object(), object(), object()]
reporter = SchedulerMetricsReporter.__new__(SchedulerMetricsReporter)
reporter.scheduler = SimpleNamespace(
disaggregation_mode=DisaggregationMode.DECODE,
disagg_decode_transfer_queue=queue,
disagg_decode_prealloc_queue=SimpleNamespace(queue=[]),
running_batch=SimpleNamespace(reqs=[]),
waiting_queue=[],
grammar_manager=[],
enable_priority_scheduling=False,
pool_stats_observer=SimpleNamespace(
get_pool_stats=lambda: SimpleNamespace(
update_scheduler_stats=lambda stats: None,
),
streaming_session_count=lambda: 0,
session_held_tokens=lambda: 0,
),
)
reporter.stats = SchedulerStats()
reporter.current_scheduler_metrics_enabled = True
collector = MagicMock()
collector.last_log_time = 0
collector.num_decode_deferred_kv_release_reqs = (
self.collector.num_decode_deferred_kv_release_reqs
)
collector.labels = self.labels
collector.log_stats.side_effect = lambda stats: (
SchedulerMetricsCollector.log_stats(collector, stats)
)
collector._log_gauge.side_effect = lambda gauge, data: (
SchedulerMetricsCollector._log_gauge(collector, gauge, data)
)
reporter.metrics_collector = collector
with (
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER",
False,
),
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg",
return_value=SimpleNamespace(
disaggregation_decode_host_receive_threshold=0
),
),
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter",
side_effect=[31.0, 31.0, 62.0, 62.0],
),
):
reporter._maybe_log_idle_metrics()
self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 3)
queue._deferred_releases.clear()
reporter._maybe_log_idle_metrics()
self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 0)
self.assertEqual(
self.collector.num_decode_deferred_kv_release_reqs.sets,
[(self.labels, 3), (self.labels, 0)],
)
def test_decode_report_logs_current_deferred_release_count(self):
# Regression: report_decode_stats reads the queue in its message section
# before the metrics block runs, so a reset inside the metrics block
# would zero the gauge on every busy-path report.
queue = DecodeTransferQueue.__new__(DecodeTransferQueue)
queue.queue = []
queue._deferred_releases = [object(), object()]
batch = SimpleNamespace(
reqs=[],
seq_lens_cpu=None,
forward_iter=1,
dp_cooperation_info=None,
)
reporter = SchedulerMetricsReporter.__new__(SchedulerMetricsReporter)
reporter.scheduler = SimpleNamespace(
disaggregation_mode=DisaggregationMode.DECODE,
disagg_decode_transfer_queue=queue,
disagg_decode_prealloc_queue=SimpleNamespace(
queue=[], retracted_queue=[], num_tokens_pre_allocated=0
),
running_batch=batch,
waiting_queue=[],
grammar_manager=[],
enable_priority_scheduling=False,
max_total_num_tokens=1,
spec_algorithm=SimpleNamespace(is_none=lambda: True),
pool_stats_observer=SimpleNamespace(
get_pool_stats=lambda: SimpleNamespace(
get_decode_usage_msg_parts=lambda: [],
update_scheduler_stats=lambda stats: None,
),
streaming_session_count=lambda: 0,
session_held_tokens=lambda: 0,
),
kv_events_publisher=SimpleNamespace(
emit_kv_metrics=lambda: None, publish_kv_events=lambda: None
),
)
reporter.stats = SchedulerStats()
reporter.current_scheduler_metrics_enabled = True
reporter.is_stats_logging_rank = False
reporter.enable_mfu_metrics = False
reporter.scheduler_status_logger = None
reporter.forward_ct_decode = 0
reporter.decode_log_interval = 1
reporter.last_decode_stats_tic = 0.0
reporter.num_generated_tokens = 0
reporter.num_retracted_reqs = 0
reporter.num_paused_reqs = 0
reporter.fwd_occupancy = 0.0
reporter._graph_backend_label = "cuda graph"
reporter._calculate_utilization = lambda: None
reporter._update_lora_metrics = lambda: None
reporter._log_hicache_stats = lambda: None
collector = MagicMock()
collector.num_decode_deferred_kv_release_reqs = (
self.collector.num_decode_deferred_kv_release_reqs
)
collector.labels = self.labels
collector.log_stats.side_effect = lambda stats: (
SchedulerMetricsCollector.log_stats(collector, stats)
)
collector._log_gauge.side_effect = lambda gauge, data: (
SchedulerMetricsCollector._log_gauge(collector, gauge, data)
)
reporter.metrics_collector = collector
with (
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER",
False,
),
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.RECORD_STEP_TIME",
False,
),
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg",
return_value=SimpleNamespace(
disaggregation_decode_host_receive_threshold=0,
language_only=False,
),
),
patch(
"sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter",
return_value=10.0,
),
):
reporter.report_decode_stats(can_run_cuda_graph=False, running_batch=batch)
self.assertEqual(reporter.stats.num_decode_deferred_kv_release_reqs, 2)
self.assertEqual(
self.collector.num_decode_deferred_kv_release_reqs.sets,
[(self.labels, 2)],
)
class TestRequestTimePerOutputToken(CustomTestCase):
def _collector(self, labels):
with get_context().override_server_args(