From 12004043b8bd900d0b6eea9b64b5d9d09cddd337 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Tue, 6 Oct 2026 13:11:29 -0700 Subject: [PATCH] [Metrics] Read scheduled prefill KV tokens from batch `prefix_lens` (#42797) --- .../srt/managers/scheduler_components/metrics_reporter.py | 4 +++- .../unit/observability/test_forward_pass_metrics.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index d4a142ff02..b8d2fdc9ce 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -418,7 +418,9 @@ class SchedulerMetricsReporter: prefill_lengths.add(len(req.origin_input_ids)) num_prefill_requests = stats.num_new_seqs if stats else len(prefill_reqs) sum_prefill_tokens = stats.log_input_tokens if stats else 0 - sum_prefill_kv_tokens = sum(len(req.prefix_indices) for req in prefill_reqs) + # Prefill reqs lead batch.reqs, so they own the head of prefix_lens; + # each req's own prefix was already advanced by result processing. + sum_prefill_kv_tokens = sum(batch.prefix_lens[: len(prefill_reqs)]) decode_kv = WelfordAccumulator() if batch.forward_mode.is_mixed(): diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index 096fff58a0..dbb27afbec 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -29,11 +29,9 @@ class _FakeReq: self, prompt_len: int, output_len: int = 0, - prefix_len: int = 0, ): self.origin_input_ids = list(range(prompt_len)) self.output_ids = list(range(output_len)) - self.prefix_indices = list(range(prefix_len)) self.seqlen = prompt_len + output_len @@ -153,6 +151,7 @@ class TestForwardPassMetrics(unittest.TestCase): forward_mode=_FakeForwardMode(), reqs=[], decoding_reqs=[], + prefix_lens=[], prefill_stats=None, seq_lens_cpu=[], fpm_start_time=100.0, @@ -164,13 +163,14 @@ class TestForwardPassMetrics(unittest.TestCase): self.scheduler._fpm_dp_rank = 3 self.scheduler.waiting_queue = [_FakeReq(6), _FakeReq(4, output_len=2)] - prefill_a = _FakeReq(10, prefix_len=2) - prefill_b = _FakeReq(14, prefix_len=3) + prefill_a = _FakeReq(10) + prefill_b = _FakeReq(14) decode_req = _FakeReq(8, output_len=3) batch = self._make_batch( forward_mode=_FakeForwardMode(is_mixed=True, is_extend=True), reqs=[prefill_a, prefill_b, decode_req], decoding_reqs=[decode_req], + prefix_lens=[2, 3, decode_req.seqlen - 1], prefill_stats=PrefillStats( log_input_tokens=12, log_hit_tokens=5,