From 5ad9ff6a5f257e66c29a11958c842c6cb93c76fc Mon Sep 17 00:00:00 2001 From: ymk Date: Thu, 12 Nov 2026 10:30:38 +0800 Subject: [PATCH] observability: prefill_batch_init wall+CPU, sub-phases and eviction coverage Parent span scheduler.loop.prefill_batch_init records wall + the scheduler thread's CPU at both existing close points (empty admission and normal return); the CPU clock is only read when loop detail is enabled and sampled, through the safe batch_telemetry.thread_cpu_ns() wrapper so a missing clock degrades to 'not measured' instead of failing the scheduling path. Fixed children, in call order: schedule_batch_init, ready_to_load_host_cache, prepare_for_extend, alloc_for_extend (+ prefix_device_indices attribute-only, sampling_batch_info), evict_to_free_tokens, evict_for_alloc, prefill_stats (+ pending_tokens), mixed_chunk_tail. * prefill_stats wraps the whole assignment so _get_num_pending_tokens() is included; mixed_chunk_tail only exists when that branch is taken. * prefill-only shape counters (n_reqs/n_new/n_cached) are snapshotted before mixed-chunk folding; a mixed batch additionally publishes decode_reqs and merged totals under explicit *_merged_* names. * eviction is timed at the call site (outside the allocator wrapper) so rank-consensus waits are included, and covers base, SWA (x2), UnifiedSWA (x3) and the extend Mamba reclaim through evict_for_alloc_observed(). Demand, shortfall and plan are separate counters; Mamba plan and realized counts are both recorded (including zero). All of it is gated by detail on a sampled loop. --- python/sglang/srt/managers/schedule_batch.py | 19 ++- python/sglang/srt/managers/scheduler.py | 130 ++++++++++++++---- python/sglang/srt/mem_cache/allocation.py | 10 +- python/sglang/srt/mem_cache/allocator/base.py | 6 +- python/sglang/srt/mem_cache/allocator/swa.py | 13 +- .../mem_cache/allocator/unified_hybrid_swa.py | 15 +- python/sglang/srt/mem_cache/common.py | 80 ++++++++++- 7 files changed, 226 insertions(+), 47 deletions(-) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 750bfa63a9..45f3ff5df4 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -120,6 +120,7 @@ from sglang.srt.mem_cache.common import ( ) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.observability import batch_telemetry from sglang.srt.model_executor.encoder_swa_replay import resets_swa_window from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -2915,9 +2916,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) self.encoder_swa_reset = [resets_swa_window(r) for r in reqs] # Allocate memory - out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend( - self - ) + with batch_telemetry.detail_phase("alloc_for_extend"): + ( + out_cache_loc, + req_pool_indices_tensor, + req_pool_indices_cpu, + ) = alloc_for_extend(self) # Set fields input_embeds = [] @@ -3133,10 +3137,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): self.prepare_encoder_info_extend(input_ids, seq_lens) # Build sampling info - self.sampling_info = SamplingBatchInfo.from_schedule_batch( - self, - self.model_config.vocab_size, - ) + with batch_telemetry.detail_phase("sampling_batch_info"): + self.sampling_info = SamplingBatchInfo.from_schedule_batch( + self, + self.model_config.vocab_size, + ) def _mamba_radix_cache_v2_req_prepare_for_extend( self, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 732a6a51b0..52955cbf2f 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4323,6 +4323,12 @@ class Scheduler( if fwd_timeline is not None: fwd_timeline.detail("prefill_scan", _pf_t2, time.monotonic_ns()) _pf_t3 = time.monotonic_ns() + _pf_cpu_t3 = ( + batch_telemetry.thread_cpu_ns() + if batch_telemetry.loop_detail_enabled() + else None + ) + _had_chunked = self.chunked_req is not None # Update waiting queue can_run_list: List[Req] = adder.can_run_list @@ -4334,7 +4340,10 @@ class Scheduler( batch_telemetry.set_admitted(0) if fwd_timeline is not None: fwd_timeline.detail( - "prefill_batch_init", _pf_t3, time.monotonic_ns() + "prefill_batch_init", + _pf_t3, + time.monotonic_ns(), + thread_cpu_ns=self._detail_cpu_delta(_pf_cpu_t3), ) return None, running_batch @@ -4359,16 +4368,17 @@ class Scheduler( set_time_batch(can_run_list, "set_forward_entry_time") # Create a new batch - new_batch = ScheduleBatch.init_new( - can_run_list, - self.req_to_token_pool, - self.token_to_kv_pool_allocator, - self.tree_cache, - self.model_config, - self.enable_overlap, - self.spec_algorithm, - chunked_req=self.chunked_req, - ) + with batch_telemetry.detail_phase("schedule_batch_init"): + new_batch = ScheduleBatch.init_new( + can_run_list, + self.req_to_token_pool, + self.token_to_kv_pool_allocator, + self.tree_cache, + self.model_config, + self.enable_overlap, + self.spec_algorithm, + chunked_req=self.chunked_req, + ) new_batch.contains_last_prefill_chunk = ( self.chunked_req is None or len(can_run_list) != 1 @@ -4377,29 +4387,35 @@ class Scheduler( if self.enable_hierarchical_cache or self.enable_unified_cache_external_linker: # todo (zhiqiang): disable cuda graph execution if hicache loading triggered - new_batch.hicache_consumer_index = ( - self.tree_cache.ready_to_load_host_cache() - ) + with batch_telemetry.detail_phase("ready_to_load_host_cache"): + new_batch.hicache_consumer_index = ( + self.tree_cache.ready_to_load_host_cache() + ) - new_batch.prepare_for_extend() + with batch_telemetry.detail_phase("prepare_for_extend"): + new_batch.prepare_for_extend() + # Snapshot the prefill-only shape before mixed-chunk folding: + # mix_with_running() appends decode rows to extend_num_tokens/prefix_lens, + # so reading them after the branch would mix two different scopes. + _prefill_n_new = int(getattr(new_batch, "extend_num_tokens", 0) or 0) + _prefill_n_cached = int(sum(getattr(new_batch, "prefix_lens", ()) or ())) if self.tp_worker.model_runner.prefill_aware_swa: for req in can_run_list: req.kv.swa_evict_floor = req.extend_end # Record prefill stats for logging after forward. - new_batch.prefill_stats = PrefillStats.from_adder( - adder, - running_batch.reqs, - self.enable_priority_scheduling, - num_pending_tokens=self.load_inquirer._get_num_pending_tokens( - chunk_deduct=( - self.chunked_req.extend_len if self.chunked_req is not None else 0 - ), - ), - ) + with batch_telemetry.detail_phase("prefill_stats"): + new_batch.prefill_stats = PrefillStats.from_adder( + adder, + running_batch.reqs, + self.enable_priority_scheduling, + num_pending_tokens=self._prefill_pending_tokens(), + ) # Mixed-style chunked prefill + _mixed_t0 = None + _mixed_cpu0 = None if ( self.is_mixed_chunk and not running_batch.is_empty() @@ -4409,6 +4425,9 @@ class Scheduler( # Beam member rows are not supported inside a mixed extend batch. and all(r.beam_group is None for r in running_batch.reqs) ): + if batch_telemetry.loop_detail_enabled(): + _mixed_t0 = time.monotonic_ns() + _mixed_cpu0 = batch_telemetry.thread_cpu_ns() # TODO (lianmin): support return_logprob + mixed chunked prefill finishing_reqs = self._reqs_finishing_inflight(running_batch) running_batch.filter_batch(reqs_to_exclude=finishing_reqs) @@ -4440,10 +4459,69 @@ class Scheduler( batch_telemetry.set_admitted(len(can_run_list)) batch_telemetry.note_new_chunk(adder.new_chunked_req is not None) + if _mixed_t0 is not None and fwd_timeline is not None: + fwd_timeline.detail( + "mixed_chunk_tail", + _mixed_t0, + time.monotonic_ns(), + thread_cpu_ns=self._detail_cpu_delta(_mixed_cpu0), + ) + if batch_telemetry.loop_detail_enabled(): + # Prefill-only scope for the three base counters, even when the + # batch was folded with running decode rows afterwards. + batch_telemetry.detail_attr("prefill_batch_n_reqs", len(can_run_list)) + batch_telemetry.detail_attr("prefill_batch_n_new", _prefill_n_new) + batch_telemetry.detail_attr("prefill_batch_n_cached", _prefill_n_cached) + batch_telemetry.detail_attr( + "prefill_batch_chunked", bool(self.chunked_req is not None) + ) + batch_telemetry.detail_attr("prefill_batch_resumed_chunked", bool(_had_chunked)) + batch_telemetry.detail_attr( + "prefill_batch_new_chunked", bool(adder.new_chunked_req is not None) + ) + batch_telemetry.detail_attr( + "prefill_batch_mixed_chunk", bool(_mixed_t0 is not None) + ) + if _mixed_t0 is not None: + # Mixed batches carry a second scope; keep it explicit instead + # of folding decode rows into the prefill counters above. + _decode_rows = len(getattr(new_batch, "decoding_reqs", ()) or ()) + batch_telemetry.detail_attr( + "prefill_batch_decode_reqs", _decode_rows + ) + batch_telemetry.detail_attr( + "prefill_batch_merged_n_reqs", len(can_run_list) + _decode_rows + ) + batch_telemetry.detail_attr( + "prefill_batch_merged_n_new", + int(getattr(new_batch, "extend_num_tokens", 0) or 0), + ) if fwd_timeline is not None: - fwd_timeline.detail("prefill_batch_init", _pf_t3, time.monotonic_ns()) + fwd_timeline.detail( + "prefill_batch_init", + _pf_t3, + time.monotonic_ns(), + thread_cpu_ns=self._detail_cpu_delta(_pf_cpu_t3), + ) return new_batch, running_batch + @staticmethod + def _detail_cpu_delta(cpu_start_ns): + """Thread CPU delta for detail spans; None when the clock is missing.""" + if cpu_start_ns is None: + return None + now = batch_telemetry.thread_cpu_ns() + return None if now is None else now - cpu_start_ns + + def _prefill_pending_tokens(self) -> int: + """Pending-token evaluation for PrefillStats, timed as its own phase.""" + with batch_telemetry.detail_phase("pending_tokens"): + return self.load_inquirer._get_num_pending_tokens( + chunk_deduct=( + self.chunked_req.extend_len if self.chunked_req is not None else 0 + ), + ) + def can_schedule_lora_req( self, req: Req, running_loras: set[Optional[str]] ) -> bool: diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index c2df820115..8ce1cb9c92 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -24,6 +24,7 @@ from sglang.srt.mem_cache.common import ( evict_from_tree_cache, ) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool +from sglang.srt.observability import batch_telemetry from sglang.srt.runtime_context import attention_backends, get_parallel from sglang.srt.utils import ( is_cpu, @@ -318,8 +319,10 @@ def ensure_mamba_capacity( supports_mamba = tree_cache is not None and tree_cache.supports_mamba() available = allocator.schedulable_available_size() if available < needed and supports_mamba: - tree_cache.evict_for_alloc( - EvictParams(num_tokens=0, mamba_num=needed - available) + from sglang.srt.mem_cache.common import evict_for_alloc_observed + + evict_for_alloc_observed( + tree_cache, EvictParams(num_tokens=0, mamba_num=needed - available) ) return allocator.schedulable_available_size() >= needed @@ -384,7 +387,8 @@ def alloc_for_extend( # free out-of-window swa tokens batch.maybe_evict_swa() - prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs] + with batch_telemetry.detail_phase("prefix_device_indices"): + prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs] reuse_kv = None if batch.is_dllm(): diff --git a/python/sglang/srt/mem_cache/allocator/base.py b/python/sglang/srt/mem_cache/allocator/base.py index 6c27e6411b..00a75861b7 100644 --- a/python/sglang/srt/mem_cache/allocator/base.py +++ b/python/sglang/srt/mem_cache/allocator/base.py @@ -165,7 +165,11 @@ class BaseTokenToKVPoolAllocator(abc.ABC): return shortfall = num_tokens - self.available_size() if shortfall > 0: - tree_cache.evict_for_alloc(EvictParams(num_tokens=shortfall)) + from sglang.srt.mem_cache.common import evict_for_alloc_observed + + evict_for_alloc_observed( + tree_cache, EvictParams(num_tokens=shortfall) + ) _evict_until_allocatable(tree_cache, self, num_tokens) def check_decode_capacity( diff --git a/python/sglang/srt/mem_cache/allocator/swa.py b/python/sglang/srt/mem_cache/allocator/swa.py index b08d973c2c..81b21feed0 100644 --- a/python/sglang/srt/mem_cache/allocator/swa.py +++ b/python/sglang/srt/mem_cache/allocator/swa.py @@ -189,10 +189,12 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): """ from sglang.srt.mem_cache.base_prefix_cache import EvictParams + from sglang.srt.mem_cache.common import evict_for_alloc_observed + available = self.swa_available_size() if available < swa_tokens: - tree_cache.evict_for_alloc( - EvictParams(swa_num_tokens=swa_tokens - available) + evict_for_alloc_observed( + tree_cache, EvictParams(swa_num_tokens=swa_tokens - available) ) available = self.swa_available_size() if available < swa_tokens: @@ -215,8 +217,11 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): full_shortfall = max(0, num_tokens - self.full_available_size()) swa_shortfall = max(0, num_tokens - self.swa_available_size()) if full_shortfall or swa_shortfall: - tree_cache.evict_for_alloc( - EvictParams(num_tokens=full_shortfall, swa_num_tokens=swa_shortfall) + from sglang.srt.mem_cache.common import evict_for_alloc_observed + + evict_for_alloc_observed( + tree_cache, + EvictParams(num_tokens=full_shortfall, swa_num_tokens=swa_shortfall), ) def full_available_size(self): diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index 3486e44ed3..dc78199432 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -1254,7 +1254,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): return full_reclaim, swa_reclaim = reclaim_plan if full_reclaim or swa_reclaim: - tree_cache.evict_for_alloc( + from sglang.srt.mem_cache.common import evict_for_alloc_observed + + evict_for_alloc_observed( + tree_cache, EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim), allocation_reclaim_satisfied=lambda: ( self.reclaim_plan(num_tokens, required_swa) == (0, 0) @@ -1909,7 +1912,10 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): # precedes state reclaim so Mamba-first tree ordering cannot skip it. state_freed = 0 if full_reclaim or swa_reclaim: - result = tree_cache.evict_for_alloc( + from sglang.srt.mem_cache.common import evict_for_alloc_observed + + result = evict_for_alloc_observed( + tree_cache, EvictParams(num_tokens=num_tokens, swa_num_tokens=num_tokens), allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied( num_tokens @@ -1930,7 +1936,10 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): ) state_quota = max(0, -(-pair_bytes // ma.entry_bytes) - state_freed) if state_quota and tree_cache.mamba_evictable_size(): - result = tree_cache.evict_for_alloc( + from sglang.srt.mem_cache.common import evict_for_alloc_observed + + result = evict_for_alloc_observed( + tree_cache, EvictParams(mamba_num=state_quota), allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied( num_tokens diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index e3e1937208..15e597f960 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -24,6 +24,7 @@ from sglang.srt.mem_cache.hicache_storage import PoolTransfer from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.observability import batch_telemetry from sglang.srt.runtime_context import get_serving, get_spec from sglang.srt.utils.common import ceil_align @@ -315,13 +316,86 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None: req.prefix_len = req.extend_end +def _rank_consensus_enabled() -> bool: + """Whether the rank-consensus checker is active (cheap, best effort).""" + try: + from sglang.srt.utils import rank_consensus_checker + + return bool(rank_consensus_checker.enabled()) + except Exception: + return False + + +def evict_for_alloc_observed(tree_cache, params, **kwargs): + """Time ``tree_cache.evict_for_alloc`` and record what it realized. + + Kept at the call site (not inside the callee) so decorator/consensus waits + are part of the measurement. ``kwargs`` forwards the unified allocators' + ``allocation_reclaim_satisfied`` predicate. + """ + with batch_telemetry.detail_phase("evict_for_alloc"): + result = tree_cache.evict_for_alloc(params, **kwargs) + if batch_telemetry.loop_detail_enabled() and result is not None: + batch_telemetry.detail_count("evict_for_alloc_calls") + batch_telemetry.detail_attr_sum( + "evict_plan_requested_tokens", int(getattr(params, "num_tokens", 0) or 0) + ) + batch_telemetry.detail_attr_sum( + "evict_plan_requested_swa_tokens", + int(getattr(params, "swa_num_tokens", 0) or 0), + ) + batch_telemetry.detail_attr_sum( + "evict_plan_requested_mamba", + int(getattr(params, "mamba_num", 0) or 0), + ) + batch_telemetry.detail_attr_sum( + "evict_evicted_tokens", + int(getattr(result, "num_tokens_evicted", 0) or 0), + ) + batch_telemetry.detail_attr_sum( + "evict_evicted_swa_tokens", + int(getattr(result, "swa_num_tokens_evicted", 0) or 0), + ) + # Record the realized Mamba count unconditionally (including zero) so + # "no Mamba demand" and "demand but nothing reclaimed" stay distinct. + batch_telemetry.detail_attr_sum( + "evict_evicted_mamba", + int(getattr(result, "mamba_num_evicted", 0) or 0), + ) + return result + + def evict_from_tree_cache( tree_cache: BasePrefixCache | None, num_tokens: int ) -> bool | None: if tree_cache is not None and tree_cache.supports_prefix_sharing(): - return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens( - tree_cache, num_tokens - ) + allocator = tree_cache.token_to_kv_pool_allocator + with batch_telemetry.detail_phase("evict_to_free_tokens"): + if batch_telemetry.loop_detail_enabled(): + available_before = allocator.available_size() + batch_telemetry.detail_count("evict_calls") + # Demand is what the batch wants allocatable; shortfall is the + # part that eviction must actually recover. Keep them apart: + # summing demand as "requested eviction" overstates eviction. + batch_telemetry.detail_attr_sum( + "evict_demand_tokens", int(num_tokens) + ) + batch_telemetry.detail_attr_sum( + "evict_shortfall_tokens", + max(0, int(num_tokens) - int(available_before)), + ) + batch_telemetry.detail_attr_setdefault( + "evict_free_tokens_before", int(available_before) + ) + batch_telemetry.detail_attr( + "evict_rank_consensus_checker", _rank_consensus_enabled() + ) + result = allocator.evict_to_free_tokens(tree_cache, num_tokens) + if batch_telemetry.loop_detail_enabled(): + batch_telemetry.detail_attr( + "evict_free_tokens_after", int(allocator.available_size()) + ) + return result def _evict_until_allocatable(