observability: prefill_batch_init wall+CPU, sub-phases and eviction coverage

Parent span scheduler.loop.prefill_batch_init records wall + the scheduler
thread's CPU at both existing close points (empty admission and normal
return); the CPU clock is only read when loop detail is enabled and sampled,
through the safe batch_telemetry.thread_cpu_ns() wrapper so a missing clock
degrades to 'not measured' instead of failing the scheduling path.

Fixed children, in call order:
  schedule_batch_init, ready_to_load_host_cache, prepare_for_extend,
  alloc_for_extend (+ prefix_device_indices attribute-only, sampling_batch_info),
  evict_to_free_tokens, evict_for_alloc, prefill_stats (+ pending_tokens),
  mixed_chunk_tail.

* prefill_stats wraps the whole assignment so _get_num_pending_tokens() is
  included; mixed_chunk_tail only exists when that branch is taken.
* prefill-only shape counters (n_reqs/n_new/n_cached) are snapshotted before
  mixed-chunk folding; a mixed batch additionally publishes decode_reqs and
  merged totals under explicit *_merged_* names.
* eviction is timed at the call site (outside the allocator wrapper) so
  rank-consensus waits are included, and covers base, SWA (x2), UnifiedSWA
  (x3) and the extend Mamba reclaim through evict_for_alloc_observed().
  Demand, shortfall and plan are separate counters; Mamba plan and realized
  counts are both recorded (including zero).

All of it is gated by detail on a sampled loop.
This commit is contained in:
2026-11-12 10:30:38 +08:00
parent fec10023e0
commit 5ad9ff6a5f
7 changed files with 226 additions and 47 deletions
+12 -7
View File
@@ -120,6 +120,7 @@ from sglang.srt.mem_cache.common import (
)
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.observability import batch_telemetry
from sglang.srt.model_executor.encoder_swa_replay import resets_swa_window
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -2915,9 +2916,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
)
self.encoder_swa_reset = [resets_swa_window(r) for r in reqs]
# Allocate memory
out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend(
self
)
with batch_telemetry.detail_phase("alloc_for_extend"):
(
out_cache_loc,
req_pool_indices_tensor,
req_pool_indices_cpu,
) = alloc_for_extend(self)
# Set fields
input_embeds = []
@@ -3133,10 +3137,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self.prepare_encoder_info_extend(input_ids, seq_lens)
# Build sampling info
self.sampling_info = SamplingBatchInfo.from_schedule_batch(
self,
self.model_config.vocab_size,
)
with batch_telemetry.detail_phase("sampling_batch_info"):
self.sampling_info = SamplingBatchInfo.from_schedule_batch(
self,
self.model_config.vocab_size,
)
def _mamba_radix_cache_v2_req_prepare_for_extend(
self,
+104 -26
View File
@@ -4323,6 +4323,12 @@ class Scheduler(
if fwd_timeline is not None:
fwd_timeline.detail("prefill_scan", _pf_t2, time.monotonic_ns())
_pf_t3 = time.monotonic_ns()
_pf_cpu_t3 = (
batch_telemetry.thread_cpu_ns()
if batch_telemetry.loop_detail_enabled()
else None
)
_had_chunked = self.chunked_req is not None
# Update waiting queue
can_run_list: List[Req] = adder.can_run_list
@@ -4334,7 +4340,10 @@ class Scheduler(
batch_telemetry.set_admitted(0)
if fwd_timeline is not None:
fwd_timeline.detail(
"prefill_batch_init", _pf_t3, time.monotonic_ns()
"prefill_batch_init",
_pf_t3,
time.monotonic_ns(),
thread_cpu_ns=self._detail_cpu_delta(_pf_cpu_t3),
)
return None, running_batch
@@ -4359,16 +4368,17 @@ class Scheduler(
set_time_batch(can_run_list, "set_forward_entry_time")
# Create a new batch
new_batch = ScheduleBatch.init_new(
can_run_list,
self.req_to_token_pool,
self.token_to_kv_pool_allocator,
self.tree_cache,
self.model_config,
self.enable_overlap,
self.spec_algorithm,
chunked_req=self.chunked_req,
)
with batch_telemetry.detail_phase("schedule_batch_init"):
new_batch = ScheduleBatch.init_new(
can_run_list,
self.req_to_token_pool,
self.token_to_kv_pool_allocator,
self.tree_cache,
self.model_config,
self.enable_overlap,
self.spec_algorithm,
chunked_req=self.chunked_req,
)
new_batch.contains_last_prefill_chunk = (
self.chunked_req is None or len(can_run_list) != 1
@@ -4377,29 +4387,35 @@ class Scheduler(
if self.enable_hierarchical_cache or self.enable_unified_cache_external_linker:
# todo (zhiqiang): disable cuda graph execution if hicache loading triggered
new_batch.hicache_consumer_index = (
self.tree_cache.ready_to_load_host_cache()
)
with batch_telemetry.detail_phase("ready_to_load_host_cache"):
new_batch.hicache_consumer_index = (
self.tree_cache.ready_to_load_host_cache()
)
new_batch.prepare_for_extend()
with batch_telemetry.detail_phase("prepare_for_extend"):
new_batch.prepare_for_extend()
# Snapshot the prefill-only shape before mixed-chunk folding:
# mix_with_running() appends decode rows to extend_num_tokens/prefix_lens,
# so reading them after the branch would mix two different scopes.
_prefill_n_new = int(getattr(new_batch, "extend_num_tokens", 0) or 0)
_prefill_n_cached = int(sum(getattr(new_batch, "prefix_lens", ()) or ()))
if self.tp_worker.model_runner.prefill_aware_swa:
for req in can_run_list:
req.kv.swa_evict_floor = req.extend_end
# Record prefill stats for logging after forward.
new_batch.prefill_stats = PrefillStats.from_adder(
adder,
running_batch.reqs,
self.enable_priority_scheduling,
num_pending_tokens=self.load_inquirer._get_num_pending_tokens(
chunk_deduct=(
self.chunked_req.extend_len if self.chunked_req is not None else 0
),
),
)
with batch_telemetry.detail_phase("prefill_stats"):
new_batch.prefill_stats = PrefillStats.from_adder(
adder,
running_batch.reqs,
self.enable_priority_scheduling,
num_pending_tokens=self._prefill_pending_tokens(),
)
# Mixed-style chunked prefill
_mixed_t0 = None
_mixed_cpu0 = None
if (
self.is_mixed_chunk
and not running_batch.is_empty()
@@ -4409,6 +4425,9 @@ class Scheduler(
# Beam member rows are not supported inside a mixed extend batch.
and all(r.beam_group is None for r in running_batch.reqs)
):
if batch_telemetry.loop_detail_enabled():
_mixed_t0 = time.monotonic_ns()
_mixed_cpu0 = batch_telemetry.thread_cpu_ns()
# TODO (lianmin): support return_logprob + mixed chunked prefill
finishing_reqs = self._reqs_finishing_inflight(running_batch)
running_batch.filter_batch(reqs_to_exclude=finishing_reqs)
@@ -4440,10 +4459,69 @@ class Scheduler(
batch_telemetry.set_admitted(len(can_run_list))
batch_telemetry.note_new_chunk(adder.new_chunked_req is not None)
if _mixed_t0 is not None and fwd_timeline is not None:
fwd_timeline.detail(
"mixed_chunk_tail",
_mixed_t0,
time.monotonic_ns(),
thread_cpu_ns=self._detail_cpu_delta(_mixed_cpu0),
)
if batch_telemetry.loop_detail_enabled():
# Prefill-only scope for the three base counters, even when the
# batch was folded with running decode rows afterwards.
batch_telemetry.detail_attr("prefill_batch_n_reqs", len(can_run_list))
batch_telemetry.detail_attr("prefill_batch_n_new", _prefill_n_new)
batch_telemetry.detail_attr("prefill_batch_n_cached", _prefill_n_cached)
batch_telemetry.detail_attr(
"prefill_batch_chunked", bool(self.chunked_req is not None)
)
batch_telemetry.detail_attr("prefill_batch_resumed_chunked", bool(_had_chunked))
batch_telemetry.detail_attr(
"prefill_batch_new_chunked", bool(adder.new_chunked_req is not None)
)
batch_telemetry.detail_attr(
"prefill_batch_mixed_chunk", bool(_mixed_t0 is not None)
)
if _mixed_t0 is not None:
# Mixed batches carry a second scope; keep it explicit instead
# of folding decode rows into the prefill counters above.
_decode_rows = len(getattr(new_batch, "decoding_reqs", ()) or ())
batch_telemetry.detail_attr(
"prefill_batch_decode_reqs", _decode_rows
)
batch_telemetry.detail_attr(
"prefill_batch_merged_n_reqs", len(can_run_list) + _decode_rows
)
batch_telemetry.detail_attr(
"prefill_batch_merged_n_new",
int(getattr(new_batch, "extend_num_tokens", 0) or 0),
)
if fwd_timeline is not None:
fwd_timeline.detail("prefill_batch_init", _pf_t3, time.monotonic_ns())
fwd_timeline.detail(
"prefill_batch_init",
_pf_t3,
time.monotonic_ns(),
thread_cpu_ns=self._detail_cpu_delta(_pf_cpu_t3),
)
return new_batch, running_batch
@staticmethod
def _detail_cpu_delta(cpu_start_ns):
"""Thread CPU delta for detail spans; None when the clock is missing."""
if cpu_start_ns is None:
return None
now = batch_telemetry.thread_cpu_ns()
return None if now is None else now - cpu_start_ns
def _prefill_pending_tokens(self) -> int:
"""Pending-token evaluation for PrefillStats, timed as its own phase."""
with batch_telemetry.detail_phase("pending_tokens"):
return self.load_inquirer._get_num_pending_tokens(
chunk_deduct=(
self.chunked_req.extend_len if self.chunked_req is not None else 0
),
)
def can_schedule_lora_req(
self, req: Req, running_loras: set[Optional[str]]
) -> bool:
+7 -3
View File
@@ -24,6 +24,7 @@ from sglang.srt.mem_cache.common import (
evict_from_tree_cache,
)
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import attention_backends, get_parallel
from sglang.srt.utils import (
is_cpu,
@@ -318,8 +319,10 @@ def ensure_mamba_capacity(
supports_mamba = tree_cache is not None and tree_cache.supports_mamba()
available = allocator.schedulable_available_size()
if available < needed and supports_mamba:
tree_cache.evict_for_alloc(
EvictParams(num_tokens=0, mamba_num=needed - available)
from sglang.srt.mem_cache.common import evict_for_alloc_observed
evict_for_alloc_observed(
tree_cache, EvictParams(num_tokens=0, mamba_num=needed - available)
)
return allocator.schedulable_available_size() >= needed
@@ -384,7 +387,8 @@ def alloc_for_extend(
# free out-of-window swa tokens
batch.maybe_evict_swa()
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
with batch_telemetry.detail_phase("prefix_device_indices"):
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
reuse_kv = None
if batch.is_dllm():
@@ -165,7 +165,11 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
return
shortfall = num_tokens - self.available_size()
if shortfall > 0:
tree_cache.evict_for_alloc(EvictParams(num_tokens=shortfall))
from sglang.srt.mem_cache.common import evict_for_alloc_observed
evict_for_alloc_observed(
tree_cache, EvictParams(num_tokens=shortfall)
)
_evict_until_allocatable(tree_cache, self, num_tokens)
def check_decode_capacity(
+9 -4
View File
@@ -189,10 +189,12 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
"""
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
from sglang.srt.mem_cache.common import evict_for_alloc_observed
available = self.swa_available_size()
if available < swa_tokens:
tree_cache.evict_for_alloc(
EvictParams(swa_num_tokens=swa_tokens - available)
evict_for_alloc_observed(
tree_cache, EvictParams(swa_num_tokens=swa_tokens - available)
)
available = self.swa_available_size()
if available < swa_tokens:
@@ -215,8 +217,11 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
full_shortfall = max(0, num_tokens - self.full_available_size())
swa_shortfall = max(0, num_tokens - self.swa_available_size())
if full_shortfall or swa_shortfall:
tree_cache.evict_for_alloc(
EvictParams(num_tokens=full_shortfall, swa_num_tokens=swa_shortfall)
from sglang.srt.mem_cache.common import evict_for_alloc_observed
evict_for_alloc_observed(
tree_cache,
EvictParams(num_tokens=full_shortfall, swa_num_tokens=swa_shortfall),
)
def full_available_size(self):
@@ -1254,7 +1254,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
return
full_reclaim, swa_reclaim = reclaim_plan
if full_reclaim or swa_reclaim:
tree_cache.evict_for_alloc(
from sglang.srt.mem_cache.common import evict_for_alloc_observed
evict_for_alloc_observed(
tree_cache,
EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim),
allocation_reclaim_satisfied=lambda: (
self.reclaim_plan(num_tokens, required_swa) == (0, 0)
@@ -1909,7 +1912,10 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
# precedes state reclaim so Mamba-first tree ordering cannot skip it.
state_freed = 0
if full_reclaim or swa_reclaim:
result = tree_cache.evict_for_alloc(
from sglang.srt.mem_cache.common import evict_for_alloc_observed
result = evict_for_alloc_observed(
tree_cache,
EvictParams(num_tokens=num_tokens, swa_num_tokens=num_tokens),
allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied(
num_tokens
@@ -1930,7 +1936,10 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
)
state_quota = max(0, -(-pair_bytes // ma.entry_bytes) - state_freed)
if state_quota and tree_cache.mamba_evictable_size():
result = tree_cache.evict_for_alloc(
from sglang.srt.mem_cache.common import evict_for_alloc_observed
result = evict_for_alloc_observed(
tree_cache,
EvictParams(mamba_num=state_quota),
allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied(
num_tokens
+77 -3
View File
@@ -24,6 +24,7 @@ from sglang.srt.mem_cache.hicache_storage import PoolTransfer
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import get_serving, get_spec
from sglang.srt.utils.common import ceil_align
@@ -315,13 +316,86 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
req.prefix_len = req.extend_end
def _rank_consensus_enabled() -> bool:
"""Whether the rank-consensus checker is active (cheap, best effort)."""
try:
from sglang.srt.utils import rank_consensus_checker
return bool(rank_consensus_checker.enabled())
except Exception:
return False
def evict_for_alloc_observed(tree_cache, params, **kwargs):
"""Time ``tree_cache.evict_for_alloc`` and record what it realized.
Kept at the call site (not inside the callee) so decorator/consensus waits
are part of the measurement. ``kwargs`` forwards the unified allocators'
``allocation_reclaim_satisfied`` predicate.
"""
with batch_telemetry.detail_phase("evict_for_alloc"):
result = tree_cache.evict_for_alloc(params, **kwargs)
if batch_telemetry.loop_detail_enabled() and result is not None:
batch_telemetry.detail_count("evict_for_alloc_calls")
batch_telemetry.detail_attr_sum(
"evict_plan_requested_tokens", int(getattr(params, "num_tokens", 0) or 0)
)
batch_telemetry.detail_attr_sum(
"evict_plan_requested_swa_tokens",
int(getattr(params, "swa_num_tokens", 0) or 0),
)
batch_telemetry.detail_attr_sum(
"evict_plan_requested_mamba",
int(getattr(params, "mamba_num", 0) or 0),
)
batch_telemetry.detail_attr_sum(
"evict_evicted_tokens",
int(getattr(result, "num_tokens_evicted", 0) or 0),
)
batch_telemetry.detail_attr_sum(
"evict_evicted_swa_tokens",
int(getattr(result, "swa_num_tokens_evicted", 0) or 0),
)
# Record the realized Mamba count unconditionally (including zero) so
# "no Mamba demand" and "demand but nothing reclaimed" stay distinct.
batch_telemetry.detail_attr_sum(
"evict_evicted_mamba",
int(getattr(result, "mamba_num_evicted", 0) or 0),
)
return result
def evict_from_tree_cache(
tree_cache: BasePrefixCache | None, num_tokens: int
) -> bool | None:
if tree_cache is not None and tree_cache.supports_prefix_sharing():
return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens(
tree_cache, num_tokens
)
allocator = tree_cache.token_to_kv_pool_allocator
with batch_telemetry.detail_phase("evict_to_free_tokens"):
if batch_telemetry.loop_detail_enabled():
available_before = allocator.available_size()
batch_telemetry.detail_count("evict_calls")
# Demand is what the batch wants allocatable; shortfall is the
# part that eviction must actually recover. Keep them apart:
# summing demand as "requested eviction" overstates eviction.
batch_telemetry.detail_attr_sum(
"evict_demand_tokens", int(num_tokens)
)
batch_telemetry.detail_attr_sum(
"evict_shortfall_tokens",
max(0, int(num_tokens) - int(available_before)),
)
batch_telemetry.detail_attr_setdefault(
"evict_free_tokens_before", int(available_before)
)
batch_telemetry.detail_attr(
"evict_rank_consensus_checker", _rank_consensus_enabled()
)
result = allocator.evict_to_free_tokens(tree_cache, num_tokens)
if batch_telemetry.loop_detail_enabled():
batch_telemetry.detail_attr(
"evict_free_tokens_after", int(allocator.available_size())
)
return result
def _evict_until_allocatable(