observability: prefill_batch_init wall+CPU, sub-phases and eviction coverage
Parent span scheduler.loop.prefill_batch_init records wall + the scheduler thread's CPU at both existing close points (empty admission and normal return); the CPU clock is only read when loop detail is enabled and sampled, through the safe batch_telemetry.thread_cpu_ns() wrapper so a missing clock degrades to 'not measured' instead of failing the scheduling path. Fixed children, in call order: schedule_batch_init, ready_to_load_host_cache, prepare_for_extend, alloc_for_extend (+ prefix_device_indices attribute-only, sampling_batch_info), evict_to_free_tokens, evict_for_alloc, prefill_stats (+ pending_tokens), mixed_chunk_tail. * prefill_stats wraps the whole assignment so _get_num_pending_tokens() is included; mixed_chunk_tail only exists when that branch is taken. * prefill-only shape counters (n_reqs/n_new/n_cached) are snapshotted before mixed-chunk folding; a mixed batch additionally publishes decode_reqs and merged totals under explicit *_merged_* names. * eviction is timed at the call site (outside the allocator wrapper) so rank-consensus waits are included, and covers base, SWA (x2), UnifiedSWA (x3) and the extend Mamba reclaim through evict_for_alloc_observed(). Demand, shortfall and plan are separate counters; Mamba plan and realized counts are both recorded (including zero). All of it is gated by detail on a sampled loop.
This commit is contained in:
@@ -120,6 +120,7 @@ from sglang.srt.mem_cache.common import (
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.model_executor.encoder_swa_replay import resets_swa_window
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -2915,9 +2916,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
self.encoder_swa_reset = [resets_swa_window(r) for r in reqs]
|
||||
# Allocate memory
|
||||
out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend(
|
||||
self
|
||||
)
|
||||
with batch_telemetry.detail_phase("alloc_for_extend"):
|
||||
(
|
||||
out_cache_loc,
|
||||
req_pool_indices_tensor,
|
||||
req_pool_indices_cpu,
|
||||
) = alloc_for_extend(self)
|
||||
|
||||
# Set fields
|
||||
input_embeds = []
|
||||
@@ -3133,10 +3137,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
self.prepare_encoder_info_extend(input_ids, seq_lens)
|
||||
|
||||
# Build sampling info
|
||||
self.sampling_info = SamplingBatchInfo.from_schedule_batch(
|
||||
self,
|
||||
self.model_config.vocab_size,
|
||||
)
|
||||
with batch_telemetry.detail_phase("sampling_batch_info"):
|
||||
self.sampling_info = SamplingBatchInfo.from_schedule_batch(
|
||||
self,
|
||||
self.model_config.vocab_size,
|
||||
)
|
||||
|
||||
def _mamba_radix_cache_v2_req_prepare_for_extend(
|
||||
self,
|
||||
|
||||
@@ -4323,6 +4323,12 @@ class Scheduler(
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("prefill_scan", _pf_t2, time.monotonic_ns())
|
||||
_pf_t3 = time.monotonic_ns()
|
||||
_pf_cpu_t3 = (
|
||||
batch_telemetry.thread_cpu_ns()
|
||||
if batch_telemetry.loop_detail_enabled()
|
||||
else None
|
||||
)
|
||||
_had_chunked = self.chunked_req is not None
|
||||
|
||||
# Update waiting queue
|
||||
can_run_list: List[Req] = adder.can_run_list
|
||||
@@ -4334,7 +4340,10 @@ class Scheduler(
|
||||
batch_telemetry.set_admitted(0)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail(
|
||||
"prefill_batch_init", _pf_t3, time.monotonic_ns()
|
||||
"prefill_batch_init",
|
||||
_pf_t3,
|
||||
time.monotonic_ns(),
|
||||
thread_cpu_ns=self._detail_cpu_delta(_pf_cpu_t3),
|
||||
)
|
||||
return None, running_batch
|
||||
|
||||
@@ -4359,16 +4368,17 @@ class Scheduler(
|
||||
set_time_batch(can_run_list, "set_forward_entry_time")
|
||||
|
||||
# Create a new batch
|
||||
new_batch = ScheduleBatch.init_new(
|
||||
can_run_list,
|
||||
self.req_to_token_pool,
|
||||
self.token_to_kv_pool_allocator,
|
||||
self.tree_cache,
|
||||
self.model_config,
|
||||
self.enable_overlap,
|
||||
self.spec_algorithm,
|
||||
chunked_req=self.chunked_req,
|
||||
)
|
||||
with batch_telemetry.detail_phase("schedule_batch_init"):
|
||||
new_batch = ScheduleBatch.init_new(
|
||||
can_run_list,
|
||||
self.req_to_token_pool,
|
||||
self.token_to_kv_pool_allocator,
|
||||
self.tree_cache,
|
||||
self.model_config,
|
||||
self.enable_overlap,
|
||||
self.spec_algorithm,
|
||||
chunked_req=self.chunked_req,
|
||||
)
|
||||
|
||||
new_batch.contains_last_prefill_chunk = (
|
||||
self.chunked_req is None or len(can_run_list) != 1
|
||||
@@ -4377,29 +4387,35 @@ class Scheduler(
|
||||
|
||||
if self.enable_hierarchical_cache or self.enable_unified_cache_external_linker:
|
||||
# todo (zhiqiang): disable cuda graph execution if hicache loading triggered
|
||||
new_batch.hicache_consumer_index = (
|
||||
self.tree_cache.ready_to_load_host_cache()
|
||||
)
|
||||
with batch_telemetry.detail_phase("ready_to_load_host_cache"):
|
||||
new_batch.hicache_consumer_index = (
|
||||
self.tree_cache.ready_to_load_host_cache()
|
||||
)
|
||||
|
||||
new_batch.prepare_for_extend()
|
||||
with batch_telemetry.detail_phase("prepare_for_extend"):
|
||||
new_batch.prepare_for_extend()
|
||||
# Snapshot the prefill-only shape before mixed-chunk folding:
|
||||
# mix_with_running() appends decode rows to extend_num_tokens/prefix_lens,
|
||||
# so reading them after the branch would mix two different scopes.
|
||||
_prefill_n_new = int(getattr(new_batch, "extend_num_tokens", 0) or 0)
|
||||
_prefill_n_cached = int(sum(getattr(new_batch, "prefix_lens", ()) or ()))
|
||||
|
||||
if self.tp_worker.model_runner.prefill_aware_swa:
|
||||
for req in can_run_list:
|
||||
req.kv.swa_evict_floor = req.extend_end
|
||||
|
||||
# Record prefill stats for logging after forward.
|
||||
new_batch.prefill_stats = PrefillStats.from_adder(
|
||||
adder,
|
||||
running_batch.reqs,
|
||||
self.enable_priority_scheduling,
|
||||
num_pending_tokens=self.load_inquirer._get_num_pending_tokens(
|
||||
chunk_deduct=(
|
||||
self.chunked_req.extend_len if self.chunked_req is not None else 0
|
||||
),
|
||||
),
|
||||
)
|
||||
with batch_telemetry.detail_phase("prefill_stats"):
|
||||
new_batch.prefill_stats = PrefillStats.from_adder(
|
||||
adder,
|
||||
running_batch.reqs,
|
||||
self.enable_priority_scheduling,
|
||||
num_pending_tokens=self._prefill_pending_tokens(),
|
||||
)
|
||||
|
||||
# Mixed-style chunked prefill
|
||||
_mixed_t0 = None
|
||||
_mixed_cpu0 = None
|
||||
if (
|
||||
self.is_mixed_chunk
|
||||
and not running_batch.is_empty()
|
||||
@@ -4409,6 +4425,9 @@ class Scheduler(
|
||||
# Beam member rows are not supported inside a mixed extend batch.
|
||||
and all(r.beam_group is None for r in running_batch.reqs)
|
||||
):
|
||||
if batch_telemetry.loop_detail_enabled():
|
||||
_mixed_t0 = time.monotonic_ns()
|
||||
_mixed_cpu0 = batch_telemetry.thread_cpu_ns()
|
||||
# TODO (lianmin): support return_logprob + mixed chunked prefill
|
||||
finishing_reqs = self._reqs_finishing_inflight(running_batch)
|
||||
running_batch.filter_batch(reqs_to_exclude=finishing_reqs)
|
||||
@@ -4440,10 +4459,69 @@ class Scheduler(
|
||||
|
||||
batch_telemetry.set_admitted(len(can_run_list))
|
||||
batch_telemetry.note_new_chunk(adder.new_chunked_req is not None)
|
||||
if _mixed_t0 is not None and fwd_timeline is not None:
|
||||
fwd_timeline.detail(
|
||||
"mixed_chunk_tail",
|
||||
_mixed_t0,
|
||||
time.monotonic_ns(),
|
||||
thread_cpu_ns=self._detail_cpu_delta(_mixed_cpu0),
|
||||
)
|
||||
if batch_telemetry.loop_detail_enabled():
|
||||
# Prefill-only scope for the three base counters, even when the
|
||||
# batch was folded with running decode rows afterwards.
|
||||
batch_telemetry.detail_attr("prefill_batch_n_reqs", len(can_run_list))
|
||||
batch_telemetry.detail_attr("prefill_batch_n_new", _prefill_n_new)
|
||||
batch_telemetry.detail_attr("prefill_batch_n_cached", _prefill_n_cached)
|
||||
batch_telemetry.detail_attr(
|
||||
"prefill_batch_chunked", bool(self.chunked_req is not None)
|
||||
)
|
||||
batch_telemetry.detail_attr("prefill_batch_resumed_chunked", bool(_had_chunked))
|
||||
batch_telemetry.detail_attr(
|
||||
"prefill_batch_new_chunked", bool(adder.new_chunked_req is not None)
|
||||
)
|
||||
batch_telemetry.detail_attr(
|
||||
"prefill_batch_mixed_chunk", bool(_mixed_t0 is not None)
|
||||
)
|
||||
if _mixed_t0 is not None:
|
||||
# Mixed batches carry a second scope; keep it explicit instead
|
||||
# of folding decode rows into the prefill counters above.
|
||||
_decode_rows = len(getattr(new_batch, "decoding_reqs", ()) or ())
|
||||
batch_telemetry.detail_attr(
|
||||
"prefill_batch_decode_reqs", _decode_rows
|
||||
)
|
||||
batch_telemetry.detail_attr(
|
||||
"prefill_batch_merged_n_reqs", len(can_run_list) + _decode_rows
|
||||
)
|
||||
batch_telemetry.detail_attr(
|
||||
"prefill_batch_merged_n_new",
|
||||
int(getattr(new_batch, "extend_num_tokens", 0) or 0),
|
||||
)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("prefill_batch_init", _pf_t3, time.monotonic_ns())
|
||||
fwd_timeline.detail(
|
||||
"prefill_batch_init",
|
||||
_pf_t3,
|
||||
time.monotonic_ns(),
|
||||
thread_cpu_ns=self._detail_cpu_delta(_pf_cpu_t3),
|
||||
)
|
||||
return new_batch, running_batch
|
||||
|
||||
@staticmethod
|
||||
def _detail_cpu_delta(cpu_start_ns):
|
||||
"""Thread CPU delta for detail spans; None when the clock is missing."""
|
||||
if cpu_start_ns is None:
|
||||
return None
|
||||
now = batch_telemetry.thread_cpu_ns()
|
||||
return None if now is None else now - cpu_start_ns
|
||||
|
||||
def _prefill_pending_tokens(self) -> int:
|
||||
"""Pending-token evaluation for PrefillStats, timed as its own phase."""
|
||||
with batch_telemetry.detail_phase("pending_tokens"):
|
||||
return self.load_inquirer._get_num_pending_tokens(
|
||||
chunk_deduct=(
|
||||
self.chunked_req.extend_len if self.chunked_req is not None else 0
|
||||
),
|
||||
)
|
||||
|
||||
def can_schedule_lora_req(
|
||||
self, req: Req, running_loras: set[Optional[str]]
|
||||
) -> bool:
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.srt.mem_cache.common import (
|
||||
evict_from_tree_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.runtime_context import attention_backends, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
is_cpu,
|
||||
@@ -318,8 +319,10 @@ def ensure_mamba_capacity(
|
||||
supports_mamba = tree_cache is not None and tree_cache.supports_mamba()
|
||||
available = allocator.schedulable_available_size()
|
||||
if available < needed and supports_mamba:
|
||||
tree_cache.evict_for_alloc(
|
||||
EvictParams(num_tokens=0, mamba_num=needed - available)
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
evict_for_alloc_observed(
|
||||
tree_cache, EvictParams(num_tokens=0, mamba_num=needed - available)
|
||||
)
|
||||
return allocator.schedulable_available_size() >= needed
|
||||
|
||||
@@ -384,7 +387,8 @@ def alloc_for_extend(
|
||||
# free out-of-window swa tokens
|
||||
batch.maybe_evict_swa()
|
||||
|
||||
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
|
||||
with batch_telemetry.detail_phase("prefix_device_indices"):
|
||||
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
|
||||
|
||||
reuse_kv = None
|
||||
if batch.is_dllm():
|
||||
|
||||
@@ -165,7 +165,11 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
|
||||
return
|
||||
shortfall = num_tokens - self.available_size()
|
||||
if shortfall > 0:
|
||||
tree_cache.evict_for_alloc(EvictParams(num_tokens=shortfall))
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
evict_for_alloc_observed(
|
||||
tree_cache, EvictParams(num_tokens=shortfall)
|
||||
)
|
||||
_evict_until_allocatable(tree_cache, self, num_tokens)
|
||||
|
||||
def check_decode_capacity(
|
||||
|
||||
@@ -189,10 +189,12 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
"""
|
||||
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
|
||||
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
available = self.swa_available_size()
|
||||
if available < swa_tokens:
|
||||
tree_cache.evict_for_alloc(
|
||||
EvictParams(swa_num_tokens=swa_tokens - available)
|
||||
evict_for_alloc_observed(
|
||||
tree_cache, EvictParams(swa_num_tokens=swa_tokens - available)
|
||||
)
|
||||
available = self.swa_available_size()
|
||||
if available < swa_tokens:
|
||||
@@ -215,8 +217,11 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
full_shortfall = max(0, num_tokens - self.full_available_size())
|
||||
swa_shortfall = max(0, num_tokens - self.swa_available_size())
|
||||
if full_shortfall or swa_shortfall:
|
||||
tree_cache.evict_for_alloc(
|
||||
EvictParams(num_tokens=full_shortfall, swa_num_tokens=swa_shortfall)
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
evict_for_alloc_observed(
|
||||
tree_cache,
|
||||
EvictParams(num_tokens=full_shortfall, swa_num_tokens=swa_shortfall),
|
||||
)
|
||||
|
||||
def full_available_size(self):
|
||||
|
||||
@@ -1254,7 +1254,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
return
|
||||
full_reclaim, swa_reclaim = reclaim_plan
|
||||
if full_reclaim or swa_reclaim:
|
||||
tree_cache.evict_for_alloc(
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
evict_for_alloc_observed(
|
||||
tree_cache,
|
||||
EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim),
|
||||
allocation_reclaim_satisfied=lambda: (
|
||||
self.reclaim_plan(num_tokens, required_swa) == (0, 0)
|
||||
@@ -1909,7 +1912,10 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
# precedes state reclaim so Mamba-first tree ordering cannot skip it.
|
||||
state_freed = 0
|
||||
if full_reclaim or swa_reclaim:
|
||||
result = tree_cache.evict_for_alloc(
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
result = evict_for_alloc_observed(
|
||||
tree_cache,
|
||||
EvictParams(num_tokens=num_tokens, swa_num_tokens=num_tokens),
|
||||
allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied(
|
||||
num_tokens
|
||||
@@ -1930,7 +1936,10 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
)
|
||||
state_quota = max(0, -(-pair_bytes // ma.entry_bytes) - state_freed)
|
||||
if state_quota and tree_cache.mamba_evictable_size():
|
||||
result = tree_cache.evict_for_alloc(
|
||||
from sglang.srt.mem_cache.common import evict_for_alloc_observed
|
||||
|
||||
result = evict_for_alloc_observed(
|
||||
tree_cache,
|
||||
EvictParams(mamba_num=state_quota),
|
||||
allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied(
|
||||
num_tokens
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.srt.mem_cache.hicache_storage import PoolTransfer
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.runtime_context import get_serving, get_spec
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
@@ -315,13 +316,86 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
req.prefix_len = req.extend_end
|
||||
|
||||
|
||||
def _rank_consensus_enabled() -> bool:
|
||||
"""Whether the rank-consensus checker is active (cheap, best effort)."""
|
||||
try:
|
||||
from sglang.srt.utils import rank_consensus_checker
|
||||
|
||||
return bool(rank_consensus_checker.enabled())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def evict_for_alloc_observed(tree_cache, params, **kwargs):
|
||||
"""Time ``tree_cache.evict_for_alloc`` and record what it realized.
|
||||
|
||||
Kept at the call site (not inside the callee) so decorator/consensus waits
|
||||
are part of the measurement. ``kwargs`` forwards the unified allocators'
|
||||
``allocation_reclaim_satisfied`` predicate.
|
||||
"""
|
||||
with batch_telemetry.detail_phase("evict_for_alloc"):
|
||||
result = tree_cache.evict_for_alloc(params, **kwargs)
|
||||
if batch_telemetry.loop_detail_enabled() and result is not None:
|
||||
batch_telemetry.detail_count("evict_for_alloc_calls")
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_plan_requested_tokens", int(getattr(params, "num_tokens", 0) or 0)
|
||||
)
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_plan_requested_swa_tokens",
|
||||
int(getattr(params, "swa_num_tokens", 0) or 0),
|
||||
)
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_plan_requested_mamba",
|
||||
int(getattr(params, "mamba_num", 0) or 0),
|
||||
)
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_evicted_tokens",
|
||||
int(getattr(result, "num_tokens_evicted", 0) or 0),
|
||||
)
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_evicted_swa_tokens",
|
||||
int(getattr(result, "swa_num_tokens_evicted", 0) or 0),
|
||||
)
|
||||
# Record the realized Mamba count unconditionally (including zero) so
|
||||
# "no Mamba demand" and "demand but nothing reclaimed" stay distinct.
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_evicted_mamba",
|
||||
int(getattr(result, "mamba_num_evicted", 0) or 0),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def evict_from_tree_cache(
|
||||
tree_cache: BasePrefixCache | None, num_tokens: int
|
||||
) -> bool | None:
|
||||
if tree_cache is not None and tree_cache.supports_prefix_sharing():
|
||||
return tree_cache.token_to_kv_pool_allocator.evict_to_free_tokens(
|
||||
tree_cache, num_tokens
|
||||
)
|
||||
allocator = tree_cache.token_to_kv_pool_allocator
|
||||
with batch_telemetry.detail_phase("evict_to_free_tokens"):
|
||||
if batch_telemetry.loop_detail_enabled():
|
||||
available_before = allocator.available_size()
|
||||
batch_telemetry.detail_count("evict_calls")
|
||||
# Demand is what the batch wants allocatable; shortfall is the
|
||||
# part that eviction must actually recover. Keep them apart:
|
||||
# summing demand as "requested eviction" overstates eviction.
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_demand_tokens", int(num_tokens)
|
||||
)
|
||||
batch_telemetry.detail_attr_sum(
|
||||
"evict_shortfall_tokens",
|
||||
max(0, int(num_tokens) - int(available_before)),
|
||||
)
|
||||
batch_telemetry.detail_attr_setdefault(
|
||||
"evict_free_tokens_before", int(available_before)
|
||||
)
|
||||
batch_telemetry.detail_attr(
|
||||
"evict_rank_consensus_checker", _rank_consensus_enabled()
|
||||
)
|
||||
result = allocator.evict_to_free_tokens(tree_cache, num_tokens)
|
||||
if batch_telemetry.loop_detail_enabled():
|
||||
batch_telemetry.detail_attr(
|
||||
"evict_free_tokens_after", int(allocator.available_size())
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _evict_until_allocatable(
|
||||
|
||||
Reference in New Issue
Block a user