observability: split flush_pending_backups host chain (off by default)

Adds the phase-3/phase-4 instrumentation from
handoff/prefill-flush-backups-validation-20261107.md:

* process_batch_result outer phases: flush_trace, load_snapshot,
  flush_backups, step_counters, metrics_report. process_residual now
  subtracts them, so the 37.5% residual is attributable.
* SGLANG_TRACE_BACKUP_DETAIL=1 (default 0) traces one flush:
  merge_ops / move_write (anchor + per-sidecar indices) / indices_cpu /
  index_sort / index_select / write_fence / build_l2 / submit_d2h
  (start-event wait, prepare, per-pool launch, ack events) / ack_enqueue,
  plus token/op/pool/device dimensions and ack queue depth.
* Per-flush resolution: one scheduler.cache_flush_backup span per
  non-empty flush, child of the loop span. Sub-phases never enter the
  loop aggregate, so the residual identity cannot double count.
* Empty vs non-empty flushes are counted on the loop.

All timings are scheduler host wall, not device transfer time.
This commit is contained in:
2026-11-07 09:23:44 +08:00
parent 7b3f34cbc0
commit aa53670fcd
6 changed files with 652 additions and 66 deletions
+108 -34
View File
@@ -43,6 +43,7 @@ from sglang.srt.layers.dp_attention import (
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_device_module
@@ -890,40 +891,104 @@ class HiCacheController:
def start_writing(self) -> None:
if len(self.write_queue) == 0:
# Empty flushes are the common case when write-through fires every
# batch: counted so a small `flush_backups` cannot be explained as
# "one big flush per loop" without noticing the empty ones.
batch_telemetry.detail_count("flush_calls")
batch_telemetry.detail_count("flush_empty")
return
op = CacheOperation.merge_ops(self.write_queue)
host_indices, device_indices, pool_transfers = self._move_write_operation(op)
self.write_queue.clear()
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
tracer = batch_telemetry.backup_flush_begin()
try:
batch_telemetry.detail_count("flush_calls")
batch_telemetry.detail_count("flush_nonempty")
batch_telemetry.flush_note(
"sglang.flush_write_queue_ops", len(self.write_queue)
)
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
batch_telemetry.flush_note(
"sglang.flush_layout", getattr(self.mem_pool_host, "layout", None)
)
batch_telemetry.flush_note(
"sglang.flush_ack_queue_before", len(self.ack_write_queue)
)
completion = self.l2_transfer_engine.submit_device_to_host(
self._l2_transfers(host_indices, device_indices, pool_transfers)
)
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
self.ack_write_queue.append(
HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
with batch_telemetry.backup_phase("merge_ops"):
op = CacheOperation.merge_ops(self.write_queue)
tokens = len(op.device_indices)
indices = op.device_indices
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
batch_telemetry.flush_note(
"sglang.flush_nodes_merged", len(op.node_ids)
)
)
batch_telemetry.flush_note(
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
)
batch_telemetry.flush_note(
"sglang.flush_indices_device", str(indices.device.type)
)
batch_telemetry.flush_note(
"sglang.flush_indices_dtype", str(indices.dtype)
)
batch_telemetry.flush_note(
"sglang.flush_indices_numel", int(indices.numel())
)
with batch_telemetry.backup_phase("move_write"):
host_indices, device_indices, pool_transfers = (
self._move_write_operation(op)
)
self.write_queue.clear()
batch_telemetry.flush_note(
"sglang.flush_host_indices_on_cpu",
bool(getattr(host_indices, "is_cuda", False) is False),
)
batch_telemetry.flush_note(
"sglang.flush_device_indices_on_cpu",
bool(getattr(device_indices, "is_cuda", False) is False),
)
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
with batch_telemetry.backup_phase("write_fence"):
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
with batch_telemetry.backup_phase("build_l2"):
transfers = self._l2_transfers(
host_indices, device_indices, pool_transfers
)
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
with batch_telemetry.backup_phase("submit_d2h"):
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
with batch_telemetry.backup_phase("set_done_event"):
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
with batch_telemetry.backup_phase("ack_enqueue"):
self.ack_write_queue.append(
HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
)
batch_telemetry.flush_note(
"sglang.flush_ack_queue_after", len(self.ack_write_queue)
)
finally:
batch_telemetry.backup_flush_end(tracer)
def _transfer_num_bytes(self, op: CacheOperation) -> int:
return len(op.device_indices) * self.mem_pool_host.size_per_token
@@ -954,15 +1019,24 @@ class HiCacheController:
# move indices to GPU if using kernels, to host if using direct indexing
if self.io_backend == "kernel":
if not host_indices.is_cuda:
host_indices = host_indices.to(self.device, non_blocking=True)
with batch_telemetry.backup_phase("indices_h2d"):
host_indices = host_indices.to(self.device, non_blocking=True)
return host_indices, device_indices
elif self.io_backend == "direct":
if self.mem_pool_host.layout == "layer_first":
device_indices = device_indices.cpu()
host_indices, idx = host_indices.sort()
return host_indices, device_indices.index_select(0, idx)
with batch_telemetry.backup_phase("indices_cpu"):
device_indices = device_indices.cpu()
with batch_telemetry.backup_phase("index_sort"):
host_indices, idx = host_indices.sort()
with batch_telemetry.backup_phase("index_select"):
return host_indices, device_indices.index_select(0, idx)
elif self.mem_pool_host.layout == "page_first_direct":
return host_indices, device_indices.cpu()
# Synchronous D2H of the token indices: the host wall time of
# this call is the scheduler blocking on the copy, not a
# device bandwidth measurement.
with batch_telemetry.backup_phase("indices_cpu"):
device_indices = device_indices.cpu()
return host_indices, device_indices
else:
raise ValueError(
f"Unsupported layout {self.mem_pool_host.layout!r} for io backend 'direct'"
+31 -18
View File
@@ -5079,15 +5079,19 @@ class Scheduler(
):
# Flush async trace ops here: in overlap mode this CPU work runs while
# the next batch's GPU forward is in flight, giving free overlap.
flush_trace_batch(batch.reqs)
snapshot = self.publish_load_snapshot(force=batch.forward_mode.is_extend())
# Router-facing gauge on the dedicated PUB socket, reusing the
# snapshot above rather than walking the queues again.
self.load_publisher.publish_load_stat(
self.load_inquirer.get_loads,
force=batch.forward_mode.is_extend(),
snapshot=snapshot,
)
with batch_telemetry.detail_phase("flush_trace"):
flush_trace_batch(batch.reqs)
with batch_telemetry.detail_phase("load_snapshot"):
snapshot = self.publish_load_snapshot(
force=batch.forward_mode.is_extend()
)
# Router-facing gauge on the dedicated PUB socket, reusing the
# snapshot above rather than walking the queues again.
self.load_publisher.publish_load_stat(
self.load_inquirer.get_loads,
force=batch.forward_mode.is_extend(),
snapshot=snapshot,
)
if batch.forward_mode.is_decode():
self.batch_result_processor.process_batch_result_decode(batch, result)
@@ -5104,19 +5108,28 @@ class Scheduler(
self.batch_result_processor.process_batch_result_idle(batch, result)
# Submit this batch's queued host backups before the next scheduler step.
self.tree_cache.flush_pending_backups()
# Tag the origin so the per-flush micro timer can separate these from
# the ones issued by check_hicache_events / write().
_flush_origin = batch_telemetry.set_flush_origin("process_result")
try:
with batch_telemetry.detail_phase("flush_backups"):
self.tree_cache.flush_pending_backups()
finally:
batch_telemetry.set_flush_origin(_flush_origin)
self._record_step_counters(batch, result)
with batch_telemetry.detail_phase("step_counters"):
self._record_step_counters(batch, result)
self.metrics_reporter.log_batch_result_stats(batch, result)
with batch_telemetry.detail_phase("metrics_report"):
self.metrics_reporter.log_batch_result_stats(batch, result)
# Emit forward pass metrics (every iteration when enabled)
if self.enable_fpm:
self.metrics_reporter._emit_forward_pass_metrics(batch, result)
# Emit forward pass metrics (every iteration when enabled)
if self.enable_fpm:
self.metrics_reporter._emit_forward_pass_metrics(batch, result)
self._maybe_clear_mm_inputs(batch)
self.maybe_send_health_check_signal()
self.metrics_reporter.update_device_timer()
self._maybe_clear_mm_inputs(batch)
self.maybe_send_health_check_signal()
self.metrics_reporter.update_device_timer()
def _record_step_counters(
self, batch: ScheduleBatch, result: GenerationBatchResult
@@ -41,6 +41,7 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.observability import batch_telemetry
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -1509,26 +1510,30 @@ class HybridCacheController(BaseHiCacheController):
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
) -> tuple[torch.Tensor, torch.Tensor]:
if uses_shared_host_layout(host_pool):
batch_telemetry.flush_count("shared_layout_pools")
return host_indices, device_indices
if write_back_jit and host_pool.can_use_write_back_jit:
if host_indices.is_cuda:
host_indices = host_indices.cpu()
with batch_telemetry.backup_phase("host_indices_cpu"):
host_indices = host_indices.cpu()
return host_indices, device_indices
return self.move_indices(host_indices, device_indices)
def move_hybrid_indices(
self, operation: CacheOperation, *, write_back_jit: bool = False
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
host_indices, device_indices = self._move_pool_indices(
self.mem_pool_host.anchor_entry.host_pool,
operation.host_indices,
operation.device_indices,
write_back_jit=write_back_jit,
)
with batch_telemetry.backup_phase("anchor_indices"):
host_indices, device_indices = self._move_pool_indices(
self.mem_pool_host.anchor_entry.host_pool,
operation.host_indices,
operation.device_indices,
write_back_jit=write_back_jit,
)
resolved_pool_transfers = None
if operation.pool_transfers:
resolved_pool_transfers = []
for transfer in operation.pool_transfers:
_pool_t0 = batch_telemetry.flush_mark()
transfer_host_indices, transfer_device_indices = (
self._move_pool_indices(
self.mem_pool_host.entry_map[transfer.name].host_pool,
@@ -1537,6 +1542,9 @@ class HybridCacheController(BaseHiCacheController):
write_back_jit=write_back_jit,
)
)
batch_telemetry.flush_phase_since(
"sidecar_indices." + str(transfer.name.value), _pool_t0
)
# Tree-owned transfers keep their original radix-tree indices.
resolved_pool_transfers.append(
replace(
+28 -7
View File
@@ -9,6 +9,7 @@ import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
from sglang.srt.observability import batch_telemetry
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
@@ -71,10 +72,13 @@ class L2TransferEngine:
original_device = transfer.device_indices.device
# The direct backend supplies CPU indices even for a CUDA pool.
# Translate alongside the v2p table, then restore the backend's device.
_t0 = batch_telemetry.flush_mark()
indices = transfer.device_indices.to(
getattr(transfer.device_pool, "device", original_device)
).contiguous()
batch_telemetry.flush_phase_since("resolve_indices.to_device", _t0)
dcp_size = getattr(transfer.host_pool, "dcp_size", 1)
_t0 = batch_telemetry.flush_mark()
if dcp_size > 1:
# The MLA host pool selects this rank and collapses logical IDs.
# Translate in local virtual space, then preserve that widened
@@ -82,16 +86,22 @@ class L2TransferEngine:
resolved = translate(indices // dcp_size) * dcp_size + indices % dcp_size
else:
resolved = translate(indices)
return resolved.to(original_device)
batch_telemetry.flush_phase_since("resolve_indices.translate", _t0)
_t0 = batch_telemetry.flush_mark()
result = resolved.to(original_device)
batch_telemetry.flush_phase_since("resolve_indices.back_to_host", _t0)
return result
def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]:
prepared = []
for transfer in transfers:
_t0 = batch_telemetry.flush_mark()
host_indices, device_indices = transfer.host_pool.prepare_transfer_indices(
transfer.host_indices,
self._resolve_device_indices(transfer),
self.io_backend,
)
batch_telemetry.flush_phase_since("prepare_transfer_indices", _t0)
prepared.append(
transfer._replace(
host_indices=host_indices,
@@ -102,44 +112,55 @@ class L2TransferEngine:
@contextmanager
def _submission(self, transfers, stream, transfer_key, start_event=None):
_t0 = batch_telemetry.flush_mark()
start_event = self._start_event(start_event)
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
completion = TransferCompletion(ack_start, ack_finish, timing_enabled)
domains = shared_host_layout_domains(t.host_pool for t in transfers)
for domain in domains:
domain.acquire_layout()
batch_telemetry.flush_phase_since("submit_setup", _t0)
finish_recorded = False
try:
with device_module.stream(stream):
# Index preparation can read producer-owned device indices.
start_event.wait(stream)
prepared = self._prepare_transfers(transfers)
ack_start.record()
with batch_telemetry.backup_phase("start_event_wait"):
start_event.wait(stream)
with batch_telemetry.backup_phase("prepare_transfers"):
prepared = self._prepare_transfers(transfers)
with batch_telemetry.backup_phase("ack_events"):
ack_start.record()
yield prepared, completion
ack_finish.record()
finish_recorded = True
self._record_stream(transfers + prepared, stream)
with batch_telemetry.backup_phase("ack_events"):
ack_finish.record()
finish_recorded = True
self._record_stream(transfers + prepared, stream)
except Exception:
stream.synchronize()
raise
finally:
_t0 = batch_telemetry.flush_mark()
for domain in reversed(domains):
domain.release_layout(
ack_finish if finish_recorded else None,
(id(self), transfer_key),
)
batch_telemetry.flush_phase_since("layout_release", _t0)
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
with self._submission(
transfers, self.device_to_host_stream, "device_to_host"
) as (transfers, completion):
for transfer in transfers:
_t0 = batch_telemetry.flush_mark()
transfer.host_pool.backup_from_device_all_layer_physical(
transfer.device_pool,
transfer.host_indices,
transfer.device_indices,
self.io_backend,
)
batch_telemetry.flush_phase_since("launch_pool", _t0)
batch_telemetry.flush_count("launch_pools")
return completion
def submit_host_to_device(
@@ -26,6 +26,9 @@ Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
loops are recorded)
SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown
(queue/source/indices/transfer/join)
SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases
(one ``scheduler.cache_flush_backup`` span;
requires SGLANG_TRACE_LOOP_DETAIL=1)
Cost model (measured on our 4xDP4 / 4-GPU instances):
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
@@ -1355,6 +1358,224 @@ class detail_phase:
return False
# ---------------------------------------------------------------------------
# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1).
#
# One tracer per ``cache_controller.start_writing()`` call (i.e. per
# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller
# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and
# must not be read as device-transfer time. The tracer keeps per-flush
# resolution by emitting ONE child span of the current loop
# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps
# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md).
#
# Disabled by default: every entry point is a single module-level bool check,
# and when off no object, dict or span is created.
# ---------------------------------------------------------------------------
_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv(
"SGLANG_TRACE_BACKUP_DETAIL", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_FLUSH_TLS = threading.local()
def backup_detail_enabled() -> bool:
"""True when per-flush micro timing is compiled in (``*_DETAIL`` is on)."""
return _BACKUP_DETAIL_ENABLED
def set_flush_origin(origin: Optional[str]) -> Optional[str]:
"""Tag the next flushes with the caller's scheduler context.
Only used to tell ``process_batch_result`` flushes (the current suspect)
from the ones issued by ``check_hicache_events`` / ``write``. Returns the
previous value so the caller can restore it.
"""
if not _BACKUP_DETAIL_ENABLED:
return None
previous = getattr(_FLUSH_TLS, "origin", None)
_FLUSH_TLS.origin = origin
return previous
def current_flush_origin() -> str:
return getattr(_FLUSH_TLS, "origin", None) or "other"
class BackupFlushTracer:
"""Sub-phase timings for one flush; emits a single span on ``end``."""
__slots__ = (
"origin",
"start_ns",
"phases",
"attrs",
"counts",
"_open",
"_prev",
)
def __init__(self, origin: str):
self.origin = origin
self.start_ns = time.monotonic_ns()
self.phases: dict[str, int] = {}
self.attrs: dict[str, Any] = {}
self.counts: dict[str, int] = {}
self._open: dict[str, int] = {}
self._prev = None
# -- phase helpers -----------------------------------------------------
def begin(self, name: str) -> int:
start = time.monotonic_ns()
self._open[name] = start
return start
def end(self, name: str, start_ns: Optional[int] = None) -> None:
start = self._open.pop(name, None)
if start is None:
start = start_ns
if start is None:
return
self.add(name, start)
def add(self, name: str, start_ns: int) -> None:
self.record(name, int(start_ns), time.monotonic_ns())
def record(self, name: str, start_ns: int, end_ns: int) -> None:
self.phases[name] = self.phases.get(name, 0) + max(
0, int(end_ns) - int(start_ns)
)
def set(self, key: str, value: Any) -> None:
self.attrs[key] = value
def inc(self, key: str, n: int = 1) -> None:
self.counts[key] = self.counts.get(key, 0) + int(n)
def total_ns(self) -> int:
return max(0, time.monotonic_ns() - self.start_ns)
# -- emission ----------------------------------------------------------
def end_and_emit(self) -> None:
now_ns = time.monotonic_ns()
total_ns = max(0, now_ns - self.start_ns)
attrs = {
"sglang.flush_origin": self.origin,
"sglang.flush_total_ms": total_ns / 1e6,
"sglang.loop_seq": current_loop_seq(),
}
for name, ns in self.phases.items():
attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6
for name, value in self.counts.items():
attrs["sglang.flush_" + name] = int(value)
attrs.update(self.attrs)
try:
from opentelemetry import trace as otel_trace
ctx = current_loop_context_for_spans()
start_wall_ns = time.time_ns() - total_ns
span = otel_trace.get_tracer("sglang.cache.flush").start_span(
"scheduler.cache_flush_backup",
context=ctx,
start_time=int(start_wall_ns),
attributes=attrs,
)
span.end(end_time=int(time.time_ns()))
except Exception:
# Telemetry must never break the cache controller.
pass
class backup_phase:
"""Context manager for one sub-phase of the active flush (no-op when off)."""
__slots__ = ("name", "tracer", "start_ns")
def __init__(self, name: str):
self.name = name
self.tracer = None
self.start_ns = None
def __enter__(self):
if not _BACKUP_DETAIL_ENABLED:
return self
self.tracer = getattr(_FLUSH_TLS, "tracer", None)
if self.tracer is not None:
self.start_ns = self.tracer.begin(self.name)
return self
def __exit__(self, exc_type, exc, tb):
if self.tracer is not None:
self.tracer.end(self.name, self.start_ns)
return False
def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]:
"""Start tracing one flush; returns None when disabled / not sampled."""
if not _BACKUP_DETAIL_ENABLED:
return None
if not loop_detail_enabled():
return None
tracer = BackupFlushTracer(origin or current_flush_origin())
tracer._prev = getattr(_FLUSH_TLS, "tracer", None)
_FLUSH_TLS.tracer = tracer
return tracer
def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None:
if tracer is None:
return
if getattr(_FLUSH_TLS, "tracer", None) is tracer:
_FLUSH_TLS.tracer = tracer._prev
try:
tracer.end_and_emit()
except Exception:
pass
def current_flush_tracer() -> Optional[BackupFlushTracer]:
if not _BACKUP_DETAIL_ENABLED:
return None
return getattr(_FLUSH_TLS, "tracer", None)
def flush_note(key: str, value: Any) -> None:
"""Attach a dimension to the active flush (no-op when disabled)."""
if not _BACKUP_DETAIL_ENABLED:
return
tracer = getattr(_FLUSH_TLS, "tracer", None)
if tracer is not None:
tracer.set(key, value)
def flush_count(key: str, n: int = 1) -> None:
"""Accumulate a counter on the active flush (no-op when disabled)."""
if not _BACKUP_DETAIL_ENABLED:
return
tracer = getattr(_FLUSH_TLS, "tracer", None)
if tracer is not None:
tracer.inc(key, n)
def flush_mark() -> Optional[int]:
"""Timestamp for a manual phase (None when no flush is being traced)."""
if not _BACKUP_DETAIL_ENABLED:
return None
if getattr(_FLUSH_TLS, "tracer", None) is None:
return None
return time.monotonic_ns()
def flush_phase_since(name: str, start_ns: Optional[int]) -> None:
"""Record ``name`` from ``start_ns`` to now (no-op when untraced)."""
if start_ns is None:
return
tracer = getattr(_FLUSH_TLS, "tracer", None)
if tracer is not None:
tracer.add(name, start_ns)
def current_loop_seq() -> int:
"""Loop sequence for cross-record correlation (0 when unavailable)."""
lt = getattr(_LOOP_TLS, "current", None)
@@ -2054,6 +2275,13 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None
if lt.process is not None:
proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0))
parts = _total(
# Outer phases of process_batch_result (scheduler.py).
"flush_trace",
"load_snapshot",
"flush_backups",
"step_counters",
"metrics_report",
# Inner disagg-prefill phases (prefill.py).
"copy_done_wait",
"auxiliary_and_logprob",
"per_request_result",
@@ -0,0 +1,242 @@
"""Unit tests for the per-flush HiCache backup micro timer.
Design: handoff/prefill-flush-backups-validation-20261107.md §3-§5.
Acceptance points:
* ``SGLANG_TRACE_BACKUP_DETAIL`` is off by default and then records nothing;
* a tracer only exists for a sampled loop (same decision as the loop span);
* sub-phases stay on the flush span, never on the loop aggregate, so the
``process_residual`` identity cannot double-count them;
* the new outer ``process_batch_result`` phases are subtracted from
``process_residual``;
* the origin tag separates process-result flushes from other callers.
"""
import importlib.util
import os
import pathlib
import sys
import time
import types
import unittest
from unittest import mock
def _module_path():
p = (
pathlib.Path(__file__).resolve().parents[4]
/ "python/sglang/srt/observability/batch_telemetry.py"
)
if p.exists():
return p
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
return pathlib.Path(spec.origin)
def _load(**env):
base = {
"SGLANG_BATCH_STAGE_TIMING": "1",
"SGLANG_TRACE_LOOP_SPAN": "1",
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
"SGLANG_TRACE_LOOP_DETAIL": "1",
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
"SGLANG_TRACE_BACKUP_DETAIL": "1",
"SGLANG_TRACE_FORWARD_SPAN": "0",
}
base.update(env)
with mock.patch.dict(os.environ, base):
spec = importlib.util.spec_from_file_location(
"batch_telemetry_backup_under_test", _module_path()
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
class _FakeCounter:
def __init__(self):
self.data = {}
def labels(self, **kw):
self._key = tuple(sorted(kw.items()))
return self
def inc(self, value=1.0):
self.data[self._key] = self.data.get(self._key, 0.0) + value
def _fake_loop_metrics():
return {
"no_batch_seconds": _FakeCounter(),
"no_batch_total": _FakeCounter(),
"lockstep_seconds": _FakeCounter(),
}
class _FakeSpan:
def __init__(self, name, **kwargs):
self.name = name
self.kwargs = kwargs
self.attributes = dict(kwargs.get("attributes") or {})
self.end_time = None
self.context = types.SimpleNamespace(is_valid=True, trace_id=1, span_id=2)
def get_span_context(self):
return self.context
def set_attributes(self, attrs):
self.attributes.update(attrs)
def end(self, end_time=None):
self.end_time = end_time
class _FakeTracer:
def __init__(self):
self.spans = []
def start_span(self, name, **kwargs):
span = _FakeSpan(name, **kwargs)
self.spans.append(span)
return span
def _fake_otel():
tracer = _FakeTracer()
trace = types.ModuleType("opentelemetry.trace")
trace.get_tracer = lambda _name: tracer
trace.set_span_in_context = lambda span: ("ctx", span)
trace.Link = lambda ctx: ("link", ctx)
package = types.ModuleType("opentelemetry")
package.trace = trace
patcher = mock.patch.dict(
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
)
return tracer, patcher
def _state(**overrides):
state = {
"waiting": 0,
"result_queue": 0,
"result_limit": 2,
"inflight": 0,
"bootstrap": 0,
"deferred": 0,
"paused": False,
"waiting_for_result": False,
"skip_batch_creation": False,
"forward_active": False,
}
state.update(overrides)
return state
def _flush_spans(tracer):
return [s for s in tracer.spans if s.name == "scheduler.cache_flush_backup"]
class BackupFlushDetailTest(unittest.TestCase):
def test_disabled_records_nothing(self):
mod = _load(SGLANG_TRACE_BACKUP_DETAIL="0")
self.assertFalse(mod.backup_detail_enabled())
self.assertIsNone(mod.backup_flush_begin())
self.assertIsNone(mod.flush_mark())
mod.flush_note("sglang.flush_tokens", 1)
mod.flush_count("launch_pools")
with mod.backup_phase("merge_ops"):
pass
# set_flush_origin is a no-op too.
self.assertIsNone(mod.set_flush_origin("process_result"))
self.assertEqual(mod.current_flush_origin(), "other")
def test_requires_compiled_detail(self):
mod = _load(SGLANG_TRACE_LOOP_DETAIL="0")
self.assertFalse(mod.backup_detail_enabled())
self.assertIsNone(mod.backup_flush_begin())
def test_unsampled_loop_creates_no_tracer(self):
mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0")
mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
self.assertTrue(mod.backup_detail_enabled())
self.assertIsNone(mod.backup_flush_begin())
def test_phases_attrs_and_counts_land_on_flush_span(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
flush = mod.backup_flush_begin()
self.assertIsNotNone(flush)
mod.flush_note("sglang.flush_tokens", 4096)
with mod.backup_phase("merge_ops"):
pass
t0 = mod.flush_mark()
mod.flush_phase_since("indices_cpu", t0)
mod.flush_count("launch_pools", 3)
mod.backup_flush_end(flush)
mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state())
flush_span = _flush_spans(tracer)[0]
attrs = flush_span.attributes
self.assertEqual(attrs["sglang.flush_origin"], "other")
self.assertEqual(attrs["sglang.flush_tokens"], 4096)
self.assertEqual(attrs["sglang.flush_launch_pools"], 3)
self.assertIn("sglang.flush_merge_ops_ms", attrs)
self.assertIn("sglang.flush_indices_cpu_ms", attrs)
self.assertGreaterEqual(attrs["sglang.flush_total_ms"], 0.0)
self.assertEqual(flush_span.kwargs["context"], lt.span_ctx)
# Sub-phases must not leak into the loop aggregate: the loop detail is
# used for the residual identity and must stay free of nested timing.
loop_span = next(s for s in tracer.spans if s.name == "scheduler.loop")
self.assertNotIn("sglang.flush_merge_ops_ms", loop_span.attributes)
def test_origin_tag_is_scoped(self):
mod = _load()
mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
previous = mod.set_flush_origin("process_result")
flush = mod.backup_flush_begin()
mod.backup_flush_end(flush)
mod.set_flush_origin(previous)
self.assertEqual(flush.origin, "process_result")
self.assertEqual(mod.current_flush_origin(), "other")
def test_nested_tracer_is_restored(self):
mod = _load()
mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
outer = mod.backup_flush_begin()
inner = mod.backup_flush_begin()
mod.backup_flush_end(inner)
self.assertIs(mod.current_flush_tracer(), outer)
mod.backup_flush_end(outer)
self.assertIsNone(mod.current_flush_tracer())
def test_outer_process_phases_are_subtracted_from_residual(self):
mod = _load()
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
tracer, patcher = _fake_otel()
with patcher:
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
t0 = time.monotonic_ns()
mod.detail_add("flush_trace", t0, t0 + 5_000_000)
mod.detail_add("load_snapshot", t0, t0 + 5_000_000)
mod.detail_add("flush_backups", t0, t0 + 20_000_000)
mod.detail_add("step_counters", t0, t0 + 5_000_000)
mod.detail_add("metrics_report", t0, t0 + 5_000_000)
mod.detail_add("per_request_result", t0, t0 + 10_000_000)
start = time.time_ns()
mod.mark_process(start, start + 100_000_000, None)
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
attrs = next(
s for s in tracer.spans if s.name == "scheduler.loop"
).attributes
self.assertAlmostEqual(attrs["sglang.flush_backups_ms"], 20.0)
# 100 ms - (5+5+20+5+5+10) ms of measured phases.
self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 50.0, delta=1.0)
if __name__ == "__main__":
unittest.main()