observability: split flush_pending_backups host chain (off by default)
Adds the phase-3/phase-4 instrumentation from handoff/prefill-flush-backups-validation-20261107.md: * process_batch_result outer phases: flush_trace, load_snapshot, flush_backups, step_counters, metrics_report. process_residual now subtracts them, so the 37.5% residual is attributable. * SGLANG_TRACE_BACKUP_DETAIL=1 (default 0) traces one flush: merge_ops / move_write (anchor + per-sidecar indices) / indices_cpu / index_sort / index_select / write_fence / build_l2 / submit_d2h (start-event wait, prepare, per-pool launch, ack events) / ack_enqueue, plus token/op/pool/device dimensions and ack queue depth. * Per-flush resolution: one scheduler.cache_flush_backup span per non-empty flush, child of the loop span. Sub-phases never enter the loop aggregate, so the residual identity cannot double count. * Empty vs non-empty flushes are counted on the loop. All timings are scheduler host wall, not device transfer time.
This commit is contained in:
@@ -43,6 +43,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
@@ -890,40 +891,104 @@ class HiCacheController:
|
||||
|
||||
def start_writing(self) -> None:
|
||||
if len(self.write_queue) == 0:
|
||||
# Empty flushes are the common case when write-through fires every
|
||||
# batch: counted so a small `flush_backups` cannot be explained as
|
||||
# "one big flush per loop" without noticing the empty ones.
|
||||
batch_telemetry.detail_count("flush_calls")
|
||||
batch_telemetry.detail_count("flush_empty")
|
||||
return
|
||||
|
||||
op = CacheOperation.merge_ops(self.write_queue)
|
||||
host_indices, device_indices, pool_transfers = self._move_write_operation(op)
|
||||
self.write_queue.clear()
|
||||
|
||||
if self.write_fence_stream is not None:
|
||||
# The copy may read KV a forward is still writing: under overlap
|
||||
# scheduling a finished request is cached while the next forward
|
||||
# writes its last token, and an evicted SWA page can belong to an
|
||||
# unfinished checkpoint. Later forwards still overlap with the copy.
|
||||
self.l2_transfer_engine.device_to_host_stream.wait_stream(
|
||||
self.write_fence_stream
|
||||
tracer = batch_telemetry.backup_flush_begin()
|
||||
try:
|
||||
batch_telemetry.detail_count("flush_calls")
|
||||
batch_telemetry.detail_count("flush_nonempty")
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_write_queue_ops", len(self.write_queue)
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_layout", getattr(self.mem_pool_host, "layout", None)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_ack_queue_before", len(self.ack_write_queue)
|
||||
)
|
||||
|
||||
completion = self.l2_transfer_engine.submit_device_to_host(
|
||||
self._l2_transfers(host_indices, device_indices, pool_transfers)
|
||||
)
|
||||
|
||||
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
|
||||
(id(self), "write"), completion.finish_event
|
||||
)
|
||||
|
||||
self.ack_write_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=completion.start_event,
|
||||
finish_event=completion.finish_event,
|
||||
node_ids=op.node_ids,
|
||||
num_tokens=len(op.device_indices),
|
||||
timing_enabled=completion.timing_enabled,
|
||||
num_tokens_by_pool=self._num_tokens_by_pool(op),
|
||||
num_bytes=self._transfer_num_bytes(op),
|
||||
with batch_telemetry.backup_phase("merge_ops"):
|
||||
op = CacheOperation.merge_ops(self.write_queue)
|
||||
tokens = len(op.device_indices)
|
||||
indices = op.device_indices
|
||||
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_nodes_merged", len(op.node_ids)
|
||||
)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_device", str(indices.device.type)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_dtype", str(indices.dtype)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_numel", int(indices.numel())
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("move_write"):
|
||||
host_indices, device_indices, pool_transfers = (
|
||||
self._move_write_operation(op)
|
||||
)
|
||||
self.write_queue.clear()
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_host_indices_on_cpu",
|
||||
bool(getattr(host_indices, "is_cuda", False) is False),
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_device_indices_on_cpu",
|
||||
bool(getattr(device_indices, "is_cuda", False) is False),
|
||||
)
|
||||
|
||||
if self.write_fence_stream is not None:
|
||||
# The copy may read KV a forward is still writing: under overlap
|
||||
# scheduling a finished request is cached while the next forward
|
||||
# writes its last token, and an evicted SWA page can belong to an
|
||||
# unfinished checkpoint. Later forwards still overlap with the copy.
|
||||
with batch_telemetry.backup_phase("write_fence"):
|
||||
self.l2_transfer_engine.device_to_host_stream.wait_stream(
|
||||
self.write_fence_stream
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("build_l2"):
|
||||
transfers = self._l2_transfers(
|
||||
host_indices, device_indices, pool_transfers
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
|
||||
|
||||
with batch_telemetry.backup_phase("submit_d2h"):
|
||||
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
|
||||
|
||||
with batch_telemetry.backup_phase("set_done_event"):
|
||||
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
|
||||
(id(self), "write"), completion.finish_event
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("ack_enqueue"):
|
||||
self.ack_write_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=completion.start_event,
|
||||
finish_event=completion.finish_event,
|
||||
node_ids=op.node_ids,
|
||||
num_tokens=len(op.device_indices),
|
||||
timing_enabled=completion.timing_enabled,
|
||||
num_tokens_by_pool=self._num_tokens_by_pool(op),
|
||||
num_bytes=self._transfer_num_bytes(op),
|
||||
)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_ack_queue_after", len(self.ack_write_queue)
|
||||
)
|
||||
finally:
|
||||
batch_telemetry.backup_flush_end(tracer)
|
||||
|
||||
def _transfer_num_bytes(self, op: CacheOperation) -> int:
|
||||
return len(op.device_indices) * self.mem_pool_host.size_per_token
|
||||
@@ -954,15 +1019,24 @@ class HiCacheController:
|
||||
# move indices to GPU if using kernels, to host if using direct indexing
|
||||
if self.io_backend == "kernel":
|
||||
if not host_indices.is_cuda:
|
||||
host_indices = host_indices.to(self.device, non_blocking=True)
|
||||
with batch_telemetry.backup_phase("indices_h2d"):
|
||||
host_indices = host_indices.to(self.device, non_blocking=True)
|
||||
return host_indices, device_indices
|
||||
elif self.io_backend == "direct":
|
||||
if self.mem_pool_host.layout == "layer_first":
|
||||
device_indices = device_indices.cpu()
|
||||
host_indices, idx = host_indices.sort()
|
||||
return host_indices, device_indices.index_select(0, idx)
|
||||
with batch_telemetry.backup_phase("indices_cpu"):
|
||||
device_indices = device_indices.cpu()
|
||||
with batch_telemetry.backup_phase("index_sort"):
|
||||
host_indices, idx = host_indices.sort()
|
||||
with batch_telemetry.backup_phase("index_select"):
|
||||
return host_indices, device_indices.index_select(0, idx)
|
||||
elif self.mem_pool_host.layout == "page_first_direct":
|
||||
return host_indices, device_indices.cpu()
|
||||
# Synchronous D2H of the token indices: the host wall time of
|
||||
# this call is the scheduler blocking on the copy, not a
|
||||
# device bandwidth measurement.
|
||||
with batch_telemetry.backup_phase("indices_cpu"):
|
||||
device_indices = device_indices.cpu()
|
||||
return host_indices, device_indices
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported layout {self.mem_pool_host.layout!r} for io backend 'direct'"
|
||||
|
||||
@@ -5079,15 +5079,19 @@ class Scheduler(
|
||||
):
|
||||
# Flush async trace ops here: in overlap mode this CPU work runs while
|
||||
# the next batch's GPU forward is in flight, giving free overlap.
|
||||
flush_trace_batch(batch.reqs)
|
||||
snapshot = self.publish_load_snapshot(force=batch.forward_mode.is_extend())
|
||||
# Router-facing gauge on the dedicated PUB socket, reusing the
|
||||
# snapshot above rather than walking the queues again.
|
||||
self.load_publisher.publish_load_stat(
|
||||
self.load_inquirer.get_loads,
|
||||
force=batch.forward_mode.is_extend(),
|
||||
snapshot=snapshot,
|
||||
)
|
||||
with batch_telemetry.detail_phase("flush_trace"):
|
||||
flush_trace_batch(batch.reqs)
|
||||
with batch_telemetry.detail_phase("load_snapshot"):
|
||||
snapshot = self.publish_load_snapshot(
|
||||
force=batch.forward_mode.is_extend()
|
||||
)
|
||||
# Router-facing gauge on the dedicated PUB socket, reusing the
|
||||
# snapshot above rather than walking the queues again.
|
||||
self.load_publisher.publish_load_stat(
|
||||
self.load_inquirer.get_loads,
|
||||
force=batch.forward_mode.is_extend(),
|
||||
snapshot=snapshot,
|
||||
)
|
||||
|
||||
if batch.forward_mode.is_decode():
|
||||
self.batch_result_processor.process_batch_result_decode(batch, result)
|
||||
@@ -5104,19 +5108,28 @@ class Scheduler(
|
||||
self.batch_result_processor.process_batch_result_idle(batch, result)
|
||||
|
||||
# Submit this batch's queued host backups before the next scheduler step.
|
||||
self.tree_cache.flush_pending_backups()
|
||||
# Tag the origin so the per-flush micro timer can separate these from
|
||||
# the ones issued by check_hicache_events / write().
|
||||
_flush_origin = batch_telemetry.set_flush_origin("process_result")
|
||||
try:
|
||||
with batch_telemetry.detail_phase("flush_backups"):
|
||||
self.tree_cache.flush_pending_backups()
|
||||
finally:
|
||||
batch_telemetry.set_flush_origin(_flush_origin)
|
||||
|
||||
self._record_step_counters(batch, result)
|
||||
with batch_telemetry.detail_phase("step_counters"):
|
||||
self._record_step_counters(batch, result)
|
||||
|
||||
self.metrics_reporter.log_batch_result_stats(batch, result)
|
||||
with batch_telemetry.detail_phase("metrics_report"):
|
||||
self.metrics_reporter.log_batch_result_stats(batch, result)
|
||||
|
||||
# Emit forward pass metrics (every iteration when enabled)
|
||||
if self.enable_fpm:
|
||||
self.metrics_reporter._emit_forward_pass_metrics(batch, result)
|
||||
# Emit forward pass metrics (every iteration when enabled)
|
||||
if self.enable_fpm:
|
||||
self.metrics_reporter._emit_forward_pass_metrics(batch, result)
|
||||
|
||||
self._maybe_clear_mm_inputs(batch)
|
||||
self.maybe_send_health_check_signal()
|
||||
self.metrics_reporter.update_device_timer()
|
||||
self._maybe_clear_mm_inputs(batch)
|
||||
self.maybe_send_health_check_signal()
|
||||
self.metrics_reporter.update_device_timer()
|
||||
|
||||
def _record_step_counters(
|
||||
self, batch: ScheduleBatch, result: GenerationBatchResult
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
|
||||
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
@@ -1509,26 +1510,30 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if uses_shared_host_layout(host_pool):
|
||||
batch_telemetry.flush_count("shared_layout_pools")
|
||||
return host_indices, device_indices
|
||||
if write_back_jit and host_pool.can_use_write_back_jit:
|
||||
if host_indices.is_cuda:
|
||||
host_indices = host_indices.cpu()
|
||||
with batch_telemetry.backup_phase("host_indices_cpu"):
|
||||
host_indices = host_indices.cpu()
|
||||
return host_indices, device_indices
|
||||
return self.move_indices(host_indices, device_indices)
|
||||
|
||||
def move_hybrid_indices(
|
||||
self, operation: CacheOperation, *, write_back_jit: bool = False
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
|
||||
host_indices, device_indices = self._move_pool_indices(
|
||||
self.mem_pool_host.anchor_entry.host_pool,
|
||||
operation.host_indices,
|
||||
operation.device_indices,
|
||||
write_back_jit=write_back_jit,
|
||||
)
|
||||
with batch_telemetry.backup_phase("anchor_indices"):
|
||||
host_indices, device_indices = self._move_pool_indices(
|
||||
self.mem_pool_host.anchor_entry.host_pool,
|
||||
operation.host_indices,
|
||||
operation.device_indices,
|
||||
write_back_jit=write_back_jit,
|
||||
)
|
||||
resolved_pool_transfers = None
|
||||
if operation.pool_transfers:
|
||||
resolved_pool_transfers = []
|
||||
for transfer in operation.pool_transfers:
|
||||
_pool_t0 = batch_telemetry.flush_mark()
|
||||
transfer_host_indices, transfer_device_indices = (
|
||||
self._move_pool_indices(
|
||||
self.mem_pool_host.entry_map[transfer.name].host_pool,
|
||||
@@ -1537,6 +1542,9 @@ class HybridCacheController(BaseHiCacheController):
|
||||
write_back_jit=write_back_jit,
|
||||
)
|
||||
)
|
||||
batch_telemetry.flush_phase_since(
|
||||
"sidecar_indices." + str(transfer.name.value), _pool_t0
|
||||
)
|
||||
# Tree-owned transfers keep their original radix-tree indices.
|
||||
resolved_pool_transfers.append(
|
||||
replace(
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -71,10 +72,13 @@ class L2TransferEngine:
|
||||
original_device = transfer.device_indices.device
|
||||
# The direct backend supplies CPU indices even for a CUDA pool.
|
||||
# Translate alongside the v2p table, then restore the backend's device.
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
indices = transfer.device_indices.to(
|
||||
getattr(transfer.device_pool, "device", original_device)
|
||||
).contiguous()
|
||||
batch_telemetry.flush_phase_since("resolve_indices.to_device", _t0)
|
||||
dcp_size = getattr(transfer.host_pool, "dcp_size", 1)
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
if dcp_size > 1:
|
||||
# The MLA host pool selects this rank and collapses logical IDs.
|
||||
# Translate in local virtual space, then preserve that widened
|
||||
@@ -82,16 +86,22 @@ class L2TransferEngine:
|
||||
resolved = translate(indices // dcp_size) * dcp_size + indices % dcp_size
|
||||
else:
|
||||
resolved = translate(indices)
|
||||
return resolved.to(original_device)
|
||||
batch_telemetry.flush_phase_since("resolve_indices.translate", _t0)
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
result = resolved.to(original_device)
|
||||
batch_telemetry.flush_phase_since("resolve_indices.back_to_host", _t0)
|
||||
return result
|
||||
|
||||
def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]:
|
||||
prepared = []
|
||||
for transfer in transfers:
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
host_indices, device_indices = transfer.host_pool.prepare_transfer_indices(
|
||||
transfer.host_indices,
|
||||
self._resolve_device_indices(transfer),
|
||||
self.io_backend,
|
||||
)
|
||||
batch_telemetry.flush_phase_since("prepare_transfer_indices", _t0)
|
||||
prepared.append(
|
||||
transfer._replace(
|
||||
host_indices=host_indices,
|
||||
@@ -102,44 +112,55 @@ class L2TransferEngine:
|
||||
|
||||
@contextmanager
|
||||
def _submission(self, transfers, stream, transfer_key, start_event=None):
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
start_event = self._start_event(start_event)
|
||||
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
|
||||
completion = TransferCompletion(ack_start, ack_finish, timing_enabled)
|
||||
domains = shared_host_layout_domains(t.host_pool for t in transfers)
|
||||
for domain in domains:
|
||||
domain.acquire_layout()
|
||||
batch_telemetry.flush_phase_since("submit_setup", _t0)
|
||||
finish_recorded = False
|
||||
try:
|
||||
with device_module.stream(stream):
|
||||
# Index preparation can read producer-owned device indices.
|
||||
start_event.wait(stream)
|
||||
prepared = self._prepare_transfers(transfers)
|
||||
ack_start.record()
|
||||
with batch_telemetry.backup_phase("start_event_wait"):
|
||||
start_event.wait(stream)
|
||||
with batch_telemetry.backup_phase("prepare_transfers"):
|
||||
prepared = self._prepare_transfers(transfers)
|
||||
with batch_telemetry.backup_phase("ack_events"):
|
||||
ack_start.record()
|
||||
yield prepared, completion
|
||||
ack_finish.record()
|
||||
finish_recorded = True
|
||||
self._record_stream(transfers + prepared, stream)
|
||||
with batch_telemetry.backup_phase("ack_events"):
|
||||
ack_finish.record()
|
||||
finish_recorded = True
|
||||
self._record_stream(transfers + prepared, stream)
|
||||
except Exception:
|
||||
stream.synchronize()
|
||||
raise
|
||||
finally:
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
for domain in reversed(domains):
|
||||
domain.release_layout(
|
||||
ack_finish if finish_recorded else None,
|
||||
(id(self), transfer_key),
|
||||
)
|
||||
batch_telemetry.flush_phase_since("layout_release", _t0)
|
||||
|
||||
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
|
||||
with self._submission(
|
||||
transfers, self.device_to_host_stream, "device_to_host"
|
||||
) as (transfers, completion):
|
||||
for transfer in transfers:
|
||||
_t0 = batch_telemetry.flush_mark()
|
||||
transfer.host_pool.backup_from_device_all_layer_physical(
|
||||
transfer.device_pool,
|
||||
transfer.host_indices,
|
||||
transfer.device_indices,
|
||||
self.io_backend,
|
||||
)
|
||||
batch_telemetry.flush_phase_since("launch_pool", _t0)
|
||||
batch_telemetry.flush_count("launch_pools")
|
||||
return completion
|
||||
|
||||
def submit_host_to_device(
|
||||
|
||||
@@ -26,6 +26,9 @@ Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs:
|
||||
loops are recorded)
|
||||
SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown
|
||||
(queue/source/indices/transfer/join)
|
||||
SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases
|
||||
(one ``scheduler.cache_flush_backup`` span;
|
||||
requires SGLANG_TRACE_LOOP_DETAIL=1)
|
||||
|
||||
Cost model (measured on our 4xDP4 / 4-GPU instances):
|
||||
* histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU.
|
||||
@@ -1355,6 +1358,224 @@ class detail_phase:
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1).
|
||||
#
|
||||
# One tracer per ``cache_controller.start_writing()`` call (i.e. per
|
||||
# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller
|
||||
# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and
|
||||
# must not be read as device-transfer time. The tracer keeps per-flush
|
||||
# resolution by emitting ONE child span of the current loop
|
||||
# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps
|
||||
# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md).
|
||||
#
|
||||
# Disabled by default: every entry point is a single module-level bool check,
|
||||
# and when off no object, dict or span is created.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv(
|
||||
"SGLANG_TRACE_BACKUP_DETAIL", "0"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
|
||||
_FLUSH_TLS = threading.local()
|
||||
|
||||
|
||||
def backup_detail_enabled() -> bool:
|
||||
"""True when per-flush micro timing is compiled in (``*_DETAIL`` is on)."""
|
||||
return _BACKUP_DETAIL_ENABLED
|
||||
|
||||
|
||||
def set_flush_origin(origin: Optional[str]) -> Optional[str]:
|
||||
"""Tag the next flushes with the caller's scheduler context.
|
||||
|
||||
Only used to tell ``process_batch_result`` flushes (the current suspect)
|
||||
from the ones issued by ``check_hicache_events`` / ``write``. Returns the
|
||||
previous value so the caller can restore it.
|
||||
"""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return None
|
||||
previous = getattr(_FLUSH_TLS, "origin", None)
|
||||
_FLUSH_TLS.origin = origin
|
||||
return previous
|
||||
|
||||
|
||||
def current_flush_origin() -> str:
|
||||
return getattr(_FLUSH_TLS, "origin", None) or "other"
|
||||
|
||||
|
||||
class BackupFlushTracer:
|
||||
"""Sub-phase timings for one flush; emits a single span on ``end``."""
|
||||
|
||||
__slots__ = (
|
||||
"origin",
|
||||
"start_ns",
|
||||
"phases",
|
||||
"attrs",
|
||||
"counts",
|
||||
"_open",
|
||||
"_prev",
|
||||
)
|
||||
|
||||
def __init__(self, origin: str):
|
||||
self.origin = origin
|
||||
self.start_ns = time.monotonic_ns()
|
||||
self.phases: dict[str, int] = {}
|
||||
self.attrs: dict[str, Any] = {}
|
||||
self.counts: dict[str, int] = {}
|
||||
self._open: dict[str, int] = {}
|
||||
self._prev = None
|
||||
|
||||
# -- phase helpers -----------------------------------------------------
|
||||
def begin(self, name: str) -> int:
|
||||
start = time.monotonic_ns()
|
||||
self._open[name] = start
|
||||
return start
|
||||
|
||||
def end(self, name: str, start_ns: Optional[int] = None) -> None:
|
||||
start = self._open.pop(name, None)
|
||||
if start is None:
|
||||
start = start_ns
|
||||
if start is None:
|
||||
return
|
||||
self.add(name, start)
|
||||
|
||||
def add(self, name: str, start_ns: int) -> None:
|
||||
self.record(name, int(start_ns), time.monotonic_ns())
|
||||
|
||||
def record(self, name: str, start_ns: int, end_ns: int) -> None:
|
||||
self.phases[name] = self.phases.get(name, 0) + max(
|
||||
0, int(end_ns) - int(start_ns)
|
||||
)
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
self.attrs[key] = value
|
||||
|
||||
def inc(self, key: str, n: int = 1) -> None:
|
||||
self.counts[key] = self.counts.get(key, 0) + int(n)
|
||||
|
||||
def total_ns(self) -> int:
|
||||
return max(0, time.monotonic_ns() - self.start_ns)
|
||||
|
||||
# -- emission ----------------------------------------------------------
|
||||
def end_and_emit(self) -> None:
|
||||
now_ns = time.monotonic_ns()
|
||||
total_ns = max(0, now_ns - self.start_ns)
|
||||
attrs = {
|
||||
"sglang.flush_origin": self.origin,
|
||||
"sglang.flush_total_ms": total_ns / 1e6,
|
||||
"sglang.loop_seq": current_loop_seq(),
|
||||
}
|
||||
for name, ns in self.phases.items():
|
||||
attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6
|
||||
for name, value in self.counts.items():
|
||||
attrs["sglang.flush_" + name] = int(value)
|
||||
attrs.update(self.attrs)
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
ctx = current_loop_context_for_spans()
|
||||
start_wall_ns = time.time_ns() - total_ns
|
||||
span = otel_trace.get_tracer("sglang.cache.flush").start_span(
|
||||
"scheduler.cache_flush_backup",
|
||||
context=ctx,
|
||||
start_time=int(start_wall_ns),
|
||||
attributes=attrs,
|
||||
)
|
||||
span.end(end_time=int(time.time_ns()))
|
||||
except Exception:
|
||||
# Telemetry must never break the cache controller.
|
||||
pass
|
||||
|
||||
|
||||
class backup_phase:
|
||||
"""Context manager for one sub-phase of the active flush (no-op when off)."""
|
||||
|
||||
__slots__ = ("name", "tracer", "start_ns")
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.tracer = None
|
||||
self.start_ns = None
|
||||
|
||||
def __enter__(self):
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return self
|
||||
self.tracer = getattr(_FLUSH_TLS, "tracer", None)
|
||||
if self.tracer is not None:
|
||||
self.start_ns = self.tracer.begin(self.name)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
if self.tracer is not None:
|
||||
self.tracer.end(self.name, self.start_ns)
|
||||
return False
|
||||
|
||||
|
||||
def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]:
|
||||
"""Start tracing one flush; returns None when disabled / not sampled."""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return None
|
||||
if not loop_detail_enabled():
|
||||
return None
|
||||
tracer = BackupFlushTracer(origin or current_flush_origin())
|
||||
tracer._prev = getattr(_FLUSH_TLS, "tracer", None)
|
||||
_FLUSH_TLS.tracer = tracer
|
||||
return tracer
|
||||
|
||||
|
||||
def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None:
|
||||
if tracer is None:
|
||||
return
|
||||
if getattr(_FLUSH_TLS, "tracer", None) is tracer:
|
||||
_FLUSH_TLS.tracer = tracer._prev
|
||||
try:
|
||||
tracer.end_and_emit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def current_flush_tracer() -> Optional[BackupFlushTracer]:
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return None
|
||||
return getattr(_FLUSH_TLS, "tracer", None)
|
||||
|
||||
|
||||
def flush_note(key: str, value: Any) -> None:
|
||||
"""Attach a dimension to the active flush (no-op when disabled)."""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return
|
||||
tracer = getattr(_FLUSH_TLS, "tracer", None)
|
||||
if tracer is not None:
|
||||
tracer.set(key, value)
|
||||
|
||||
|
||||
def flush_count(key: str, n: int = 1) -> None:
|
||||
"""Accumulate a counter on the active flush (no-op when disabled)."""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return
|
||||
tracer = getattr(_FLUSH_TLS, "tracer", None)
|
||||
if tracer is not None:
|
||||
tracer.inc(key, n)
|
||||
|
||||
|
||||
def flush_mark() -> Optional[int]:
|
||||
"""Timestamp for a manual phase (None when no flush is being traced)."""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return None
|
||||
if getattr(_FLUSH_TLS, "tracer", None) is None:
|
||||
return None
|
||||
return time.monotonic_ns()
|
||||
|
||||
|
||||
def flush_phase_since(name: str, start_ns: Optional[int]) -> None:
|
||||
"""Record ``name`` from ``start_ns`` to now (no-op when untraced)."""
|
||||
if start_ns is None:
|
||||
return
|
||||
tracer = getattr(_FLUSH_TLS, "tracer", None)
|
||||
if tracer is not None:
|
||||
tracer.add(name, start_ns)
|
||||
|
||||
|
||||
def current_loop_seq() -> int:
|
||||
"""Loop sequence for cross-record correlation (0 when unavailable)."""
|
||||
lt = getattr(_LOOP_TLS, "current", None)
|
||||
@@ -2054,6 +2275,13 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None
|
||||
if lt.process is not None:
|
||||
proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0))
|
||||
parts = _total(
|
||||
# Outer phases of process_batch_result (scheduler.py).
|
||||
"flush_trace",
|
||||
"load_snapshot",
|
||||
"flush_backups",
|
||||
"step_counters",
|
||||
"metrics_report",
|
||||
# Inner disagg-prefill phases (prefill.py).
|
||||
"copy_done_wait",
|
||||
"auxiliary_and_logprob",
|
||||
"per_request_result",
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Unit tests for the per-flush HiCache backup micro timer.
|
||||
|
||||
Design: handoff/prefill-flush-backups-validation-20261107.md §3-§5.
|
||||
|
||||
Acceptance points:
|
||||
|
||||
* ``SGLANG_TRACE_BACKUP_DETAIL`` is off by default and then records nothing;
|
||||
* a tracer only exists for a sampled loop (same decision as the loop span);
|
||||
* sub-phases stay on the flush span, never on the loop aggregate, so the
|
||||
``process_residual`` identity cannot double-count them;
|
||||
* the new outer ``process_batch_result`` phases are subtracted from
|
||||
``process_residual``;
|
||||
* the origin tag separates process-result flushes from other callers.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def _module_path():
|
||||
p = (
|
||||
pathlib.Path(__file__).resolve().parents[4]
|
||||
/ "python/sglang/srt/observability/batch_telemetry.py"
|
||||
)
|
||||
if p.exists():
|
||||
return p
|
||||
spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry")
|
||||
return pathlib.Path(spec.origin)
|
||||
|
||||
|
||||
def _load(**env):
|
||||
base = {
|
||||
"SGLANG_BATCH_STAGE_TIMING": "1",
|
||||
"SGLANG_TRACE_LOOP_SPAN": "1",
|
||||
"SGLANG_TRACE_LOOP_SAMPLE": "1.0",
|
||||
"SGLANG_TRACE_LOOP_DETAIL": "1",
|
||||
"SGLANG_TRACE_MOONCAKE_DETAIL": "0",
|
||||
"SGLANG_TRACE_BACKUP_DETAIL": "1",
|
||||
"SGLANG_TRACE_FORWARD_SPAN": "0",
|
||||
}
|
||||
base.update(env)
|
||||
with mock.patch.dict(os.environ, base):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"batch_telemetry_backup_under_test", _module_path()
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
class _FakeCounter:
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
|
||||
def labels(self, **kw):
|
||||
self._key = tuple(sorted(kw.items()))
|
||||
return self
|
||||
|
||||
def inc(self, value=1.0):
|
||||
self.data[self._key] = self.data.get(self._key, 0.0) + value
|
||||
|
||||
|
||||
def _fake_loop_metrics():
|
||||
return {
|
||||
"no_batch_seconds": _FakeCounter(),
|
||||
"no_batch_total": _FakeCounter(),
|
||||
"lockstep_seconds": _FakeCounter(),
|
||||
}
|
||||
|
||||
|
||||
class _FakeSpan:
|
||||
def __init__(self, name, **kwargs):
|
||||
self.name = name
|
||||
self.kwargs = kwargs
|
||||
self.attributes = dict(kwargs.get("attributes") or {})
|
||||
self.end_time = None
|
||||
self.context = types.SimpleNamespace(is_valid=True, trace_id=1, span_id=2)
|
||||
|
||||
def get_span_context(self):
|
||||
return self.context
|
||||
|
||||
def set_attributes(self, attrs):
|
||||
self.attributes.update(attrs)
|
||||
|
||||
def end(self, end_time=None):
|
||||
self.end_time = end_time
|
||||
|
||||
|
||||
class _FakeTracer:
|
||||
def __init__(self):
|
||||
self.spans = []
|
||||
|
||||
def start_span(self, name, **kwargs):
|
||||
span = _FakeSpan(name, **kwargs)
|
||||
self.spans.append(span)
|
||||
return span
|
||||
|
||||
|
||||
def _fake_otel():
|
||||
tracer = _FakeTracer()
|
||||
trace = types.ModuleType("opentelemetry.trace")
|
||||
trace.get_tracer = lambda _name: tracer
|
||||
trace.set_span_in_context = lambda span: ("ctx", span)
|
||||
trace.Link = lambda ctx: ("link", ctx)
|
||||
package = types.ModuleType("opentelemetry")
|
||||
package.trace = trace
|
||||
patcher = mock.patch.dict(
|
||||
sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace}
|
||||
)
|
||||
return tracer, patcher
|
||||
|
||||
|
||||
def _state(**overrides):
|
||||
state = {
|
||||
"waiting": 0,
|
||||
"result_queue": 0,
|
||||
"result_limit": 2,
|
||||
"inflight": 0,
|
||||
"bootstrap": 0,
|
||||
"deferred": 0,
|
||||
"paused": False,
|
||||
"waiting_for_result": False,
|
||||
"skip_batch_creation": False,
|
||||
"forward_active": False,
|
||||
}
|
||||
state.update(overrides)
|
||||
return state
|
||||
|
||||
|
||||
def _flush_spans(tracer):
|
||||
return [s for s in tracer.spans if s.name == "scheduler.cache_flush_backup"]
|
||||
|
||||
|
||||
class BackupFlushDetailTest(unittest.TestCase):
|
||||
def test_disabled_records_nothing(self):
|
||||
mod = _load(SGLANG_TRACE_BACKUP_DETAIL="0")
|
||||
self.assertFalse(mod.backup_detail_enabled())
|
||||
self.assertIsNone(mod.backup_flush_begin())
|
||||
self.assertIsNone(mod.flush_mark())
|
||||
mod.flush_note("sglang.flush_tokens", 1)
|
||||
mod.flush_count("launch_pools")
|
||||
with mod.backup_phase("merge_ops"):
|
||||
pass
|
||||
# set_flush_origin is a no-op too.
|
||||
self.assertIsNone(mod.set_flush_origin("process_result"))
|
||||
self.assertEqual(mod.current_flush_origin(), "other")
|
||||
|
||||
def test_requires_compiled_detail(self):
|
||||
mod = _load(SGLANG_TRACE_LOOP_DETAIL="0")
|
||||
self.assertFalse(mod.backup_detail_enabled())
|
||||
self.assertIsNone(mod.backup_flush_begin())
|
||||
|
||||
def test_unsampled_loop_creates_no_tracer(self):
|
||||
mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0")
|
||||
mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
self.assertTrue(mod.backup_detail_enabled())
|
||||
self.assertIsNone(mod.backup_flush_begin())
|
||||
|
||||
def test_phases_attrs_and_counts_land_on_flush_span(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
flush = mod.backup_flush_begin()
|
||||
self.assertIsNotNone(flush)
|
||||
mod.flush_note("sglang.flush_tokens", 4096)
|
||||
with mod.backup_phase("merge_ops"):
|
||||
pass
|
||||
t0 = mod.flush_mark()
|
||||
mod.flush_phase_since("indices_cpu", t0)
|
||||
mod.flush_count("launch_pools", 3)
|
||||
mod.backup_flush_end(flush)
|
||||
mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state())
|
||||
|
||||
flush_span = _flush_spans(tracer)[0]
|
||||
attrs = flush_span.attributes
|
||||
self.assertEqual(attrs["sglang.flush_origin"], "other")
|
||||
self.assertEqual(attrs["sglang.flush_tokens"], 4096)
|
||||
self.assertEqual(attrs["sglang.flush_launch_pools"], 3)
|
||||
self.assertIn("sglang.flush_merge_ops_ms", attrs)
|
||||
self.assertIn("sglang.flush_indices_cpu_ms", attrs)
|
||||
self.assertGreaterEqual(attrs["sglang.flush_total_ms"], 0.0)
|
||||
self.assertEqual(flush_span.kwargs["context"], lt.span_ctx)
|
||||
|
||||
# Sub-phases must not leak into the loop aggregate: the loop detail is
|
||||
# used for the residual identity and must stay free of nested timing.
|
||||
loop_span = next(s for s in tracer.spans if s.name == "scheduler.loop")
|
||||
self.assertNotIn("sglang.flush_merge_ops_ms", loop_span.attributes)
|
||||
|
||||
def test_origin_tag_is_scoped(self):
|
||||
mod = _load()
|
||||
mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
previous = mod.set_flush_origin("process_result")
|
||||
flush = mod.backup_flush_begin()
|
||||
mod.backup_flush_end(flush)
|
||||
mod.set_flush_origin(previous)
|
||||
self.assertEqual(flush.origin, "process_result")
|
||||
self.assertEqual(mod.current_flush_origin(), "other")
|
||||
|
||||
def test_nested_tracer_is_restored(self):
|
||||
mod = _load()
|
||||
mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
outer = mod.backup_flush_begin()
|
||||
inner = mod.backup_flush_begin()
|
||||
mod.backup_flush_end(inner)
|
||||
self.assertIs(mod.current_flush_tracer(), outer)
|
||||
mod.backup_flush_end(outer)
|
||||
self.assertIsNone(mod.current_flush_tracer())
|
||||
|
||||
def test_outer_process_phases_are_subtracted_from_residual(self):
|
||||
mod = _load()
|
||||
mod._ensure_loop_metrics = lambda: _fake_loop_metrics()
|
||||
tracer, patcher = _fake_otel()
|
||||
with patcher:
|
||||
lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False)
|
||||
t0 = time.monotonic_ns()
|
||||
mod.detail_add("flush_trace", t0, t0 + 5_000_000)
|
||||
mod.detail_add("load_snapshot", t0, t0 + 5_000_000)
|
||||
mod.detail_add("flush_backups", t0, t0 + 20_000_000)
|
||||
mod.detail_add("step_counters", t0, t0 + 5_000_000)
|
||||
mod.detail_add("metrics_report", t0, t0 + 5_000_000)
|
||||
mod.detail_add("per_request_result", t0, t0 + 10_000_000)
|
||||
start = time.time_ns()
|
||||
mod.mark_process(start, start + 100_000_000, None)
|
||||
mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state())
|
||||
|
||||
attrs = next(
|
||||
s for s in tracer.spans if s.name == "scheduler.loop"
|
||||
).attributes
|
||||
self.assertAlmostEqual(attrs["sglang.flush_backups_ms"], 20.0)
|
||||
# 100 ms - (5+5+20+5+5+10) ms of measured phases.
|
||||
self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 50.0, delta=1.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user