diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 256ae63311..208aeb4851 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -43,6 +43,7 @@ from sglang.srt.layers.dp_attention import ( from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.mem_cache.utils import get_storage_hash_str +from sglang.srt.observability import batch_telemetry from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_device_module @@ -890,40 +891,104 @@ class HiCacheController: def start_writing(self) -> None: if len(self.write_queue) == 0: + # Empty flushes are the common case when write-through fires every + # batch: counted so a small `flush_backups` cannot be explained as + # "one big flush per loop" without noticing the empty ones. + batch_telemetry.detail_count("flush_calls") + batch_telemetry.detail_count("flush_empty") return - op = CacheOperation.merge_ops(self.write_queue) - host_indices, device_indices, pool_transfers = self._move_write_operation(op) - self.write_queue.clear() - - if self.write_fence_stream is not None: - # The copy may read KV a forward is still writing: under overlap - # scheduling a finished request is cached while the next forward - # writes its last token, and an evicted SWA page can belong to an - # unfinished checkpoint. Later forwards still overlap with the copy. - self.l2_transfer_engine.device_to_host_stream.wait_stream( - self.write_fence_stream + tracer = batch_telemetry.backup_flush_begin() + try: + batch_telemetry.detail_count("flush_calls") + batch_telemetry.detail_count("flush_nonempty") + batch_telemetry.flush_note( + "sglang.flush_write_queue_ops", len(self.write_queue) + ) + batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend) + batch_telemetry.flush_note( + "sglang.flush_layout", getattr(self.mem_pool_host, "layout", None) + ) + batch_telemetry.flush_note( + "sglang.flush_ack_queue_before", len(self.ack_write_queue) ) - completion = self.l2_transfer_engine.submit_device_to_host( - self._l2_transfers(host_indices, device_indices, pool_transfers) - ) - - self.mem_pool_device_allocator.set_hicache_transfer_done_event( - (id(self), "write"), completion.finish_event - ) - - self.ack_write_queue.append( - HiCacheAck( - start_event=completion.start_event, - finish_event=completion.finish_event, - node_ids=op.node_ids, - num_tokens=len(op.device_indices), - timing_enabled=completion.timing_enabled, - num_tokens_by_pool=self._num_tokens_by_pool(op), - num_bytes=self._transfer_num_bytes(op), + with batch_telemetry.backup_phase("merge_ops"): + op = CacheOperation.merge_ops(self.write_queue) + tokens = len(op.device_indices) + indices = op.device_indices + batch_telemetry.flush_note("sglang.flush_tokens", int(tokens)) + batch_telemetry.flush_note( + "sglang.flush_nodes_merged", len(op.node_ids) ) - ) + batch_telemetry.flush_note( + "sglang.flush_sidecar_pools", len(op.pool_transfers or []) + ) + batch_telemetry.flush_note( + "sglang.flush_indices_device", str(indices.device.type) + ) + batch_telemetry.flush_note( + "sglang.flush_indices_dtype", str(indices.dtype) + ) + batch_telemetry.flush_note( + "sglang.flush_indices_numel", int(indices.numel()) + ) + + with batch_telemetry.backup_phase("move_write"): + host_indices, device_indices, pool_transfers = ( + self._move_write_operation(op) + ) + self.write_queue.clear() + batch_telemetry.flush_note( + "sglang.flush_host_indices_on_cpu", + bool(getattr(host_indices, "is_cuda", False) is False), + ) + batch_telemetry.flush_note( + "sglang.flush_device_indices_on_cpu", + bool(getattr(device_indices, "is_cuda", False) is False), + ) + + if self.write_fence_stream is not None: + # The copy may read KV a forward is still writing: under overlap + # scheduling a finished request is cached while the next forward + # writes its last token, and an evicted SWA page can belong to an + # unfinished checkpoint. Later forwards still overlap with the copy. + with batch_telemetry.backup_phase("write_fence"): + self.l2_transfer_engine.device_to_host_stream.wait_stream( + self.write_fence_stream + ) + + with batch_telemetry.backup_phase("build_l2"): + transfers = self._l2_transfers( + host_indices, device_indices, pool_transfers + ) + batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers)) + + with batch_telemetry.backup_phase("submit_d2h"): + completion = self.l2_transfer_engine.submit_device_to_host(transfers) + + with batch_telemetry.backup_phase("set_done_event"): + self.mem_pool_device_allocator.set_hicache_transfer_done_event( + (id(self), "write"), completion.finish_event + ) + + with batch_telemetry.backup_phase("ack_enqueue"): + self.ack_write_queue.append( + HiCacheAck( + start_event=completion.start_event, + finish_event=completion.finish_event, + node_ids=op.node_ids, + num_tokens=len(op.device_indices), + timing_enabled=completion.timing_enabled, + num_tokens_by_pool=self._num_tokens_by_pool(op), + num_bytes=self._transfer_num_bytes(op), + ) + ) + batch_telemetry.flush_note( + "sglang.flush_ack_queue_after", len(self.ack_write_queue) + ) + finally: + batch_telemetry.backup_flush_end(tracer) def _transfer_num_bytes(self, op: CacheOperation) -> int: return len(op.device_indices) * self.mem_pool_host.size_per_token @@ -954,15 +1019,24 @@ class HiCacheController: # move indices to GPU if using kernels, to host if using direct indexing if self.io_backend == "kernel": if not host_indices.is_cuda: - host_indices = host_indices.to(self.device, non_blocking=True) + with batch_telemetry.backup_phase("indices_h2d"): + host_indices = host_indices.to(self.device, non_blocking=True) return host_indices, device_indices elif self.io_backend == "direct": if self.mem_pool_host.layout == "layer_first": - device_indices = device_indices.cpu() - host_indices, idx = host_indices.sort() - return host_indices, device_indices.index_select(0, idx) + with batch_telemetry.backup_phase("indices_cpu"): + device_indices = device_indices.cpu() + with batch_telemetry.backup_phase("index_sort"): + host_indices, idx = host_indices.sort() + with batch_telemetry.backup_phase("index_select"): + return host_indices, device_indices.index_select(0, idx) elif self.mem_pool_host.layout == "page_first_direct": - return host_indices, device_indices.cpu() + # Synchronous D2H of the token indices: the host wall time of + # this call is the scheduler blocking on the copy, not a + # device bandwidth measurement. + with batch_telemetry.backup_phase("indices_cpu"): + device_indices = device_indices.cpu() + return host_indices, device_indices else: raise ValueError( f"Unsupported layout {self.mem_pool_host.layout!r} for io backend 'direct'" diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index a317abc1ad..47eaefd6f0 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5079,15 +5079,19 @@ class Scheduler( ): # Flush async trace ops here: in overlap mode this CPU work runs while # the next batch's GPU forward is in flight, giving free overlap. - flush_trace_batch(batch.reqs) - snapshot = self.publish_load_snapshot(force=batch.forward_mode.is_extend()) - # Router-facing gauge on the dedicated PUB socket, reusing the - # snapshot above rather than walking the queues again. - self.load_publisher.publish_load_stat( - self.load_inquirer.get_loads, - force=batch.forward_mode.is_extend(), - snapshot=snapshot, - ) + with batch_telemetry.detail_phase("flush_trace"): + flush_trace_batch(batch.reqs) + with batch_telemetry.detail_phase("load_snapshot"): + snapshot = self.publish_load_snapshot( + force=batch.forward_mode.is_extend() + ) + # Router-facing gauge on the dedicated PUB socket, reusing the + # snapshot above rather than walking the queues again. + self.load_publisher.publish_load_stat( + self.load_inquirer.get_loads, + force=batch.forward_mode.is_extend(), + snapshot=snapshot, + ) if batch.forward_mode.is_decode(): self.batch_result_processor.process_batch_result_decode(batch, result) @@ -5104,19 +5108,28 @@ class Scheduler( self.batch_result_processor.process_batch_result_idle(batch, result) # Submit this batch's queued host backups before the next scheduler step. - self.tree_cache.flush_pending_backups() + # Tag the origin so the per-flush micro timer can separate these from + # the ones issued by check_hicache_events / write(). + _flush_origin = batch_telemetry.set_flush_origin("process_result") + try: + with batch_telemetry.detail_phase("flush_backups"): + self.tree_cache.flush_pending_backups() + finally: + batch_telemetry.set_flush_origin(_flush_origin) - self._record_step_counters(batch, result) + with batch_telemetry.detail_phase("step_counters"): + self._record_step_counters(batch, result) - self.metrics_reporter.log_batch_result_stats(batch, result) + with batch_telemetry.detail_phase("metrics_report"): + self.metrics_reporter.log_batch_result_stats(batch, result) - # Emit forward pass metrics (every iteration when enabled) - if self.enable_fpm: - self.metrics_reporter._emit_forward_pass_metrics(batch, result) + # Emit forward pass metrics (every iteration when enabled) + if self.enable_fpm: + self.metrics_reporter._emit_forward_pass_metrics(batch, result) - self._maybe_clear_mm_inputs(batch) - self.maybe_send_health_check_signal() - self.metrics_reporter.update_device_timer() + self._maybe_clear_mm_inputs(batch) + self.maybe_send_health_check_signal() + self.metrics_reporter.update_device_timer() def _record_step_counters( self, batch: ScheduleBatch, result: GenerationBatchResult diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index dde54ef69d..e74880af6e 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -41,6 +41,7 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.observability import batch_telemetry if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator @@ -1509,26 +1510,30 @@ class HybridCacheController(BaseHiCacheController): self, host_pool, host_indices, device_indices, *, write_back_jit: bool ) -> tuple[torch.Tensor, torch.Tensor]: if uses_shared_host_layout(host_pool): + batch_telemetry.flush_count("shared_layout_pools") return host_indices, device_indices if write_back_jit and host_pool.can_use_write_back_jit: if host_indices.is_cuda: - host_indices = host_indices.cpu() + with batch_telemetry.backup_phase("host_indices_cpu"): + host_indices = host_indices.cpu() return host_indices, device_indices return self.move_indices(host_indices, device_indices) def move_hybrid_indices( self, operation: CacheOperation, *, write_back_jit: bool = False ) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]: - host_indices, device_indices = self._move_pool_indices( - self.mem_pool_host.anchor_entry.host_pool, - operation.host_indices, - operation.device_indices, - write_back_jit=write_back_jit, - ) + with batch_telemetry.backup_phase("anchor_indices"): + host_indices, device_indices = self._move_pool_indices( + self.mem_pool_host.anchor_entry.host_pool, + operation.host_indices, + operation.device_indices, + write_back_jit=write_back_jit, + ) resolved_pool_transfers = None if operation.pool_transfers: resolved_pool_transfers = [] for transfer in operation.pool_transfers: + _pool_t0 = batch_telemetry.flush_mark() transfer_host_indices, transfer_device_indices = ( self._move_pool_indices( self.mem_pool_host.entry_map[transfer.name].host_pool, @@ -1537,6 +1542,9 @@ class HybridCacheController(BaseHiCacheController): write_back_jit=write_back_jit, ) ) + batch_telemetry.flush_phase_since( + "sidecar_indices." + str(transfer.name.value), _pool_t0 + ) # Tree-owned transfers keep their original radix-tree indices. resolved_pool_transfers.append( replace( diff --git a/python/sglang/srt/mem_cache/l2_transfer.py b/python/sglang/srt/mem_cache/l2_transfer.py index 82f67bc86b..39e6d0a119 100644 --- a/python/sglang/srt/mem_cache/l2_transfer.py +++ b/python/sglang/srt/mem_cache/l2_transfer.py @@ -9,6 +9,7 @@ import torch from sglang.srt.environ import envs from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains +from sglang.srt.observability import batch_telemetry from sglang.srt.utils import get_device_module logger = logging.getLogger(__name__) @@ -71,10 +72,13 @@ class L2TransferEngine: original_device = transfer.device_indices.device # The direct backend supplies CPU indices even for a CUDA pool. # Translate alongside the v2p table, then restore the backend's device. + _t0 = batch_telemetry.flush_mark() indices = transfer.device_indices.to( getattr(transfer.device_pool, "device", original_device) ).contiguous() + batch_telemetry.flush_phase_since("resolve_indices.to_device", _t0) dcp_size = getattr(transfer.host_pool, "dcp_size", 1) + _t0 = batch_telemetry.flush_mark() if dcp_size > 1: # The MLA host pool selects this rank and collapses logical IDs. # Translate in local virtual space, then preserve that widened @@ -82,16 +86,22 @@ class L2TransferEngine: resolved = translate(indices // dcp_size) * dcp_size + indices % dcp_size else: resolved = translate(indices) - return resolved.to(original_device) + batch_telemetry.flush_phase_since("resolve_indices.translate", _t0) + _t0 = batch_telemetry.flush_mark() + result = resolved.to(original_device) + batch_telemetry.flush_phase_since("resolve_indices.back_to_host", _t0) + return result def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]: prepared = [] for transfer in transfers: + _t0 = batch_telemetry.flush_mark() host_indices, device_indices = transfer.host_pool.prepare_transfer_indices( transfer.host_indices, self._resolve_device_indices(transfer), self.io_backend, ) + batch_telemetry.flush_phase_since("prepare_transfer_indices", _t0) prepared.append( transfer._replace( host_indices=host_indices, @@ -102,44 +112,55 @@ class L2TransferEngine: @contextmanager def _submission(self, transfers, stream, transfer_key, start_event=None): + _t0 = batch_telemetry.flush_mark() start_event = self._start_event(start_event) ack_start, ack_finish, timing_enabled = make_timing_event_pair() completion = TransferCompletion(ack_start, ack_finish, timing_enabled) domains = shared_host_layout_domains(t.host_pool for t in transfers) for domain in domains: domain.acquire_layout() + batch_telemetry.flush_phase_since("submit_setup", _t0) finish_recorded = False try: with device_module.stream(stream): # Index preparation can read producer-owned device indices. - start_event.wait(stream) - prepared = self._prepare_transfers(transfers) - ack_start.record() + with batch_telemetry.backup_phase("start_event_wait"): + start_event.wait(stream) + with batch_telemetry.backup_phase("prepare_transfers"): + prepared = self._prepare_transfers(transfers) + with batch_telemetry.backup_phase("ack_events"): + ack_start.record() yield prepared, completion - ack_finish.record() - finish_recorded = True - self._record_stream(transfers + prepared, stream) + with batch_telemetry.backup_phase("ack_events"): + ack_finish.record() + finish_recorded = True + self._record_stream(transfers + prepared, stream) except Exception: stream.synchronize() raise finally: + _t0 = batch_telemetry.flush_mark() for domain in reversed(domains): domain.release_layout( ack_finish if finish_recorded else None, (id(self), transfer_key), ) + batch_telemetry.flush_phase_since("layout_release", _t0) def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion: with self._submission( transfers, self.device_to_host_stream, "device_to_host" ) as (transfers, completion): for transfer in transfers: + _t0 = batch_telemetry.flush_mark() transfer.host_pool.backup_from_device_all_layer_physical( transfer.device_pool, transfer.host_indices, transfer.device_indices, self.io_backend, ) + batch_telemetry.flush_phase_since("launch_pool", _t0) + batch_telemetry.flush_count("launch_pools") return completion def submit_host_to_device( diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index 9f3f148851..9caad04834 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -26,6 +26,9 @@ Enable with ``SGLANG_BATCH_TELEMETRY=1``. Knobs: loops are recorded) SGLANG_TRACE_MOONCAKE_DETAIL=1 per-chunk transfer worker breakdown (queue/source/indices/transfer/join) + SGLANG_TRACE_BACKUP_DETAIL=1 per-flush HiCache backup sub-phases + (one ``scheduler.cache_flush_backup`` span; + requires SGLANG_TRACE_LOOP_DETAIL=1) Cost model (measured on our 4xDP4 / 4-GPU instances): * histograms: ~1-2 us per batch; P does ~17 batches/s total -> <0.01% CPU. @@ -1355,6 +1358,224 @@ class detail_phase: return False +# --------------------------------------------------------------------------- +# HiCache flush micro timing (SGLANG_TRACE_BACKUP_DETAIL=1). +# +# One tracer per ``cache_controller.start_writing()`` call (i.e. per +# ``flush_pending_backups``). Sub-phases are host-wall timings on the caller +# thread; a synchronous ``.cpu()`` therefore shows up as a long host phase and +# must not be read as device-transfer time. The tracer keeps per-flush +# resolution by emitting ONE child span of the current loop +# (``scheduler.cache_flush_backup``), because the loop-level detail only keeps +# per-loop sums (design: handoff/prefill-flush-backups-validation-20261107.md). +# +# Disabled by default: every entry point is a single module-level bool check, +# and when off no object, dict or span is created. +# --------------------------------------------------------------------------- + +_BACKUP_DETAIL_ENABLED = _DETAIL_ENABLED and os.getenv( + "SGLANG_TRACE_BACKUP_DETAIL", "0" +).strip().lower() not in ("", "0", "false", "no", "off") + +_FLUSH_TLS = threading.local() + + +def backup_detail_enabled() -> bool: + """True when per-flush micro timing is compiled in (``*_DETAIL`` is on).""" + return _BACKUP_DETAIL_ENABLED + + +def set_flush_origin(origin: Optional[str]) -> Optional[str]: + """Tag the next flushes with the caller's scheduler context. + + Only used to tell ``process_batch_result`` flushes (the current suspect) + from the ones issued by ``check_hicache_events`` / ``write``. Returns the + previous value so the caller can restore it. + """ + if not _BACKUP_DETAIL_ENABLED: + return None + previous = getattr(_FLUSH_TLS, "origin", None) + _FLUSH_TLS.origin = origin + return previous + + +def current_flush_origin() -> str: + return getattr(_FLUSH_TLS, "origin", None) or "other" + + +class BackupFlushTracer: + """Sub-phase timings for one flush; emits a single span on ``end``.""" + + __slots__ = ( + "origin", + "start_ns", + "phases", + "attrs", + "counts", + "_open", + "_prev", + ) + + def __init__(self, origin: str): + self.origin = origin + self.start_ns = time.monotonic_ns() + self.phases: dict[str, int] = {} + self.attrs: dict[str, Any] = {} + self.counts: dict[str, int] = {} + self._open: dict[str, int] = {} + self._prev = None + + # -- phase helpers ----------------------------------------------------- + def begin(self, name: str) -> int: + start = time.monotonic_ns() + self._open[name] = start + return start + + def end(self, name: str, start_ns: Optional[int] = None) -> None: + start = self._open.pop(name, None) + if start is None: + start = start_ns + if start is None: + return + self.add(name, start) + + def add(self, name: str, start_ns: int) -> None: + self.record(name, int(start_ns), time.monotonic_ns()) + + def record(self, name: str, start_ns: int, end_ns: int) -> None: + self.phases[name] = self.phases.get(name, 0) + max( + 0, int(end_ns) - int(start_ns) + ) + + def set(self, key: str, value: Any) -> None: + self.attrs[key] = value + + def inc(self, key: str, n: int = 1) -> None: + self.counts[key] = self.counts.get(key, 0) + int(n) + + def total_ns(self) -> int: + return max(0, time.monotonic_ns() - self.start_ns) + + # -- emission ---------------------------------------------------------- + def end_and_emit(self) -> None: + now_ns = time.monotonic_ns() + total_ns = max(0, now_ns - self.start_ns) + attrs = { + "sglang.flush_origin": self.origin, + "sglang.flush_total_ms": total_ns / 1e6, + "sglang.loop_seq": current_loop_seq(), + } + for name, ns in self.phases.items(): + attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6 + for name, value in self.counts.items(): + attrs["sglang.flush_" + name] = int(value) + attrs.update(self.attrs) + try: + from opentelemetry import trace as otel_trace + + ctx = current_loop_context_for_spans() + start_wall_ns = time.time_ns() - total_ns + span = otel_trace.get_tracer("sglang.cache.flush").start_span( + "scheduler.cache_flush_backup", + context=ctx, + start_time=int(start_wall_ns), + attributes=attrs, + ) + span.end(end_time=int(time.time_ns())) + except Exception: + # Telemetry must never break the cache controller. + pass + + +class backup_phase: + """Context manager for one sub-phase of the active flush (no-op when off).""" + + __slots__ = ("name", "tracer", "start_ns") + + def __init__(self, name: str): + self.name = name + self.tracer = None + self.start_ns = None + + def __enter__(self): + if not _BACKUP_DETAIL_ENABLED: + return self + self.tracer = getattr(_FLUSH_TLS, "tracer", None) + if self.tracer is not None: + self.start_ns = self.tracer.begin(self.name) + return self + + def __exit__(self, exc_type, exc, tb): + if self.tracer is not None: + self.tracer.end(self.name, self.start_ns) + return False + + +def backup_flush_begin(origin: Optional[str] = None) -> Optional[BackupFlushTracer]: + """Start tracing one flush; returns None when disabled / not sampled.""" + if not _BACKUP_DETAIL_ENABLED: + return None + if not loop_detail_enabled(): + return None + tracer = BackupFlushTracer(origin or current_flush_origin()) + tracer._prev = getattr(_FLUSH_TLS, "tracer", None) + _FLUSH_TLS.tracer = tracer + return tracer + + +def backup_flush_end(tracer: Optional[BackupFlushTracer]) -> None: + if tracer is None: + return + if getattr(_FLUSH_TLS, "tracer", None) is tracer: + _FLUSH_TLS.tracer = tracer._prev + try: + tracer.end_and_emit() + except Exception: + pass + + +def current_flush_tracer() -> Optional[BackupFlushTracer]: + if not _BACKUP_DETAIL_ENABLED: + return None + return getattr(_FLUSH_TLS, "tracer", None) + + +def flush_note(key: str, value: Any) -> None: + """Attach a dimension to the active flush (no-op when disabled).""" + if not _BACKUP_DETAIL_ENABLED: + return + tracer = getattr(_FLUSH_TLS, "tracer", None) + if tracer is not None: + tracer.set(key, value) + + +def flush_count(key: str, n: int = 1) -> None: + """Accumulate a counter on the active flush (no-op when disabled).""" + if not _BACKUP_DETAIL_ENABLED: + return + tracer = getattr(_FLUSH_TLS, "tracer", None) + if tracer is not None: + tracer.inc(key, n) + + +def flush_mark() -> Optional[int]: + """Timestamp for a manual phase (None when no flush is being traced).""" + if not _BACKUP_DETAIL_ENABLED: + return None + if getattr(_FLUSH_TLS, "tracer", None) is None: + return None + return time.monotonic_ns() + + +def flush_phase_since(name: str, start_ns: Optional[int]) -> None: + """Record ``name`` from ``start_ns`` to now (no-op when untraced).""" + if start_ns is None: + return + tracer = getattr(_FLUSH_TLS, "tracer", None) + if tracer is not None: + tracer.add(name, start_ns) + + def current_loop_seq() -> int: """Loop sequence for cross-record correlation (0 when unavailable).""" lt = getattr(_LOOP_TLS, "current", None) @@ -2054,6 +2275,13 @@ def _emit_loop_detail(lt: LoopTimings, attrs: dict, child, dp_rank: int) -> None if lt.process is not None: proc_ms = float(lt.process[2].get("sglang.process_result_ms", 0.0)) parts = _total( + # Outer phases of process_batch_result (scheduler.py). + "flush_trace", + "load_snapshot", + "flush_backups", + "step_counters", + "metrics_report", + # Inner disagg-prefill phases (prefill.py). "copy_done_wait", "auxiliary_and_logprob", "per_request_result", diff --git a/test/registered/unit/observability/test_backup_flush_detail.py b/test/registered/unit/observability/test_backup_flush_detail.py new file mode 100644 index 0000000000..8ee008be44 --- /dev/null +++ b/test/registered/unit/observability/test_backup_flush_detail.py @@ -0,0 +1,242 @@ +"""Unit tests for the per-flush HiCache backup micro timer. + +Design: handoff/prefill-flush-backups-validation-20261107.md §3-§5. + +Acceptance points: + + * ``SGLANG_TRACE_BACKUP_DETAIL`` is off by default and then records nothing; + * a tracer only exists for a sampled loop (same decision as the loop span); + * sub-phases stay on the flush span, never on the loop aggregate, so the + ``process_residual`` identity cannot double-count them; + * the new outer ``process_batch_result`` phases are subtracted from + ``process_residual``; + * the origin tag separates process-result flushes from other callers. +""" +import importlib.util +import os +import pathlib +import sys +import time +import types +import unittest +from unittest import mock + + +def _module_path(): + p = ( + pathlib.Path(__file__).resolve().parents[4] + / "python/sglang/srt/observability/batch_telemetry.py" + ) + if p.exists(): + return p + spec = importlib.util.find_spec("sglang.srt.observability.batch_telemetry") + return pathlib.Path(spec.origin) + + +def _load(**env): + base = { + "SGLANG_BATCH_STAGE_TIMING": "1", + "SGLANG_TRACE_LOOP_SPAN": "1", + "SGLANG_TRACE_LOOP_SAMPLE": "1.0", + "SGLANG_TRACE_LOOP_DETAIL": "1", + "SGLANG_TRACE_MOONCAKE_DETAIL": "0", + "SGLANG_TRACE_BACKUP_DETAIL": "1", + "SGLANG_TRACE_FORWARD_SPAN": "0", + } + base.update(env) + with mock.patch.dict(os.environ, base): + spec = importlib.util.spec_from_file_location( + "batch_telemetry_backup_under_test", _module_path() + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class _FakeCounter: + def __init__(self): + self.data = {} + + def labels(self, **kw): + self._key = tuple(sorted(kw.items())) + return self + + def inc(self, value=1.0): + self.data[self._key] = self.data.get(self._key, 0.0) + value + + +def _fake_loop_metrics(): + return { + "no_batch_seconds": _FakeCounter(), + "no_batch_total": _FakeCounter(), + "lockstep_seconds": _FakeCounter(), + } + + +class _FakeSpan: + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs + self.attributes = dict(kwargs.get("attributes") or {}) + self.end_time = None + self.context = types.SimpleNamespace(is_valid=True, trace_id=1, span_id=2) + + def get_span_context(self): + return self.context + + def set_attributes(self, attrs): + self.attributes.update(attrs) + + def end(self, end_time=None): + self.end_time = end_time + + +class _FakeTracer: + def __init__(self): + self.spans = [] + + def start_span(self, name, **kwargs): + span = _FakeSpan(name, **kwargs) + self.spans.append(span) + return span + + +def _fake_otel(): + tracer = _FakeTracer() + trace = types.ModuleType("opentelemetry.trace") + trace.get_tracer = lambda _name: tracer + trace.set_span_in_context = lambda span: ("ctx", span) + trace.Link = lambda ctx: ("link", ctx) + package = types.ModuleType("opentelemetry") + package.trace = trace + patcher = mock.patch.dict( + sys.modules, {"opentelemetry": package, "opentelemetry.trace": trace} + ) + return tracer, patcher + + +def _state(**overrides): + state = { + "waiting": 0, + "result_queue": 0, + "result_limit": 2, + "inflight": 0, + "bootstrap": 0, + "deferred": 0, + "paused": False, + "waiting_for_result": False, + "skip_batch_creation": False, + "forward_active": False, + } + state.update(overrides) + return state + + +def _flush_spans(tracer): + return [s for s in tracer.spans if s.name == "scheduler.cache_flush_backup"] + + +class BackupFlushDetailTest(unittest.TestCase): + def test_disabled_records_nothing(self): + mod = _load(SGLANG_TRACE_BACKUP_DETAIL="0") + self.assertFalse(mod.backup_detail_enabled()) + self.assertIsNone(mod.backup_flush_begin()) + self.assertIsNone(mod.flush_mark()) + mod.flush_note("sglang.flush_tokens", 1) + mod.flush_count("launch_pools") + with mod.backup_phase("merge_ops"): + pass + # set_flush_origin is a no-op too. + self.assertIsNone(mod.set_flush_origin("process_result")) + self.assertEqual(mod.current_flush_origin(), "other") + + def test_requires_compiled_detail(self): + mod = _load(SGLANG_TRACE_LOOP_DETAIL="0") + self.assertFalse(mod.backup_detail_enabled()) + self.assertIsNone(mod.backup_flush_begin()) + + def test_unsampled_loop_creates_no_tracer(self): + mod = _load(SGLANG_TRACE_LOOP_SAMPLE="0.0") + mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + self.assertTrue(mod.backup_detail_enabled()) + self.assertIsNone(mod.backup_flush_begin()) + + def test_phases_attrs_and_counts_land_on_flush_span(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + flush = mod.backup_flush_begin() + self.assertIsNotNone(flush) + mod.flush_note("sglang.flush_tokens", 4096) + with mod.backup_phase("merge_ops"): + pass + t0 = mod.flush_mark() + mod.flush_phase_since("indices_cpu", t0) + mod.flush_count("launch_pools", 3) + mod.backup_flush_end(flush) + mod.end_loop(lt, dp_rank=1, has_batch=True, state=_state()) + + flush_span = _flush_spans(tracer)[0] + attrs = flush_span.attributes + self.assertEqual(attrs["sglang.flush_origin"], "other") + self.assertEqual(attrs["sglang.flush_tokens"], 4096) + self.assertEqual(attrs["sglang.flush_launch_pools"], 3) + self.assertIn("sglang.flush_merge_ops_ms", attrs) + self.assertIn("sglang.flush_indices_cpu_ms", attrs) + self.assertGreaterEqual(attrs["sglang.flush_total_ms"], 0.0) + self.assertEqual(flush_span.kwargs["context"], lt.span_ctx) + + # Sub-phases must not leak into the loop aggregate: the loop detail is + # used for the residual identity and must stay free of nested timing. + loop_span = next(s for s in tracer.spans if s.name == "scheduler.loop") + self.assertNotIn("sglang.flush_merge_ops_ms", loop_span.attributes) + + def test_origin_tag_is_scoped(self): + mod = _load() + mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + previous = mod.set_flush_origin("process_result") + flush = mod.backup_flush_begin() + mod.backup_flush_end(flush) + mod.set_flush_origin(previous) + self.assertEqual(flush.origin, "process_result") + self.assertEqual(mod.current_flush_origin(), "other") + + def test_nested_tracer_is_restored(self): + mod = _load() + mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + outer = mod.backup_flush_begin() + inner = mod.backup_flush_begin() + mod.backup_flush_end(inner) + self.assertIs(mod.current_flush_tracer(), outer) + mod.backup_flush_end(outer) + self.assertIsNone(mod.current_flush_tracer()) + + def test_outer_process_phases_are_subtracted_from_residual(self): + mod = _load() + mod._ensure_loop_metrics = lambda: _fake_loop_metrics() + tracer, patcher = _fake_otel() + with patcher: + lt = mod.begin_loop("prefill", pre_has_work=True, forward_active=False) + t0 = time.monotonic_ns() + mod.detail_add("flush_trace", t0, t0 + 5_000_000) + mod.detail_add("load_snapshot", t0, t0 + 5_000_000) + mod.detail_add("flush_backups", t0, t0 + 20_000_000) + mod.detail_add("step_counters", t0, t0 + 5_000_000) + mod.detail_add("metrics_report", t0, t0 + 5_000_000) + mod.detail_add("per_request_result", t0, t0 + 10_000_000) + start = time.time_ns() + mod.mark_process(start, start + 100_000_000, None) + mod.end_loop(lt, dp_rank=0, has_batch=True, state=_state()) + + attrs = next( + s for s in tracer.spans if s.name == "scheduler.loop" + ).attributes + self.assertAlmostEqual(attrs["sglang.flush_backups_ms"], 20.0) + # 100 ms - (5+5+20+5+5+10) ms of measured phases. + self.assertAlmostEqual(attrs["sglang.process_residual_ms"], 50.0, delta=1.0) + + +if __name__ == "__main__": + unittest.main()