Files
sglang/python/sglang/srt/mem_cache/l2_transfer.py
T
minke.yu aa53670fcd observability: split flush_pending_backups host chain (off by default)
Adds the phase-3/phase-4 instrumentation from
handoff/prefill-flush-backups-validation-20261107.md:

* process_batch_result outer phases: flush_trace, load_snapshot,
  flush_backups, step_counters, metrics_report. process_residual now
  subtracts them, so the 37.5% residual is attributable.
* SGLANG_TRACE_BACKUP_DETAIL=1 (default 0) traces one flush:
  merge_ops / move_write (anchor + per-sidecar indices) / indices_cpu /
  index_sort / index_select / write_fence / build_l2 / submit_d2h
  (start-event wait, prepare, per-pool launch, ack events) / ack_enqueue,
  plus token/op/pool/device dimensions and ack queue depth.
* Per-flush resolution: one scheduler.cache_flush_backup span per
  non-empty flush, child of the loop span. Sub-phases never enter the
  loop aggregate, so the residual identity cannot double count.
* Empty vs non-empty flushes are counted on the loop.

All timings are scheduler host wall, not device transfer time.
2026-11-07 09:23:44 +08:00

225 lines
8.9 KiB
Python

from __future__ import annotations
import logging
from contextlib import contextmanager
from functools import cache
from typing import Any, Callable, NamedTuple, Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
from sglang.srt.observability import batch_telemetry
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
device_module = get_device_module()
@cache
def _timing_events_supported() -> bool:
try:
device_module.Event(enable_timing=True)
return True
except (TypeError, NotImplementedError):
logger.warning(
"%s.Event does not support timing; L2 transfer timing is disabled",
device_module.__name__,
)
return False
def make_timing_event_pair():
timing_enabled = _timing_events_supported()
kwargs = {"enable_timing": True} if timing_enabled else {}
return device_module.Event(**kwargs), device_module.Event(**kwargs), timing_enabled
class L2Transfer(NamedTuple):
host_pool: Any
device_pool: Any
host_indices: torch.Tensor
device_indices: torch.Tensor
layer_mapper: Optional[Callable[[int], Optional[int]]] = None
is_draft: bool = False
class TransferCompletion(NamedTuple):
start_event: Any
finish_event: Any
timing_enabled: bool
class L2TransferEngine:
"""Runs resolved device↔host transfers without owning cache state."""
def __init__(self, io_backend: str):
self.io_backend = io_backend
self.device_to_host_stream = device_module.Stream()
self.host_to_device_stream = device_module.Stream()
@staticmethod
def _resolve_device_indices(transfer: L2Transfer) -> torch.Tensor:
"""Resolve controller IDs into this pool's device-buffer indices.
Call on the transfer stream after the producer event. The host-transfer
move gate prevents relocation until the transfer completes.
"""
# Mamba state pools do not inherit KVCache's default attributes.
translate = getattr(transfer.device_pool, "host_transfer_translate", None)
if translate is None:
return transfer.device_indices
original_device = transfer.device_indices.device
# The direct backend supplies CPU indices even for a CUDA pool.
# Translate alongside the v2p table, then restore the backend's device.
_t0 = batch_telemetry.flush_mark()
indices = transfer.device_indices.to(
getattr(transfer.device_pool, "device", original_device)
).contiguous()
batch_telemetry.flush_phase_since("resolve_indices.to_device", _t0)
dcp_size = getattr(transfer.host_pool, "dcp_size", 1)
_t0 = batch_telemetry.flush_mark()
if dcp_size > 1:
# The MLA host pool selects this rank and collapses logical IDs.
# Translate in local virtual space, then preserve that widened
# interface (including token order) for the host pool.
resolved = translate(indices // dcp_size) * dcp_size + indices % dcp_size
else:
resolved = translate(indices)
batch_telemetry.flush_phase_since("resolve_indices.translate", _t0)
_t0 = batch_telemetry.flush_mark()
result = resolved.to(original_device)
batch_telemetry.flush_phase_since("resolve_indices.back_to_host", _t0)
return result
def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]:
prepared = []
for transfer in transfers:
_t0 = batch_telemetry.flush_mark()
host_indices, device_indices = transfer.host_pool.prepare_transfer_indices(
transfer.host_indices,
self._resolve_device_indices(transfer),
self.io_backend,
)
batch_telemetry.flush_phase_since("prepare_transfer_indices", _t0)
prepared.append(
transfer._replace(
host_indices=host_indices,
device_indices=device_indices,
)
)
return prepared
@contextmanager
def _submission(self, transfers, stream, transfer_key, start_event=None):
_t0 = batch_telemetry.flush_mark()
start_event = self._start_event(start_event)
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
completion = TransferCompletion(ack_start, ack_finish, timing_enabled)
domains = shared_host_layout_domains(t.host_pool for t in transfers)
for domain in domains:
domain.acquire_layout()
batch_telemetry.flush_phase_since("submit_setup", _t0)
finish_recorded = False
try:
with device_module.stream(stream):
# Index preparation can read producer-owned device indices.
with batch_telemetry.backup_phase("start_event_wait"):
start_event.wait(stream)
with batch_telemetry.backup_phase("prepare_transfers"):
prepared = self._prepare_transfers(transfers)
with batch_telemetry.backup_phase("ack_events"):
ack_start.record()
yield prepared, completion
with batch_telemetry.backup_phase("ack_events"):
ack_finish.record()
finish_recorded = True
self._record_stream(transfers + prepared, stream)
except Exception:
stream.synchronize()
raise
finally:
_t0 = batch_telemetry.flush_mark()
for domain in reversed(domains):
domain.release_layout(
ack_finish if finish_recorded else None,
(id(self), transfer_key),
)
batch_telemetry.flush_phase_since("layout_release", _t0)
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
with self._submission(
transfers, self.device_to_host_stream, "device_to_host"
) as (transfers, completion):
for transfer in transfers:
_t0 = batch_telemetry.flush_mark()
transfer.host_pool.backup_from_device_all_layer_physical(
transfer.device_pool,
transfer.host_indices,
transfer.device_indices,
self.io_backend,
)
batch_telemetry.flush_phase_since("launch_pool", _t0)
batch_telemetry.flush_count("launch_pools")
return completion
def submit_host_to_device(
self,
transfers: list[L2Transfer],
*,
transfer_layer_id_max: int,
start_event=None,
on_layer_done=None,
) -> TransferCompletion:
with self._submission(
transfers, self.host_to_device_stream, "host_to_device", start_event
) as (transfers, completion):
primary = transfers[0] if transfers else None
delay_cycles = envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.get()
for layer_id in range(transfer_layer_id_max):
if delay_cycles:
torch.cuda._sleep(delay_cycles)
for transfer in transfers:
local_layer_id = (
transfer.layer_mapper(layer_id)
if transfer.layer_mapper is not None
else layer_id
)
if local_layer_id is None or (
transfer is not primary
and transfer.layer_mapper is None
and layer_id >= transfer.host_pool.layer_num
):
continue
transfer.host_pool.load_to_device_per_layer_physical(
transfer.device_pool,
transfer.host_indices,
transfer.device_indices,
local_layer_id,
self.io_backend,
is_draft=transfer.is_draft,
)
if on_layer_done is not None:
on_layer_done(layer_id)
return completion
@staticmethod
def _start_event(start_event):
if start_event is None:
start_event = device_module.Event()
start_event.record()
return start_event
@staticmethod
def _record_stream(transfers: list[L2Transfer], stream, resolved=()) -> None:
tensors = []
for transfer in transfers:
tensors.extend((transfer.host_indices, transfer.device_indices))
# Keep temporary translated indices alive until the transfer completes.
tensors.extend(resolved)
for indices in tensors:
if indices is not None and (
indices.is_cuda or indices.device.type == "xpu"
):
indices.record_stream(stream)