observability: GPU-window probe + forward/prefill sub-phase timing

Phase 1/3 of the async-backup handoff instrumentation plan:

* new fwd_timeline probe (SGLANG_FWD_TIMELINE=1): one CUDA event pair around
  the model execute call per forward, resolved without blocking and written as
  wall-clock JSONL per DP rank. Joined with loop spans it separates host time
  spent while the GPU is busy (overlapped) from GPU-idle host stalls.
* detail phases: fwd_init_new, fwd_prepare, fwd_execute, fwd_graph_load,
  fwd_graph_replay, fwd_post, fwd_runner_tail,
  prefill_policy, prefill_adder_init, prefill_scan, prefill_batch_init.

All spans ride the existing SGLANG_TRACE_LOOP_DETAIL path; the GPU probe is
off by default.
This commit is contained in:
2026-11-08 01:19:09 +08:00
parent a5fbe834e8
commit 32e5e0f30f
6 changed files with 242 additions and 0 deletions
@@ -921,6 +921,12 @@ class SchedulerDisaggregationPrefillMixin:
if self.gracefully_exit:
break
try:
from sglang.srt.observability import fwd_timeline
fwd_timeline.drain()
except Exception:
pass
_lt = None
if batch_telemetry.loop_accounting_enabled():
_lt = batch_telemetry.begin_loop(
+20
View File
@@ -4119,11 +4119,18 @@ class Scheduler(
return None, running_batch
# Get priority queue
_pf_t0 = time.monotonic_ns()
self.policy.calc_priority(
self.waiting_queue,
running_batch,
processed_tokens=self.processed_tokens_counter,
)
try:
from sglang.srt.observability import fwd_timeline
fwd_timeline.detail("prefill_policy", _pf_t0, time.monotonic_ns())
except Exception:
fwd_timeline = None
if TEST_RETRACT and running_bs > TEST_RETRACT_NO_PREFILL_BS:
# If we are testing retraction and the running batch size exceeds
@@ -4147,6 +4154,7 @@ class Scheduler(
else:
prefill_tile_block_m = 64 # Fallback for non-Triton backends
_pf_t1 = time.monotonic_ns()
adder = self.get_prefill_adder_class()(
self.page_size,
self.tree_cache,
@@ -4189,9 +4197,12 @@ class Scheduler(
running_batch.reqs,
)
if fwd_timeline is not None:
fwd_timeline.detail("prefill_adder_init", _pf_t1, time.monotonic_ns())
mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None)
if mamba_allocator is not None:
mamba_allocator.alloc_group_begin(len(self.waiting_queue))
_pf_t2 = time.monotonic_ns()
buffer_pipeline = self.tree_cache.buffer_pipeline
# Get requests from the waiting queue to a new prefill batch
for req in self.waiting_queue:
@@ -4304,6 +4315,9 @@ class Scheduler(
if mamba_allocator is not None:
mamba_allocator.alloc_group_end()
if fwd_timeline is not None:
fwd_timeline.detail("prefill_scan", _pf_t2, time.monotonic_ns())
_pf_t3 = time.monotonic_ns()
# Update waiting queue
can_run_list: List[Req] = adder.can_run_list
@@ -4313,6 +4327,10 @@ class Scheduler(
for req in adder.preempt_list:
self._add_request_to_queue(req)
batch_telemetry.set_admitted(0)
if fwd_timeline is not None:
fwd_timeline.detail(
"prefill_batch_init", _pf_t3, time.monotonic_ns()
)
return None, running_batch
can_run_set = set(can_run_list)
@@ -4417,6 +4435,8 @@ class Scheduler(
batch_telemetry.set_admitted(len(can_run_list))
batch_telemetry.note_new_chunk(adder.new_chunked_req is not None)
if fwd_timeline is not None:
fwd_timeline.detail("prefill_batch_init", _pf_t3, time.monotonic_ns())
return new_batch, running_batch
def can_schedule_lora_req(
+10
View File
@@ -16,6 +16,7 @@
from __future__ import annotations
import logging
import time
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, List, Optional, Tuple
@@ -643,12 +644,21 @@ class TpModelWorker(BaseTpWorker):
# Replay reads restored main/indexer KV before the normal extend.
run_encoder_swa_replay(self, batch)
try:
from sglang.srt.observability import fwd_timeline
_ft_t = time.monotonic_ns()
except Exception: # pragma: no cover
fwd_timeline = None
_ft_t = 0
forward_batch = ForwardBatch.init_new(
batch if folded is None else folded.batch,
self.model_runner,
capture_hidden_mode=capture_hidden_mode,
return_hidden_states_before_norm=False,
)
if fwd_timeline is not None:
fwd_timeline.detail("fwd_init_new", _ft_t, time.monotonic_ns())
if folded is not None:
apply_folded_extend(folded, forward_batch)
else:
@@ -1848,6 +1848,7 @@ class ModelRunner:
else contextlib.nullcontext()
)
_fwd_tail_t = time.monotonic_ns()
with (
canary_ctx,
step_span_ctx,
@@ -1909,6 +1910,12 @@ class ModelRunner:
):
self.maybe_join_ep_ranks()
try:
from sglang.srt.observability import fwd_timeline as _ft
_ft.detail("fwd_runner_tail", _fwd_tail_t, time.monotonic_ns())
except Exception:
pass
return output
def _maybe_execute_deferred_mamba_cow_and_clear(
@@ -2009,6 +2016,7 @@ class ModelRunner:
)
return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph)
_fwd_prep_t = time.monotonic_ns()
# DP / MLP-sync padding + attn-tp normalization. Only the decode
# cuda-graph path above pre-pads its static buffers and returns
# early; split prefill, the prefill cuda graph, and the eager
@@ -2025,6 +2033,21 @@ class ModelRunner:
if dwdp_mgr is not None:
dwdp_mgr.prefetch_first_layers()
try:
from sglang.srt.observability import fwd_timeline
except Exception: # pragma: no cover
fwd_timeline = None
if fwd_timeline is not None:
fwd_timeline.detail(
"fwd_prepare", _fwd_prep_t, time.monotonic_ns()
)
_fwd_exec_t = time.monotonic_ns()
_fwd_ticket = (
fwd_timeline.before_execute()
if fwd_timeline is not None
else None
)
if forward_batch.forward_mode.is_split_prefill():
# Layer-split mode; stays on ModelRunner, not the eager runner.
ret = self.forward_split_prefill(
@@ -2063,11 +2086,20 @@ class ModelRunner:
forward_batch, pp_proxy_tensors=pp_proxy_tensors
)
if fwd_timeline is not None:
fwd_timeline.after_execute(_fwd_ticket)
fwd_timeline.detail(
"fwd_execute", _fwd_exec_t, time.monotonic_ns()
)
_fwd_post_t = time.monotonic_ns()
if (
forward_batch.global_num_tokens_cpu is not None
and self.pp_group.is_last_rank
):
forward_batch.post_forward_mlp_sync_batch(ret)
if fwd_timeline is not None:
fwd_timeline.detail("fwd_post", _fwd_post_t, time.monotonic_ns())
return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph)
@@ -38,6 +38,7 @@ Backend selection comes from cuda_graph_config.prefill:
from __future__ import annotations
import copy
import time
import dataclasses
import inspect
import logging
@@ -2338,9 +2339,18 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
self._validate_capture_hidden_mode(forward_batch)
with self.model_context(), self.backend.replay_session():
try:
from sglang.srt.observability import fwd_timeline
_gl_t = time.monotonic_ns()
except Exception: # pragma: no cover
fwd_timeline = None
static_forward_batch, shared_read_ends = self.load_batch(
forward_batch, **kwargs
)
if fwd_timeline is not None:
fwd_timeline.detail("fwd_graph_load", _gl_t, time.monotonic_ns())
_gr_t = time.monotonic_ns()
static_num_tokens = len(static_forward_batch.input_ids)
raw_num_tokens = self.raw_num_tokens
shape_key = self._shape_key(static_num_tokens, forward_batch)
@@ -2382,4 +2392,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
raw_num_tokens,
**kwargs,
)
if fwd_timeline is not None:
fwd_timeline.detail(
"fwd_graph_replay", _gr_t, time.monotonic_ns()
)
return self._finalize_execute_output(output)
@@ -0,0 +1,160 @@
"""Opt-in GPU busy-window probe for prefill forwards (SGLANG_FWD_TIMELINE=1).
The scheduler/loop spans tell us *host* time; they cannot tell whether the GPU
was busy while the host was working. This probe records one CUDA event pair
around the model execute call and writes the resolved wall-clock window of the
GPU work as one JSONL line per forward::
{"seq": 123, "rank": 0, "a_ns": ..., "b_ns": ..., "dur_ms": 42.1}
``a_ns``/``b_ns`` are ``time.time_ns()``-based wall clock nanoseconds, anchored
to a reference event recorded once (after a ``cudaSynchronize``) so they can be
joined with the loop/pass spans from OpenSearch. Everything before ``a_ns``
and after ``b_ns`` in the same loop is GPU-idle (i.e. host-serial) time.
Design constraints:
* never raise into the scheduler -- any error disables the probe permanently;
* the resolve step only reads events that are already complete
(``Event.query()``), so it never blocks the scheduler thread;
* file is per DP rank under ``/tmp`` (the container is hostPath-mounted for
other reasons; ``kubectl exec ... cat`` is enough to collect it).
"""
from __future__ import annotations
import json
import os
import time
_ENABLED = os.environ.get("SGLANG_FWD_TIMELINE", "0") == "1"
try: # pragma: no cover - import guard only
import torch
except Exception: # pragma: no cover
torch = None
_REF = None # reference CUDA event
_REF_WALL_NS = 0
_RANK = -1
_PENDING = [] # [(seq, ev_a, ev_b)]
_FILE = None
_FAILED = False
def enabled() -> bool:
return _ENABLED and not _FAILED
def _init() -> bool:
"""Record the reference event; returns False when the probe must stay off."""
global _REF, _REF_WALL_NS, _RANK, _FILE, _FAILED, _ENABLED
if _REF is not None:
return True
if not _ENABLED or torch is None:
return False
try:
from sglang.srt.distributed import get_parallel
_RANK = int(get_parallel().attn_dp_rank or 0)
except Exception:
_RANK = 0
try:
torch.cuda.synchronize()
_REF_WALL_NS = time.time_ns()
_REF = torch.cuda.Event(enable_timing=True)
_REF.record()
path = f"/tmp/fwd_timeline_rank{_RANK}.jsonl"
_FILE = open(path, "a", buffering=1)
_FILE.write(
json.dumps({"header": True, "rank": _RANK, "ref_ns": _REF_WALL_NS}) + "\n"
)
return True
except Exception:
_ENABLED = False
_FAILED = True
return False
def before_execute():
"""Record the GPU-start marker; returns an opaque ticket (or None)."""
if not enabled():
return None
try:
if not _init():
return None
ev = torch.cuda.Event(enable_timing=True)
ev.record()
return ev
except Exception:
_disable()
return None
def after_execute(ticket) -> None:
if ticket is None or not enabled():
return
try:
ev = torch.cuda.Event(enable_timing=True)
ev.record()
seq = 0
try:
from sglang.srt.observability import batch_telemetry
seq = int(batch_telemetry.current_loop_seq() or 0)
except Exception:
pass
_PENDING.append((seq, ticket, ev))
except Exception:
_disable()
def detail(name: str, start_ns: int, end_ns: int) -> None:
"""Attach a host phase window to the current loop span (absolute times)."""
try:
from sglang.srt.observability import batch_telemetry
batch_telemetry.detail_add(name, start_ns, end_ns)
except Exception:
pass
def drain() -> None:
"""Resolve completed windows (never blocks) and append them to the file."""
if not _PENDING or _FILE is None:
return
try:
keep = []
lines = []
for seq, ev_a, ev_b in _PENDING:
try:
ready = ev_b.query()
except Exception:
ready = False
if not ready:
keep.append((seq, ev_a, ev_b))
continue
a_ns = _REF_WALL_NS + int(_REF.elapsed_time(ev_a) * 1e6)
b_ns = _REF_WALL_NS + int(_REF.elapsed_time(ev_b) * 1e6)
lines.append(
json.dumps(
{
"seq": seq,
"rank": _RANK,
"a_ns": a_ns,
"b_ns": b_ns,
"dur_ms": round((b_ns - a_ns) / 1e6, 3),
}
)
)
_PENDING[:] = keep
if lines:
_FILE.write("\n".join(lines) + "\n")
except Exception:
_disable()
def _disable() -> None:
global _ENABLED, _FAILED
_ENABLED = False
_FAILED = True