observability: GPU-window probe + forward/prefill sub-phase timing
Phase 1/3 of the async-backup handoff instrumentation plan: * new fwd_timeline probe (SGLANG_FWD_TIMELINE=1): one CUDA event pair around the model execute call per forward, resolved without blocking and written as wall-clock JSONL per DP rank. Joined with loop spans it separates host time spent while the GPU is busy (overlapped) from GPU-idle host stalls. * detail phases: fwd_init_new, fwd_prepare, fwd_execute, fwd_graph_load, fwd_graph_replay, fwd_post, fwd_runner_tail, prefill_policy, prefill_adder_init, prefill_scan, prefill_batch_init. All spans ride the existing SGLANG_TRACE_LOOP_DETAIL path; the GPU probe is off by default.
This commit is contained in:
@@ -921,6 +921,12 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
try:
|
||||
from sglang.srt.observability import fwd_timeline
|
||||
|
||||
fwd_timeline.drain()
|
||||
except Exception:
|
||||
pass
|
||||
_lt = None
|
||||
if batch_telemetry.loop_accounting_enabled():
|
||||
_lt = batch_telemetry.begin_loop(
|
||||
|
||||
@@ -4119,11 +4119,18 @@ class Scheduler(
|
||||
return None, running_batch
|
||||
|
||||
# Get priority queue
|
||||
_pf_t0 = time.monotonic_ns()
|
||||
self.policy.calc_priority(
|
||||
self.waiting_queue,
|
||||
running_batch,
|
||||
processed_tokens=self.processed_tokens_counter,
|
||||
)
|
||||
try:
|
||||
from sglang.srt.observability import fwd_timeline
|
||||
|
||||
fwd_timeline.detail("prefill_policy", _pf_t0, time.monotonic_ns())
|
||||
except Exception:
|
||||
fwd_timeline = None
|
||||
|
||||
if TEST_RETRACT and running_bs > TEST_RETRACT_NO_PREFILL_BS:
|
||||
# If we are testing retraction and the running batch size exceeds
|
||||
@@ -4147,6 +4154,7 @@ class Scheduler(
|
||||
else:
|
||||
prefill_tile_block_m = 64 # Fallback for non-Triton backends
|
||||
|
||||
_pf_t1 = time.monotonic_ns()
|
||||
adder = self.get_prefill_adder_class()(
|
||||
self.page_size,
|
||||
self.tree_cache,
|
||||
@@ -4189,9 +4197,12 @@ class Scheduler(
|
||||
running_batch.reqs,
|
||||
)
|
||||
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("prefill_adder_init", _pf_t1, time.monotonic_ns())
|
||||
mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None)
|
||||
if mamba_allocator is not None:
|
||||
mamba_allocator.alloc_group_begin(len(self.waiting_queue))
|
||||
_pf_t2 = time.monotonic_ns()
|
||||
buffer_pipeline = self.tree_cache.buffer_pipeline
|
||||
# Get requests from the waiting queue to a new prefill batch
|
||||
for req in self.waiting_queue:
|
||||
@@ -4304,6 +4315,9 @@ class Scheduler(
|
||||
|
||||
if mamba_allocator is not None:
|
||||
mamba_allocator.alloc_group_end()
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("prefill_scan", _pf_t2, time.monotonic_ns())
|
||||
_pf_t3 = time.monotonic_ns()
|
||||
|
||||
# Update waiting queue
|
||||
can_run_list: List[Req] = adder.can_run_list
|
||||
@@ -4313,6 +4327,10 @@ class Scheduler(
|
||||
for req in adder.preempt_list:
|
||||
self._add_request_to_queue(req)
|
||||
batch_telemetry.set_admitted(0)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail(
|
||||
"prefill_batch_init", _pf_t3, time.monotonic_ns()
|
||||
)
|
||||
return None, running_batch
|
||||
|
||||
can_run_set = set(can_run_list)
|
||||
@@ -4417,6 +4435,8 @@ class Scheduler(
|
||||
|
||||
batch_telemetry.set_admitted(len(can_run_list))
|
||||
batch_telemetry.note_new_chunk(adder.new_chunked_req is not None)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("prefill_batch_init", _pf_t3, time.monotonic_ns())
|
||||
return new_batch, running_batch
|
||||
|
||||
def can_schedule_lora_req(
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
@@ -643,12 +644,21 @@ class TpModelWorker(BaseTpWorker):
|
||||
# Replay reads restored main/indexer KV before the normal extend.
|
||||
run_encoder_swa_replay(self, batch)
|
||||
|
||||
try:
|
||||
from sglang.srt.observability import fwd_timeline
|
||||
|
||||
_ft_t = time.monotonic_ns()
|
||||
except Exception: # pragma: no cover
|
||||
fwd_timeline = None
|
||||
_ft_t = 0
|
||||
forward_batch = ForwardBatch.init_new(
|
||||
batch if folded is None else folded.batch,
|
||||
self.model_runner,
|
||||
capture_hidden_mode=capture_hidden_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("fwd_init_new", _ft_t, time.monotonic_ns())
|
||||
if folded is not None:
|
||||
apply_folded_extend(folded, forward_batch)
|
||||
else:
|
||||
|
||||
@@ -1848,6 +1848,7 @@ class ModelRunner:
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
|
||||
_fwd_tail_t = time.monotonic_ns()
|
||||
with (
|
||||
canary_ctx,
|
||||
step_span_ctx,
|
||||
@@ -1909,6 +1910,12 @@ class ModelRunner:
|
||||
):
|
||||
self.maybe_join_ep_ranks()
|
||||
|
||||
try:
|
||||
from sglang.srt.observability import fwd_timeline as _ft
|
||||
|
||||
_ft.detail("fwd_runner_tail", _fwd_tail_t, time.monotonic_ns())
|
||||
except Exception:
|
||||
pass
|
||||
return output
|
||||
|
||||
def _maybe_execute_deferred_mamba_cow_and_clear(
|
||||
@@ -2009,6 +2016,7 @@ class ModelRunner:
|
||||
)
|
||||
return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph)
|
||||
|
||||
_fwd_prep_t = time.monotonic_ns()
|
||||
# DP / MLP-sync padding + attn-tp normalization. Only the decode
|
||||
# cuda-graph path above pre-pads its static buffers and returns
|
||||
# early; split prefill, the prefill cuda graph, and the eager
|
||||
@@ -2025,6 +2033,21 @@ class ModelRunner:
|
||||
if dwdp_mgr is not None:
|
||||
dwdp_mgr.prefetch_first_layers()
|
||||
|
||||
try:
|
||||
from sglang.srt.observability import fwd_timeline
|
||||
except Exception: # pragma: no cover
|
||||
fwd_timeline = None
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail(
|
||||
"fwd_prepare", _fwd_prep_t, time.monotonic_ns()
|
||||
)
|
||||
_fwd_exec_t = time.monotonic_ns()
|
||||
_fwd_ticket = (
|
||||
fwd_timeline.before_execute()
|
||||
if fwd_timeline is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if forward_batch.forward_mode.is_split_prefill():
|
||||
# Layer-split mode; stays on ModelRunner, not the eager runner.
|
||||
ret = self.forward_split_prefill(
|
||||
@@ -2063,11 +2086,20 @@ class ModelRunner:
|
||||
forward_batch, pp_proxy_tensors=pp_proxy_tensors
|
||||
)
|
||||
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.after_execute(_fwd_ticket)
|
||||
fwd_timeline.detail(
|
||||
"fwd_execute", _fwd_exec_t, time.monotonic_ns()
|
||||
)
|
||||
_fwd_post_t = time.monotonic_ns()
|
||||
|
||||
if (
|
||||
forward_batch.global_num_tokens_cpu is not None
|
||||
and self.pp_group.is_last_rank
|
||||
):
|
||||
forward_batch.post_forward_mlp_sync_batch(ret)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("fwd_post", _fwd_post_t, time.monotonic_ns())
|
||||
|
||||
return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph)
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ Backend selection comes from cuda_graph_config.prefill:
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import time
|
||||
import dataclasses
|
||||
import inspect
|
||||
import logging
|
||||
@@ -2338,9 +2339,18 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
self._validate_capture_hidden_mode(forward_batch)
|
||||
with self.model_context(), self.backend.replay_session():
|
||||
try:
|
||||
from sglang.srt.observability import fwd_timeline
|
||||
|
||||
_gl_t = time.monotonic_ns()
|
||||
except Exception: # pragma: no cover
|
||||
fwd_timeline = None
|
||||
static_forward_batch, shared_read_ends = self.load_batch(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail("fwd_graph_load", _gl_t, time.monotonic_ns())
|
||||
_gr_t = time.monotonic_ns()
|
||||
static_num_tokens = len(static_forward_batch.input_ids)
|
||||
raw_num_tokens = self.raw_num_tokens
|
||||
shape_key = self._shape_key(static_num_tokens, forward_batch)
|
||||
@@ -2382,4 +2392,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
raw_num_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
if fwd_timeline is not None:
|
||||
fwd_timeline.detail(
|
||||
"fwd_graph_replay", _gr_t, time.monotonic_ns()
|
||||
)
|
||||
return self._finalize_execute_output(output)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Opt-in GPU busy-window probe for prefill forwards (SGLANG_FWD_TIMELINE=1).
|
||||
|
||||
The scheduler/loop spans tell us *host* time; they cannot tell whether the GPU
|
||||
was busy while the host was working. This probe records one CUDA event pair
|
||||
around the model execute call and writes the resolved wall-clock window of the
|
||||
GPU work as one JSONL line per forward::
|
||||
|
||||
{"seq": 123, "rank": 0, "a_ns": ..., "b_ns": ..., "dur_ms": 42.1}
|
||||
|
||||
``a_ns``/``b_ns`` are ``time.time_ns()``-based wall clock nanoseconds, anchored
|
||||
to a reference event recorded once (after a ``cudaSynchronize``) so they can be
|
||||
joined with the loop/pass spans from OpenSearch. Everything before ``a_ns``
|
||||
and after ``b_ns`` in the same loop is GPU-idle (i.e. host-serial) time.
|
||||
|
||||
Design constraints:
|
||||
|
||||
* never raise into the scheduler -- any error disables the probe permanently;
|
||||
* the resolve step only reads events that are already complete
|
||||
(``Event.query()``), so it never blocks the scheduler thread;
|
||||
* file is per DP rank under ``/tmp`` (the container is hostPath-mounted for
|
||||
other reasons; ``kubectl exec ... cat`` is enough to collect it).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
_ENABLED = os.environ.get("SGLANG_FWD_TIMELINE", "0") == "1"
|
||||
|
||||
try: # pragma: no cover - import guard only
|
||||
import torch
|
||||
except Exception: # pragma: no cover
|
||||
torch = None
|
||||
|
||||
_REF = None # reference CUDA event
|
||||
_REF_WALL_NS = 0
|
||||
_RANK = -1
|
||||
_PENDING = [] # [(seq, ev_a, ev_b)]
|
||||
_FILE = None
|
||||
_FAILED = False
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return _ENABLED and not _FAILED
|
||||
|
||||
|
||||
def _init() -> bool:
|
||||
"""Record the reference event; returns False when the probe must stay off."""
|
||||
global _REF, _REF_WALL_NS, _RANK, _FILE, _FAILED, _ENABLED
|
||||
if _REF is not None:
|
||||
return True
|
||||
if not _ENABLED or torch is None:
|
||||
return False
|
||||
try:
|
||||
from sglang.srt.distributed import get_parallel
|
||||
|
||||
_RANK = int(get_parallel().attn_dp_rank or 0)
|
||||
except Exception:
|
||||
_RANK = 0
|
||||
try:
|
||||
torch.cuda.synchronize()
|
||||
_REF_WALL_NS = time.time_ns()
|
||||
_REF = torch.cuda.Event(enable_timing=True)
|
||||
_REF.record()
|
||||
path = f"/tmp/fwd_timeline_rank{_RANK}.jsonl"
|
||||
_FILE = open(path, "a", buffering=1)
|
||||
_FILE.write(
|
||||
json.dumps({"header": True, "rank": _RANK, "ref_ns": _REF_WALL_NS}) + "\n"
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
_ENABLED = False
|
||||
_FAILED = True
|
||||
return False
|
||||
|
||||
|
||||
def before_execute():
|
||||
"""Record the GPU-start marker; returns an opaque ticket (or None)."""
|
||||
if not enabled():
|
||||
return None
|
||||
try:
|
||||
if not _init():
|
||||
return None
|
||||
ev = torch.cuda.Event(enable_timing=True)
|
||||
ev.record()
|
||||
return ev
|
||||
except Exception:
|
||||
_disable()
|
||||
return None
|
||||
|
||||
|
||||
def after_execute(ticket) -> None:
|
||||
if ticket is None or not enabled():
|
||||
return
|
||||
try:
|
||||
ev = torch.cuda.Event(enable_timing=True)
|
||||
ev.record()
|
||||
seq = 0
|
||||
try:
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
|
||||
seq = int(batch_telemetry.current_loop_seq() or 0)
|
||||
except Exception:
|
||||
pass
|
||||
_PENDING.append((seq, ticket, ev))
|
||||
except Exception:
|
||||
_disable()
|
||||
|
||||
|
||||
def detail(name: str, start_ns: int, end_ns: int) -> None:
|
||||
"""Attach a host phase window to the current loop span (absolute times)."""
|
||||
try:
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
|
||||
batch_telemetry.detail_add(name, start_ns, end_ns)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def drain() -> None:
|
||||
"""Resolve completed windows (never blocks) and append them to the file."""
|
||||
if not _PENDING or _FILE is None:
|
||||
return
|
||||
try:
|
||||
keep = []
|
||||
lines = []
|
||||
for seq, ev_a, ev_b in _PENDING:
|
||||
try:
|
||||
ready = ev_b.query()
|
||||
except Exception:
|
||||
ready = False
|
||||
if not ready:
|
||||
keep.append((seq, ev_a, ev_b))
|
||||
continue
|
||||
a_ns = _REF_WALL_NS + int(_REF.elapsed_time(ev_a) * 1e6)
|
||||
b_ns = _REF_WALL_NS + int(_REF.elapsed_time(ev_b) * 1e6)
|
||||
lines.append(
|
||||
json.dumps(
|
||||
{
|
||||
"seq": seq,
|
||||
"rank": _RANK,
|
||||
"a_ns": a_ns,
|
||||
"b_ns": b_ns,
|
||||
"dur_ms": round((b_ns - a_ns) / 1e6, 3),
|
||||
}
|
||||
)
|
||||
)
|
||||
_PENDING[:] = keep
|
||||
if lines:
|
||||
_FILE.write("\n".join(lines) + "\n")
|
||||
except Exception:
|
||||
_disable()
|
||||
|
||||
|
||||
def _disable() -> None:
|
||||
global _ENABLED, _FAILED
|
||||
_ENABLED = False
|
||||
_FAILED = True
|
||||
Reference in New Issue
Block a user