diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index c2846102e5..cecc957537 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -921,6 +921,12 @@ class SchedulerDisaggregationPrefillMixin: if self.gracefully_exit: break + try: + from sglang.srt.observability import fwd_timeline + + fwd_timeline.drain() + except Exception: + pass _lt = None if batch_telemetry.loop_accounting_enabled(): _lt = batch_telemetry.begin_loop( diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e9442b330b..908dfbd9b0 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4119,11 +4119,18 @@ class Scheduler( return None, running_batch # Get priority queue + _pf_t0 = time.monotonic_ns() self.policy.calc_priority( self.waiting_queue, running_batch, processed_tokens=self.processed_tokens_counter, ) + try: + from sglang.srt.observability import fwd_timeline + + fwd_timeline.detail("prefill_policy", _pf_t0, time.monotonic_ns()) + except Exception: + fwd_timeline = None if TEST_RETRACT and running_bs > TEST_RETRACT_NO_PREFILL_BS: # If we are testing retraction and the running batch size exceeds @@ -4147,6 +4154,7 @@ class Scheduler( else: prefill_tile_block_m = 64 # Fallback for non-Triton backends + _pf_t1 = time.monotonic_ns() adder = self.get_prefill_adder_class()( self.page_size, self.tree_cache, @@ -4189,9 +4197,12 @@ class Scheduler( running_batch.reqs, ) + if fwd_timeline is not None: + fwd_timeline.detail("prefill_adder_init", _pf_t1, time.monotonic_ns()) mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None) if mamba_allocator is not None: mamba_allocator.alloc_group_begin(len(self.waiting_queue)) + _pf_t2 = time.monotonic_ns() buffer_pipeline = self.tree_cache.buffer_pipeline # Get requests from the waiting queue to a new prefill batch for req in self.waiting_queue: @@ -4304,6 +4315,9 @@ class Scheduler( if mamba_allocator is not None: mamba_allocator.alloc_group_end() + if fwd_timeline is not None: + fwd_timeline.detail("prefill_scan", _pf_t2, time.monotonic_ns()) + _pf_t3 = time.monotonic_ns() # Update waiting queue can_run_list: List[Req] = adder.can_run_list @@ -4313,6 +4327,10 @@ class Scheduler( for req in adder.preempt_list: self._add_request_to_queue(req) batch_telemetry.set_admitted(0) + if fwd_timeline is not None: + fwd_timeline.detail( + "prefill_batch_init", _pf_t3, time.monotonic_ns() + ) return None, running_batch can_run_set = set(can_run_list) @@ -4417,6 +4435,8 @@ class Scheduler( batch_telemetry.set_admitted(len(can_run_list)) batch_telemetry.note_new_chunk(adder.new_chunked_req is not None) + if fwd_timeline is not None: + fwd_timeline.detail("prefill_batch_init", _pf_t3, time.monotonic_ns()) return new_batch, running_batch def can_schedule_lora_req( diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index ce98e3fdfd..fb24090b45 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging +import time from abc import ABC, abstractmethod from typing import TYPE_CHECKING, List, Optional, Tuple @@ -643,12 +644,21 @@ class TpModelWorker(BaseTpWorker): # Replay reads restored main/indexer KV before the normal extend. run_encoder_swa_replay(self, batch) + try: + from sglang.srt.observability import fwd_timeline + + _ft_t = time.monotonic_ns() + except Exception: # pragma: no cover + fwd_timeline = None + _ft_t = 0 forward_batch = ForwardBatch.init_new( batch if folded is None else folded.batch, self.model_runner, capture_hidden_mode=capture_hidden_mode, return_hidden_states_before_norm=False, ) + if fwd_timeline is not None: + fwd_timeline.detail("fwd_init_new", _ft_t, time.monotonic_ns()) if folded is not None: apply_folded_extend(folded, forward_batch) else: diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 609b19ead0..f5af4ab793 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1848,6 +1848,7 @@ class ModelRunner: else contextlib.nullcontext() ) + _fwd_tail_t = time.monotonic_ns() with ( canary_ctx, step_span_ctx, @@ -1909,6 +1910,12 @@ class ModelRunner: ): self.maybe_join_ep_ranks() + try: + from sglang.srt.observability import fwd_timeline as _ft + + _ft.detail("fwd_runner_tail", _fwd_tail_t, time.monotonic_ns()) + except Exception: + pass return output def _maybe_execute_deferred_mamba_cow_and_clear( @@ -2009,6 +2016,7 @@ class ModelRunner: ) return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph) + _fwd_prep_t = time.monotonic_ns() # DP / MLP-sync padding + attn-tp normalization. Only the decode # cuda-graph path above pre-pads its static buffers and returns # early; split prefill, the prefill cuda graph, and the eager @@ -2025,6 +2033,21 @@ class ModelRunner: if dwdp_mgr is not None: dwdp_mgr.prefetch_first_layers() + try: + from sglang.srt.observability import fwd_timeline + except Exception: # pragma: no cover + fwd_timeline = None + if fwd_timeline is not None: + fwd_timeline.detail( + "fwd_prepare", _fwd_prep_t, time.monotonic_ns() + ) + _fwd_exec_t = time.monotonic_ns() + _fwd_ticket = ( + fwd_timeline.before_execute() + if fwd_timeline is not None + else None + ) + if forward_batch.forward_mode.is_split_prefill(): # Layer-split mode; stays on ModelRunner, not the eager runner. ret = self.forward_split_prefill( @@ -2063,11 +2086,20 @@ class ModelRunner: forward_batch, pp_proxy_tensors=pp_proxy_tensors ) + if fwd_timeline is not None: + fwd_timeline.after_execute(_fwd_ticket) + fwd_timeline.detail( + "fwd_execute", _fwd_exec_t, time.monotonic_ns() + ) + _fwd_post_t = time.monotonic_ns() + if ( forward_batch.global_num_tokens_cpu is not None and self.pp_group.is_last_rank ): forward_batch.post_forward_mlp_sync_batch(ret) + if fwd_timeline is not None: + fwd_timeline.detail("fwd_post", _fwd_post_t, time.monotonic_ns()) return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 54b7e9f230..d82196783f 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -38,6 +38,7 @@ Backend selection comes from cuda_graph_config.prefill: from __future__ import annotations import copy +import time import dataclasses import inspect import logging @@ -2338,9 +2339,18 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): ) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]: self._validate_capture_hidden_mode(forward_batch) with self.model_context(), self.backend.replay_session(): + try: + from sglang.srt.observability import fwd_timeline + + _gl_t = time.monotonic_ns() + except Exception: # pragma: no cover + fwd_timeline = None static_forward_batch, shared_read_ends = self.load_batch( forward_batch, **kwargs ) + if fwd_timeline is not None: + fwd_timeline.detail("fwd_graph_load", _gl_t, time.monotonic_ns()) + _gr_t = time.monotonic_ns() static_num_tokens = len(static_forward_batch.input_ids) raw_num_tokens = self.raw_num_tokens shape_key = self._shape_key(static_num_tokens, forward_batch) @@ -2382,4 +2392,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): raw_num_tokens, **kwargs, ) + if fwd_timeline is not None: + fwd_timeline.detail( + "fwd_graph_replay", _gr_t, time.monotonic_ns() + ) return self._finalize_execute_output(output) diff --git a/python/sglang/srt/observability/fwd_timeline.py b/python/sglang/srt/observability/fwd_timeline.py new file mode 100644 index 0000000000..47e1d20cd8 --- /dev/null +++ b/python/sglang/srt/observability/fwd_timeline.py @@ -0,0 +1,160 @@ +"""Opt-in GPU busy-window probe for prefill forwards (SGLANG_FWD_TIMELINE=1). + +The scheduler/loop spans tell us *host* time; they cannot tell whether the GPU +was busy while the host was working. This probe records one CUDA event pair +around the model execute call and writes the resolved wall-clock window of the +GPU work as one JSONL line per forward:: + + {"seq": 123, "rank": 0, "a_ns": ..., "b_ns": ..., "dur_ms": 42.1} + +``a_ns``/``b_ns`` are ``time.time_ns()``-based wall clock nanoseconds, anchored +to a reference event recorded once (after a ``cudaSynchronize``) so they can be +joined with the loop/pass spans from OpenSearch. Everything before ``a_ns`` +and after ``b_ns`` in the same loop is GPU-idle (i.e. host-serial) time. + +Design constraints: + +* never raise into the scheduler -- any error disables the probe permanently; +* the resolve step only reads events that are already complete + (``Event.query()``), so it never blocks the scheduler thread; +* file is per DP rank under ``/tmp`` (the container is hostPath-mounted for + other reasons; ``kubectl exec ... cat`` is enough to collect it). +""" + +from __future__ import annotations + +import json +import os +import time + +_ENABLED = os.environ.get("SGLANG_FWD_TIMELINE", "0") == "1" + +try: # pragma: no cover - import guard only + import torch +except Exception: # pragma: no cover + torch = None + +_REF = None # reference CUDA event +_REF_WALL_NS = 0 +_RANK = -1 +_PENDING = [] # [(seq, ev_a, ev_b)] +_FILE = None +_FAILED = False + + +def enabled() -> bool: + return _ENABLED and not _FAILED + + +def _init() -> bool: + """Record the reference event; returns False when the probe must stay off.""" + global _REF, _REF_WALL_NS, _RANK, _FILE, _FAILED, _ENABLED + if _REF is not None: + return True + if not _ENABLED or torch is None: + return False + try: + from sglang.srt.distributed import get_parallel + + _RANK = int(get_parallel().attn_dp_rank or 0) + except Exception: + _RANK = 0 + try: + torch.cuda.synchronize() + _REF_WALL_NS = time.time_ns() + _REF = torch.cuda.Event(enable_timing=True) + _REF.record() + path = f"/tmp/fwd_timeline_rank{_RANK}.jsonl" + _FILE = open(path, "a", buffering=1) + _FILE.write( + json.dumps({"header": True, "rank": _RANK, "ref_ns": _REF_WALL_NS}) + "\n" + ) + return True + except Exception: + _ENABLED = False + _FAILED = True + return False + + +def before_execute(): + """Record the GPU-start marker; returns an opaque ticket (or None).""" + if not enabled(): + return None + try: + if not _init(): + return None + ev = torch.cuda.Event(enable_timing=True) + ev.record() + return ev + except Exception: + _disable() + return None + + +def after_execute(ticket) -> None: + if ticket is None or not enabled(): + return + try: + ev = torch.cuda.Event(enable_timing=True) + ev.record() + seq = 0 + try: + from sglang.srt.observability import batch_telemetry + + seq = int(batch_telemetry.current_loop_seq() or 0) + except Exception: + pass + _PENDING.append((seq, ticket, ev)) + except Exception: + _disable() + + +def detail(name: str, start_ns: int, end_ns: int) -> None: + """Attach a host phase window to the current loop span (absolute times).""" + try: + from sglang.srt.observability import batch_telemetry + + batch_telemetry.detail_add(name, start_ns, end_ns) + except Exception: + pass + + +def drain() -> None: + """Resolve completed windows (never blocks) and append them to the file.""" + if not _PENDING or _FILE is None: + return + try: + keep = [] + lines = [] + for seq, ev_a, ev_b in _PENDING: + try: + ready = ev_b.query() + except Exception: + ready = False + if not ready: + keep.append((seq, ev_a, ev_b)) + continue + a_ns = _REF_WALL_NS + int(_REF.elapsed_time(ev_a) * 1e6) + b_ns = _REF_WALL_NS + int(_REF.elapsed_time(ev_b) * 1e6) + lines.append( + json.dumps( + { + "seq": seq, + "rank": _RANK, + "a_ns": a_ns, + "b_ns": b_ns, + "dur_ms": round((b_ns - a_ns) / 1e6, 3), + } + ) + ) + _PENDING[:] = keep + if lines: + _FILE.write("\n".join(lines) + "\n") + except Exception: + _disable() + + +def _disable() -> None: + global _ENABLED, _FAILED + _ENABLED = False + _FAILED = True