diff --git a/python/sglang/srt/managers/load_snapshot.py b/python/sglang/srt/managers/load_snapshot.py index 4d65df608a..b1cb54bbea 100644 --- a/python/sglang/srt/managers/load_snapshot.py +++ b/python/sglang/srt/managers/load_snapshot.py @@ -201,6 +201,8 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True): num_running_reqs: int = 0 num_waiting_reqs: int = 0 num_waiting_uncached_tokens: int = 0 + queued_total_input_tokens: int = 0 + drain_rate_tokens_per_s: float = 0.0 num_used_tokens: int = 0 num_total_tokens: int = 0 # num_total_tokens minus tokens still awaiting a KV transfer (equal to it diff --git a/python/sglang/srt/managers/scheduler_components/load_inquirer.py b/python/sglang/srt/managers/scheduler_components/load_inquirer.py index d475721757..5d1dfaed8d 100644 --- a/python/sglang/srt/managers/scheduler_components/load_inquirer.py +++ b/python/sglang/srt/managers/scheduler_components/load_inquirer.py @@ -226,6 +226,13 @@ class SchedulerLoadInquirer: totals = self.get_decode_moment_totals() decode_moments = list(totals) if totals[0] > 0 else None + _waiting_uncached = self.get_num_waiting_uncached_tokens() + _busy_us = self.get_total_prefill_busy_us() + _drain_rate = ( + round(self.get_total_prefill_uncached_tokens() * 1e6 / _busy_us, 2) + if _busy_us > 0 + else 0.0 + ) return LoadSnapshot( dp_rank=int(get_parallel().dp_rank) if get_parallel().dp_rank is not None @@ -233,7 +240,9 @@ class SchedulerLoadInquirer: timestamp=time.time(), num_running_reqs=num_running_reqs, num_waiting_reqs=num_waiting_reqs, - num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(), + num_waiting_uncached_tokens=_waiting_uncached, + queued_total_input_tokens=_waiting_uncached, + drain_rate_tokens_per_s=_drain_rate, num_used_tokens=num_used_tokens, num_total_tokens=num_total_tokens, num_active_tokens=num_active_tokens, diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 76473c12bc..baceb54b93 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3,6 +3,7 @@ from __future__ import annotations import concurrent.futures import functools import logging +import os import time from array import array from contextlib import contextmanager, nullcontext @@ -273,6 +274,17 @@ def get_mhc_ops() -> MhcOps: logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Optional vision-tower timing (env-gated; off by default). +# SGLANG_MM_TRACE=1 emit one OTel span + one log line per get_image_feature call +# SGLANG_MM_TRACE_SYNC=1 synchronize CUDA around ViT/aligner so the times are GPU time +# (only for short vision-focused runs; leave 0 under load) +# --------------------------------------------------------------------------- +_MM_TRACE = os.getenv("SGLANG_MM_TRACE", "0").strip().lower() not in ("", "0", "false", "no", "off") +_MM_TRACE_SYNC = os.getenv("SGLANG_MM_TRACE_SYNC", "0").strip().lower() not in ("", "0", "false", "no", "off") +_MM_TRACE_RIDS = None +_MM_TRACE_ACC = None + _FP8_WO_A_GEMM = envs.SGLANG_OPT_FP8_WO_A_GEMM.get() _FP8_WO_A_UE8M0 = _FP8_WO_A_GEMM and DEEPGEMM_SCALE_UE8M0 @@ -4754,6 +4766,62 @@ class DeepseekV4ForCausalLM(nn.Module): return pad_mm_tokens(input_ids, mm_inputs) def get_image_feature(self, items): + if not _MM_TRACE: + return self._get_image_feature_impl(items) + + global _MM_TRACE_ACC + from opentelemetry import trace as _otel_trace + + parallel = get_parallel() + rids = list(_MM_TRACE_RIDS or []) + acc = { + "vit_s": 0.0, + "aligner_s": 0.0, + "items": 0, + "patches": 0, + "cuda": torch.cuda.is_available(), + } + prev_acc, _MM_TRACE_ACC = _MM_TRACE_ACC, acc + tracer = _otel_trace.get_tracer("sglang.mm") + try: + if _MM_TRACE_SYNC and acc["cuda"]: + torch.cuda.synchronize() + t0 = time.perf_counter() + with tracer.start_as_current_span("mm.vision_tower") as span: + try: + out = self._get_image_feature_impl(items) + finally: + if _MM_TRACE_SYNC and acc["cuda"]: + torch.cuda.synchronize() + total_s = time.perf_counter() - t0 + dp_rank = int(getattr(parallel, "attn_dp_rank", -1)) + span.set_attribute("sglang.dp_rank", dp_rank) + span.set_attribute("sglang.n_items", int(acc["items"])) + span.set_attribute("sglang.n_patches", int(acc["patches"])) + span.set_attribute("sglang.vit_ms", round(acc["vit_s"] * 1e3, 3)) + span.set_attribute("sglang.aligner_ms", round(acc["aligner_s"] * 1e3, 3)) + span.set_attribute("sglang.total_ms", round(total_s * 1e3, 3)) + span.set_attribute("sglang.timing", "gpu_sync" if _MM_TRACE_SYNC else "cpu_wall_launch") + span.set_attribute("sglang.n_rids", len(rids)) + if rids: + span.set_attribute("sglang.rids", ",".join(map(str, rids[:8]))) + logger.info( + "[MMTrace] dp=%d items=%d patches=%d vit_ms=%.3f aligner_ms=%.3f " + "total_ms=%.3f timing=%s rids=%s", + dp_rank, + acc["items"], + acc["patches"], + acc["vit_s"] * 1e3, + acc["aligner_s"] * 1e3, + total_s * 1e3, + "gpu_sync" if _MM_TRACE_SYNC else "cpu_wall_launch", + ",".join(map(str, rids[:4])), + ) + return out + finally: + _MM_TRACE_ACC = prev_acc + + def _get_image_feature_impl(self, items): """Return complete spans for the shared MM cache and chunk scheduler.""" spans = [] @@ -4768,7 +4836,21 @@ class DeepseekV4ForCausalLM(nn.Module): if plan is not None else pixels.to(dtype) ) - features = self.aligner(self.vision(patches, h, w), h, w) + if _MM_TRACE_ACC is not None: + t_vit = time.perf_counter() + vision_features = self.vision(patches, h, w) + if _MM_TRACE_SYNC and _MM_TRACE_ACC["cuda"]: + torch.cuda.synchronize() + _MM_TRACE_ACC["vit_s"] += time.perf_counter() - t_vit + t_aligner = time.perf_counter() + features = self.aligner(vision_features, h, w) + if _MM_TRACE_SYNC and _MM_TRACE_ACC["cuda"]: + torch.cuda.synchronize() + _MM_TRACE_ACC["aligner_s"] += time.perf_counter() - t_aligner + _MM_TRACE_ACC["items"] += 1 + _MM_TRACE_ACC["patches"] += int(h) * int(w) + else: + features = self.aligner(self.vision(patches, h, w), h, w) r = self.config.vision_downsample_ratio types = image_token_types((h + r - 1) // r, (w + r - 1) // r).to(device) span = torch.empty( @@ -4782,18 +4864,25 @@ class DeepseekV4ForCausalLM(nn.Module): return spans def _prepare_mm_embeddings(self, input_ids, forward_batch): - # Keep scheduler hash IDs intact: the shared embedder clamps its input in place. - input_embeds, _ = embed_mm_inputs( - mm_inputs_list=[ - item if item is not None else MultimodalInputs(mm_items=[]) - for item in forward_batch.mm_inputs - ], - extend_prefix_lens=forward_batch.extend_prefix_lens_cpu, - extend_seq_lens=forward_batch.extend_seq_lens_cpu, - input_ids=input_ids.clone(), - input_embedding=self.get_input_embeddings(), - multimodal_model=self, - ) + global _MM_TRACE_RIDS + prev_rids = _MM_TRACE_RIDS + if _MM_TRACE: + _MM_TRACE_RIDS = list(getattr(forward_batch, "rids", None) or []) + try: + # Keep scheduler hash IDs intact: the shared embedder clamps its input in place. + input_embeds, _ = embed_mm_inputs( + mm_inputs_list=[ + item if item is not None else MultimodalInputs(mm_items=[]) + for item in forward_batch.mm_inputs + ], + extend_prefix_lens=forward_batch.extend_prefix_lens_cpu, + extend_seq_lens=forward_batch.extend_seq_lens_cpu, + input_ids=input_ids.clone(), + input_embedding=self.get_input_embeddings(), + multimodal_model=self, + ) + finally: + _MM_TRACE_RIDS = prev_rids forward_batch.mm_input_embeds = input_embeds return input_embeds