deploy: bake the live hostPath overrides into the tree

Copies the three files that the live cw mounts over the image:
- managers/load_snapshot.py: queued_total_input_tokens + drain_rate_tokens_per_s
- managers/scheduler_components/load_inquirer.py: fills the two new fields
- models/deepseek_v4.py: env-gated MM trace (SGLANG_MM_TRACE, off by default)

After this commit every runtime file the P/D pods need is in the tree, so the
hostPath file mounts can be dropped once the image is rebuilt.
This commit is contained in:
ymk
2026-11-09 06:31:48 +00:00
parent 883f7d08e1
commit 15218b2f65
3 changed files with 114 additions and 14 deletions
@@ -201,6 +201,8 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
num_running_reqs: int = 0
num_waiting_reqs: int = 0
num_waiting_uncached_tokens: int = 0
queued_total_input_tokens: int = 0
drain_rate_tokens_per_s: float = 0.0
num_used_tokens: int = 0
num_total_tokens: int = 0
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
@@ -226,6 +226,13 @@ class SchedulerLoadInquirer:
totals = self.get_decode_moment_totals()
decode_moments = list(totals) if totals[0] > 0 else None
_waiting_uncached = self.get_num_waiting_uncached_tokens()
_busy_us = self.get_total_prefill_busy_us()
_drain_rate = (
round(self.get_total_prefill_uncached_tokens() * 1e6 / _busy_us, 2)
if _busy_us > 0
else 0.0
)
return LoadSnapshot(
dp_rank=int(get_parallel().dp_rank)
if get_parallel().dp_rank is not None
@@ -233,7 +240,9 @@ class SchedulerLoadInquirer:
timestamp=time.time(),
num_running_reqs=num_running_reqs,
num_waiting_reqs=num_waiting_reqs,
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
num_waiting_uncached_tokens=_waiting_uncached,
queued_total_input_tokens=_waiting_uncached,
drain_rate_tokens_per_s=_drain_rate,
num_used_tokens=num_used_tokens,
num_total_tokens=num_total_tokens,
num_active_tokens=num_active_tokens,
+102 -13
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import concurrent.futures
import functools
import logging
import os
import time
from array import array
from contextlib import contextmanager, nullcontext
@@ -273,6 +274,17 @@ def get_mhc_ops() -> MhcOps:
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Optional vision-tower timing (env-gated; off by default).
# SGLANG_MM_TRACE=1 emit one OTel span + one log line per get_image_feature call
# SGLANG_MM_TRACE_SYNC=1 synchronize CUDA around ViT/aligner so the times are GPU time
# (only for short vision-focused runs; leave 0 under load)
# ---------------------------------------------------------------------------
_MM_TRACE = os.getenv("SGLANG_MM_TRACE", "0").strip().lower() not in ("", "0", "false", "no", "off")
_MM_TRACE_SYNC = os.getenv("SGLANG_MM_TRACE_SYNC", "0").strip().lower() not in ("", "0", "false", "no", "off")
_MM_TRACE_RIDS = None
_MM_TRACE_ACC = None
_FP8_WO_A_GEMM = envs.SGLANG_OPT_FP8_WO_A_GEMM.get()
_FP8_WO_A_UE8M0 = _FP8_WO_A_GEMM and DEEPGEMM_SCALE_UE8M0
@@ -4754,6 +4766,62 @@ class DeepseekV4ForCausalLM(nn.Module):
return pad_mm_tokens(input_ids, mm_inputs)
def get_image_feature(self, items):
if not _MM_TRACE:
return self._get_image_feature_impl(items)
global _MM_TRACE_ACC
from opentelemetry import trace as _otel_trace
parallel = get_parallel()
rids = list(_MM_TRACE_RIDS or [])
acc = {
"vit_s": 0.0,
"aligner_s": 0.0,
"items": 0,
"patches": 0,
"cuda": torch.cuda.is_available(),
}
prev_acc, _MM_TRACE_ACC = _MM_TRACE_ACC, acc
tracer = _otel_trace.get_tracer("sglang.mm")
try:
if _MM_TRACE_SYNC and acc["cuda"]:
torch.cuda.synchronize()
t0 = time.perf_counter()
with tracer.start_as_current_span("mm.vision_tower") as span:
try:
out = self._get_image_feature_impl(items)
finally:
if _MM_TRACE_SYNC and acc["cuda"]:
torch.cuda.synchronize()
total_s = time.perf_counter() - t0
dp_rank = int(getattr(parallel, "attn_dp_rank", -1))
span.set_attribute("sglang.dp_rank", dp_rank)
span.set_attribute("sglang.n_items", int(acc["items"]))
span.set_attribute("sglang.n_patches", int(acc["patches"]))
span.set_attribute("sglang.vit_ms", round(acc["vit_s"] * 1e3, 3))
span.set_attribute("sglang.aligner_ms", round(acc["aligner_s"] * 1e3, 3))
span.set_attribute("sglang.total_ms", round(total_s * 1e3, 3))
span.set_attribute("sglang.timing", "gpu_sync" if _MM_TRACE_SYNC else "cpu_wall_launch")
span.set_attribute("sglang.n_rids", len(rids))
if rids:
span.set_attribute("sglang.rids", ",".join(map(str, rids[:8])))
logger.info(
"[MMTrace] dp=%d items=%d patches=%d vit_ms=%.3f aligner_ms=%.3f "
"total_ms=%.3f timing=%s rids=%s",
dp_rank,
acc["items"],
acc["patches"],
acc["vit_s"] * 1e3,
acc["aligner_s"] * 1e3,
total_s * 1e3,
"gpu_sync" if _MM_TRACE_SYNC else "cpu_wall_launch",
",".join(map(str, rids[:4])),
)
return out
finally:
_MM_TRACE_ACC = prev_acc
def _get_image_feature_impl(self, items):
"""Return complete spans for the shared MM cache and chunk scheduler."""
spans = []
@@ -4768,7 +4836,21 @@ class DeepseekV4ForCausalLM(nn.Module):
if plan is not None
else pixels.to(dtype)
)
features = self.aligner(self.vision(patches, h, w), h, w)
if _MM_TRACE_ACC is not None:
t_vit = time.perf_counter()
vision_features = self.vision(patches, h, w)
if _MM_TRACE_SYNC and _MM_TRACE_ACC["cuda"]:
torch.cuda.synchronize()
_MM_TRACE_ACC["vit_s"] += time.perf_counter() - t_vit
t_aligner = time.perf_counter()
features = self.aligner(vision_features, h, w)
if _MM_TRACE_SYNC and _MM_TRACE_ACC["cuda"]:
torch.cuda.synchronize()
_MM_TRACE_ACC["aligner_s"] += time.perf_counter() - t_aligner
_MM_TRACE_ACC["items"] += 1
_MM_TRACE_ACC["patches"] += int(h) * int(w)
else:
features = self.aligner(self.vision(patches, h, w), h, w)
r = self.config.vision_downsample_ratio
types = image_token_types((h + r - 1) // r, (w + r - 1) // r).to(device)
span = torch.empty(
@@ -4782,18 +4864,25 @@ class DeepseekV4ForCausalLM(nn.Module):
return spans
def _prepare_mm_embeddings(self, input_ids, forward_batch):
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
input_embeds, _ = embed_mm_inputs(
mm_inputs_list=[
item if item is not None else MultimodalInputs(mm_items=[])
for item in forward_batch.mm_inputs
],
extend_prefix_lens=forward_batch.extend_prefix_lens_cpu,
extend_seq_lens=forward_batch.extend_seq_lens_cpu,
input_ids=input_ids.clone(),
input_embedding=self.get_input_embeddings(),
multimodal_model=self,
)
global _MM_TRACE_RIDS
prev_rids = _MM_TRACE_RIDS
if _MM_TRACE:
_MM_TRACE_RIDS = list(getattr(forward_batch, "rids", None) or [])
try:
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
input_embeds, _ = embed_mm_inputs(
mm_inputs_list=[
item if item is not None else MultimodalInputs(mm_items=[])
for item in forward_batch.mm_inputs
],
extend_prefix_lens=forward_batch.extend_prefix_lens_cpu,
extend_seq_lens=forward_batch.extend_seq_lens_cpu,
input_ids=input_ids.clone(),
input_embedding=self.get_input_embeddings(),
multimodal_model=self,
)
finally:
_MM_TRACE_RIDS = prev_rids
forward_batch.mm_input_embeds = input_embeds
return input_embeds