deploy: bake the live hostPath overrides into the tree
Copies the three files that the live cw mounts over the image: - managers/load_snapshot.py: queued_total_input_tokens + drain_rate_tokens_per_s - managers/scheduler_components/load_inquirer.py: fills the two new fields - models/deepseek_v4.py: env-gated MM trace (SGLANG_MM_TRACE, off by default) After this commit every runtime file the P/D pods need is in the tree, so the hostPath file mounts can be dropped once the image is rebuilt.
This commit is contained in:
@@ -201,6 +201,8 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
|
||||
num_running_reqs: int = 0
|
||||
num_waiting_reqs: int = 0
|
||||
num_waiting_uncached_tokens: int = 0
|
||||
queued_total_input_tokens: int = 0
|
||||
drain_rate_tokens_per_s: float = 0.0
|
||||
num_used_tokens: int = 0
|
||||
num_total_tokens: int = 0
|
||||
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
|
||||
|
||||
@@ -226,6 +226,13 @@ class SchedulerLoadInquirer:
|
||||
totals = self.get_decode_moment_totals()
|
||||
decode_moments = list(totals) if totals[0] > 0 else None
|
||||
|
||||
_waiting_uncached = self.get_num_waiting_uncached_tokens()
|
||||
_busy_us = self.get_total_prefill_busy_us()
|
||||
_drain_rate = (
|
||||
round(self.get_total_prefill_uncached_tokens() * 1e6 / _busy_us, 2)
|
||||
if _busy_us > 0
|
||||
else 0.0
|
||||
)
|
||||
return LoadSnapshot(
|
||||
dp_rank=int(get_parallel().dp_rank)
|
||||
if get_parallel().dp_rank is not None
|
||||
@@ -233,7 +240,9 @@ class SchedulerLoadInquirer:
|
||||
timestamp=time.time(),
|
||||
num_running_reqs=num_running_reqs,
|
||||
num_waiting_reqs=num_waiting_reqs,
|
||||
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
|
||||
num_waiting_uncached_tokens=_waiting_uncached,
|
||||
queued_total_input_tokens=_waiting_uncached,
|
||||
drain_rate_tokens_per_s=_drain_rate,
|
||||
num_used_tokens=num_used_tokens,
|
||||
num_total_tokens=num_total_tokens,
|
||||
num_active_tokens=num_active_tokens,
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import concurrent.futures
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from array import array
|
||||
from contextlib import contextmanager, nullcontext
|
||||
@@ -273,6 +274,17 @@ def get_mhc_ops() -> MhcOps:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional vision-tower timing (env-gated; off by default).
|
||||
# SGLANG_MM_TRACE=1 emit one OTel span + one log line per get_image_feature call
|
||||
# SGLANG_MM_TRACE_SYNC=1 synchronize CUDA around ViT/aligner so the times are GPU time
|
||||
# (only for short vision-focused runs; leave 0 under load)
|
||||
# ---------------------------------------------------------------------------
|
||||
_MM_TRACE = os.getenv("SGLANG_MM_TRACE", "0").strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_MM_TRACE_SYNC = os.getenv("SGLANG_MM_TRACE_SYNC", "0").strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_MM_TRACE_RIDS = None
|
||||
_MM_TRACE_ACC = None
|
||||
|
||||
_FP8_WO_A_GEMM = envs.SGLANG_OPT_FP8_WO_A_GEMM.get()
|
||||
_FP8_WO_A_UE8M0 = _FP8_WO_A_GEMM and DEEPGEMM_SCALE_UE8M0
|
||||
|
||||
@@ -4754,6 +4766,62 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items):
|
||||
if not _MM_TRACE:
|
||||
return self._get_image_feature_impl(items)
|
||||
|
||||
global _MM_TRACE_ACC
|
||||
from opentelemetry import trace as _otel_trace
|
||||
|
||||
parallel = get_parallel()
|
||||
rids = list(_MM_TRACE_RIDS or [])
|
||||
acc = {
|
||||
"vit_s": 0.0,
|
||||
"aligner_s": 0.0,
|
||||
"items": 0,
|
||||
"patches": 0,
|
||||
"cuda": torch.cuda.is_available(),
|
||||
}
|
||||
prev_acc, _MM_TRACE_ACC = _MM_TRACE_ACC, acc
|
||||
tracer = _otel_trace.get_tracer("sglang.mm")
|
||||
try:
|
||||
if _MM_TRACE_SYNC and acc["cuda"]:
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
with tracer.start_as_current_span("mm.vision_tower") as span:
|
||||
try:
|
||||
out = self._get_image_feature_impl(items)
|
||||
finally:
|
||||
if _MM_TRACE_SYNC and acc["cuda"]:
|
||||
torch.cuda.synchronize()
|
||||
total_s = time.perf_counter() - t0
|
||||
dp_rank = int(getattr(parallel, "attn_dp_rank", -1))
|
||||
span.set_attribute("sglang.dp_rank", dp_rank)
|
||||
span.set_attribute("sglang.n_items", int(acc["items"]))
|
||||
span.set_attribute("sglang.n_patches", int(acc["patches"]))
|
||||
span.set_attribute("sglang.vit_ms", round(acc["vit_s"] * 1e3, 3))
|
||||
span.set_attribute("sglang.aligner_ms", round(acc["aligner_s"] * 1e3, 3))
|
||||
span.set_attribute("sglang.total_ms", round(total_s * 1e3, 3))
|
||||
span.set_attribute("sglang.timing", "gpu_sync" if _MM_TRACE_SYNC else "cpu_wall_launch")
|
||||
span.set_attribute("sglang.n_rids", len(rids))
|
||||
if rids:
|
||||
span.set_attribute("sglang.rids", ",".join(map(str, rids[:8])))
|
||||
logger.info(
|
||||
"[MMTrace] dp=%d items=%d patches=%d vit_ms=%.3f aligner_ms=%.3f "
|
||||
"total_ms=%.3f timing=%s rids=%s",
|
||||
dp_rank,
|
||||
acc["items"],
|
||||
acc["patches"],
|
||||
acc["vit_s"] * 1e3,
|
||||
acc["aligner_s"] * 1e3,
|
||||
total_s * 1e3,
|
||||
"gpu_sync" if _MM_TRACE_SYNC else "cpu_wall_launch",
|
||||
",".join(map(str, rids[:4])),
|
||||
)
|
||||
return out
|
||||
finally:
|
||||
_MM_TRACE_ACC = prev_acc
|
||||
|
||||
def _get_image_feature_impl(self, items):
|
||||
"""Return complete spans for the shared MM cache and chunk scheduler."""
|
||||
|
||||
spans = []
|
||||
@@ -4768,7 +4836,21 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
if plan is not None
|
||||
else pixels.to(dtype)
|
||||
)
|
||||
features = self.aligner(self.vision(patches, h, w), h, w)
|
||||
if _MM_TRACE_ACC is not None:
|
||||
t_vit = time.perf_counter()
|
||||
vision_features = self.vision(patches, h, w)
|
||||
if _MM_TRACE_SYNC and _MM_TRACE_ACC["cuda"]:
|
||||
torch.cuda.synchronize()
|
||||
_MM_TRACE_ACC["vit_s"] += time.perf_counter() - t_vit
|
||||
t_aligner = time.perf_counter()
|
||||
features = self.aligner(vision_features, h, w)
|
||||
if _MM_TRACE_SYNC and _MM_TRACE_ACC["cuda"]:
|
||||
torch.cuda.synchronize()
|
||||
_MM_TRACE_ACC["aligner_s"] += time.perf_counter() - t_aligner
|
||||
_MM_TRACE_ACC["items"] += 1
|
||||
_MM_TRACE_ACC["patches"] += int(h) * int(w)
|
||||
else:
|
||||
features = self.aligner(self.vision(patches, h, w), h, w)
|
||||
r = self.config.vision_downsample_ratio
|
||||
types = image_token_types((h + r - 1) // r, (w + r - 1) // r).to(device)
|
||||
span = torch.empty(
|
||||
@@ -4782,18 +4864,25 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
return spans
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch):
|
||||
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
|
||||
input_embeds, _ = embed_mm_inputs(
|
||||
mm_inputs_list=[
|
||||
item if item is not None else MultimodalInputs(mm_items=[])
|
||||
for item in forward_batch.mm_inputs
|
||||
],
|
||||
extend_prefix_lens=forward_batch.extend_prefix_lens_cpu,
|
||||
extend_seq_lens=forward_batch.extend_seq_lens_cpu,
|
||||
input_ids=input_ids.clone(),
|
||||
input_embedding=self.get_input_embeddings(),
|
||||
multimodal_model=self,
|
||||
)
|
||||
global _MM_TRACE_RIDS
|
||||
prev_rids = _MM_TRACE_RIDS
|
||||
if _MM_TRACE:
|
||||
_MM_TRACE_RIDS = list(getattr(forward_batch, "rids", None) or [])
|
||||
try:
|
||||
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
|
||||
input_embeds, _ = embed_mm_inputs(
|
||||
mm_inputs_list=[
|
||||
item if item is not None else MultimodalInputs(mm_items=[])
|
||||
for item in forward_batch.mm_inputs
|
||||
],
|
||||
extend_prefix_lens=forward_batch.extend_prefix_lens_cpu,
|
||||
extend_seq_lens=forward_batch.extend_seq_lens_cpu,
|
||||
input_ids=input_ids.clone(),
|
||||
input_embedding=self.get_input_embeddings(),
|
||||
multimodal_model=self,
|
||||
)
|
||||
finally:
|
||||
_MM_TRACE_RIDS = prev_rids
|
||||
forward_batch.mm_input_embeds = input_embeds
|
||||
return input_embeds
|
||||
|
||||
|
||||
Reference in New Issue
Block a user