fix: restore VLM nightly regression coverage (#34662)
This commit is contained in:
@@ -2806,10 +2806,9 @@ class ServerArgs:
|
||||
Optional[Literal["cpu", "cuda_ipc", "cuda_vmm"]],
|
||||
"Transport multimodal features through CPU memory, a bounded CUDA IPC "
|
||||
"pool, or a bounded CUDA VMM pool. "
|
||||
"Unset resolves automatically: multimodal models on single-node CUDA "
|
||||
"deployments (without disaggregation) use cuda_ipc; validated multi-node "
|
||||
"GB200/GB300 MNNVL models use cuda_vmm when an IMEX channel is available; "
|
||||
"all other deployments use cpu. GPU transports reserve "
|
||||
"Unset uses cpu except for validated multi-node GB200/GB300 MNNVL models, "
|
||||
"which use cuda_vmm when an IMEX channel is available. Select cuda_ipc "
|
||||
"explicitly for single-node GPU transport. GPU transports reserve "
|
||||
"SGLANG_MM_FEATURE_CACHE_MB (default 1024 MiB) on the base GPU and fall "
|
||||
"back to CPU transport when the pool is full.",
|
||||
NS("mm"),
|
||||
@@ -7749,10 +7748,10 @@ class ServerArgs:
|
||||
def _handle_multimodal_feature_transport(self):
|
||||
"""Resolve multimodal feature transport before tokenizer workers start.
|
||||
|
||||
GPU transports use a fixed pool on ``base_gpu_id`` and therefore reduce
|
||||
the memory left for model/KV-cache allocations. The legacy CUDA IPC flag
|
||||
and environment variable remain supported so existing deployments map
|
||||
to this single policy.
|
||||
CUDA IPC is opt-in because its fixed pool on ``base_gpu_id`` reduces the
|
||||
memory left for model/KV-cache allocations. Multi-node MNNVL deployments
|
||||
may still auto-select CUDA VMM. The legacy CUDA IPC flag and environment
|
||||
variable remain supported so existing deployments map to this policy.
|
||||
"""
|
||||
requested_transport = self.mm_feature_transport
|
||||
legacy_ipc_is_set = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.is_set()
|
||||
@@ -7792,16 +7791,12 @@ class ServerArgs:
|
||||
and self.disaggregation_mode == "null"
|
||||
):
|
||||
# A full GPU pool always degrades to CPU transport per tensor.
|
||||
# CUDA IPC is intra-node; multi-node auto-selection is limited
|
||||
# to GB200/GB300 systems where the runtime already enables the
|
||||
# MNNVL/IMEX communication stack.
|
||||
# Keep CUDA IPC opt-in because even an idle pool consumes HBM
|
||||
# that would otherwise back the KV cache. Multi-node
|
||||
# auto-selection is limited to GB200/GB300 systems where the
|
||||
# runtime already enables the MNNVL/IMEX communication stack.
|
||||
if self.nnodes == 1:
|
||||
requested_transport = "cuda_ipc"
|
||||
logger.info(
|
||||
"Multimodal feature transport auto-resolved to cuda_ipc "
|
||||
"(single-node CUDA). Pass --mm-feature-transport=cpu to "
|
||||
"opt out."
|
||||
)
|
||||
requested_transport = "cpu"
|
||||
elif is_mnnvl_fabric_device() and os.path.exists(
|
||||
"/dev/nvidia-caps-imex-channels/channel0"
|
||||
):
|
||||
|
||||
@@ -2223,12 +2223,6 @@ class ModelLaunchSettings:
|
||||
self.extra_args.append(fixed_arg)
|
||||
|
||||
|
||||
class ModelEvalMetrics:
|
||||
def __init__(self, accuracy: float, eval_time: float):
|
||||
self.accuracy = accuracy
|
||||
self.eval_time = eval_time
|
||||
|
||||
|
||||
def extract_trace_link_from_bench_one_batch_server_output(output: str) -> str:
|
||||
match = re.search(r"\[Profile\]\((.*?)\)", output)
|
||||
if match:
|
||||
|
||||
Reference in New Issue
Block a user