Files
sglang/python/sglang/test/test_utils.py
T

2826 lines
90 KiB
Python

"""Common utilities for testing and benchmarking"""
import argparse
import asyncio
import contextlib
import copy
import doctest
import importlib.util
import inspect
import json
import logging
import os
import random
import re
import shlex
import subprocess
import sys
import threading
import time
import unittest
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from functools import wraps
from io import BytesIO
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any, Awaitable, Callable, List, Optional, Tuple
import aiohttp
import msgspec
import numpy as np
import psutil
import requests
import torch
import torch.nn.functional as F
from PIL import Image
from sglang.benchmark.serving import run_benchmark
from sglang.lang.global_config import global_config
from sglang.srt.environ import envs
from sglang.srt.utils import (
get_bool_env_var,
get_device,
is_blackwell,
is_cuda,
is_xpu,
kill_process_tree,
retry,
)
from sglang.srt.utils.network import is_port_available
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.utils import normalize_base_url
# General test models
DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct"
DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE = "meta-llama/Llama-3.2-1B"
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE = "Qwen/Qwen3-Reranker-0.6B"
DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1"
DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B"
DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat"
# MLA test models
DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST = "cross-encoder/ms-marco-MiniLM-L6-v2"
DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"
DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8"
DEFAULT_MODEL_NAME_FOR_TEST_MLA = "lmsys/sglang-ci-dsv3-test"
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = "lmsys/sglang-ci-dsv3-test-NextN"
# Hybrid Mamba models
DEFAULT_HYBRID_MAMBA_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-Next-80B-A3B-Instruct"
# Small GDN-hybrid (gated delta net) model that fits a single GPU
DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST = "Qwen/Qwen3.5-4B"
# VL test models
DEFAULT_MODEL_NAME_FOR_TEST_VL_PP = "Qwen/Qwen3-VL-2B-Thinking"
DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP = "zai-org/GLM-4.1V-9B-Thinking"
DEFAULT_MODEL_NAME_FOR_TEST_GEMMA4_PP = "google/gemma-4-26B-A4B-it"
DEFAULT_MODEL_NAME_FOR_TEST_GEMMA4_PLE_PP = "google/gemma-4-E4B-it"
# NVFP4 models
DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST = "nvidia/DeepSeek-V3-0324-FP4"
DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4 = "nvidia/Qwen3-30B-A3B-FP4"
# FP8 models
DEFAULT_MODEL_NAME_FOR_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8"
DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8"
DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8 = (
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic"
)
DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8 = (
"nvidia/Llama-3.1-8B-Instruct-FP8"
)
DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8 = "Qwen/Qwen3-1.7B-FP8"
DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE = "gaunernst/DeepSeek-V2-Lite-Chat-FP8"
# MXFP4 models
# Standard MXFP4 MoE test model
DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE = "openai/gpt-oss-20b"
# W8A8 models
DEFAULT_MODEL_NAME_FOR_TEST_W8A8 = "RedHatAI/Llama-3.2-3B-quantized.w8a8"
DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE = "nytopop/Qwen3-30B-A3B.w8a8"
# INT4 models
DEFAULT_MODEL_NAME_FOR_TEST_AWQ_INT4 = (
"hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4"
)
# EAGLE2 algorithm models
DEFAULT_TARGET_MODEL_EAGLE = "meta-llama/Llama-2-7b-chat-hf"
DEFAULT_DRAFT_MODEL_EAGLE = "lmsys/sglang-EAGLE-llama2-chat-7B"
# EAGLE3 model
DEFAULT_TARGET_MODEL_EAGLE3 = "meta-llama/Llama-3.1-8B-Instruct"
DEFAULT_DRAFT_MODEL_EAGLE3 = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B"
# DFLASH model
DEFAULT_TARGET_MODEL_DFLASH = "meta-llama/Llama-3.1-8B-Instruct"
DEFAULT_DRAFT_MODEL_DFLASH = "z-lab/LLaMA3.1-8B-Instruct-DFlash-UltraChat"
# EAGLE2 with DP-Attention models
DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN = "Qwen/Qwen3-30B-A3B"
DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN = "Tengyunw/qwen3_30b_moe_eagle3"
# Standalone speculative decoding models
DEFAULT_TARGET_MODEL_STANDALONE = "meta-llama/Llama-3.1-8B-Instruct"
DEFAULT_DRAFT_MODEL_STANDALONE = "meta-llama/Llama-3.2-1B-Instruct"
# N-gram speculative decoding models
DEFAULT_TARGET_MODEL_NGRAM = "Qwen/Qwen2.5-Coder-7B-Instruct"
# Other use cases
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST = (
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", # auto_round:auto_gptq
"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", # auto_round:auto_awq
)
DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION = (
"meta-llama/Llama-4-Scout-17B-16E-Instruct"
)
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct"
DEFAULT_REASONING_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-V3-0324"
DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST_NEXTN = "lmsys/DeepSeek-V3-NextN"
DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST = (
"hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4"
)
DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B"
DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8"
DEFAULT_ENABLE_ROUTED_EXPERTS_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B"
# Nightly tests
# Deliberate omission: a model another registered suite already uses as its base
# model is left out, since a regression there surfaces in that suite instead.
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = (
"meta-llama/Llama-3.1-70B-Instruct,Qwen/Qwen2-57B-A14B-Instruct"
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1 = "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4,hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4,hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4"
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN = "Qwen/Qwen2.5-1.5B-Instruct"
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST = "Qwen/Qwen2.5-VL-3B-Instruct"
DEFAULT_IMAGE_URL = "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png"
DEFAULT_VIDEO_URL = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4"
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 600
def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Image:
for i in range(max_retries):
try:
response = requests.get(image_url, timeout=30)
response.raise_for_status()
image = Image.open(BytesIO(response.content))
image.load()
return image
except Exception as e:
if i == max_retries - 1:
raise RuntimeError(
f"Failed to download image after {max_retries} retries: {image_url}"
) from e
time.sleep(2**i)
def build_vlm_image_prompt(processor, question: str) -> str:
# Take the image placeholder from the model's own HF chat template: a
# hand-written one silently degrades to a text-only prompt on any model
# whose placeholder differs.
return processor.apply_chat_template(
[
{
"role": "user",
"content": [{"type": "image"}, {"type": "text", "text": question}],
}
],
tokenize=False,
add_generation_prompt=True,
)
def is_in_ci():
"""Return whether it is in CI runner."""
return get_bool_env_var("SGLANG_IS_IN_CI")
def is_in_amd_ci():
"""Return whether it is in an AMD CI runner."""
return get_bool_env_var("SGLANG_IS_IN_CI_AMD")
def is_blackwell_system():
"""Same CUDA capability + toolkit semantics as ``sglang.srt.utils.is_blackwell``."""
return is_blackwell()
def is_h200_system():
"""Return whether it is running on an H200 system."""
return envs.IS_H200.get()
def is_rust_server_built():
"""Return whether the embedded Rust server extension (``SGLANG_RUST_SERVER``)
is importable.
The ``sglang.srt.rust_extensions`` Python package is always present; the
private ``_server`` module exists only when the PyO3 extension was built.
"""
return importlib.util.find_spec("sglang.srt.rust_extensions._server") is not None
def _use_cached_default_models(model_repo: str):
cache_dir = os.getenv("DEFAULT_MODEL_CACHE_DIR")
if cache_dir and model_repo:
model_path = os.path.join(cache_dir, model_repo)
if os.path.isdir(model_path):
return os.path.abspath(model_path)
return ""
if is_in_ci():
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
10000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 2000
)
else:
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
20000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 1000
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
if is_in_amd_ci():
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600 # Match H200 timeout for large models
if is_blackwell_system():
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3000
if is_h200_system():
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600
if is_in_ci() and is_xpu():
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 1800
def find_available_port(base_port: int):
port = base_port + random.randint(100, 1000)
while True:
if is_port_available(port):
return port
if port < 60000:
port += 42
else:
port -= 43
def auto_config_device() -> str:
"""Auto-config available device platform"""
try:
device = get_device()
except (RuntimeError, ImportError) as e:
print(f"Warning: {e} - Falling back to CPU")
device = "cpu"
return device
def add_common_sglang_args_and_parse(parser: argparse.ArgumentParser):
parser.add_argument("--parallel", type=int, default=64)
parser.add_argument("--host", type=str, default="127.0.0.1")
parser.add_argument("--port", type=int, default=30000)
parser.add_argument("--backend", type=str, default="srt")
parser.add_argument(
"--device",
type=str,
default="auto",
choices=["auto", "cuda", "rocm", "cpu"],
help="Device type (auto/cuda/rocm/cpu). Auto will detect available platforms",
)
parser.add_argument("--result-file", type=str, default="result.jsonl")
parser.add_argument("--raw-result-file", type=str)
args = parser.parse_args()
return args
def select_sglang_backend(args: argparse.Namespace):
from sglang.lang.backend.openai import OpenAI
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
if args.backend.startswith("srt"):
if args.backend == "srt-no-parallel":
global_config.enable_parallel_encoding = False
backend = RuntimeEndpoint(normalize_base_url(args.host, args.port))
elif args.backend.startswith("gpt-"):
backend = OpenAI(args.backend)
else:
raise ValueError(f"Invalid backend: {args.backend}")
return backend
def _get_default_models():
import inspect
current_module = inspect.getmodule(_get_default_models)
default_models = set()
for name, value in current_module.__dict__.items():
if (
isinstance(name, str)
and "DEFAULT_" in name
and "MODEL_" in name
and isinstance(value, str)
):
if "," in value:
parts = [part.strip() for part in value.split(",")]
default_models.update(parts)
else:
default_models.add(value.strip())
return json.dumps(list(default_models))
def try_cached_model(model_repo: str):
model_dir = _use_cached_default_models(model_repo)
return model_dir if model_dir else model_repo
def popen_with_error_check(command: list[str]):
process = subprocess.Popen(command, stdout=None, stderr=None)
def _run_and_check():
process.wait()
if process.returncode == -9:
return
if process.returncode != 0:
raise Exception(
f"{shlex.join(command)} exited with code {process.returncode}"
)
t = threading.Thread(target=_run_and_check, daemon=True)
t.start()
return process
def start_subprocess_fail_fast_watcher(
named_procs: list[tuple[str, subprocess.Popen]],
) -> threading.Event:
"""Abort the test runner the moment any watched subprocess exits non-zero.
Caller must `.set()` the returned Event before intentional teardown."""
stop = threading.Event()
def watcher():
while not stop.is_set():
for name, proc in named_procs:
rc = proc.poll() if proc else None
if rc is None or rc == 0:
continue
if stop.is_set():
return
sys.stderr.write(
f"[FIXTURE FAIL-FAST] {name} (pid={proc.pid}) exited "
f"rc={rc}; aborting.\n"
)
sys.stderr.flush()
for _, sib in named_procs:
if sib and sib is not proc:
try:
kill_process_tree(sib.pid, wait_timeout=10)
except Exception:
pass
# POSIX: signal N -> 128+N (os._exit masks negatives via & 0xff).
os._exit(rc if rc >= 0 else 128 + (-rc))
time.sleep(0.1)
threading.Thread(target=watcher, daemon=True, name="SubprocFailFastWatcher").start()
return stop
def _try_enable_offline_mode_if_cache_complete(
model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None
) -> Optional[str]:
"""Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the
per-run marker path, or None if offline mode was not enabled.
"""
other_args = other_args or []
# Skip offline mode for LoRA scenarios (dynamic adapter loading may need online access)
is_lora_enabled = "--enable-lora" in other_args or "--lora-paths" in other_args
if is_lora_enabled:
print(f"CI_OFFLINE: LoRA enabled, skip offline mode - {model_name_or_path}")
return None
# Fast-path: If subprocess env already has HF_HUB_OFFLINE=1, skip
if env.get("HF_HUB_OFFLINE") == "1":
print(
f"CI_OFFLINE: Subprocess env already has HF_HUB_OFFLINE=1, skip - {model_name_or_path}"
)
return None
# Detect before the marker check so the current launch's requirements are known.
requires_hf_quant_config = False
for i, arg in enumerate(other_args):
if arg == "--quantization" and i + 1 < len(other_args):
quant_value = other_args[i + 1].lower()
if quant_value in ["modelopt_fp4", "modelopt_fp8", "modelopt"]:
requires_hf_quant_config = True
break
marker_path = _validate_cache_for_offline(
model_name_or_path, requires_hf_quant_config
)
if marker_path is not None:
env["HF_HUB_OFFLINE"] = "1"
return marker_path
def _validate_cache_for_offline(
model_name_or_path: str, requires_hf_quant_config: bool
) -> Optional[str]:
"""Return the per-run marker path if the model's local cache is complete
enough to load offline, else None.
Markers are per-run and not shared across runners. A marker is only a hint;
the current launch's requirements (e.g. hf_quant_config.json) are revalidated.
"""
from sglang.srt.model_loader.ci_weight_validation import (
_get_per_run_marker_path,
_read_per_run_marker,
_write_per_run_marker,
validate_cache_lightweight,
)
from sglang.srt.utils import find_local_repo_dir
# Skip if already a local path
if os.path.isdir(model_name_or_path):
return None
# Try to find local snapshot
try:
snapshot_dir = find_local_repo_dir(model_name_or_path, revision=None)
if not snapshot_dir or not os.path.isdir(snapshot_dir):
return None
except Exception:
return None
# Check per-run marker (fast hint - snapshot validated earlier in this run)
per_run_marker = _read_per_run_marker(snapshot_dir)
if per_run_marker is not None:
# Still validate: a marker from a launch without --quantization must not
# enable offline mode for a later launch that needs hf_quant_config.json.
is_valid = validate_cache_lightweight(snapshot_dir, requires_hf_quant_config)
if not is_valid:
# Current launch requirements not met, ignore marker
print(
f"CI_OFFLINE: Per-run marker found but current validation failed "
f"(requires_hf_quant_config={requires_hf_quant_config}), "
f"will use online mode - {model_name_or_path}"
)
return None
# Marker exists and current validation passed
marker_path = _get_per_run_marker_path(snapshot_dir)
print(
f"CI_OFFLINE: Per-run marker found and current validation passed "
f"(requires_hf_quant_config={requires_hf_quant_config}), "
f"enabling offline mode - {model_name_or_path}"
)
return marker_path
# No per-run marker - perform lightweight validation
is_valid = validate_cache_lightweight(snapshot_dir, requires_hf_quant_config)
if not is_valid:
# Validation failed - cache is incomplete on this runner
print(
f"CI_OFFLINE: Cache validation failed "
f"(requires_hf_quant_config={requires_hf_quant_config}), "
f"will use online mode - {model_name_or_path}"
)
return None
# Write per-run marker for subsequent tests in this run
_write_per_run_marker(snapshot_dir, model_name_or_path)
# Return marker path for potential invalidation if offline launch fails
marker_path = _get_per_run_marker_path(snapshot_dir)
snapshot_basename = os.path.basename(snapshot_dir)
print(
f"CI_OFFLINE: Enabled HF_HUB_OFFLINE=1 for subprocess - "
f"validation passed for {model_name_or_path} "
f"(snapshot={snapshot_basename}, requires_hf_quant_config={requires_hf_quant_config})"
)
return marker_path
@contextlib.contextmanager
def _hf_offline_if_cache_complete(engine_kwargs: dict):
"""Run an in-process Engine startup with HF_HUB_OFFLINE=1 when every Hub
repo it loads validates in the local cache.
A scheduler that fails at startup kills the test process, so there is no
online retry as in popen_launch_server; any doubt keeps the startup online.
"""
import huggingface_hub.constants
if "server_args" in engine_kwargs:
server_args = engine_kwargs["server_args"]
model_path = server_args.model_path
tokenizer_path = server_args.tokenizer_path
draft_path = server_args.speculative_draft_model_path
quantization = server_args.quantization
uses_lora = server_args.enable_lora or bool(server_args.lora_paths)
else:
model_path = engine_kwargs.get("model_path")
tokenizer_path = engine_kwargs.get("tokenizer_path")
draft_path = engine_kwargs.get("speculative_draft_model_path")
quantization = engine_kwargs.get("quantization")
uses_lora = bool(
engine_kwargs.get("enable_lora") or engine_kwargs.get("lora_paths")
)
# Adapters can be loaded by repo id after startup, from an offline scheduler.
if uses_lora or os.environ.get("HF_HUB_OFFLINE") == "1":
yield
return
requires_hf_quant_config = quantization in [
"modelopt_fp4",
"modelopt_fp8",
"modelopt",
]
repo_ids = {
path
for path in (model_path, tokenizer_path, draft_path)
if path is not None and not os.path.isdir(path)
}
if not repo_ids or not all(
_validate_cache_for_offline(
repo_id, requires_hf_quant_config and repo_id == model_path
)
for repo_id in repo_ids
):
yield
return
# Spawned children read the env at import; this process already imported
# huggingface_hub, which reads it once into a module constant.
prev_env = os.environ.get("HF_HUB_OFFLINE")
prev_constant = huggingface_hub.constants.HF_HUB_OFFLINE
os.environ["HF_HUB_OFFLINE"] = "1"
huggingface_hub.constants.HF_HUB_OFFLINE = True
try:
yield
finally:
if prev_env is None:
os.environ.pop("HF_HUB_OFFLINE", None)
else:
os.environ["HF_HUB_OFFLINE"] = prev_env
huggingface_hub.constants.HF_HUB_OFFLINE = prev_constant
def _install_ci_engine_offline_hook():
"""In CI, wrap Engine.__init__ once per process with _hf_offline_if_cache_complete."""
try:
from sglang.srt.entrypoints.engine import Engine
except ImportError as e:
# A test that never builds an Engine must not fail on this.
print(f"CI_OFFLINE: Engine import failed, skip the offline hook: {e}")
return
orig_init = Engine.__init__
if getattr(orig_init, "_ci_offline_wrapped", False):
return
@wraps(orig_init)
def init(self, **kwargs):
with _hf_offline_if_cache_complete(kwargs):
orig_init(self, **kwargs)
init._ci_offline_wrapped = True
Engine.__init__ = init
def _create_clean_subprocess_env(env: dict) -> dict:
"""Create a clean subprocess environment without internal CI keys."""
child_env = env.copy()
keys_to_remove = [
k for k in child_env if k.startswith(("_CI_OFFLINE_", "CI_OFFLINE_"))
]
for k in keys_to_remove:
del child_env[k]
return child_env
def _subprocess_popen_with_outputs(
command: list,
env: Optional[dict],
return_stdout_stderr: Optional[tuple],
) -> subprocess.Popen:
# Release allocator-cached GPU memory to the driver before spawning a
# server: cached blocks stay cudaMalloc'd and shrink the child's memory.
if torch.cuda.is_initialized():
torch.cuda.empty_cache()
if not return_stdout_stderr:
return subprocess.Popen(command, stdout=None, stderr=None, env=env)
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
text=True,
bufsize=1,
)
def _dump(src, sinks):
for line in iter(src.readline, ""):
for sink in sinks:
sink.write(line)
sink.flush()
src.close()
threading.Thread(
target=_dump,
args=(process.stdout, [return_stdout_stderr[0], sys.stdout]),
daemon=True,
).start()
threading.Thread(
target=_dump,
args=(process.stderr, [return_stdout_stderr[1], sys.stderr]),
daemon=True,
).start()
return process
def _launch_server_process(
command: List[str],
env: dict,
return_stdout_stderr: Optional[tuple],
model: str,
) -> subprocess.Popen:
"""Launch server subprocess with clean environment.
`return_stdout_stderr` is an optional (stdout_file, stderr_file) pair.
"""
child_env = _create_clean_subprocess_env(env)
hf_hub_offline = child_env.get("HF_HUB_OFFLINE", "0")
print(f"CI_OFFLINE: Launching server HF_HUB_OFFLINE={hf_hub_offline} model={model}")
return _subprocess_popen_with_outputs(
command=command,
env=child_env,
return_stdout_stderr=return_stdout_stderr,
)
def _wait_for_server_health(
proc: subprocess.Popen,
base_url: str,
api_key: Optional[str],
timeout_duration: float,
) -> Tuple[bool, Optional[str]]:
"""Wait for server health check to pass; return (success, error_message)."""
start_time = time.perf_counter()
with requests.Session() as session:
while time.perf_counter() - start_time < timeout_duration:
return_code = proc.poll()
if return_code is not None:
return False, f"Server process exited with code {return_code}"
try:
headers = {
"Content-Type": "application/json; charset=utf-8",
"Authorization": f"Bearer {api_key}",
}
response = session.get(
f"{base_url}/health_generate",
headers=headers,
timeout=5,
)
if response.status_code == 200:
return True, None
except requests.RequestException:
pass
return_code = proc.poll()
if return_code is not None:
return False, f"Server unexpectedly exited (return_code={return_code})"
time.sleep(10)
return False, "Server failed to start within the timeout period"
def unified_radix_tree_server_env(
tree_core_backend: str, **extra_env: str
) -> dict[str, str]:
return {
**os.environ,
**extra_env,
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND": tree_core_backend,
}
def popen_launch_server(
model: str,
base_url: str,
timeout: float,
api_key: Optional[str] = None,
other_args: Optional[list[str]] = None,
env: Optional[dict] = None,
return_stdout_stderr: Optional[tuple] = None,
device: str = "auto",
pd_separated: bool = False,
num_replicas: Optional[int] = None,
):
"""Launch a server process with automatic device detection and offline/online retry.
`device` is one of "auto", "cuda", "rocm", "cpu";
`num_replicas` applies to mixed PD mode.
"""
other_args = other_args or []
# Auto-detect device if needed
if device == "auto":
device = auto_config_device()
other_args = list(other_args)
other_args += ["--device", str(device)]
# CI-specific: Validate cache and enable offline mode if complete
if env is None:
env = os.environ.copy()
else:
merged = os.environ.copy()
merged.update(env)
env = merged
# A dying predecessor can hold the derived port plan past
# kill_process_tree() while GPU teardown completes; give CI launches
# teardown-sized patience (see wait_port_available).
env.setdefault("SGLANG_WAIT_PORT_TIMEOUT", "120")
# Store per-run marker path for potential invalidation
per_run_marker_path = None
try:
from sglang.utils import is_in_ci
if is_in_ci():
per_run_marker_path = _try_enable_offline_mode_if_cache_complete(
model, env, other_args
)
except Exception as e:
print(f"CI cache validation failed (non-fatal): {e}")
# Build server command
_, host, port = base_url.split(":")
host = host[2:]
use_mixed_pd_engine = not pd_separated and num_replicas is not None
if pd_separated or use_mixed_pd_engine:
command = [
"python3",
"-m",
"sglang.launch_pd_server",
"--model-path",
model,
*[str(x) for x in other_args],
]
else:
command = [
"sglang",
"serve",
"--model-path",
model,
*[str(x) for x in other_args],
]
if pd_separated or use_mixed_pd_engine:
command.extend(["--lb-host", host, "--lb-port", port])
else:
command.extend(["--host", host, "--port", port])
if use_mixed_pd_engine:
command.extend(["--mixed", "--num-replicas", str(num_replicas)])
if api_key:
command += ["--api-key", api_key]
print(f"command={shlex.join(command)}")
# Track if offline mode was enabled for potential retry
offline_enabled = env.get("HF_HUB_OFFLINE") == "1"
# First launch attempt
process = _launch_server_process(command, env, return_stdout_stderr, model)
success, error_msg = _wait_for_server_health(process, base_url, api_key, timeout)
# If offline launch failed and offline was enabled, retry with online mode
if not success and offline_enabled:
print(
f"CI_OFFLINE: Offline launch failed ({error_msg}), retrying with online mode..."
)
# Kill failed process
try:
if process.poll() is None:
kill_process_tree(process.pid)
else:
process.wait(timeout=5)
except Exception as e:
print(f"CI_OFFLINE: Error cleaning up failed offline process: {e}")
# Invalidate per-run marker to prevent subsequent tests from using offline
if per_run_marker_path and os.path.exists(per_run_marker_path):
try:
os.remove(per_run_marker_path)
print("CI_OFFLINE: Invalidated per-run marker due to offline failure")
except Exception as e:
print(f"CI_OFFLINE: Failed to remove per-run marker: {e}")
# Retry with online mode
env["HF_HUB_OFFLINE"] = "0"
process = _launch_server_process(command, env, return_stdout_stderr, model)
success, error_msg = _wait_for_server_health(
process, base_url, api_key, timeout
)
if success:
print("CI_OFFLINE: Online retry succeeded")
return process
# Online retry also failed
try:
kill_process_tree(process.pid)
except Exception as e:
print(f"CI_OFFLINE: Error killing process after online retry failure: {e}")
if "exited" in error_msg:
raise Exception(error_msg + ". Check server logs for errors.")
raise TimeoutError(error_msg)
# First attempt succeeded or offline was not enabled
if success:
return process
# First attempt failed and offline was not enabled
try:
kill_process_tree(process.pid)
except Exception as e:
print(f"CI_OFFLINE: Error killing process after first attempt failure: {e}")
if "exited" in error_msg:
raise Exception(error_msg + ". Check server logs for errors.")
raise TimeoutError(error_msg)
def terminate_and_kill_process_tree(
process,
terminate_timeout: float = 60,
**kill_kwargs,
) -> None:
"""Shut a launched server down gracefully, then SIGKILL whatever is left.
A bare ``kill_process_tree`` leaves the kernel to unwind the CUDA context
and unpin the host memory during process reclaim, which can hold GPU memory
for minutes on a busy host -- long enough to trip the per-class GPU-idle
gate in the next ``setUpClass``. SIGTERM first so the server releases those
resources in userspace, then wait for the memory to come back:
a reaped tree does not mean the driver is done with it.
"""
pids = collect_process_tree_pids(process.pid)
process.terminate()
try:
process.wait(timeout=terminate_timeout)
except subprocess.TimeoutExpired:
pass
kill_process_tree(process.pid, **kill_kwargs)
wait_for_gpu_release(pids)
def popen_launch_pd_server(
model: str,
base_url: str,
timeout: float,
api_key: Optional[str] = None,
other_args: list[str] = (),
env: Optional[dict] = None,
return_stdout_stderr: Optional[tuple] = None,
):
_, host, port = base_url.split(":")
host = host[2:]
command = "sglang.launch_server"
command = [
"python3",
"-m",
command,
"--model-path",
model,
*[str(x) for x in other_args],
]
command.extend(
[
"--host",
host,
"--port",
port,
]
)
if api_key:
command += ["--api-key", api_key]
print(f"command={' '.join(command)}")
# Merge so caller-supplied env adds to, not replaces, PATH / PYTHONPATH / etc.
if env is not None:
env = {**os.environ, **env}
return _subprocess_popen_with_outputs(
command=command,
env=env,
return_stdout_stderr=return_stdout_stderr,
)
def get_similarities(vec1, vec2):
return F.cosine_similarity(torch.tensor(vec1), torch.tensor(vec2), dim=0)
def get_benchmark_args(
base_url="",
backend="sglang",
dataset_name="",
dataset_path="",
tokenizer="",
num_prompts=500,
sharegpt_output_len=None,
random_input_len=4096,
random_output_len=2048,
sharegpt_context_len=None,
request_rate=float("inf"),
disable_stream=False,
disable_ignore_eos=False,
seed: int = 0,
device="auto",
pd_separated: bool = False,
lora_name=None,
lora_request_distribution="uniform",
lora_zipf_alpha=1.5,
gsp_num_groups=4,
gsp_prompts_per_group=4,
gsp_system_prompt_len=128,
gsp_question_len=32,
gsp_output_len=32,
gsp_num_turns=1,
header=None,
max_concurrency=None,
):
return SimpleNamespace(
backend=backend,
base_url=base_url,
host=None,
port=None,
dataset_name=dataset_name,
dataset_path=dataset_path,
model=None,
tokenizer=tokenizer,
num_prompts=num_prompts,
sharegpt_output_len=sharegpt_output_len,
sharegpt_context_len=sharegpt_context_len,
random_input_len=random_input_len,
random_output_len=random_output_len,
random_range_ratio=0.0,
request_rate=request_rate,
multi=None,
output_file=None,
disable_tqdm=False,
disable_stream=disable_stream,
return_logprob=False,
return_routed_experts=False,
seed=seed,
disable_ignore_eos=disable_ignore_eos,
extra_request_body=None,
apply_chat_template=False,
profile=None,
lora_name=lora_name,
lora_request_distribution=lora_request_distribution,
lora_zipf_alpha=lora_zipf_alpha,
prompt_suffix="",
device=device,
pd_separated=pd_separated,
gsp_num_groups=gsp_num_groups,
gsp_prompts_per_group=gsp_prompts_per_group,
gsp_system_prompt_len=gsp_system_prompt_len,
gsp_question_len=gsp_question_len,
gsp_output_len=gsp_output_len,
gsp_num_turns=gsp_num_turns,
header=header,
max_concurrency=max_concurrency,
ready_check_timeout_sec=0,
)
def run_bench_serving(
model,
num_prompts,
request_rate,
other_server_args,
dataset_name="random",
dataset_path="",
tokenizer=None,
random_input_len=4096,
random_output_len=2048,
sharegpt_context_len=None,
disable_stream=False,
disable_ignore_eos=False,
need_warmup=False,
seed: int = 0,
device="auto",
background_task: Optional[Callable[[str, asyncio.Event], Awaitable[None]]] = None,
lora_name: Optional[str] = None,
):
if device == "auto":
device = auto_config_device()
# Launch the server
base_url = DEFAULT_URL_FOR_TEST
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_server_args,
)
# Prefer the local snapshot so the client's AutoTokenizer skips the HF Hub API,
# which can stall for minutes in CI.
bench_tokenizer = tokenizer
if bench_tokenizer is None:
try:
from sglang.srt.utils import find_local_repo_dir
local_dir = find_local_repo_dir(model, revision=None)
if local_dir and os.path.isdir(local_dir):
bench_tokenizer = local_dir
except Exception:
pass
# Run benchmark
args = get_benchmark_args(
base_url=base_url,
dataset_name=dataset_name,
dataset_path=dataset_path,
tokenizer=bench_tokenizer,
num_prompts=num_prompts,
random_input_len=random_input_len,
random_output_len=random_output_len,
sharegpt_context_len=sharegpt_context_len,
request_rate=request_rate,
disable_stream=disable_stream,
disable_ignore_eos=disable_ignore_eos,
seed=seed,
device=device,
lora_name=lora_name,
)
async def _run():
if need_warmup:
warmup_args = copy.deepcopy(args)
warmup_args.num_prompts = 16
await asyncio.to_thread(run_benchmark, warmup_args)
start_event = asyncio.Event()
stop_event = asyncio.Event()
task_handle = (
asyncio.create_task(background_task(base_url, start_event, stop_event))
if background_task
else None
)
try:
start_event.set()
result = await asyncio.to_thread(run_benchmark, args)
finally:
if task_handle:
stop_event.set()
await task_handle
return result
try:
res = asyncio.run(_run())
finally:
kill_process_tree(process.pid)
assert res["completed"] == num_prompts
return res
async def _run_api_benchmark_requests(
base_url: str,
endpoint: str,
test_requests: List[dict],
num_requests: int,
response_validator: Callable[[dict], bool],
):
"""Run API benchmark requests and return a dict of metrics."""
start_time = time.monotonic()
successful_requests = 0
total_latency = 0
latencies = []
async with aiohttp.ClientSession() as session:
for request_data in test_requests:
try:
request_start = time.monotonic()
async with session.post(
f"{base_url}{endpoint}",
json=request_data,
timeout=aiohttp.ClientTimeout(total=30),
) as response:
if response.status == 200:
response_data = await response.json()
request_end = time.monotonic()
if response_validator(response_data):
latency_ms = (request_end - request_start) * 1000
latencies.append(latency_ms)
total_latency += latency_ms
successful_requests += 1
except Exception:
continue
end_time = time.monotonic()
total_time = end_time - start_time
if successful_requests > 0:
throughput = successful_requests / total_time
avg_latency = total_latency / successful_requests
p95_latency = np.percentile(latencies, 95) if latencies else 0
return {
"completed": successful_requests,
"total_requests": num_requests,
"throughput": throughput,
"avg_latency_ms": avg_latency,
"p95_latency_ms": p95_latency,
"successful_requests": successful_requests,
}
else:
return {
"completed": 0,
"total_requests": num_requests,
"throughput": 0,
"avg_latency_ms": 0,
"p95_latency_ms": 0,
"successful_requests": 0,
}
def run_score_benchmark(
model,
num_requests=100,
batch_size=5,
other_server_args=None,
need_warmup=False,
device="auto",
):
"""Score API benchmark function compatible with run_bench_serving pattern"""
return run_score_benchmark_multi(
model,
[batch_size],
num_requests=num_requests,
other_server_args=other_server_args,
need_warmup=need_warmup,
device=device,
)[0]
def run_score_benchmark_multi(
model,
batch_sizes,
num_requests=100,
other_server_args=None,
need_warmup=False,
device="auto",
):
"""One server, one benchmark per batch size.
Batch size is a property of the request, not of the server, so the launch
is shared rather than repeated per size.
"""
if other_server_args is None:
other_server_args = []
if device == "auto":
device = auto_config_device()
# Launch the server (consistent with run_bench_serving)
base_url = DEFAULT_URL_FOR_TEST
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_server_args,
)
async def _run_benchmark(batch_size, warmup):
# Load tokenizer for generating test data
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
tokenizer = get_tokenizer(model)
# Score API configuration
score_query_tokens = 120
score_item_tokens = 180
score_label_token_ids = [9454, 2753] # Yes/No token IDs
special_token = "<|im_start|>"
def generate_text_with_token_count(num_tokens):
"""Generate text with precise token count using replicated token."""
text = special_token * num_tokens
actual_tokens = len(tokenizer.encode(text, add_special_tokens=False))
if actual_tokens != num_tokens:
text = special_token * (
num_tokens
// len(tokenizer.encode(special_token, add_special_tokens=False))
)
return text
if warmup:
warmup_data = {
"query": generate_text_with_token_count(score_query_tokens),
"items": [
generate_text_with_token_count(score_item_tokens) for _ in range(3)
],
"label_token_ids": score_label_token_ids,
"model": model,
"apply_softmax": True,
}
async with aiohttp.ClientSession() as session:
try:
await session.post(
f"{base_url}/v1/score",
json=warmup_data,
timeout=aiohttp.ClientTimeout(total=30),
)
except Exception:
pass # Ignore warmup errors
test_requests = []
for i in range(num_requests):
query = generate_text_with_token_count(score_query_tokens)
items = [
generate_text_with_token_count(score_item_tokens)
for _ in range(batch_size)
]
score_data = {
"query": query,
"items": items,
"label_token_ids": score_label_token_ids,
"model": model,
"apply_softmax": True,
}
test_requests.append(score_data)
# Run benchmark requests using shared helper
return await _run_api_benchmark_requests(
base_url=base_url,
endpoint="/v1/score",
test_requests=test_requests,
num_requests=num_requests,
response_validator=lambda resp: "scores" in resp or "logprobs" in resp,
)
try:
results = [
asyncio.run(_run_benchmark(bs, need_warmup and i == 0))
for i, bs in enumerate(batch_sizes)
]
finally:
kill_process_tree(process.pid)
for res in results:
assert res["completed"] == res["successful_requests"]
return results
def run_embeddings_benchmark(
model,
num_requests=100,
batch_size=1,
input_tokens=500,
other_server_args=None,
need_warmup=False,
device="auto",
):
"""Embeddings API benchmark function compatible with run_bench_serving pattern"""
return run_embeddings_benchmark_multi(
model,
[batch_size],
num_requests=num_requests,
input_tokens=input_tokens,
other_server_args=other_server_args,
need_warmup=need_warmup,
device=device,
)[0]
def run_embeddings_benchmark_multi(
model,
batch_sizes,
num_requests=100,
input_tokens=500,
other_server_args=None,
need_warmup=False,
device="auto",
):
"""One server, one benchmark per batch size. See run_score_benchmark_multi."""
if other_server_args is None:
other_server_args = []
if device == "auto":
device = auto_config_device()
# Add --is-embedding flag for embedding models
server_args = ["--is-embedding"] + other_server_args
# Launch the server (consistent with run_bench_serving)
base_url = DEFAULT_URL_FOR_TEST
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=server_args,
)
async def _run_benchmark(batch_size, warmup):
def generate_text_with_token_count(num_tokens):
"""Generate text with precise token count using special tokens."""
special_token = "<|im_start|>"
text = special_token * num_tokens
return text
# Generate input text
input_text = generate_text_with_token_count(input_tokens)
if warmup:
warmup_data = {
"input": input_text,
"model": model,
}
async with aiohttp.ClientSession() as session:
try:
await session.post(
f"{base_url}/v1/embeddings",
json=warmup_data,
timeout=aiohttp.ClientTimeout(total=30),
)
except Exception:
pass # Ignore warmup errors
test_requests = []
for i in range(num_requests):
if batch_size == 1:
input_data = input_text
else:
input_data = [input_text for _ in range(batch_size)]
embeddings_data = {
"input": input_data,
"model": model,
}
test_requests.append(embeddings_data)
# Run benchmark requests using shared helper
return await _run_api_benchmark_requests(
base_url=base_url,
endpoint="/v1/embeddings",
test_requests=test_requests,
num_requests=num_requests,
response_validator=lambda resp: "data" in resp,
)
try:
results = [
asyncio.run(_run_benchmark(bs, need_warmup and i == 0))
for i, bs in enumerate(batch_sizes)
]
finally:
kill_process_tree(process.pid)
for res in results:
assert res["completed"] == res["successful_requests"]
return results
def run_bench_serving_multi(
model,
base_url,
other_server_args,
benchmark_args,
need_warmup=False,
pd_separated=False,
):
# Launch the server
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_server_args,
pd_separated=pd_separated,
)
# run benchmark for all
res_l = []
try:
for args in benchmark_args:
if need_warmup:
warmup_args = copy.deepcopy(args)
warmup_args.num_prompts = 16
run_benchmark(warmup_args)
res = run_benchmark(args)
res_l.append((args, res))
finally:
kill_process_tree(process.pid)
return res_l
def run_bench_one_batch(model, other_args):
"""Launch an offline process with automatic device detection."""
# Auto-detect device if needed
device = auto_config_device()
print(f"Auto-configed device: {device}", flush=True)
other_args += ["--device", str(device)]
command = [
"python3",
"-m",
"sglang.benchmark.one_batch",
"--batch-size",
"1",
"--input",
"128",
"--output",
"8",
*[str(x) for x in other_args],
]
if model is not None:
command += ["--model-path", model]
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
prefill_latency = None
decode_throughput = None
decode_latency = None
try:
stdout, stderr = process.communicate()
output = stdout.decode(errors="backslashreplace")
error = stderr.decode(errors="backslashreplace")
print(f"Output: {output}", flush=True)
print(f"Error: {error}", flush=True)
# Return prefill_latency, decode_throughput, decode_latency
pattern = r"Benchmark[\s\S]*Total"
match = re.search(pattern, output)
bench_output = match[0] if match else ""
pattern = r".*?latency: (?P<latency>\d+\.\d+).*?throughput:\s*(?P<throughput>\d+\.\d+)"
match = re.search(r"Prefill." + pattern, bench_output)
if match:
prefill_latency = float(match.group("latency"))
match = re.search(r"Decode." + pattern, bench_output)
if match:
decode_latency = float(match.group("latency"))
decode_throughput = float(match.group("throughput"))
finally:
kill_process_tree(process.pid)
if prefill_latency is None or decode_throughput is None or decode_latency is None:
raise RuntimeError(
f"Failed to parse benchmark output. "
f"prefill_latency={prefill_latency}, decode_throughput={decode_throughput}, decode_latency={decode_latency}"
)
return prefill_latency, decode_throughput, decode_latency
def run_bench_offline_throughput(model, other_args):
command = [
"python3",
"-m",
"sglang.benchmark.offline_throughput",
"--num-prompts",
"1",
"--dataset-name",
"random",
"--random-input-len",
"256",
"--random-output-len",
"256",
"--model-path",
model,
*[str(x) for x in other_args],
]
print(f"command={' '.join(command)}")
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
stdout, stderr = process.communicate()
output = stdout.decode(errors="backslashreplace")
error = stderr.decode(errors="backslashreplace")
print(f"Output: {output}", flush=True)
print(f"Error: {error}", flush=True)
output_throughput = -1
for line in output.split("\n"):
if "Last generation throughput (tok/s):" in line:
output_throughput = float(line.split(":")[-1])
finally:
kill_process_tree(process.pid)
return output_throughput
def run_bench_one_batch_server(
model,
base_url,
server_args,
bench_args,
other_server_args,
simulate_spec_acc_lens=None,
):
from sglang.bench_one_batch_server import run_benchmark
if simulate_spec_acc_lens is not None:
env = {**os.environ, "SIMULATE_ACC_LEN": str(simulate_spec_acc_lens)}
else:
env = None
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_server_args,
env=env,
)
try:
run_benchmark(server_args=server_args, bench_args=bench_args)
finally:
kill_process_tree(process.pid)
def lcs(X, Y):
m = len(X)
n = len(Y)
L = [[0] * (n + 1) for _ in range(m + 1)]
for i in range(m + 1):
for j in range(n + 1):
if i == 0 or j == 0:
L[i][j] = 0
elif X[i - 1] == Y[j - 1]:
L[i][j] = L[i - 1][j - 1] + 1
else:
L[i][j] = max(L[i - 1][j], L[i][j - 1])
return L[m][n]
def calculate_rouge_l(output_strs_list1, output_strs_list2):
"""calculate the ROUGE-L score"""
rouge_l_scores = []
for s1, s2 in zip(output_strs_list1, output_strs_list2):
lcs_len = lcs(s1, s2)
precision = lcs_len / len(s1) if len(s1) > 0 else 0
recall = lcs_len / len(s2) if len(s2) > 0 else 0
if precision + recall > 0:
fmeasure = (2 * precision * recall) / (precision + recall)
else:
fmeasure = 0.0
rouge_l_scores.append(fmeasure)
return rouge_l_scores
STDERR_FILENAME = "/tmp/stderr.txt"
STDOUT_FILENAME = "/tmp/stdout.txt"
def read_output(output_lines: List[str], filename: str = STDERR_FILENAME):
"""Print the output in real time with another thread."""
while not os.path.exists(filename):
time.sleep(0.01)
pt = 0
while pt >= 0:
if pt > 0 and not os.path.exists(filename):
break
try:
lines = open(filename).readlines()
except FileNotFoundError:
print(f"{pt=}, {os.path.exists(filename)=}")
raise
for line in lines[pt:]:
print(line, end="", flush=True)
output_lines.append(line)
pt += 1
time.sleep(0.1)
def run_and_check_memory_leak(
workload_func,
disable_radix_cache,
enable_mixed_chunk,
disable_overlap,
chunked_prefill_size,
assert_has_abort,
api_key: Optional[str] = None,
):
other_args = [
"--chunked-prefill-size",
str(chunked_prefill_size),
"--log-level",
"debug",
]
if disable_radix_cache:
other_args += ["--disable-radix-cache"]
if enable_mixed_chunk:
other_args += ["--enable-mixed-chunk"]
if disable_overlap:
other_args += ["--disable-overlap-schedule"]
model = DEFAULT_MODEL_NAME_FOR_TEST
port = random.randint(4000, 5000)
base_url = f"http://127.0.0.1:{port}"
# Create files and launch the server
stdout = open(STDOUT_FILENAME, "w")
stderr = open(STDERR_FILENAME, "w")
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
return_stdout_stderr=(stdout, stderr),
api_key=api_key,
)
# Launch a thread to stream the output
output_lines = []
t = threading.Thread(target=read_output, args=(output_lines,))
t.start()
# Run the workload
workload_func(base_url, model)
# Clean up everything
kill_process_tree(process.pid)
stdout.close()
stderr.close()
if os.path.exists(STDOUT_FILENAME):
os.remove(STDOUT_FILENAME)
if os.path.exists(STDERR_FILENAME):
os.remove(STDERR_FILENAME)
kill_process_tree(process.pid)
t.join()
# Assert success
has_new_server = False
has_leak = False
has_abort = False
for line in output_lines:
if "Uvicorn running" in line:
has_new_server = True
if "leak" in line:
has_leak = True
if "Abort" in line:
has_abort = True
assert has_new_server
assert not has_leak
if assert_has_abort:
assert has_abort
def run_command_and_capture_output(command, env: Optional[dict] = None):
stdout = open(STDOUT_FILENAME, "w")
stderr = open(STDERR_FILENAME, "w")
process = subprocess.Popen(
command, stdout=stdout, stderr=stdout, env=env, text=True
)
# Launch a thread to stream the output
output_lines = []
t = threading.Thread(target=read_output, args=(output_lines, STDOUT_FILENAME))
t.start()
# Join the process
process.wait()
stdout.close()
stderr.close()
if os.path.exists(STDOUT_FILENAME):
os.remove(STDOUT_FILENAME)
if os.path.exists(STDERR_FILENAME):
os.remove(STDERR_FILENAME)
kill_process_tree(process.pid)
t.join()
return output_lines
def run_mmlu_test(
disable_radix_cache=False,
enable_mixed_chunk=False,
disable_overlap=False,
chunked_prefill_size=32,
):
def workload_func(base_url, model):
# Run the eval
args = SimpleNamespace(
base_url=base_url,
model=model,
eval_name="mmlu",
num_examples=128,
num_threads=128,
)
try:
metrics = run_sgl_eval(args)
assert metrics["score"] >= 0.65, f"{metrics=}"
finally:
pass
run_and_check_memory_leak(
workload_func,
disable_radix_cache,
enable_mixed_chunk,
disable_overlap,
chunked_prefill_size,
assert_has_abort=False,
)
def run_mulit_request_test(
disable_radix_cache=False,
enable_mixed_chunk=False,
enable_overlap=False,
chunked_prefill_size=32,
):
def workload_func(base_url, model):
def run_one(_):
prompt = """
System: You are a helpful assistant.
User: What is the capital of France?
Assistant: The capital of France is
"""
response = requests.post(
f"{base_url}/generate",
json={
"text": prompt,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 8,
},
},
)
response.json()
with ThreadPoolExecutor(2) as executor:
list(executor.map(run_one, list(range(4))))
run_and_check_memory_leak(
workload_func,
disable_radix_cache,
enable_mixed_chunk,
enable_overlap,
chunked_prefill_size,
assert_has_abort=False,
)
def write_github_step_summary(content):
if not os.environ.get("GITHUB_STEP_SUMMARY"):
logging.warning("GITHUB_STEP_SUMMARY environment variable not set")
return
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(content)
def run_logprob_check(self: unittest.TestCase, arg: Tuple):
(
input_len,
output_len,
temperature,
logprob_start_len,
return_logprob,
top_logprobs_num,
) = arg
input_ids = list(range(input_len))
response = requests.post(
self.base_url + "/generate",
json={
"input_ids": input_ids,
"sampling_params": {
"temperature": temperature,
"max_new_tokens": output_len,
"ignore_eos": True,
},
"return_logprob": return_logprob,
"logprob_start_len": logprob_start_len,
"top_logprobs_num": top_logprobs_num,
},
)
response_json = response.json()
res = response_json
self.assertEqual(res["meta_info"]["prompt_tokens"], input_len)
self.assertEqual(res["meta_info"]["completion_tokens"], output_len)
# Test the number of tokens are correct
if return_logprob:
self.assertEqual(
len(res["meta_info"]["input_token_logprobs"]) + logprob_start_len,
res["meta_info"]["prompt_tokens"],
)
self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), output_len)
if top_logprobs_num:
self.assertEqual(
len(res["meta_info"]["input_top_logprobs"]) + logprob_start_len,
res["meta_info"]["prompt_tokens"],
)
self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), output_len)
for i in range(output_len):
self.assertEqual(
len(res["meta_info"]["output_top_logprobs"][i]),
top_logprobs_num,
)
# Test the top-1 tokens are the same as output tokens if temperature == 0
if temperature == 0:
rank = 0
while rank < len(res["meta_info"]["output_top_logprobs"][i]):
try:
self.assertListEqual(
res["meta_info"]["output_token_logprobs"][i],
res["meta_info"]["output_top_logprobs"][i][rank],
)
break
except AssertionError:
# There's a tie. Allow the second item in this case.
if (
res["meta_info"]["output_top_logprobs"][i][rank][0]
== res["meta_info"]["output_top_logprobs"][i][rank + 1][
0
]
):
rank += 1
else:
raise
def send_generate_requests(base_url: str, num_requests: int) -> List[str]:
"""Sends generate request serially and returns status codes. Max concurrency is 1."""
def generate():
prompt = """
System: You are a helpful assistant.
User: What is the capital of France?
Assistant: The capital of France is
"""
response = requests.post(
f"{base_url}/generate",
json={
"text": prompt,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 500,
},
},
)
return response.status_code
return [generate() for _ in range(num_requests)]
async def send_concurrent_generate_requests(
base_url: str, num_requests: int
) -> List[str]:
"""Sends generate request concurrently and returns status codes. Max concurrency is num_requests."""
async def async_generate():
async with aiohttp.ClientSession() as session:
prompt = """
System: You are a helpful assistant.
User: What is the capital of France?
Assistant: The capital of France is
"""
async with session.post(
f"{base_url}/generate",
json={
"text": prompt,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 500,
},
},
) as response:
return response.status
tasks = [asyncio.create_task(async_generate()) for _ in range(num_requests)]
return await asyncio.gather(*tasks)
async def send_concurrent_generate_requests_with_custom_params(
base_url: str,
custom_params: List[dict[str, Any]],
) -> Tuple[int, Any]:
"""Sends generate request concurrently with custom parameters and returns status code and response json tuple. Max concurrency is num_requests."""
base_payload = {
"text": """
System: You are a helpful assistant.
User: What is the capital of France?
Assistant: The capital of France is
""",
"sampling_params": {
"temperature": 0,
"max_new_tokens": 500,
},
}
async def async_generate_with_priority(req):
async with aiohttp.ClientSession() as session:
async with session.post(
f"{base_url}/generate",
json=req,
) as response:
resp_json = await response.json()
return (response.status, resp_json)
tasks = []
for c in custom_params:
req = base_payload.copy()
req.update(c)
tasks.append(asyncio.create_task(async_generate_with_priority(req)))
return await asyncio.gather(*tasks)
def run_distributed_test(func, world_size=2, backend="nccl", **kwargs):
"""Spawn ``world_size`` processes, initialise torch.distributed in each,
run *func(rank, **kwargs)*, and propagate any worker exception to the caller.
"""
import torch.multiprocessing as mp
ctx = mp.get_context("spawn")
result_queue = ctx.Queue()
port = find_available_port(29500)
processes = []
for rank in range(world_size):
p = ctx.Process(
target=_distributed_worker,
args=(rank, world_size, backend, port, func, result_queue, kwargs),
)
p.start()
processes.append(p)
for p in processes:
p.join()
errors = [result_queue.get() for _ in range(world_size)]
errors = [e for e in errors if e]
if errors:
raise AssertionError("\n".join(errors))
def _distributed_worker(rank, world_size, backend, port, func, result_queue, kwargs):
import traceback
import torch.distributed as dist
if backend == "nccl":
torch.cuda.set_device(rank)
dist.init_process_group(
backend=backend,
init_method=f"tcp://127.0.0.1:{port}",
world_size=world_size,
rank=rank,
)
try:
func(rank, **kwargs)
result_queue.put(None)
except Exception as e:
result_queue.put(f"Rank {rank}: {e}\n{traceback.format_exc()}")
finally:
dist.destroy_process_group()
def maybe_stub_sgl_kernel():
"""Stub sgl_kernel if it cannot be imported (e.g. no GPU).
Must be called before any import that transitively depends on sgl_kernel.
On machines with a working sgl_kernel this is a no-op.
"""
try:
import sgl_kernel # noqa: F401
return
except (ImportError, OSError):
pass
import importlib.abc
import importlib.machinery
class _SglKernelLoader(importlib.abc.Loader):
def create_module(self, spec):
return None
def exec_module(self, module):
from unittest.mock import MagicMock
module.__getattr__ = lambda name: MagicMock()
class _SglKernelFinder(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path, target=None):
if fullname == "sgl_kernel" or fullname.startswith("sgl_kernel."):
return importlib.machinery.ModuleSpec(
fullname,
_SglKernelLoader(),
is_package=True,
)
return None
sys.meta_path.insert(0, _SglKernelFinder())
@contextlib.contextmanager
def published_topology(role: str = "test", *, ranks=None, **server_args_fields):
"""Publish a test topology, defaulting to WORLD rank zero.
``ranks`` overrides the launcher placement. Reset the context before
publication and on exit, including when the test fails.
"""
from sglang.srt.runtime_context import SpawnRanks, publish, reset_context
from sglang.srt.server_args import ServerArgs
bundle = dict(world_rank=0, dp_rank=None)
bundle.update(ranks or {})
server_args = ServerArgs(model_path="dummy", **server_args_fields)
reset_context()
publish(server_args, role=role, ranks=SpawnRanks(**bundle))
try:
yield server_args
finally:
reset_context()
def publish_build_topology(*, world_rank: int = 0, **server_args_fields):
"""Publish the topology for a subsequent ``initialize_model_parallel`` call.
Preserve an existing WORLD group across the context reset. The caller is
responsible for tearing down groups and resetting the context afterward.
"""
from sglang.srt.distributed import parallel_state
from sglang.srt.runtime_context import (
SpawnRanks,
get_parallel,
publish,
reset_context,
)
from sglang.srt.server_args import ServerArgs
reset_context()
publish(
ServerArgs(model_path="dummy", **server_args_fields),
role="test",
ranks=SpawnRanks(world_rank=world_rank),
)
# Restore the existing WORLD handle after resetting the context.
if parallel_state._WORLD is not None:
get_parallel().override_permanently(world_group=parallel_state._WORLD)
_GPU_IDLE_TIMEOUT_SECS = 30.0
_GPU_IDLE_POLL_INTERVAL_SECS = 2.0
_GPU_IDLE_USED_MEMORY_THRESHOLD = 2 << 30 # 2 GiB
_GPU_RELEASE_TIMEOUT_SECS = 60.0
_GPU_RELEASE_POLL_INTERVAL_SECS = 0.5
_GPU_RELEASE_REPORT_THRESHOLD_SECS = 1.0
def _format_gib(num_bytes: Optional[int]) -> str:
if num_bytes is None:
return "N/A"
return f"{num_bytes / (1 << 30):.2f} GiB"
def _visible_gpu_indices(pynvml) -> List[int]:
num_gpus = pynvml.nvmlDeviceGetCount()
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
if visible is None:
return list(range(num_gpus))
entries = [entry.strip() for entry in visible.split(",") if entry.strip()]
if not all(entry.isdigit() for entry in entries):
# UUID-style entries; fall back to checking all GPUs.
return list(range(num_gpus))
return [int(entry) for entry in entries if int(entry) < num_gpus]
def _collect_busy_gpu_reports(pynvml, gpu_indices: List[int]) -> List[str]:
self_pid = os.getpid()
reports = []
for index in gpu_indices:
handle = pynvml.nvmlDeviceGetHandleByIndex(index)
used_bytes = pynvml.nvmlDeviceGetMemoryInfo(handle).used
if used_bytes < _GPU_IDLE_USED_MEMORY_THRESHOLD:
continue
try:
procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
except pynvml.NVMLError:
procs = None
if procs is not None:
# Discount our own usage: the caching allocator retains memory
# across test classes, and waiting on ourselves never succeeds.
self_used = sum(
proc.usedGpuMemory or 0 for proc in procs if proc.pid == self_pid
)
if used_bytes - self_used < _GPU_IDLE_USED_MEMORY_THRESHOLD:
continue
proc_info = ", ".join(
f"pid={proc.pid} {_format_gib(proc.usedGpuMemory)}"
for proc in procs
if proc.pid != self_pid
) or (
f"no other compute processes;"
f" self pid={self_pid} holds {_format_gib(self_used)}"
)
else:
proc_info = ""
reports.append(
f"GPU {index} uses {_format_gib(used_bytes)}"
f" ({proc_info or 'no compute processes found'})"
)
return reports
def _wait_for_gpu_idle_in_ci(
timeout: float = _GPU_IDLE_TIMEOUT_SECS,
poll_interval: float = _GPU_IDLE_POLL_INTERVAL_SECS,
) -> None:
"""Wait until visible GPUs release residual memory from earlier tests.
Killed server processes return GPU memory asynchronously; launching the
next server too early makes memory profiling over-commit the KV cache and
OOM. Abort with the offending processes if the memory is never returned.
"""
if not is_in_ci():
return
try:
import pynvml
pynvml.nvmlInit()
except Exception:
# Non-NVIDIA runner (CPU/AMD) or NVML unavailable; nothing to check.
return
try:
gpu_indices = _visible_gpu_indices(pynvml)
deadline = time.monotonic() + timeout
while True:
busy_reports = _collect_busy_gpu_reports(pynvml, gpu_indices)
if not busy_reports:
return
if time.monotonic() >= deadline:
raise RuntimeError(
f"GPU(s) still not idle after waiting {timeout:.0f}s "
f"before setUpClass: {'; '.join(busy_reports)}"
)
print(
f"[CI GPU Idle] Waiting for GPU to become idle: "
f"{'; '.join(busy_reports)}",
flush=True,
)
time.sleep(poll_interval)
finally:
try:
pynvml.nvmlShutdown()
except Exception:
pass
def collect_process_tree_pids(pid: int, include_parent: bool = True) -> List[int]:
"""Snapshot a process tree's pids, for a later ``wait_for_gpu_release``.
Call it BEFORE the kill; afterwards the tree cannot be walked.
"""
try:
pids = [child.pid for child in psutil.Process(pid).children(recursive=True)]
except psutil.Error:
pids = []
if include_parent:
pids.append(pid)
return pids
def _gpu_memory_holders(pynvml, gpu_indices: List[int], pids: set) -> List[str]:
reports = []
for index in gpu_indices:
handle = pynvml.nvmlDeviceGetHandleByIndex(index)
try:
procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
except pynvml.NVMLError:
# No per-pid enumeration in this container; nothing to wait on.
continue
reports.extend(
f"GPU {index} pid={proc.pid} {_format_gib(proc.usedGpuMemory)}"
for proc in procs
if proc.pid in pids
)
return reports
def wait_for_gpu_release(
pids: List[int],
timeout: float = _GPU_RELEASE_TIMEOUT_SECS,
poll_interval: float = _GPU_RELEASE_POLL_INTERVAL_SECS,
) -> None:
"""Block until none of ``pids`` is still charged device memory.
Killing a server only queues the driver-side teardown,
so the next launch can OOM against memory charged to a reaped process.
Waiting on these pids, rather than on an idle GPU,
keeps this usable while other servers of the same test still run.
Best effort: a timeout or a dead NVML warns, never raises.
"""
if not pids:
return
try:
import pynvml
pynvml.nvmlInit()
except Exception:
# Non-NVIDIA runner (CPU/AMD) or NVML unavailable; nothing to check.
return
try:
gpu_indices = _visible_gpu_indices(pynvml)
pending = set(pids)
start = time.monotonic()
deadline = start + timeout
while True:
holders = _gpu_memory_holders(pynvml, gpu_indices, pending)
if not holders:
# Without this, a wait is indistinguishable from no wait.
waited = time.monotonic() - start
if waited >= _GPU_RELEASE_REPORT_THRESHOLD_SECS:
print(
f"[CI GPU Release] Waited {waited:.1f}s for"
f" {len(pending)} pid(s) to release.",
flush=True,
)
return
if time.monotonic() >= deadline:
print(
f"[CI GPU Release] Still charged after {timeout:.0f}s:"
f" {'; '.join(holders)}",
flush=True,
)
return
time.sleep(poll_interval)
except Exception as e:
# NVML can go away after a successful init (GPU lost, driver reset).
# Raising here would fail a teardown whose test already passed.
print(f"[CI GPU Release] Giving up, {type(e).__name__}: {e}", flush=True)
finally:
try:
pynvml.nvmlShutdown()
except Exception:
pass
# Names the runner kits stamp onto a record that are not members of it.
# `ModelRunner` computes `use_mla_backend` on itself; the kits copy that bool
# onto the record they hand the runner, and `hasattr` cannot see it.
_RUNNER_WRITTEN_NAMES = frozenset({"use_mla_backend"})
def server_args_variant(server_args, **fields):
"""A modified deep copy of a config, for a test double whose fixture
differs from the (possibly published, read-only) config it starts from.
The receiver is untouched; the copy keeps its read-only guard.
A name may also be one the kits stamp on rather than a field (see
``_RUNNER_WRITTEN_NAMES``); names that exist nowhere fail loudly."""
variant = copy.deepcopy(server_args)
cls = type(variant)
unknown = {
name
for name in fields
if name not in cls.__struct_fields__
and not hasattr(cls, name)
and name not in _RUNNER_WRITTEN_NAMES
}
if unknown:
raise ValueError(f"unknown ServerArgs field(s): {sorted(unknown)}")
# Reach the stash as well as the fields (the bags project from raw input
# + declarations); through `object` because the copy keeps its read-only
# guard.
stash = getattr(variant, "_resolved_overrides", None)
if stash is None:
stash = []
msgspec.Struct.__setattr__(variant, "_resolved_overrides", stash)
declared = {
name: value for name, value in fields.items() if name in cls.__struct_fields__
}
if declared:
stash.append(("server_args_variant", dict(declared)))
for name, value in fields.items():
msgspec.Struct.__setattr__(variant, name, value)
return variant
def enter_override(test_case, override):
"""Install a scoped context override for the length of one test.
Stands in for `unittest.TestCase.enterContext`, which is 3.11+ while this package
supports 3.10; on 3.10 it raises AttributeError, so tests pass locally on newer
interpreters but fail in CI.
"""
installed = override.install()
test_case.addCleanup(override.restore)
return installed
def enter_scope(test_case, scope):
"""Enter a context manager for the length of one test.
The `with`-statement form of `enter_override` above, and 3.10-safe for the
same reason: `enterContext` arrived in 3.11.
"""
entered = scope.__enter__()
test_case.addCleanup(scope.__exit__, None, None, None)
return entered
class CustomTestCase(unittest.TestCase):
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# unittest skips tearDownClass when setUpClass raises, leaking ports and
# processes; wrap setUpClass so tearDownClass still runs.
setup = cls.setUpClass
if getattr(setup, "_safe_setup_wrapped", False):
return
orig_func = setup.__func__
def safe_setUpClass(klass):
try:
_wait_for_gpu_idle_in_ci()
if is_in_ci():
_install_ci_engine_offline_hook()
orig_func(klass)
except Exception:
# Best-effort cleanup; suppress teardown errors so the
# original setUpClass exception propagates clearly.
try:
klass.tearDownClass()
except Exception:
pass
raise
# Set sentinel on the raw function so that bound method attribute
# lookup (which delegates to __func__) can detect it in subclasses.
safe_setUpClass._safe_setup_wrapped = True
cls.setUpClass = classmethod(safe_setUpClass)
def _callTestMethod(self, method):
max_retry = envs.SGLANG_TEST_MAX_RETRY.get()
if max_retry is None:
max_retry = 1 if is_in_ci() else 0
retry(
lambda: super(CustomTestCase, self)._callTestMethod(method),
max_retry=max_retry,
)
def setUp(self):
print(
f"[CI Test Method] {self.__class__.__name__}.{self._testMethodName}",
flush=True,
)
class ModelLaunchSettings:
def __init__(
self,
model_path: str,
tp_size: int = 1,
extra_args: Optional[List[str]] = None,
env: Optional[dict] = None,
variant: Optional[str] = None,
launch_timeout: Optional[float] = None,
):
self.model_path = model_path
self.tp_size = tp_size
self.extra_args = list(extra_args) if extra_args else []
self.env = env
self.variant = variant
self.launch_timeout = launch_timeout
if self.tp_size > 1 and "--tp" not in self.extra_args:
self.extra_args.extend(["--tp", str(self.tp_size)])
fixed_args = ["--enable-multimodal", "--trust-remote-code"]
for fixed_arg in fixed_args:
if fixed_arg not in self.extra_args:
self.extra_args.append(fixed_arg)
def extract_trace_link_from_bench_one_batch_server_output(output: str) -> str:
match = re.search(r"\[Profile\]\((.*?)\)", output)
if match:
trace_link = match.group(1)
return trace_link
return None
def parse_models(model_string: str):
return [model.strip() for model in model_string.split(",") if model.strip()]
def check_evaluation_test_results(
results,
test_name,
model_accuracy_thresholds,
model_latency_thresholds=None,
model_count=None,
):
"""
results: list of tuple of (model_path, accuracy, latency) or (model_path, accuracy, latency, error)
"""
failed_models = []
if model_latency_thresholds is not None:
summary = " | model | status | score | score_threshold | latency | latency_threshold | error | \n"
summary += "| ----- | ------ | ----- | --------------- | ------- | ----------------- | ----- | \n"
else:
summary = " | model | status | score | score_threshold | error | \n"
summary += "| ----- | ------ | ----- | --------------- | ----- | \n"
results_dict = {
res[0]: (res[1], res[2], res[3] if len(res) == 4 else None) for res in results
}
for model, accuracy_threshold in sorted(model_accuracy_thresholds.items()):
latency_threshold = (
model_latency_thresholds.get(model)
if model_latency_thresholds is not None
else 1e9
)
# check for error here
error = (
results_dict.get(model, (None, None, None))[2]
if model in results_dict
else None
)
if model in results_dict and error is None:
accuracy, latency, _ = results_dict[model]
is_success = accuracy >= accuracy_threshold and latency <= latency_threshold
status_emoji = "✅" if is_success else "❌"
if not is_success:
if accuracy < accuracy_threshold:
failed_models.append(
f"\nScore Check Failed: {model}\n"
f"Model {model} score ({accuracy:.4f}) is below threshold ({accuracy_threshold:.4f})"
)
if latency > latency_threshold:
failed_models.append(
f"\nLatency Check Failed: {model}\n"
f"Model {model} latency ({latency:.4f}) is above threshold ({latency_threshold:.4f})"
)
if model_latency_thresholds is not None:
line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | {latency} | {latency_threshold} | - |\n"
else:
line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | - |\n"
else:
status_emoji = "❌"
error_display = error if error else "Model not evaluated"
failed_models.append(f"Model failed to launch or be evaluated: {model}")
if model_latency_thresholds is not None:
line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | N/A | {latency_threshold} | {error_display} |\n"
else:
line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | {error_display} |\n"
summary += line
print(summary)
if is_in_ci():
write_github_step_summary(f"## {test_name}\n{summary}")
if failed_models:
print("Some models failed the evaluation.")
raise AssertionError("\n".join(failed_models))
# Bench knobs for bench_one_batch_server (override by env)
def _parse_int_list_env(name: str, default_val: str):
val = os.environ.get(name, default_val)
return [int(x) for x in val.split(",") if x]
# Return filenames
def find_traces_under_path(path: str) -> List[str]:
results = []
for _, dirs, files in os.walk(path):
for file in files:
if file.endswith(".trace.json.gz"):
results.append(f"{file}")
return results
def write_results_to_json(model, metrics, mode="a"):
result = {
"timestamp": datetime.now().isoformat(),
"model": model,
"metrics": metrics,
"score": metrics["score"],
}
if "latency" in metrics:
result["latency"] = (metrics.get("latency"),)
existing_results = []
if mode == "a" and os.path.exists("results.json"):
try:
with open("results.json", "r") as f:
existing_results = json.load(f)
except json.JSONDecodeError:
existing_results = []
if isinstance(existing_results, list):
existing_results.append(result)
else:
existing_results = [result]
with open("results.json", "w") as f:
json.dump(existing_results, f, indent=2)
def intel_amx_benchmark(extra_args=None, min_throughput=None):
def decorator(test_func):
@wraps(test_func)
def wrapper(self):
common_args = [
"--attention-backend",
"intel_amx",
"--disable-radix",
"--trust-remote-code",
]
full_args = common_args + (extra_args or [])
model = test_func(self)
prefill_latency, decode_throughput, decode_latency = run_bench_one_batch(
model, full_args
)
print(f"{model=}")
print(f"{prefill_latency=}")
print(f"{decode_throughput=}")
print(f"{decode_latency=}")
if is_in_ci() and min_throughput is not None:
self.assertGreater(decode_throughput, min_throughput)
return wrapper
return decorator
def get_gpu_count():
if get_device() == "cpu":
gpu_count = 0
else:
gpu_count = torch.accelerator.device_count()
return gpu_count
def empty_gpu_cache():
"""
Unified empty_cache for PyTorch 2.8 (no torch.accelerator)
and PyTorch 2.9+ (where torch.accelerator.empty_cache() exists).
"""
if hasattr(torch, "accelerator") and hasattr(torch.accelerator, "empty_cache"):
return torch.accelerator.empty_cache()
# CUDA
if hasattr(torch, "cuda") and torch.cuda.is_available():
torch.cuda.empty_cache()
return
# XPU (Intel)
if hasattr(torch, "xpu") and torch.xpu.is_available():
torch.xpu.empty_cache()
return
return
def get_gpu_memory_gb():
if is_cuda():
return torch.cuda.device_memory_used() / 1024**3
elif is_xpu():
return torch.xpu.memory_allocated() / 1024**3
else:
return 0
def run_doctests(obj: Callable[..., Any] | ModuleType):
mod = inspect.getmodule(obj)
globals = dict(mod.__dict__)
finder = doctest.DocTestFinder()
runner = doctest.DocTestRunner(verbose=True)
tests = finder.find(obj, obj.__name__, globs=globals)
assert len(tests) >= 1, f"No tests found for {obj.__name__}"
for test in tests:
result = runner.run(test)
assert result.failed == 0, f"Test {test.name} failed"
def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
"""
Output test metric to JSONL and stdout for CI performance tracking.
Schema (v1):
- Required: filename, test_case, metric_name, value
- Optional fields supported: ts, labels
- ts is emitted by default for convenience
- labels preferred as dict; if not JSON-serializable, stored as string
Value types (v1 contract):
- Supported: int, float, str
- Input may be bool (will be coerced to int: True=1, False=0)
- Others: best-effort conversion to float, fallback to str
Output channels:
- JSONL: ${SGLANG_TEST_METRICS_OUTPUT}.${pid}.jsonl (if env var set)
- stdout: [METRIC] metric_name=value [labels=...]
This function never fails tests - all errors are silently caught.
"""
try:
# 1. Capture test context
filename, test_case = _get_test_context()
# 2. Convert value to int/float/str
# First unwrap numpy/torch scalars
if hasattr(value, "item"):
value = value.item()
# Now convert, ensuring no bool in final result
if isinstance(value, bool):
converted_value = int(value) # True->1, False->0
elif isinstance(value, (int, float, str)):
converted_value = value
else:
try:
converted_value = float(value)
except (ValueError, TypeError):
converted_value = str(value)
# 3. Build record
record = {
"filename": filename,
"test_case": test_case,
"metric_name": metric_name,
"value": converted_value,
"ts": time.time(),
}
# 4. Handle labels (best-effort JSON serialization)
labels_for_output = None
if labels:
try:
json.dumps(labels, ensure_ascii=False) # Test serializability
record["labels"] = labels
labels_for_output = labels
except (TypeError, ValueError):
# If not serializable, stringify
stringified = str(labels)
record["labels"] = stringified
labels_for_output = stringified
# 5. Write JSONL
base_path = os.getenv("SGLANG_TEST_METRICS_OUTPUT")
if base_path:
try:
jsonl_path = f"{base_path}.{os.getpid()}.jsonl"
with open(jsonl_path, "a", encoding="utf-8") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
except Exception as e:
logging.warning(
f"sglang.test.dump_metric: failed to write to {jsonl_path}: {e}"
)
# 6. Output to stdout (use same labels as record)
if labels_for_output:
if isinstance(labels_for_output, str):
labels_str = f" labels='{labels_for_output}'"
else:
labels_str = (
f" labels={json.dumps(labels_for_output, ensure_ascii=False)}"
)
else:
labels_str = ""
print(f"[METRIC] {metric_name}={converted_value}{labels_str}")
except Exception as e:
# Silent failure - never break tests
logging.warning(
f"sglang.test.dump_metric: failed to dump metric '{metric_name}': {e}",
exc_info=True,
)
def _get_test_context() -> tuple[str, str]:
"""Get current test's filename and test_case, via PYTEST_CURRENT_TEST or stack."""
# 1. Try parsing PYTEST_CURRENT_TEST
pytest_current = os.getenv("PYTEST_CURRENT_TEST")
if pytest_current:
# Format: "path/to/test.py::TestClass::test_method (call)"
parts = pytest_current.split(" ")[0].split("::", 1)
if len(parts) == 2:
filename = _repo_relative_path(parts[0])
test_case = parts[1].replace("::", ".")
return filename, test_case
# 2. Fallback to inspect
import inspect
frame = inspect.currentframe()
# Assumes direct callsite: frame -> _get_test_context -> dump_metric -> caller
# If dump_metric gets wrapped, may need to scan upward
if frame and frame.f_back and frame.f_back.f_back:
caller = frame.f_back.f_back
filename = _repo_relative_path(caller.f_code.co_filename)
# Try to get class name from self
test_self = caller.f_locals.get("self")
if test_self and hasattr(test_self, "__class__"):
test_case = f"{test_self.__class__.__name__}.{caller.f_code.co_name}"
else:
test_case = caller.f_code.co_name
return filename, test_case
return "unknown.py", "unknown_test"
def _repo_relative_path(filepath: str) -> str:
"""Convert absolute path to repo-relative, preferring GITHUB_WORKSPACE"""
# Path is imported at module top (line 20)
try:
abs_path = Path(filepath).resolve()
# Try GITHUB_WORKSPACE first
workspace = os.getenv("GITHUB_WORKSPACE")
if workspace:
try:
return str(abs_path.relative_to(Path(workspace).resolve()))
except ValueError:
pass
# Fallback to cwd
try:
return str(abs_path.relative_to(Path.cwd()))
except ValueError:
return abs_path.name
except Exception:
return Path(filepath).name