diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index e082532e95..f609d2705a 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -785,15 +785,15 @@ The command evaluates all 57 subjects. Historical results below use the removed - **Benchmark Command:** ```shell Command -python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --thinking-mode deepseek-v3 +sgl-eval run gpqa --base-url http://127.0.0.1:30000/v1 --num-examples 198 --max-tokens 128000 --n-repeats 8 --thinking ``` -- **Test Results** (model: `deepseek-ai/DeepSeek-V3.2-Exp`, 8×B200): +- **Historical Test Results** (model: `deepseek-ai/DeepSeek-V3.2-Exp`, 8×B200; measured with the earlier `sglang.test.run_eval` GPQA command): - Default (`temperature=0`): mean **0.797** over 8 runs — closely matches the official GPQA-Diamond score of **79.9** for DeepSeek-V3.2-Exp reported in its [model card](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) - With `temperature=1.0, top_p=0.95` (as recommended by DeepSeek): ```shell Command -python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --top-p 0.95 --temperature 1.0 --thinking-mode deepseek-v3 +sgl-eval run gpqa --base-url http://127.0.0.1:30000/v1 --num-examples 198 --max-tokens 128000 --n-repeats 8 --top-p 0.95 --temperature 1.0 --thinking ``` ``` diff --git a/docs/docs/developer_guide/evaluating_new_models.mdx b/docs/docs/developer_guide/evaluating_new_models.mdx index e20a2f54f9..33a0b7f143 100644 --- a/docs/docs/developer_guide/evaluating_new_models.mdx +++ b/docs/docs/developer_guide/evaluating_new_models.mdx @@ -45,16 +45,15 @@ python benchmark/hellaswag/bench_sglang.py \ **GPQA** ```bash Command -python -m sglang.test.run_eval \ - --eval-name gpqa \ - --port 30000 \ +sgl-eval run gpqa \ + --base-url http://127.0.0.1:30000/v1 \ --num-examples 198 \ --max-tokens 120000 \ - --repeat 8 + --n-repeats 8 ``` -For reasoning models, add `--thinking-mode `. Supported values are `deepseek-v3`, `qwen-3`, `glm-45`, and `kimi-k2`. You may skip it if the model has forced thinking enabled. +For reasoning models, add `--thinking` (DeepSeek-V3 / Kimi-K2 style `thinking` key) or `--chat-template-kwarg enable_thinking=true` (Qwen3 / GLM style). You may skip it if the model has forced thinking enabled. **HumanEval** diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py index fb7c5d6f01..412ca716ac 100644 --- a/python/sglang/test/accuracy_test_runner.py +++ b/python/sglang/test/accuracy_test_runner.py @@ -4,6 +4,7 @@ from typing import List, Optional, Tuple from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -31,9 +32,6 @@ class AccuracyTestParams: repeat: Optional[int] = None api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval seed: Optional[int] = None # pin for reproducibility when temperature > 0 - # sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead - # of letting _run_sgl_eval infer it from the model name. - sgl_eval_thinking: Optional[bool] = None num_shots: Optional[int] = None # few-shot count; None = run_eval's default @@ -56,13 +54,7 @@ def write_accuracy_github_summary( dataset: str, results: List[AccuracyTestResult], ) -> None: - """Write accuracy test results to GitHub step summary. - - Args: - test_name: Name of the test - dataset: Dataset name used for evaluation - results: List of AccuracyTestResult objects - """ + """Write accuracy test results to GitHub step summary.""" summary = f"#### {test_name} - Accuracy ({dataset})\n" summary += "| config | status | score | baseline | error |\n" summary += "| ------ | ------ | ----- | -------- | ----- |\n" @@ -95,13 +87,8 @@ def _run_simple_eval( repeat: Optional[int] = None, api: Optional[str] = None, seed: Optional[int] = None, - sgl_eval_thinking: Optional[bool] = None, ) -> Tuple[bool, Optional[str], Optional[dict]]: - """Run evaluation using simple_eval backend (run_eval.py). - - Returns: - Tuple of (success, error_message, metrics_dict) - """ + """Run ``dataset`` through run_sgl_eval (sgl-eval benchmarks) or run_eval.""" process = None try: process = popen_launch_server( @@ -150,10 +137,8 @@ def _run_simple_eval( if seed is not None: args.seed = seed - if sgl_eval_thinking is not None: - args.sgl_eval_thinking = sgl_eval_thinking - - result = run_eval(args) + evaluate = run_sgl_eval if dataset in SGL_EVAL_BENCHMARKS else run_eval + result = evaluate(args) # Handle result format (run_eval can return metrics or (metrics, latency)) if return_latency and isinstance(result, tuple): @@ -177,16 +162,7 @@ def run_accuracy_test( params: AccuracyTestParams, base_url: Optional[str] = None, ) -> AccuracyTestResult: - """Run accuracy test for a single model. - - Args: - model: ModelLaunchSettings with model config - params: AccuracyTestParams with dataset, baseline, and optional settings - base_url: Server base URL (default: DEFAULT_URL_FOR_TEST) - - Returns: - AccuracyTestResult with test outcome - """ + """Run accuracy test for a single model.""" base_url = base_url or DEFAULT_URL_FOR_TEST print(f"\n{'=' * 60}") @@ -211,7 +187,6 @@ def run_accuracy_test( repeat=params.repeat, api=params.api, seed=params.seed, - sgl_eval_thinking=params.sgl_eval_thinking, ) if not success: diff --git a/python/sglang/test/ascend/test_ascend_utils.py b/python/sglang/test/ascend/test_ascend_utils.py index ddbeb5b7b0..a64473ea34 100644 --- a/python/sglang/test/ascend/test_ascend_utils.py +++ b/python/sglang/test/ascend/test_ascend_utils.py @@ -1,14 +1,6 @@ -""" -Common utilities for testing and benchmarking on NPU. +"""Common utilities for testing and benchmarking on NPU. -This file contains the following weight path categories: -- LLM model weights path -- VLM model weights path -- Embedding model weights path -- Rerank model weights path -- Reward model weights path - -Please remember to sort by variable name within each section. +Keep the weight-path constants sorted by variable name within each section. """ import asyncio @@ -26,7 +18,7 @@ import requests from sglang.benchmark.serving import run_benchmark from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -395,15 +387,7 @@ CONFIG_YAML_PATH = ( class ModelTestConfig(NamedTuple): - """ - Configuration for model testing. - - Attributes: - model_path: Path to the model weights directory - mmlu_score: Weight for MMLU benchmark score - gsm8k_accuracy: Weight for GSM8K benchmark score - mmmu_accuracy: Weight for MMMU benchmark score - """ + """Model weights path and the expected MMLU / GSM8K / MMMU scores.""" model_path: str mmlu_score: Optional[float] = None @@ -436,16 +420,7 @@ DEFAULT_WEIGHTS_FOR_TEST = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_FOR_TEST def run_command(cmd, shell=True): - """Execute system command and return stdout - - parameter: - cmd: command to execute - shell: - True, Execute command in shell - False, Commands are invoked directly without shell parsing - return: - The result of executing the command - """ + """Run ``cmd`` and return its stdout.""" try: result = subprocess.run( cmd, shell=shell, capture_output=True, text=True, check=True @@ -556,37 +531,7 @@ def run_bench_serving( lora_name: Optional[str] = None, timeout_for_server_launch=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, ): - """Start the service and obtain the inference results. - - Parameters: - model: Model name - num_prompts: Total number of test requests - request_rate: Request rate - other_server_args: Additional configuration when starting the service - dataset_name: Data set name - dataset_path: Dataset path - tokenizer: tokenizer - random_input_len: The length of the randomly generated input prompt - random_output_len: The length of the randomly generated output prompt - sharegpt_context_len: Sharegpt dataset context length - disable_stream: Disable streaming output - disable_ignore_eos: Should eos_token be ignored? - need_warmup: Preheating required - seed: random seed - device: Device type - gsp_num_groups: Grouped Sequence Parallelism - gsp_prompts_per_group: Number of parallel prompts within each group - gsp_system_prompt_len: GSP system prompts length - gsp_question_len: GSP question length - gsp_output_len: GSP output length - max_concurrency: Maximum number of concurrent requests - background_task: Background tasks - lora_name: LoRA fine-tuning model path - timeout_for_server_launch: Raise the service timeout period - Returns: - res: Number of requests successfully completed - - """ + """Launch the server, run bench_serving against it, and return the result.""" if device == "auto": device = auto_config_device() @@ -658,17 +603,7 @@ def run_bench_serving( # hook factory def create_attention_monitor_hook_factory(config): - """ - Factory function to create a forward hook for monitoring self-attention layer states. - This hook records input/output statistics during model forward propagation. - - Args: - config (dict): Configuration dictionary containing hook parameters - layer_index (int): Index of the target attention layer to monitor - - Returns: - function: Forward hook function to be registered on the target module - """ + """Return a forward hook that logs self-attention states of ``config["layer_index"]``.""" # Get target layer index from config, default to 0 if not specified layer_index = config.get("layer_index", 0) @@ -681,18 +616,7 @@ def create_attention_monitor_hook_factory(config): ) def attention_monitor_hook(module, inputs, output): - """ - Forward hook function that monitors and logs the internal states of a self-attention layer. - Executed automatically during the forward pass of the module it is registered to. - - Args: - module (torch.nn.Module): The module this hook is attached to - inputs (tuple): Input tensors passed to the module's forward method - output (torch.Tensor): Output tensor returned by the module's forward method - - Returns: - torch.Tensor: Unmodified output tensor to preserve model computation flow - """ + """Log the attention layer's input/output statistics; returns ``output`` unchanged.""" # Record current timestamp for time-series tracking timestamp = time.time() @@ -832,7 +756,7 @@ def run_mmlu_test( ) try: - metrics = run_eval(args) + metrics = run_sgl_eval(args) assert metrics["score"] >= 0.65, f"{metrics=}" finally: pass @@ -856,26 +780,10 @@ def send_concurrent_requests( temperature: float = 0.0, request_timeout: int = 60, ) -> list: - """Send multiple concurrent HTTP POST requests to the /generate endpoint. + """Send ``num_requests`` concurrent POSTs to /generate; returns one result dict per request. - Uses threading (NOT asyncio + blocking calls) to achieve true concurrency. - asyncio.gather() combined with synchronous requests.post() does not produce - real parallelism; threading is required for concurrent blocking I/O. - - Parameters: - base_url: Server base URL, e.g. "http://127.0.0.1:30000" - num_requests: Total number of requests to send - num_concurrent: Maximum in-flight requests at any given time (semaphore) - input_text: Text prompt sent to every request - max_new_tokens: Maximum new tokens to generate per request - temperature: Sampling temperature (0 = greedy / deterministic) - request_timeout: Per-request HTTP timeout in seconds; raises on exceed - - Returns: - Unsorted list of result dicts, one per request, each with: - task_id (int) -- zero-based request index - status_code (int)-- HTTP status code, or -1 on exception - text (str) -- response body, or exception message on failure + Uses threads: asyncio.gather() over synchronous requests.post() gives no real + parallelism for blocking I/O. """ results: list = [] diff --git a/python/sglang/test/ascend/test_mmlu.py b/python/sglang/test/ascend/test_mmlu.py index 8a46b5ba1e..91b28b59dd 100644 --- a/python/sglang/test/ascend/test_mmlu.py +++ b/python/sglang/test/ascend/test_mmlu.py @@ -2,7 +2,7 @@ import subprocess from types import SimpleNamespace from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval class TestMMLU: @@ -28,7 +28,7 @@ class TestMMLU: num_threads=32, ) print("Starting mmlu test...") - metrics = run_eval(args) + metrics = run_sgl_eval(args) model_metrics["accuracy"] = metrics["score"] model_metrics["latency"] = metrics.get("latency", "-") model_metrics["output_throughput"] = metrics.get("output_throughput", "-") diff --git a/python/sglang/test/gpt_oss_common.py b/python/sglang/test/gpt_oss_common.py index 1954330485..6a0a830156 100644 --- a/python/sglang/test/gpt_oss_common.py +++ b/python/sglang/test/gpt_oss_common.py @@ -7,7 +7,7 @@ from typing import Dict, List, Literal, Optional import requests from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -66,8 +66,7 @@ class BaseTestGptOss(CustomTestCase): try: self._check_streaming_responses_api_request(model) - # run multiple tests in parallel since we are mostly bound by the longest generate sequence - # instead of the number of questions + # Parallel: bound by the longest generation, not the question count. with ThreadPoolExecutor(max_workers=4) as executor: list( executor.map( @@ -131,8 +130,7 @@ class BaseTestGptOss(CustomTestCase): num_threads=198, # sgl-eval's gpqa defaults to n_repeats=8. repeat=1, - # TODO 4k is still not enough, we need e.g. 64k token, but that is super slow - # otherwise a lot of questions are not answered + # Too small (many answers truncate); ~64k would fit but is too slow. max_tokens=4096, # Arbitrary; non-zero so a tier is not scored on one greedy path. temperature=0.1, @@ -142,7 +140,7 @@ class BaseTestGptOss(CustomTestCase): setup = f"model={model} reasoning_effort={reasoning_effort} expected_score={expected_score}" print(f"Evaluation start: {setup}") - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation end: {setup} {metrics=}") self.assertGreaterEqual(metrics["score"], expected_score) diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index 81eedf8b7f..04afac8cbe 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -4,6 +4,7 @@ from typing import Optional import requests from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary _THRESHOLD_NOT_SET = float("nan") @@ -51,14 +52,8 @@ def _run_accuracy_eval( ): """Shared driver for the accuracy mixins below. - Runs ``run_eval`` for ``eval_name`` against the test class's server - (``base_url`` / ``model``), records a CI step summary, asserts the score - meets ``score_threshold``, and checks the speculative accept length. - - ``eval_overrides`` (e.g. ``api``, ``max_tokens``, ``temperature``, - ``top_p``, ``num_shots``) are forwarded to ``run_eval`` only when not - ``None``, so the common case stays identical to ``run_eval``'s defaults. - Returns the metrics dict. + ``eval_overrides`` are forwarded only when not ``None``, so unset knobs keep + the evaluator's defaults. """ assert score_threshold == score_threshold, ( f"{type(test_case).__name__} must set the {eval_name} score threshold" @@ -74,7 +69,8 @@ def _run_accuracy_eval( ) kwargs.update({k: v for k, v in eval_overrides.items() if v is not None}) - metrics = run_eval(SimpleNamespace(**kwargs)) + evaluate = run_sgl_eval if eval_name in SGL_EVAL_BENCHMARKS else run_eval + metrics = evaluate(SimpleNamespace(**kwargs)) print(f"{eval_name} {metrics=}") _finalize_eval( test_case, @@ -92,80 +88,53 @@ def _run_sgl_eval( *, eval_name: str, score_threshold: float, - metric: str = "score", - n_repeats: int = 1, num_examples: Optional[int] = None, num_threads: int = 512, thinking: bool = True, chat_template_kwargs: Optional[dict] = None, - reasoning_effort: Optional[str] = None, max_tokens: Optional[int] = None, temperature: Optional[float] = None, - top_p: Optional[float] = None, accept_length_thres: Optional[float] = None, - summary_label: Optional[str] = None, ): - """Shared sgl-eval driver for the reasoning mixins and the ``sgl_eval`` backend. + """Shared sgl-eval driver for the MMLU sanity gate and the GSM8K ``sgl_eval`` backend. - Runs ``eval_name`` via the sgl-eval Python API (``registry.get`` -> - ``EvalSpec.run``) against the test class's server, records a CI step summary, - asserts the score meets ``score_threshold``, and checks the speculative accept - length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": - True}`` so the server separates reasoning from the final answer. Skips the test - if sgl-eval is not installed. Returns the RunResult. + ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": True}`` + so the server separates reasoning from the final answer. """ assert score_threshold == score_threshold, ( f"{type(test_case).__name__} must set the {eval_name} score threshold" ) try: - from sgl_eval.registry import get as get_eval_spec - from sgl_eval.sampler import ChatCompletionSampler - from sgl_eval.types import GenConfig + import sgl_eval # noqa: F401 except ImportError: test_case.skipTest("sgl-eval not installed; pip install 'sglang[test]'") - base_url = test_case.base_url.rstrip("/") - if not base_url.endswith("/v1"): - base_url += "/v1" - sampler = ChatCompletionSampler( - base_url=base_url, model=getattr(test_case, "model", None), api_key="EMPTY" - ) - - gen_kwargs = dict( + args = SimpleNamespace( + eval_name=eval_name, + base_url=test_case.base_url, + model=getattr(test_case, "model", None), + num_examples=num_examples, + num_threads=num_threads, max_tokens=max_tokens, - reasoning_effort=reasoning_effort, + temperature=temperature, chat_template_kwargs=( chat_template_kwargs if chat_template_kwargs is not None else ({"thinking": True} if thinking else None) ), ) - if temperature is not None: - gen_kwargs["temperature"] = temperature - if top_p is not None: - gen_kwargs["top_p"] = top_p - - result = get_eval_spec(eval_name).run( - sampler=sampler, - gen=GenConfig(**gen_kwargs), - n_repeats=n_repeats, - num_examples=num_examples, - num_threads=num_threads, - predictions_writer=None, - load_examples=None, - ) - score = result.aggregate[metric] - print(f"{eval_name} sgl-eval {metric}={score:.4f}") + metrics = run_sgl_eval(args) + score = metrics["score"] + print(f"{eval_name} sgl-eval score={score:.4f}") _finalize_eval( test_case, eval_name=eval_name, score=score, score_threshold=score_threshold, accept_length_thres=accept_length_thres, - summary_label=summary_label, ) - return result + return metrics class MMLUSanityMixin: @@ -192,18 +161,9 @@ class MMLUSanityMixin: class GSM8KMixin: """Mixin for GSM8K evaluation. - Backend is selectable via ``gsm8k_backend`` (default ``"run_eval"``: OpenAI - completion API, 5-shot; or ``"sgl_eval"``: sgl-eval chat + boxed/sympy grader, - skipped if sgl-eval is not installed). The canonical threshold/count knobs are - ``gsm8k_score_threshold`` / ``gsm8k_num_examples``; the legacy - ``gsm8k_accuracy_thres`` / ``gsm8k_num_questions`` are still honored. - - Required attributes on the test class: - base_url: str - gsm8k_score_threshold: float - - Optional attributes: - model: str (if not set, auto-detected from server) + ``"run_eval"`` backend: OpenAI completion API, 5-shot; ``"sgl_eval"``: sgl-eval + chat + boxed/sympy grader. The legacy ``gsm8k_accuracy_thres`` / + ``gsm8k_num_questions`` are honored when the canonical knobs are unset. """ gsm8k_score_threshold: float = _THRESHOLD_NOT_SET @@ -216,7 +176,6 @@ class GSM8KMixin: gsm8k_backend: str = "run_eval" # "run_eval" | "sgl_eval" gsm8k_thinking: bool = False # sgl_eval backend gsm8k_max_tokens: Optional[int] = None # sgl_eval backend - gsm8k_n_repeats: int = 1 # sgl_eval backend # None keeps run_eval's greedy default; set both to route the run through # the sampling path. gsm8k_temperature: Optional[float] = None @@ -237,7 +196,6 @@ class GSM8KMixin: self, eval_name="gsm8k", score_threshold=threshold, - n_repeats=self.gsm8k_n_repeats, num_examples=num_examples, num_threads=self.gsm8k_num_threads, thinking=self.gsm8k_thinking, @@ -261,60 +219,29 @@ class GSM8KMixin: class MMLUMixin: - """Mixin for MMLU evaluation. - - Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it - in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch - picks the call mechanism, not the grader. - - Required attributes on the test class: - base_url: str - model: str - mmlu_score_threshold: float - """ + """Mixin for MMLU evaluation via sgl-eval (2048-token cap, no thinking).""" mmlu_score_threshold: float = _THRESHOLD_NOT_SET mmlu_accept_length_thres: Optional[float] = None mmlu_num_examples: int = 5000 mmlu_num_threads: int = 1024 - mmlu_backend: str = "run_eval" # "run_eval" | "sgl_eval" - mmlu_thinking: bool = False # sgl_eval backend - mmlu_n_repeats: int = 1 # sgl_eval backend def test_mmlu(self): - if self.mmlu_backend == "sgl_eval": - _run_sgl_eval( - self, - eval_name="mmlu", - score_threshold=self.mmlu_score_threshold, - n_repeats=self.mmlu_n_repeats, - num_examples=self.mmlu_num_examples, - num_threads=self.mmlu_num_threads, - thinking=self.mmlu_thinking, - accept_length_thres=self.mmlu_accept_length_thres, - ) - else: - _run_accuracy_eval( - self, - eval_name="mmlu", - score_threshold=self.mmlu_score_threshold, - num_examples=self.mmlu_num_examples, - num_threads=self.mmlu_num_threads, - accept_length_thres=self.mmlu_accept_length_thres, - ) + _run_accuracy_eval( + self, + eval_name="mmlu", + score_threshold=self.mmlu_score_threshold, + num_examples=self.mmlu_num_examples, + num_threads=self.mmlu_num_threads, + accept_length_thres=self.mmlu_accept_length_thres, + ) class MMMUProMixin: """Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval. - The model preset supplies the endpoint model and all generation settings. - Leaving those values to sgl-eval is important for reasoning models whose - recommended token budget and sampling settings differ from run_eval defaults. - - Required attributes on the test class: - base_url: str - mmmu_pro_score_threshold: float - mmmu_pro_load_preset_from_model_id: str + The model preset supplies the endpoint model and all generation settings; + reasoning models' token budget and sampling differ from run_eval defaults. """ mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET @@ -339,110 +266,8 @@ class MMMUProMixin: ) -class GPQAMixin: - """Mixin for GPQA-Diamond evaluation (graduate-level multiple choice). - - Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not - installed). ``gpqa_thinking`` defaults to True, which - enables per-request thinking so the server separates reasoning from the final - answer. - - Required attributes on the test class: - base_url: str - model: str - gpqa_score_threshold: float - - Optional sampling knobs (default to sgl-eval's defaults when unset). Set these - for reasoning models -- e.g. DeepSeek-V4 Think-Max wants - gpqa_reasoning_effort="max", gpqa_max_tokens=200000, gpqa_temperature=1.0, - gpqa_top_p=1.0. GPQA-Diamond is 198 questions; raise gpqa_n_repeats (e.g. 16) - for a stable number. - """ - - gpqa_score_threshold: float = _THRESHOLD_NOT_SET - gpqa_accept_length_thres: Optional[float] = None - gpqa_num_examples: Optional[int] = None - gpqa_num_threads: int = 1024 - gpqa_n_repeats: int = 1 - gpqa_thinking: bool = True - gpqa_reasoning_effort: Optional[str] = None - gpqa_max_tokens: Optional[int] = None - gpqa_temperature: Optional[float] = None - gpqa_top_p: Optional[float] = None - - def test_gpqa(self): - _run_sgl_eval( - self, - eval_name="gpqa", - score_threshold=self.gpqa_score_threshold, - n_repeats=self.gpqa_n_repeats, - num_examples=self.gpqa_num_examples, - num_threads=self.gpqa_num_threads, - thinking=self.gpqa_thinking, - reasoning_effort=self.gpqa_reasoning_effort, - max_tokens=self.gpqa_max_tokens, - temperature=self.gpqa_temperature, - top_p=self.gpqa_top_p, - accept_length_thres=self.gpqa_accept_length_thres, - ) - - -class AIME25Mixin: - """Mixin for AIME 2025 evaluation (competition math, integer answers). - - Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not - installed). ``aime25_thinking`` defaults to True, which - enables per-request thinking so the server separates reasoning from the final - answer. - - Required attributes on the test class: - base_url: str - model: str - aime25_score_threshold: float - - Optional sampling knobs (default to sgl-eval's defaults when unset). Set these - for reasoning models -- e.g. DeepSeek-V4 Think-Max wants - aime25_reasoning_effort="max", aime25_max_tokens=200000, aime25_temperature=1.0, - aime25_top_p=1.0. AIME25 has only 30 problems, so it is high variance; raise - aime25_n_repeats (e.g. 16) for a stable number. - """ - - aime25_score_threshold: float = _THRESHOLD_NOT_SET - aime25_accept_length_thres: Optional[float] = None - aime25_num_examples: Optional[int] = None - aime25_num_threads: int = 1024 - aime25_n_repeats: int = 1 - aime25_thinking: bool = True - aime25_reasoning_effort: Optional[str] = None - aime25_max_tokens: Optional[int] = None - aime25_temperature: Optional[float] = None - aime25_top_p: Optional[float] = None - - def test_aime25(self): - _run_sgl_eval( - self, - eval_name="aime25", - score_threshold=self.aime25_score_threshold, - n_repeats=self.aime25_n_repeats, - num_examples=self.aime25_num_examples, - num_threads=self.aime25_num_threads, - thinking=self.aime25_thinking, - reasoning_effort=self.aime25_reasoning_effort, - max_tokens=self.aime25_max_tokens, - temperature=self.aime25_temperature, - top_p=self.aime25_top_p, - accept_length_thres=self.aime25_accept_length_thres, - ) - - class HumanEvalMixin: - """Mixin for HumanEval evaluation. - - Required attributes on the test class: - base_url: str - model: str - humaneval_score_threshold: float - """ + """Mixin for HumanEval evaluation.""" humaneval_score_threshold: float = _THRESHOLD_NOT_SET humaneval_score_threshold_amd: Optional[float] = None @@ -464,13 +289,7 @@ class HumanEvalMixin: class MGSMEnMixin: - """Mixin for MGSM English evaluation. - - Required attributes on the test class: - base_url: str - model: str - mgsm_en_score_threshold: float - """ + """Mixin for MGSM English evaluation.""" mgsm_en_score_threshold: float = _THRESHOLD_NOT_SET mgsm_en_num_examples: Optional[int] = None diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index 5b74d93fdd..37d17d4036 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -1,18 +1,17 @@ """ Usage: -python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10 +python3 -m sglang.test.run_eval --port 30000 --eval-name gsm8k --num-examples 10 """ import argparse import json import os import statistics -import subprocess import time -import uuid -from pathlib import Path +import warnings from sglang.test.sgl_eval_utils import ( + SGL_EVAL_BENCHMARKS, THINKING_MODE_CHOICES, get_thinking_kwargs, parse_json_object, @@ -89,125 +88,6 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict: return result, latency, sampler -def _run_sgl_eval(eval_name, args) -> dict: - # Returns a metrics dict (score, latency, output_throughput) so the - # existing write_results_to_json + threshold gate keep working. - from sglang.test.test_utils import dump_metric - - base_url = ( - f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1" - ) - out_parent = Path( - getattr(args, "sgl_eval_out_dir", None) - or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex) - ).expanduser() - out_parent.mkdir(parents=True, exist_ok=True) - - model_preset_id = getattr(args, "load_preset_from_model_id", None) - cmd = [ - "sgl-eval", - "run", - eval_name, - "--base-url", - base_url, - "--out-dir", - str(out_parent), - ] - if model_preset_id: - cmd += ["--load-preset-from-model-id", model_preset_id] - if getattr(args, "model", None): - cmd += ["--model", args.model] - if getattr(args, "num_examples", None) is not None: - cmd += ["--num-examples", str(args.num_examples)] - if getattr(args, "num_threads", None) is not None: - cmd += ["--num-threads", str(args.num_threads)] - if getattr(args, "temperature", None) is not None: - cmd += ["--temperature", str(args.temperature)] - elif not model_preset_id: - cmd += ["--temperature", "0.0"] - if getattr(args, "top_p", None) is not None: - cmd += ["--top-p", str(args.top_p)] - elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False): - cmd += ["--top-p", "1.0"] - # Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it. - if getattr(args, "seed", None) is not None: - cmd += ["--seed", str(args.seed)] - # gpt-oss grades one score per effort tier, so dropping this collapses every - # tier onto the served model's default. - if getattr(args, "reasoning_effort", None) is not None: - cmd += ["--reasoning-effort", str(args.reasoning_effort)] - if getattr(args, "repeat", None) is not None: - cmd += ["--n-repeats", str(args.repeat)] - # Bound generation length so long-reasoning models don't stall the eval. - if getattr(args, "max_tokens", None) is not None: - cmd += ["--max-tokens", str(args.max_tokens)] - elif not model_preset_id: - cmd += ["--max-tokens", "2048"] - # Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel; - # without --thinking their message.content is empty and sgl-eval scores 0. - sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None) - if sgl_eval_thinking is None: - if not model_preset_id: - model_l = (getattr(args, "model", None) or "").lower() - if "qwen3.5" in model_l or "qwen3-thinking" in model_l: - cmd += ["--thinking"] - elif sgl_eval_thinking: - cmd += ["--thinking"] - - try: - completed = subprocess.run( - cmd, - text=True, - capture_output=True, - check=False, - timeout=getattr(args, "sgl_eval_timeout", None), - ) - except subprocess.TimeoutExpired as e: - raise TimeoutError( - f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n" - f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}" - ) from e - - if completed.returncode != 0: - raise RuntimeError( - f"sgl-eval failed with exit code {completed.returncode}: " - f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" - ) - - metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json")) - if len(metrics_files) != 1: - raise FileNotFoundError( - f"Expected exactly one metrics.json under {out_parent}, " - f"found {len(metrics_files)}" - ) - payload = json.loads(metrics_files[0].read_text()) - aggregate = payload.get("aggregate") - if not isinstance(aggregate, dict) or "score" not in aggregate: - raise KeyError(f"{metrics_files[0]} missing aggregate.score") - - metrics = dict(aggregate) - metrics["latency"] = payload.get("latency_seconds", 0.0) - metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0) - metrics["sgl_eval_metrics_path"] = str(metrics_files[0]) - - model = payload.get("model") or getattr(args, "model", None) - dump_metric( - f"{eval_name}_score", - metrics["score"], - labels={"model": model, "eval": eval_name}, - ) - dump_metric( - f"{eval_name}_latency", - metrics["latency"], - labels={"model": model, "eval": eval_name}, - ) - print(f"Score: {metrics['score']:.3f}") - print(f"Total latency: {metrics['latency']:.3f} s") - print(f"Output throughput: {metrics['output_throughput']:.3f} token/s") - print(f"sgl-eval metrics: {metrics_files[0]}") - return metrics - - def print_accept_length_summary(samplers: list) -> None: accept_lengths = [ m["spec_accept_length"] @@ -236,6 +116,23 @@ def run_eval(args): # Lazy import to avoid circular dependency with test_utils from sglang.test.test_utils import dump_metric + warnings.warn( + "sglang.test.run_eval is deprecated; its legacy scorers will be removed. " + "Use sglang.test.sgl_eval_utils.run_sgl_eval or `sgl-eval run` instead.", + DeprecationWarning, + stacklevel=2, + ) + if ( + args.eval_name in SGL_EVAL_BENCHMARKS + or args.eval_name == "mmmu-pro" + or getattr(args, "api", None) == "sgl_eval" + or getattr(args, "load_preset_from_model_id", None) + ): + raise ValueError( + f"{args.eval_name} is scored by sgl-eval; call " + "sglang.test.sgl_eval_utils.run_sgl_eval instead of run_eval." + ) + set_ulimit() if "OPENAI_API_KEY" not in os.environ: @@ -245,18 +142,10 @@ def run_eval(args): f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1" ) - if args.eval_name == "mmlu": - # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a - # caller's threshold has to be measured against it, not inherited. - return _run_sgl_eval("mmlu", args) - elif args.eval_name == "mgsm_en": + if args.eval_name == "mgsm_en": from sglang.test.simple_eval_mgsm import MGSMEval eval_obj = MGSMEval(args.num_examples, args.num_threads, languages=["en"]) - elif args.eval_name == "gpqa": - # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a - # caller's threshold has to be measured against it, not inherited. - return _run_sgl_eval("gpqa", args) elif args.eval_name == "humaneval": from sglang.test.simple_eval_humaneval import HumanEval @@ -270,25 +159,7 @@ def run_eval(args): args.num_threads, response_answer_regex=getattr(args, "response_answer_regex", None), ) - elif args.eval_name in ("mmmu_pro", "mmmu-pro"): - # Canonical sgl-eval name for MMMU-Pro's standard 10-option split. - return _run_sgl_eval("mmmu_pro", args) - elif args.eval_name == "mmmu_pro_vision": - # sgl-eval owns this benchmark's dataset, prompt and grader; there is no - # simple_eval implementation to fall back to. - return _run_sgl_eval("mmmu_pro_vision", args) - elif args.eval_name == "aime25": - return _run_sgl_eval("aime25", args) - elif args.eval_name == "aime26": - return _run_sgl_eval("aime26", args) elif args.eval_name == "gsm8k": - if getattr(args, "api", None) == "sgl_eval": - # Only the nightly correctness eval opts into sgl-eval (zero-shot - # chat, \boxed{}, math_verify). Every other gsm8k caller — spec - # decoding perf/accuracy, disaggregation, quant, model e2e — uses - # the 5-shot completion last-number scorer and relies on - # max_tokens/throughput behavior sgl-eval cannot provide. - return _run_sgl_eval("gsm8k", args) from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval eval_obj = GSM8KEval( @@ -428,16 +299,10 @@ if __name__ == "__main__": type=str, help="Name or path of the model. If not set, the default model will request /v1/models for conf.", ) - parser.add_argument( - "--load-preset-from-model-id", - type=str, - default=None, - help="Load repository-maintained sgl-eval generation defaults for this model ID.", - ) parser.add_argument( "--repeat", type=int, default=1, help="repeat the evaluation n times" ) - parser.add_argument("--eval-name", type=str, default="mmlu") + parser.add_argument("--eval-name", type=str, default="gsm8k") parser.add_argument( "--api", type=str, @@ -498,6 +363,5 @@ if __name__ == "__main__": ) args = parser.parse_args() - args._sgl_eval_from_cli = True run_eval(args) diff --git a/python/sglang/test/sgl_eval_utils.py b/python/sglang/test/sgl_eval_utils.py index 01f7c651ce..abc9a95339 100644 --- a/python/sglang/test/sgl_eval_utils.py +++ b/python/sglang/test/sgl_eval_utils.py @@ -6,13 +6,20 @@ import multiprocessing import os import threading import uuid +import warnings from concurrent.futures import ProcessPoolExecutor from dataclasses import replace from itertools import islice from pathlib import Path +from types import SimpleNamespace THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"] +# Benchmarks scored only by sgl-eval; run_eval does not serve them. +SGL_EVAL_BENCHMARKS = frozenset( + {"mmlu", "gpqa", "mmmu_pro", "mmmu_pro_vision", "aime25", "aime26"} +) + def get_thinking_kwargs(args): thinking_mode = getattr(args, "thinking_mode", None) @@ -85,9 +92,35 @@ def run_sgl_eval(args): from sglang.test.test_utils import dump_metric spec = get(args.eval_name) - overrides = {"max_tokens": 2048} + model = getattr(args, "model", None) + default_gen = spec.default_gen + model_preset_id = getattr(args, "load_preset_from_model_id", None) + if model_preset_id: + from sgl_eval.model_preset import load_model_preset + from sgl_eval.preset import apply_to_gen + + model_preset = load_model_preset(model_preset_id) + unset = dict.fromkeys( + ("thinking", "temperature", "top_p", "max_tokens", "reasoning_effort") + ) + default_gen = apply_to_gen( + default_gen, None, SimpleNamespace(**unset), model_preset + ) + model = model or model_preset.model + + overrides = {} + # Omitted max_tokens keeps the 2048 CI cap; an explicit None defers to the server. + if hasattr(args, "max_tokens"): + if args.max_tokens is None: + warnings.warn( + f"sgl-eval {spec.name}: max_tokens=None leaves the output length " + "uncapped; generation stops only at the server limit.", + stacklevel=2, + ) + overrides["max_tokens"] = args.max_tokens + elif not model_preset_id: + overrides["max_tokens"] = 2048 for key in ( - "max_tokens", "temperature", "top_p", "min_p", @@ -107,12 +140,12 @@ def run_sgl_eval(args): chat_kwargs.setdefault("enable_thinking", thinking) if chat_kwargs: overrides["chat_template_kwargs"] = { - **(spec.default_gen.chat_template_kwargs or {}), + **(default_gen.chat_template_kwargs or {}), **chat_kwargs, } if getattr(args, "top_k", None) is not None: overrides["extra_body"] = {"top_k": args.top_k} - gen = replace(spec.default_gen, **overrides) + gen = replace(default_gen, **overrides) generation = { key: getattr(gen, key) for key in ("max_tokens", "temperature", "top_p", "min_p", "seed") @@ -120,7 +153,7 @@ def run_sgl_eval(args): print(f"sgl-eval {spec.name} generation: {generation}", flush=True) sampler = ChatCompletionSampler( base_url=api_base_url(args), - model=getattr(args, "model", None), + model=model, api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"), ) out_parent = Path( @@ -129,7 +162,7 @@ def run_sgl_eval(args): ).expanduser() out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}" out_dir.mkdir(parents=True, exist_ok=True) - repeats = getattr(args, "repeat", 1) + repeats = getattr(args, "repeat", None) or spec.default_n_repeats writer = PredictionsWriter(out_dir, repeats, spec.pred_schema) try: result = spec.run( diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 521f330cb4..1d745a118e 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -48,7 +48,7 @@ from sglang.srt.utils import ( retry, ) from sglang.srt.utils.network import is_port_available -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.utils import normalize_base_url # General test models @@ -399,24 +399,11 @@ def start_subprocess_fail_fast_watcher( def _try_enable_offline_mode_if_cache_complete( model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None ) -> Optional[str]: - """ - CI helper: Check if model cache is complete and enable offline mode. + """Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the + per-run marker path, or None if offline mode was not enabled. - Uses per-run validation markers that are NOT shared across runners. - Each runner independently validates its cache using lightweight checks - before enabling offline mode. - - IMPORTANT: Even if a per-run marker exists, this function ALWAYS validates - the current launch's requirements (e.g., hf_quant_config.json for modelopt). - The marker is only a hint that this snapshot was validated earlier in the run. - - Args: - model_name_or_path: Model identifier or path - env: Environment dict to modify (will add HF_HUB_OFFLINE=1 if validation passes) - other_args: Launch command arguments (used to detect quantization requirement) - - Returns: - Per-run marker path if offline mode was enabled, None otherwise + Markers are per-run and not shared across runners. A marker is only a hint; + the current launch's requirements (e.g. hf_quant_config.json) are revalidated. """ from sglang.srt.model_loader.ci_weight_validation import ( _get_per_run_marker_path, @@ -453,8 +440,7 @@ def _try_enable_offline_mode_if_cache_complete( except Exception: return None - # Detect if quantization requires hf_quant_config.json - # Do this BEFORE checking marker to ensure current launch requirements are known + # Detect before the marker check so the current launch's requirements are known. requires_hf_quant_config = False for i, arg in enumerate(other_args): if arg == "--quantization" and i + 1 < len(other_args): @@ -466,9 +452,8 @@ def _try_enable_offline_mode_if_cache_complete( # Check per-run marker (fast hint - snapshot validated earlier in this run) per_run_marker = _read_per_run_marker(snapshot_dir) if per_run_marker is not None: - # Marker exists, but STILL validate for current launch requirements - # This prevents a test without --quantization from enabling offline - # for a later test with --quantization that needs hf_quant_config.json + # Still validate: a marker from a launch without --quantization must not + # enable offline mode for a later launch that needs hf_quant_config.json. is_valid = validate_cache_lightweight(snapshot_dir, requires_hf_quant_config) if not is_valid: @@ -522,17 +507,7 @@ def _try_enable_offline_mode_if_cache_complete( def _create_clean_subprocess_env(env: dict) -> dict: - """Create a clean subprocess environment without internal CI keys. - - Removes all keys starting with '_CI_OFFLINE_' or 'CI_OFFLINE' to prevent - leaking implementation details to the server subprocess. - - Args: - env: Source environment dict - - Returns: - Clean copy of environment dict - """ + """Create a clean subprocess environment without internal CI keys.""" child_env = env.copy() keys_to_remove = [ k for k in child_env if k.startswith(("_CI_OFFLINE_", "CI_OFFLINE_")) @@ -592,14 +567,7 @@ def _launch_server_process( ) -> subprocess.Popen: """Launch server subprocess with clean environment. - Args: - command: Command list for subprocess - env: Environment dict (will be cleaned before use) - return_stdout_stderr: Optional tuple of (stdout_file, stderr_file) for output capture - model: Model name for logging - - Returns: - Started subprocess.Popen object + `return_stdout_stderr` is an optional (stdout_file, stderr_file) pair. """ child_env = _create_clean_subprocess_env(env) @@ -619,17 +587,7 @@ def _wait_for_server_health( api_key: Optional[str], timeout_duration: float, ) -> Tuple[bool, Optional[str]]: - """Wait for server health check to pass. - - Args: - proc: Server subprocess - base_url: Base URL for health check - api_key: Optional API key for authorization - timeout_duration: Maximum wait time in seconds - - Returns: - Tuple of (success, error_message) - """ + """Wait for server health check to pass; return (success, error_message).""" start_time = time.perf_counter() with requests.Session() as session: while time.perf_counter() - start_time < timeout_duration: @@ -686,20 +644,8 @@ def popen_launch_server( ): """Launch a server process with automatic device detection and offline/online retry. - Args: - model: Model path or identifier - base_url: Base URL for the server - timeout: Timeout for server startup - api_key: Optional API key for authentication - other_args: Additional command line arguments - env: Environment dict for subprocess - return_stdout_stderr: Optional tuple for output capture - device: Device type ("auto", "cuda", "rocm" or "cpu") - pd_separated: Whether to use PD separated mode - num_replicas: Number of replicas for mixed PD mode - - Returns: - Started subprocess.Popen object + `device` is one of "auto", "cuda", "rocm", "cpu"; + `num_replicas` applies to mixed PD mode. """ other_args = other_args or [] @@ -897,9 +843,7 @@ def popen_launch_pd_server( print(f"command={' '.join(command)}") - # Merge with os.environ so caller-supplied env adds to (not replaces) - # PATH / PYTHONPATH / HF_HOME / etc. When env is None, Popen inherits - # parent's environment automatically. + # Merge so caller-supplied env adds to, not replaces, PATH / PYTHONPATH / etc. if env is not None: env = {**os.environ, **env} @@ -1018,9 +962,8 @@ def run_bench_serving( other_args=other_server_args, ) - # Resolve tokenizer to local snapshot path when available, so the benchmark - # client's AutoTokenizer.from_pretrained uses the local path directly instead - # of calling the HF Hub API (which can stall for minutes in CI). + # Prefer the local snapshot so the client's AutoTokenizer skips the HF Hub API, + # which can stall for minutes in CI. bench_tokenizer = tokenizer if bench_tokenizer is None: try: @@ -1090,19 +1033,7 @@ async def _run_api_benchmark_requests( num_requests: int, response_validator: Callable[[dict], bool], ): - """ - Helper function to run API benchmark requests and collect metrics. - - Args: - base_url: The base URL of the server - endpoint: The API endpoint to test (e.g., "/v1/score", "/v1/embeddings") - test_requests: List of request payloads to send - num_requests: Total number of requests expected - response_validator: Function to validate if response contains expected data - - Returns: - Dictionary with benchmark metrics - """ + """Run API benchmark requests and return a dict of metrics.""" start_time = time.monotonic() successful_requests = 0 total_latency = 0 @@ -1433,12 +1364,7 @@ def run_bench_serving_multi( def run_bench_one_batch(model, other_args): - """Launch a offline process with automatic device detection. - - Args: - device: Device type ("auto", "cuda", "rocm" or "cpu"). - If "auto", will detect available platforms automatically. - """ + """Launch an offline process with automatic device detection.""" # Auto-detect device if needed device = auto_config_device() @@ -1740,7 +1666,7 @@ def run_mmlu_test( ) try: - metrics = run_eval(args) + metrics = run_sgl_eval(args) assert metrics["score"] >= 0.65, f"{metrics=}" finally: pass @@ -2340,11 +2266,9 @@ def server_args_variant(server_args, **fields): def enter_override(test_case, override): """Install a scoped context override for the length of one test. - `unittest.TestCase.enterContext` does exactly this in one call, but it is - Python 3.11+ and this package supports 3.10 (`requires-python = ">=3.10"`). - On 3.10 it raises `AttributeError: ... has no attribute 'enterContext'` -- - and only there, so a developer on a newer interpreter sees every test pass - while CI does not. + Stands in for `unittest.TestCase.enterContext`, which is 3.11+ while this package + supports 3.10; on 3.10 it raises AttributeError, so tests pass locally on newer + interpreters but fail in CI. """ installed = override.install() test_case.addCleanup(override.restore) @@ -2366,9 +2290,8 @@ class CustomTestCase(unittest.TestCase): def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - # Wrap the effective setUpClass so that tearDownClass is called - # even when setUpClass fails. Python's unittest skips tearDownClass - # if setUpClass raises, which can leak resources (ports, processes). + # unittest skips tearDownClass when setUpClass raises, leaking ports and + # processes; wrap setUpClass so tearDownClass still runs. setup = cls.setUpClass if getattr(setup, "_safe_setup_wrapped", False): return @@ -2669,11 +2592,6 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None): - stdout: [METRIC] metric_name=value [labels=...] This function never fails tests - all errors are silently caught. - - Args: - metric_name: Metric name (e.g., "gsm8k_accuracy", "cache_hit_rate") - value: Metric value - labels: Optional label dict (e.g., {"backend": "fa3"}) """ try: # 1. Capture test context @@ -2750,11 +2668,7 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None): def _get_test_context() -> tuple[str, str]: - """ - Get current test's filename and test_case. - - Tries PYTEST_CURRENT_TEST first, falls back to inspect.stack(). - """ + """Get current test's filename and test_case, via PYTEST_CURRENT_TEST or stack.""" # 1. Try parsing PYTEST_CURRENT_TEST pytest_current = os.getenv("PYTEST_CURRENT_TEST") if pytest_current: diff --git a/test/manual/dsv4/_common.py b/test/manual/dsv4/_common.py index 19aefd08a0..40b357bf0d 100644 --- a/test/manual/dsv4/_common.py +++ b/test/manual/dsv4/_common.py @@ -1,90 +1,65 @@ -"""Shared fixture for DeepSeek-V4 cookbook launch-command tests. +"""Shared fixture for DeepSeek-V4 cookbook launch-command tests (manual, not CI). -Each sibling ``test__.py`` declares ONE -``hardware x model_size`` cell from the cookbook (e.g. B200 x Flash) -and contains one ``CustomTestCase`` subclass per recipe -(Low-Latency / Balanced / Max-Throughput / CP, where supported). - -Each subclass launches the server with the cookbook's exact flags and -runs two sgl-eval evaluations (https://github.com/sgl-project/sgl-eval): -- ``test_smoke_gsm8k`` — short, cheap GSM8K pass to verify the server - can produce coherent math answers at all (sanity gate). -- ``test_aime25`` — full AIME25 accuracy run (heavy; 16 repeats default). +Each sibling ``test__.py`` covers one cookbook cell +(e.g. B200 x Flash) with one ``CustomTestCase`` subclass per recipe. Each +launches the server with the cookbook's exact flags and runs +``test_smoke_gsm8k`` (sanity gate) and ``test_aime25`` (full accuracy run) +through sgl-eval, which must be installed. Cookbook reference: https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4 -These are MANUAL tests (not CI). ``sgl-eval`` must be on PATH. - -Per-variant defaults (set on the Flash/Pro intermediate base classes): - Flash recipes -> AIME25 score threshold 0.93 - Pro recipes -> AIME25 score threshold 0.95 -GSM8K sanity threshold (0.93) is shared across Flash and Pro. - -AIME25 knobs (env vars): - DSV4_AIME25_NUM_REPEATS (default 16 -> --n-repeats) - DSV4_AIME25_TEMPERATURE (default 1.0 -> --temperature) - DSV4_AIME25_TOP_P (default 1.0 -> --top-p) - DSV4_AIME25_MAX_TOKENS (default 65536 -> --max-tokens) - DSV4_AIME25_NUM_THREADS (default 512 -> --num-threads) - DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate") +AIME25 knobs (env vars); the score threshold is 0.93 for Flash, 0.95 for Pro: + DSV4_AIME25_NUM_REPEATS (default 16) + DSV4_AIME25_TEMPERATURE (default 1.0) + DSV4_AIME25_TOP_P (default 1.0) + DSV4_AIME25_MAX_TOKENS (default 65536) + DSV4_AIME25_NUM_THREADS (default 512) + DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval aggregate key) DSV4_AIME25_SCORE_THRESHOLD (default 0; >0 overrides per-variant default) GSM8K sanity knobs (env vars): - DSV4_GSM8K_NUM_EXAMPLES (default 50 -> --num-examples) - DSV4_GSM8K_N_REPEATS (default 1 -> --n-repeats) - DSV4_GSM8K_TEMPERATURE (default 0.6 -> --temperature) - DSV4_GSM8K_TOP_P (default 0.95 -> --top-p) - DSV4_GSM8K_MAX_TOKENS (default 8192 -> --max-tokens) - DSV4_GSM8K_NUM_THREADS (default 64 -> --num-threads) - DSV4_GSM8K_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate") + DSV4_GSM8K_NUM_EXAMPLES (default 50) + DSV4_GSM8K_N_REPEATS (default 1) + DSV4_GSM8K_TEMPERATURE (default 0.6) + DSV4_GSM8K_TOP_P (default 0.95) + DSV4_GSM8K_MAX_TOKENS (default 8192) + DSV4_GSM8K_NUM_THREADS (default 64) + DSV4_GSM8K_SCORE_METRIC (default "score"; sgl-eval aggregate key) DSV4_GSM8K_SCORE_THRESHOLD (default 0.93; set to 0 to skip the assertion) Shared knobs: - DSV4_SGL_EVAL_OUT_DIR (default /tmp/sgl-eval-out -> --out-dir) - DSV4_SGL_EVAL_BIN (default "sgl-eval"; override path to the CLI) - DSV4_SERVER_LAUNCH_TIMEOUT (default 3600s; the sglang 600s default is - too short for DSV4 model load + DeepGEMM - warmup. 1800s is also tight for the heavier - recipes (DP-attn + DeepEP); 3600s is the - safe default. Bump again for first-run - model downloads if needed.) + DSV4_SGL_EVAL_OUT_DIR (default /tmp/sgl-eval-out) + DSV4_SERVER_LAUNCH_TIMEOUT (default 3600s; bump for first-run downloads) -Multi-node knobs (only consumed by multi-node test classes; if either -is unset, those classes ``SkipTest``): +Multi-node knobs (multi-node classes ``SkipTest`` if either is unset): DSV4_NODE_RANK (per-node rank for --node-rank) DSV4_DIST_INIT_ADDR (e.g. 10.0.0.1:20000 for --dist-init-addr) -Always-on env (set by the base class for every recipe; per-recipe EXTRA_ENV -wins on key conflict): - SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 skip the slow DeepGEMM warmup grid +Every recipe also gets SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 (skips the slow +DeepGEMM warmup grid); per-recipe EXTRA_ENV wins on key conflict. """ -import json import os -import shutil -import subprocess import unittest -from pathlib import Path +from types import SimpleNamespace from typing import ClassVar, Dict, List, Optional from sglang.srt.utils import kill_process_tree +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, ) -SGL_EVAL_BIN = os.environ.get("DSV4_SGL_EVAL_BIN", "sgl-eval") SGL_EVAL_OUT_DIR = os.environ.get("DSV4_SGL_EVAL_OUT_DIR", "/tmp/sgl-eval-out") -# DSV4 server launch needs more than the 600s sglang default: model load alone -# can take 5+ min and DeepGEMM warmup another ~5 min. First-run model download -# adds ~10-30 min on top. 1800s covers steady-state; bump via env for downloads. +# Model load (5+ min) plus DeepGEMM warmup (~5 min) exceed sglang's 600s default, +# and 1800s is tight for DP-attn + DeepEP recipes; downloads add ~10-30 min. SERVER_LAUNCH_TIMEOUT = int(os.environ.get("DSV4_SERVER_LAUNCH_TIMEOUT", "3600")) -# Defaults applied to every recipe's EXTRA_ENV. Per-recipe EXTRA_ENV wins on key -# conflict. +# Defaults for every recipe; per-recipe EXTRA_ENV wins on key conflict. BASE_ENV: Dict[str, str] = { # Skip the slow exhaustive DeepGEMM warmup grid; covers the shapes DSV4 # actually hits and shaves several minutes off server startup. @@ -108,8 +83,8 @@ GSM8K_NUM_THREADS = int(os.environ.get("DSV4_GSM8K_NUM_THREADS", "64")) GSM8K_SCORE_METRIC = os.environ.get("DSV4_GSM8K_SCORE_METRIC", "score") GSM8K_SCORE_THRESHOLD = float(os.environ.get("DSV4_GSM8K_SCORE_THRESHOLD", "0.93")) -# DeepEP "large SMS" config — appears as `--deepep-config '{...}'` in every -# DeepEP recipe except multi-node ones (where it is gated off in the JSX). +# Cookbook `--deepep-config` of every DeepEP recipe except multi-node, +# where the JSX gates it off. DEEPEP_LARGE_SMS_CONFIG = ( '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' ) @@ -118,9 +93,7 @@ DEEPEP_LARGE_SMS_CONFIG = ( def multinode_args(nnodes: int) -> List[str]: """Return CLI args for a multi-node launch, or skip the test. - Reads DSV4_NODE_RANK and DSV4_DIST_INIT_ADDR from the env. Raises - ``unittest.SkipTest`` when either is missing — call from inside - ``setUpClass`` so the whole class skips cleanly. + Call from ``setUpClass`` so the whole class skips cleanly. """ rank = os.environ.get("DSV4_NODE_RANK") addr = os.environ.get("DSV4_DIST_INIT_ADDR") @@ -141,9 +114,6 @@ def multinode_args(nnodes: int) -> List[str]: class DSV4Aime25TestBase(CustomTestCase): """Subclass via ``DSV4FlashAime25TestBase`` or ``DSV4ProAime25TestBase``, not directly. Per-recipe subclasses set MODEL / OTHER_ARGS / EXTRA_ENV. - - SCORE_THRESHOLD is set by the Flash/Pro intermediate base classes: - Flash 0.93, Pro 0.95. """ MODEL: ClassVar[str] = "" @@ -220,52 +190,18 @@ class DSV4Aime25TestBase(CustomTestCase): metric, threshold, ): - if shutil.which(SGL_EVAL_BIN) is None: - self.skipTest(f"{SGL_EVAL_BIN!r} not found on PATH") - - out_dir = Path(SGL_EVAL_OUT_DIR) - out_dir.mkdir(parents=True, exist_ok=True) - glob_pattern = f"sgl_eval_{eval_name}_*.json" - before = set(out_dir.glob(glob_pattern)) - - cmd = [ - SGL_EVAL_BIN, - "run", - eval_name, - "--base-url", - f"{self.base_url}/v1", - "--n-repeats", - str(n_repeats), - "--temperature", - str(temperature), - "--top-p", - str(top_p), - "--max-tokens", - str(max_tokens), - "--num-threads", - str(num_threads), - "--out-dir", - str(out_dir), - ] - if num_examples is not None: - cmd += ["--num-examples", str(num_examples)] - - print(f"[{type(self).__name__}] + {' '.join(cmd)}", flush=True) - subprocess.run(cmd, check=True) - - new = sorted(set(out_dir.glob(glob_pattern)) - before) - if not new: - self.fail(f"sgl-eval produced no new {eval_name} JSON in {out_dir}") - result_path = new[-1] - with open(result_path) as f: - result = json.load(f) - print( - f"[{type(self).__name__}] sgl-eval {eval_name} result " - f"({result_path.name}): {json.dumps(result, indent=2)}", - flush=True, + args = SimpleNamespace( + eval_name=eval_name, + base_url=self.base_url, + repeat=n_repeats, + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + num_threads=num_threads, + num_examples=num_examples, + sgl_eval_out_dir=SGL_EVAL_OUT_DIR, ) - - score = self._extract_score(result, metric) + score = run_sgl_eval(args)[metric] if threshold > 0: self.assertGreaterEqual( score, @@ -273,30 +209,6 @@ class DSV4Aime25TestBase(CustomTestCase): f"{eval_name} {metric}={score} below threshold {threshold}", ) - @staticmethod - def _extract_score(result, metric): - """Find ``metric`` (e.g. "pass@1") anywhere in the sgl-eval JSON tree.""" - - def walk(o): - if isinstance(o, dict): - if metric in o and isinstance(o[metric], (int, float)): - return float(o[metric]) - for v in o.values(): - s = walk(v) - if s is not None: - return s - elif isinstance(o, list): - for v in o: - s = walk(v) - if s is not None: - return s - return None - - score = walk(result) - if score is None: - raise AssertionError(f"metric {metric!r} not found in sgl-eval result JSON") - return score - class DSV4FlashAime25TestBase(DSV4Aime25TestBase): """Base for DeepSeek-V4-Flash recipes: AIME25 threshold 0.93.""" diff --git a/test/manual/ep/test_eplb.py b/test/manual/ep/test_eplb.py index 5dd97364ee..f0c5cce2e4 100755 --- a/test/manual/ep/test_eplb.py +++ b/test/manual/ep/test_eplb.py @@ -8,7 +8,7 @@ from types import SimpleNamespace import sglang as sgl from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -96,7 +96,7 @@ class _BaseTestDynamicEPLB(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.5) @@ -123,8 +123,6 @@ class TestStaticEPLB(CustomTestCase): tp_size=2, dp_size=2, log_level="info", - # TODO pr-chain: enable later - # enable_expert_distribution_metrics=True, ) engine_kwargs.update(get_a2a_backend_kwargs()) diff --git a/test/manual/ep/test_moe_deepep.py b/test/manual/ep/test_moe_deepep.py index 495a3aefff..ce7074b817 100644 --- a/test/manual/ep/test_moe_deepep.py +++ b/test/manual/ep/test_moe_deepep.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -46,7 +46,7 @@ class TestPureTP(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.5) @@ -108,7 +108,7 @@ class TestDPAttn(unittest.TestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.5) diff --git a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py index 0b990f3b35..9693e7f530 100644 --- a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py +++ b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py @@ -1,13 +1,7 @@ -""" -Usage: -python -m unittest test_moe_deepep_eval_accuracy_large.TestMoEDeepEPEvalAccuracyLarge.test_mmlu -""" - import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, @@ -65,7 +59,7 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Eval accuracy of MMLU: {metrics=}") self.assertGreater(metrics["score"], 0.87) diff --git a/test/manual/quant/test_autoround.py b/test/manual/quant/test_autoround.py index 9d879330d2..cd7dba0fd8 100644 --- a/test/manual/quant/test_autoround.py +++ b/test/manual/quant/test_autoround.py @@ -1,13 +1,8 @@ -""" -Usage: -python3 -m unittest test_autoround.TestAutoRound.test_mmlu -""" - import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -48,7 +43,7 @@ class TestAutoRound(CustomTestCase): num_threads=32, device=device, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) if "Llama" in model: self.assertGreaterEqual(metrics["score"], 0.6) else: diff --git a/test/manual/quant/test_awq_archived.py b/test/manual/quant/test_awq_archived.py index 64a42acf4b..2e558006ef 100644 --- a/test/manual/quant/test_awq_archived.py +++ b/test/manual/quant/test_awq_archived.py @@ -1,15 +1,10 @@ -"""Archived test classes split out of test/registered/quant/test_awq.py. - -Originally registered with `register_cuda_ci(...)`. Moved here as part of -the per-commit pruning effort to keep the code reachable manually. -Run with `python3 test/manual/quant/test_awq_archived.py`. -""" +"""Archived test classes split out of test/registered/quant/test_awq.py; not in CI.""" import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -21,10 +16,6 @@ from sglang.test.test_utils import ( @unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs") class TestAWQMarlinFloat16(CustomTestCase): - """ - Verify that the model can be loaded with float16 dtype and awq_marlin quantization - """ - @classmethod def setUpClass(cls): cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ" @@ -49,7 +40,7 @@ class TestAWQMarlinFloat16(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.85) diff --git a/test/manual/quant/test_eval_fp8_accuracy.py b/test/manual/quant/test_eval_fp8_accuracy.py index d6b8b9eea1..f69af9dc5f 100644 --- a/test/manual/quant/test_eval_fp8_accuracy.py +++ b/test/manual/quant/test_eval_fp8_accuracy.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8, DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, @@ -37,7 +37,7 @@ class TestEvalFP8Accuracy(CustomTestCase): temperature=0.1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) if is_hip(): # Another threshold for AMD because fp8 dtype is difference self.assertGreaterEqual(metrics["score"], 0.60) @@ -67,7 +67,7 @@ class TestEvalFP8DynamicQuantAccuracy(CustomTestCase): temperature=0.1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], expected_score) finally: kill_process_tree(process.pid) @@ -86,7 +86,6 @@ class TestEvalFP8DynamicQuantAccuracy(CustomTestCase): model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, other_args=["--quantization", "w8a8_fp8"], # inference will use sgl kernel w/ online quant override - # we observed that the accuracy is higher then offline only expected_score=0.64, ) @@ -95,7 +94,6 @@ class TestEvalFP8DynamicQuantAccuracy(CustomTestCase): self._run_test( model=DEFAULT_MODEL_NAME_FOR_TEST, # inference will use sgl kernel w/ online quantization only - # we observed that the accuracy is higher then offline only other_args=["--quantization", "w8a8_fp8"], expected_score=0.64, ) diff --git a/test/manual/quant/test_fp8_kvcache.py b/test/manual/quant/test_fp8_kvcache.py index 669926f5db..9ba9aed00a 100644 --- a/test/manual/quant/test_fp8_kvcache.py +++ b/test/manual/quant/test_fp8_kvcache.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, @@ -71,7 +72,7 @@ class TestFp8KvcacheLlama(TestFp8KvcacheBase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.65) @@ -106,7 +107,7 @@ class TestFp8KvcacheQwen(TestFp8KvcacheBase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.3) diff --git a/test/manual/spec/eagle/test_eagle3_basic.py b/test/manual/spec/eagle/test_eagle3_basic.py index b0c1956d28..ec7f735db3 100644 --- a/test/manual/spec/eagle/test_eagle3_basic.py +++ b/test/manual/spec/eagle/test_eagle3_basic.py @@ -4,8 +4,8 @@ from types import SimpleNamespace import requests from sglang.srt.utils import is_hip -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.eagle_fixture import EagleServerBase +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, DEFAULT_TARGET_MODEL_EAGLE3, @@ -44,7 +44,7 @@ class TestEagle3Basic(EagleServerBase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.72) server_info = requests.get(self.base_url + "/server_info").json() diff --git a/test/manual/test_modelopt.py b/test/manual/test_modelopt.py index 69a90e3851..7951fdc0c3 100644 --- a/test/manual/test_modelopt.py +++ b/test/manual/test_modelopt.py @@ -4,7 +4,7 @@ from types import SimpleNamespace import torch from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8, DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8_REVISION, @@ -37,7 +37,7 @@ class TestEvalFP8ModelOptQuantAccuracy(CustomTestCase): temperature=0.1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], expected_score) finally: kill_process_tree(process.pid) diff --git a/test/manual/test_two_batch_overlap.py b/test/manual/test_two_batch_overlap.py index 410872166a..2786aa6f90 100644 --- a/test/manual/test_two_batch_overlap.py +++ b/test/manual/test_two_batch_overlap.py @@ -10,7 +10,7 @@ from sglang.srt.batch_overlap.two_batch_overlap import ( from sglang.srt.environ import envs from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST, DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -71,7 +71,7 @@ class TestTwoBatchOverlap(unittest.TestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.5) diff --git a/test/manual/test_wave_attention_backend.py b/test/manual/test_wave_attention_backend.py index 5feab45955..dd38ad48dc 100644 --- a/test/manual/test_wave_attention_backend.py +++ b/test/manual/test_wave_attention_backend.py @@ -1,13 +1,8 @@ -""" -Usage: -python3 -m unittest test_wave_attention_backend.TestWaveAttnBackend.test_mmlu -""" - import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -51,7 +46,7 @@ class TestWaveAttnBackend(unittest.TestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.65) finally: kill_process_tree(process.pid) diff --git a/test/registered/accuracy/models/test_text_models_gsm8k_eval.py b/test/registered/accuracy/models/test_text_models_gsm8k_eval.py index 359ffe6351..0a2c9d37aa 100644 --- a/test/registered/accuracy/models/test_text_models_gsm8k_eval.py +++ b/test/registered/accuracy/models/test_text_models_gsm8k_eval.py @@ -5,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, @@ -82,12 +82,11 @@ class TestNightlyGsm8KEval(unittest.TestCase): base_url=self.base_url, model=model_setup.model_path, eval_name="gsm8k", - api="sgl_eval", num_examples=None, num_threads=1024, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print( f"{'=' * 42}\n{model_setup.model_path} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" ) diff --git a/test/registered/attention/test_torch_native_attention_backend.py b/test/registered/attention/test_torch_native_attention_backend.py index 4e9f8ad304..f97c374183 100644 --- a/test/registered/attention/test_torch_native_attention_backend.py +++ b/test/registered/attention/test_torch_native_attention_backend.py @@ -1,14 +1,9 @@ -""" -Usage: -python3 -m unittest test_torch_native_attention_backend.TestTorchNativeAttnBackend.test_mmlu -""" - import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -42,7 +37,7 @@ class TestTorchNativeAttnBackend(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.64) finally: kill_process_tree(process.pid) diff --git a/test/registered/attention/test_triton_sliding_window.py b/test/registered/attention/test_triton_sliding_window.py index 24f88fcd1e..77eff242ed 100644 --- a/test/registered/attention/test_triton_sliding_window.py +++ b/test/registered/attention/test_triton_sliding_window.py @@ -5,7 +5,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -59,7 +59,7 @@ class TestSlidingWindowAttentionTriton(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"MMLU metrics with sliding window: {metrics}") # gemma-3-4b-it scores 0.59 over 256 questions under sgl-eval's grader, diff --git a/test/registered/cpu/quant/test_autoround.py b/test/registered/cpu/quant/test_autoround.py index 7e954af714..f428044c9d 100644 --- a/test/registered/cpu/quant/test_autoround.py +++ b/test/registered/cpu/quant/test_autoround.py @@ -1,12 +1,5 @@ -""" -Usage: -SGLANG_USE_CPU_ENGINE=1 python3 -m unittest test_autoround - -CPU accuracy test for AutoRound INT4 checkpoints. Covers both AutoRound packing -formats (auto_round:auto_gptq / auto_round:auto_awq) by launching a server and -running an MMLU eval. AutoRound INT4 CPU inference uses the Intel AMX backend, -so the test is skipped on AMD CPUs and other non-AMX CPU hosts. -""" +"""CPU accuracy test for AutoRound INT4 checkpoints in both packing formats +(auto_round:auto_gptq / auto_round:auto_awq).""" import os import unittest @@ -15,7 +8,7 @@ from types import SimpleNamespace from sglang.srt.layers.quantization.auto_round import AutoRoundConfig from sglang.srt.utils import cpu_has_amx_support, kill_process_tree from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -89,7 +82,7 @@ class TestAutoRoundCPU(CustomTestCase): num_threads=32, device=device, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.25) finally: kill_process_tree(process.pid) diff --git a/test/registered/cpu/test_cpu_graph.py b/test/registered/cpu/test_cpu_graph.py index c9511f3b71..f3bdef6d48 100644 --- a/test/registered/cpu/test_cpu_graph.py +++ b/test/registered/cpu/test_cpu_graph.py @@ -1,8 +1,3 @@ -""" -Usage: -python3 -m unittest test_cpu_graph.TestCPUGraph.test_mmlu_torch_compile_cpu -""" - import copy import os import unittest @@ -10,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.utils import get_cpu_ids_by_node, kill_process_tree from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -79,7 +74,7 @@ class TestCPUGraph(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) if is_in_ci(): self.assertGreater(metrics["score"], 0.45) finally: diff --git a/test/registered/cpu/test_intel_amx_attention_backend_a.py b/test/registered/cpu/test_intel_amx_attention_backend_a.py index c9f0ea2ce6..23a27c8269 100644 --- a/test/registered/cpu/test_intel_amx_attention_backend_a.py +++ b/test/registered/cpu/test_intel_amx_attention_backend_a.py @@ -1,15 +1,10 @@ -""" -Usage: -python3 -m unittest test_intel_amx_attention_backend.TestIntelAMXAttnBackend.test_latency_default_model -""" - import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST, @@ -65,7 +60,7 @@ class TestIntelAMXAttnBackend(CustomTestCase): num_examples=64, num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) if is_in_ci(): self.assertGreater(metrics["score"], 0.45) finally: diff --git a/test/registered/disaggregation/test_disaggregation_decode_offload.py b/test/registered/disaggregation/test_disaggregation_decode_offload.py index f4dfd02198..3091583089 100644 --- a/test/registered/disaggregation/test_disaggregation_decode_offload.py +++ b/test/registered/disaggregation/test_disaggregation_decode_offload.py @@ -4,10 +4,10 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -15,8 +15,7 @@ from sglang.test.test_utils import ( terminate_and_kill_process_tree, ) -# Registering the test for CUDA CI with appropriate parameters -# Increasing estimated time since we run evaluation twice +# est_time covers two eval rounds. register_cuda_ci( est_time=600, stage="base-b", @@ -25,10 +24,7 @@ register_cuda_ci( class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): - """ - Test class for verifying KV cache offloading on the decode side in a - prefill-decode disaggregation setup. - """ + """Decode-side KV cache offloading in a PD disaggregation setup.""" @classmethod def setUpClass(cls): @@ -128,13 +124,8 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): ) def test_mmlu_double_eval(self): - """ - Run two rounds of MMLU evaluation: - 1. First round: Decode node offloads KV cache back to disk (HiCache). - 2. Restart All Nodes to clear memory cache. - 3. Second round: Prefill node loads KV cache from disk (HiCache). - Verify that both rounds produce consistent scores. - """ + """Round 1 offloads decode KV to HiCache disk; after restarting all nodes, + round 2 loads it on prefill. Both rounds must score consistently.""" args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", model=self.model, @@ -143,7 +134,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): num_threads=32, ) - metrics1 = run_eval(args) + metrics1 = run_sgl_eval(args) # Ensure all offloads are committed to disk import time @@ -160,7 +151,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): self.wait_server_ready(self.prefill_url + "/health") self.wait_server_ready(self.decode_url + "/health") - metrics2 = run_eval(args) + metrics2 = run_sgl_eval(args) # Assert score is above a minimum threshold for both rounds self.assertGreater(metrics1["score"], 0.64) diff --git a/test/registered/e2e/models/test_transformers_models.py b/test/registered/e2e/models/test_transformers_models.py index 826cf6e528..107a0b6a6f 100644 --- a/test/registered/e2e/models/test_transformers_models.py +++ b/test/registered/e2e/models/test_transformers_models.py @@ -10,7 +10,6 @@ import torch from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( @@ -52,7 +51,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase): num_examples=256, num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound) def test_gsm8k(self): diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py index 589ff28010..9554584c04 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py @@ -62,7 +62,6 @@ class TestKimiK25Nvfp4Dp(CustomTestCase): max_tokens=32768, temperature=0.7, seed=0, - sgl_eval_thinking=False, ), performance_params=PerformanceTestParams( batch_sizes=[16], diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_tp.py b/test/registered/gb300/test_kimi_k25_nvfp4_tp.py index 36c49c4502..4974ec169c 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4_tp.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4_tp.py @@ -60,7 +60,6 @@ class TestKimiK25Nvfp4Tp(CustomTestCase): max_tokens=32768, temperature=0.7, seed=0, - sgl_eval_thinking=False, ), performance_params=PerformanceTestParams( batch_sizes=[1, 8], diff --git a/test/registered/gb300/test_qwen35_fp8_dp.py b/test/registered/gb300/test_qwen35_fp8_dp.py index e7c5e31da6..380c713f8a 100644 --- a/test/registered/gb300/test_qwen35_fp8_dp.py +++ b/test/registered/gb300/test_qwen35_fp8_dp.py @@ -59,7 +59,6 @@ class TestQwen35Fp8Dp(CustomTestCase): max_tokens=32768, temperature=0.7, seed=0, - sgl_eval_thinking=False, ), performance_params=PerformanceTestParams( batch_sizes=[16], diff --git a/test/registered/gb300/test_qwen35_fp8_tp.py b/test/registered/gb300/test_qwen35_fp8_tp.py index 2e6f17b9d8..310e91737c 100644 --- a/test/registered/gb300/test_qwen35_fp8_tp.py +++ b/test/registered/gb300/test_qwen35_fp8_tp.py @@ -57,7 +57,6 @@ class TestQwen35Fp8Tp(CustomTestCase): max_tokens=32768, temperature=0.7, seed=0, - sgl_eval_thinking=False, ), performance_params=PerformanceTestParams( batch_sizes=[1, 8], diff --git a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py index 3b36d5eb4e..946891b48b 100644 --- a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py +++ b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py @@ -5,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -49,7 +49,7 @@ class Test00(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -86,7 +86,7 @@ class Test01(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -123,7 +123,7 @@ class Test02(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -160,7 +160,7 @@ class Test03(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -199,7 +199,7 @@ class Test04(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -238,7 +238,7 @@ class Test05(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -276,7 +276,7 @@ class Test06(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -314,7 +314,7 @@ class Test07(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -354,7 +354,7 @@ class Test08(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -394,7 +394,7 @@ class Test09(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -432,7 +432,7 @@ class Test10(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -473,7 +473,7 @@ class Test11(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -514,7 +514,7 @@ class Test12(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -554,7 +554,7 @@ class Test13(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -597,7 +597,7 @@ class Test14(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -640,7 +640,7 @@ class Test15(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -682,7 +682,7 @@ class Test16(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -724,7 +724,7 @@ class Test17(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -768,7 +768,7 @@ class Test18(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -812,7 +812,7 @@ class Test19(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -848,7 +848,7 @@ class Test20(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -887,7 +887,7 @@ class Test21(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -926,7 +926,7 @@ class Test22(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -964,7 +964,7 @@ class Test23(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1005,7 +1005,7 @@ class Test24(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1046,7 +1046,7 @@ class Test25(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1086,7 +1086,7 @@ class Test26(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1126,7 +1126,7 @@ class Test27(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1168,7 +1168,7 @@ class Test28(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1210,7 +1210,7 @@ class Test29(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1254,7 +1254,7 @@ class Test30(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1301,7 +1301,7 @@ class Test31(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1348,7 +1348,7 @@ class Test32(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1394,7 +1394,7 @@ class Test33(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1443,7 +1443,7 @@ class Test34(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1492,7 +1492,7 @@ class Test35(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1540,7 +1540,7 @@ class Test36(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1588,7 +1588,7 @@ class Test37(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1638,7 +1638,7 @@ class Test38(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1688,7 +1688,7 @@ class Test39(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1738,7 +1738,7 @@ class Test40(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1791,7 +1791,7 @@ class Test41(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1844,7 +1844,7 @@ class Test42(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1896,7 +1896,7 @@ class Test43(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -1951,7 +1951,7 @@ class Test44(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2006,7 +2006,7 @@ class Test45(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2060,7 +2060,7 @@ class Test46(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2114,7 +2114,7 @@ class Test47(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2170,7 +2170,7 @@ class Test48(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2226,7 +2226,7 @@ class Test49(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2272,7 +2272,7 @@ class Test50(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2321,7 +2321,7 @@ class Test51(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2370,7 +2370,7 @@ class Test52(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2418,7 +2418,7 @@ class Test53(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2469,7 +2469,7 @@ class Test54(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2520,7 +2520,7 @@ class Test55(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2570,7 +2570,7 @@ class Test56(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2620,7 +2620,7 @@ class Test57(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2672,7 +2672,7 @@ class Test58(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) @@ -2724,7 +2724,7 @@ class Test59(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.48) diff --git a/test/registered/moe/test_torch_compile_moe.py b/test/registered/moe/test_torch_compile_moe.py index de6308adbc..a1d3fa99d7 100644 --- a/test/registered/moe/test_torch_compile_moe.py +++ b/test/registered/moe/test_torch_compile_moe.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import is_cuda, kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -45,7 +45,7 @@ class TestTorchCompileMoe(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) # 0.48 measured, minus the 0.05 margin the other eval thresholds use. self.assertGreaterEqual(metrics["score"], 0.43) diff --git a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py index df93ae543e..e876a758b1 100644 --- a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py +++ b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -54,7 +54,7 @@ class TestAscendSamplingBackend(CustomTestCase): temperature=0.1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.65) def test_greedy(self): diff --git a/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py index f5d28eae58..e885e3334f 100644 --- a/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py +++ b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ascend.test_ascend_utils import LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -17,7 +17,7 @@ register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) class TestNoChunkedPrefill(CustomTestCase): - """Testcase: Verify Llama-3.1-8B-Instruct accuracy ≥ 0.65 and serving normal with chunked prefill disabled. + """Testcase: Verify Llama-3.1-8B-Instruct accuracy >= 0.65 and serving normal with chunked prefill disabled. [Test Category] Parameter [Test Target] --chunked-prefill-size @@ -56,7 +56,7 @@ class TestNoChunkedPrefill(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.65) def test_no_chunked_prefill_without_radix_cache(self): diff --git a/test/registered/quant/test_autoround_quantization.py b/test/registered/quant/test_autoround_quantization.py index 7ea13b35f6..2d122cbd03 100644 --- a/test/registered/quant/test_autoround_quantization.py +++ b/test/registered/quant/test_autoround_quantization.py @@ -1,8 +1,3 @@ -""" -Usage: -python3 -m unittest test_autoround_quantization -""" - import os import shutil import tempfile @@ -15,7 +10,7 @@ from sglang.srt.configs.model_config import ModelConfig from sglang.srt.model_loader.loader import get_model_loader from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -62,7 +57,7 @@ class TestAutoRoundQuantization(CustomTestCase): num_threads=MMLU_NUM_THREADS, device="auto", ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], MMLU_SCORE_THRESHOLD) finally: kill_process_tree(process.pid) diff --git a/test/registered/quant/test_awq.py b/test/registered/quant/test_awq.py index 4ef1bfa471..f07e62ad0b 100644 --- a/test/registered/quant/test_awq.py +++ b/test/registered/quant/test_awq.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -42,16 +42,12 @@ class TestAWQ(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.64) @unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs") class TestAWQMarlinBfloat16(CustomTestCase): - """ - Verify that the model can be loaded with bfloat16 dtype and awq_marlin quantization - """ - @classmethod def setUpClass(cls): cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ" @@ -76,7 +72,7 @@ class TestAWQMarlinBfloat16(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.80) diff --git a/test/registered/sampling/test_pytorch_sampling_backend.py b/test/registered/sampling/test_pytorch_sampling_backend.py index 3973412b47..0f5a61fe88 100644 --- a/test/registered/sampling/test_pytorch_sampling_backend.py +++ b/test/registered/sampling/test_pytorch_sampling_backend.py @@ -5,7 +5,7 @@ import requests from sglang.srt.utils import is_hip, kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -49,7 +49,7 @@ class TestPyTorchSamplingBackend(CustomTestCase): temperature=0.1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.64) @unittest.skipIf( diff --git a/test/registered/scheduler/test_retract_decode.py b/test/registered/scheduler/test_retract_decode.py index 541671c6d7..d6bba96525 100644 --- a/test/registered/scheduler/test_retract_decode.py +++ b/test/registered/scheduler/test_retract_decode.py @@ -7,7 +7,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -55,7 +55,7 @@ class TestRetractDecode(CustomTestCase): num_threads=32, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreaterEqual(metrics["score"], 0.64) time.sleep(1) # wait for mem check diff --git a/test/registered/unit/bench/test_simple_eval_gsm8k.py b/test/registered/unit/bench/test_simple_eval_gsm8k.py deleted file mode 100644 index 8314691de3..0000000000 --- a/test/registered/unit/bench/test_simple_eval_gsm8k.py +++ /dev/null @@ -1,269 +0,0 @@ -import json -import subprocess -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.run_eval import _run_sgl_eval, run_eval -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=6, suite="stage-a-test-cpu-intel") - - -def _write_fake_metrics(out_parent: Path, eval_name: str, payload: dict) -> None: - run_dir = out_parent / f"sgl_eval_{eval_name}_20260101-000000" - run_dir.mkdir(parents=True, exist_ok=True) - (run_dir / "metrics.json").write_text(json.dumps(payload)) - - -class TestRunSglEval(CustomTestCase): - """sgl-eval is a black box, so these mock subprocess.run and assert the shim - builds the CLI and parses metrics.json's aggregate.score (not top-level).""" - - def _args(self, out_dir: str, **overrides): - defaults = dict( - base_url="http://127.0.0.1:30000", - model="test-model", - num_examples=7, - num_threads=8, - temperature=0.0, - sgl_eval_out_dir=out_dir, - ) - defaults.update(overrides) - return SimpleNamespace(**defaults) - - def _fake_run_factory(self, out_dir: Path, eval_name: str, payload: dict): - def fake_run(cmd, **kwargs): - _write_fake_metrics(out_dir, eval_name, payload) - return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") - - return fake_run - - def test_parses_aggregate_score_and_maps_latency(self): - payload = { - "name": "gsm8k", - "model": "test-model", - "num_examples": 7, - "n_repeats": 1, - "latency_seconds": 12.5, - "output_throughput_tps": 34.0, - "aggregate": {"score": 0.75, "no_answer": 0.1}, - } - with tempfile.TemporaryDirectory() as td: - out_dir = Path(td) - args = self._args(td) - with patch( - "sglang.test.run_eval.subprocess.run", - side_effect=self._fake_run_factory(out_dir, "gsm8k", payload), - ): - metrics = _run_sgl_eval("gsm8k", args) - - self.assertAlmostEqual(metrics["score"], 0.75) - self.assertAlmostEqual(metrics["latency"], 12.5) - self.assertAlmostEqual(metrics["output_throughput"], 34.0) - self.assertEqual(metrics["no_answer"], 0.1) - self.assertTrue(metrics["sgl_eval_metrics_path"].endswith("metrics.json")) - - def test_builds_cli_with_required_flags(self): - captured = {} - - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd - out_dir = Path(captured["cmd"][captured["cmd"].index("--out-dir") + 1]) - _write_fake_metrics( - out_dir, - "gsm8k", - { - "name": "gsm8k", - "model": "test-model", - "latency_seconds": 1.0, - "output_throughput_tps": 1.0, - "aggregate": {"score": 0.5}, - }, - ) - return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") - - with tempfile.TemporaryDirectory() as td: - args = self._args(td) - with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): - _run_sgl_eval("gsm8k", args) - - cmd = captured["cmd"] - self.assertEqual(cmd[0:3], ["sgl-eval", "run", "gsm8k"]) - self.assertIn("--base-url", cmd) - self.assertIn("http://127.0.0.1:30000/v1", cmd) - self.assertIn("--model", cmd) - self.assertIn("test-model", cmd) - self.assertIn("--num-threads", cmd) - self.assertIn("8", cmd) - self.assertIn("--temperature", cmd) - self.assertIn("0.0", cmd) - self.assertIn("--num-examples", cmd) - self.assertIn("7", cmd) - - def test_omits_num_examples_when_none(self): - captured = {} - - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd - out_dir = Path(cmd[cmd.index("--out-dir") + 1]) - _write_fake_metrics( - out_dir, - "gsm8k", - { - "model": "test-model", - "latency_seconds": 1.0, - "output_throughput_tps": 1.0, - "aggregate": {"score": 0.5}, - }, - ) - return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") - - with tempfile.TemporaryDirectory() as td: - args = self._args(td, num_examples=None) - with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): - _run_sgl_eval("gsm8k", args) - - self.assertNotIn("--num-examples", captured["cmd"]) - - def _capture_cmd(self, eval_name="gsm8k", **overrides): - captured = {} - - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd - out_dir = Path(cmd[cmd.index("--out-dir") + 1]) - _write_fake_metrics( - out_dir, - eval_name, - { - "model": "test-model", - "latency_seconds": 1.0, - "output_throughput_tps": 1.0, - "aggregate": {"score": 0.5}, - }, - ) - return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") - - with tempfile.TemporaryDirectory() as td: - args = self._args(td, **overrides) - with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): - _run_sgl_eval(eval_name, args) - return captured["cmd"] - - def test_omits_sampling_flags_when_unset(self): - """Unset top_p / seed / repeat must not reach the CLI -- sgl-eval's own - defaults differ from a forced value (seed unset != seed 0).""" - cmd = self._capture_cmd() - for flag in ("--top-p", "--seed", "--n-repeats"): - self.assertNotIn(flag, cmd) - - def test_forwards_sampling_flags_when_set(self): - cmd = self._capture_cmd(top_p=0.95, seed=0, repeat=1) - for flag, value in (("--top-p", "0.95"), ("--seed", "0"), ("--n-repeats", "1")): - self.assertIn(flag, cmd) - self.assertEqual(cmd[cmd.index(flag) + 1], value) - - def test_model_preset_owns_model_and_sampling_defaults(self): - cmd = self._capture_cmd( - eval_name="mmmu_pro", - model=None, - num_examples=300, - num_threads=None, - temperature=None, - load_preset_from_model_id="moonshotai/Kimi-K3", - ) - - self.assertEqual(cmd[:3], ["sgl-eval", "run", "mmmu_pro"]) - self.assertIn("--load-preset-from-model-id", cmd) - self.assertEqual( - cmd[cmd.index("--load-preset-from-model-id") + 1], - "moonshotai/Kimi-K3", - ) - self.assertEqual(cmd[cmd.index("--num-examples") + 1], "300") - for flag in ( - "--model", - "--num-threads", - "--temperature", - "--top-p", - "--max-tokens", - "--thinking", - ): - self.assertNotIn(flag, cmd) - - def test_non_preset_cli_keeps_legacy_top_p_default(self): - cmd = self._capture_cmd(top_p=None, _sgl_eval_from_cli=True) - - self.assertIn("--top-p", cmd) - self.assertEqual(cmd[cmd.index("--top-p") + 1], "1.0") - - @patch("sglang.test.run_eval._run_sgl_eval", return_value={"score": 0.8}) - def test_run_eval_dispatches_hyphenated_mmmu_pro_name(self, mock_sgl_eval): - args = SimpleNamespace( - base_url="http://127.0.0.1:30000", - eval_name="mmmu-pro", - ) - - try: - result = run_eval(args) - except ValueError as exc: - self.fail(f"mmmu-pro must dispatch to sgl-eval: {exc}") - self.assertEqual(result, {"score": 0.8}) - mock_sgl_eval.assert_called_once_with("mmmu_pro", args) - - def test_thinking_auto_detected_from_model_name(self): - self.assertIn( - "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8") - ) - - def test_explicit_thinking_false_suppresses_auto_detect(self): - """A caller matching a harness that sent no chat_template_kwargs has to be - able to turn the model-name heuristic off.""" - cmd = self._capture_cmd( - model="Qwen/Qwen3.5-397B-A17B-FP8", sgl_eval_thinking=False - ) - self.assertNotIn("--thinking", cmd) - - def test_raises_on_nonzero_exit(self): - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom") - - with tempfile.TemporaryDirectory() as td: - args = self._args(td) - with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): - with self.assertRaises(RuntimeError) as cm: - _run_sgl_eval("gsm8k", args) - self.assertIn("exit code 2", str(cm.exception)) - - def test_raises_when_metrics_json_missing(self): - def fake_run(cmd, **kwargs): - return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") - - with tempfile.TemporaryDirectory() as td: - args = self._args(td) - with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): - with self.assertRaises(FileNotFoundError): - _run_sgl_eval("gsm8k", args) - - def test_raises_when_aggregate_score_missing(self): - payload = { - "name": "gsm8k", - "latency_seconds": 1.0, - "output_throughput_tps": 1.0, - "aggregate": {"no_answer": 0.5}, # no score key - } - with tempfile.TemporaryDirectory() as td: - out_dir = Path(td) - args = self._args(td) - with patch( - "sglang.test.run_eval.subprocess.run", - side_effect=self._fake_run_factory(out_dir, "gsm8k", payload), - ): - with self.assertRaises(KeyError): - _run_sgl_eval("gsm8k", args) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py b/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py deleted file mode 100644 index 9154663716..0000000000 --- a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Unit tests for sgl-eval-backed accuracy mixin dispatch. - -Hermetic (no server, no real sgl-eval install). These guard the behavior that -existing consumers rely on -- not the sgl-eval happy path, which the live -accuracy runs already cover: - - 1. The default GSM8K backend stays on ``run_eval`` (OpenAI completion API); - the ~47 existing GSM8K consumers must never be silently rerouted. - 2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail - gate when the canonical ``gsm8k_score_threshold`` is unset. - 3. MMMU-Pro delegates model and sampling selection to a built-in model preset. - 4. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent, - so CI without the optional dependency stays green. -""" - -import sys -import unittest -from unittest.mock import MagicMock, patch - -import requests - -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.kits import eval_accuracy_kit as kit -from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin, MMMUProMixin -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=12, suite="base-a-test-cpu") - - -def _fake_get(url, *args, **kwargs): - # flush_cache must succeed (GSM8K calls it unguarded); /server_info is probed - # by _check_accept_length, which swallows RequestException. - if str(url).endswith("/flush_cache"): - return MagicMock() - raise requests.RequestException() - - -def _make_host(mixin, method): - """Build a throwaway mixin host bound to ``method``. - - Created dynamically (never bound at module scope) so it is collected by - neither runner: CI executes this file via ``python3 `` -> - ``unittest.main()``, whose loader ignores pytest's ``__test__`` flag, and - pytest only collects module-level ``Test*`` classes. The host runs only when - a test below instantiates and drives it directly. - """ - return type(f"_{mixin.__name__}Host", (mixin, CustomTestCase), {})(method) - - -class TestEvalKitBackendDispatch(CustomTestCase): - def _run_gsm8k_default(self, score, **attrs): - """Run GSM8K on the default (run_eval) backend with run_eval faked to - return ``score``; returns the SimpleNamespace args run_eval received.""" - captured = {} - - def fake_run_eval(args): - captured["args"] = args - return {"score": score} - - host = _make_host(GSM8KMixin, "test_gsm8k") - host.base_url = "http://127.0.0.1:0" - host.model = "m" - for k, v in attrs.items(): - setattr(host, k, v) - with ( - patch.object(kit, "run_eval", side_effect=fake_run_eval), - patch.object(kit.requests, "get", side_effect=_fake_get), - ): - host.test_gsm8k() - return captured["args"] - - def test_default_backend_uses_run_eval_completion(self): - # The default path that all existing GSM8K consumers rely on must stay on - # run_eval's OpenAI completion API -- it must not touch sgl-eval. - args = self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.5) - self.assertEqual(args.eval_name, "gsm8k") - self.assertEqual(args.api, "completion") - - def test_legacy_accuracy_thres_alias_gates_score(self): - # Canonical gsm8k_score_threshold left unset (NaN) -> the legacy - # gsm8k_accuracy_thres must still be the pass/fail gate. - self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.90) # above -> passes - with self.assertRaises(AssertionError): - self._run_gsm8k_default(0.80, gsm8k_accuracy_thres=0.90) # below -> fails - - def test_sgl_eval_path_skips_when_not_installed(self): - # GPQA/AIME25 and the sgl_eval backend must skip -- not error -- when - # sgl-eval is not installed. None in sys.modules makes the import raise. - host = _make_host(GPQAMixin, "test_gpqa") - host.base_url = "http://127.0.0.1:0" - host.model = "m" - host.gpqa_score_threshold = 0.5 - absent = { - k: None for k in ("sgl_eval.registry", "sgl_eval.sampler", "sgl_eval.types") - } - with patch.dict(sys.modules, absent): - with self.assertRaises(unittest.SkipTest): - host.test_gpqa() - - def _run_mmmu_pro(self, score): - captured = {} - - def fake_run_eval(args): - captured["args"] = args - return {"score": score} - - host = _make_host(MMMUProMixin, "test_mmmu_pro") - host.base_url = "http://127.0.0.1:0" - host.model = "deployment-model" - host.mmmu_pro_score_threshold = 0.75 - host.mmmu_pro_load_preset_from_model_id = "moonshotai/Kimi-K3" - with ( - patch.object(kit, "run_eval", side_effect=fake_run_eval), - patch.object(kit.requests, "get", side_effect=_fake_get), - ): - host.test_mmmu_pro() - return captured["args"] - - def test_mmmu_pro_uses_kimi_preset_and_300_examples(self): - args = self._run_mmmu_pro(0.80) - - self.assertEqual(args.eval_name, "mmmu_pro") - self.assertEqual(args.load_preset_from_model_id, "moonshotai/Kimi-K3") - self.assertEqual(args.num_examples, 300) - self.assertIsNone(args.num_threads) - self.assertIsNone(args.model) - for attr in ("temperature", "top_p", "max_tokens", "reasoning_effort"): - self.assertFalse(hasattr(args, attr)) - - def test_mmmu_pro_score_threshold_gates_result(self): - with self.assertRaises(AssertionError): - self._run_mmmu_pro(0.74) - - -if __name__ == "__main__": - unittest.main()