diff --git a/.github/workflows/_pr-test-check-changes.yml b/.github/workflows/_pr-test-check-changes.yml index f8c5ebba2..9262678ea 100644 --- a/.github/workflows/_pr-test-check-changes.yml +++ b/.github/workflows/_pr-test-check-changes.yml @@ -113,6 +113,7 @@ jobs: - "test/registered/kernels/ops/diffusion/**" - "test/registered/kernels/benchmark/diffusion/**" - "test/registered/kernel/diffusion/**" + - "test/registered/unit/diffusion/**" - "python/sglang/cli/**" jit_kernel: - ".github/workflows/pr-test.yml" diff --git a/python/sglang/test/kits/hellaswag_kit.py b/python/sglang/test/kits/hellaswag_kit.py deleted file mode 100644 index cc79533f4..000000000 --- a/python/sglang/test/kits/hellaswag_kit.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Hellaswag sanity kit. - -Runs hellaswag via the sgl frontend DSL bound to ``self.base_url`` and -asserts accuracy above a threshold. Catches systematic regressions that -pass every cheap single-prompt probe but tank multi-choice reasoning. - -Mix into a ``CustomTestCase`` subclass exposing ``self.base_url``. -""" - - -class HellaswagMixin: - """Assert hellaswag accuracy > threshold.""" - - hellaswag_accuracy_threshold: float = 0.60 - - def test_accuracy_floor(self): - import sglang as sgl - from sglang.test.test_programs import test_hellaswag_select - - sgl.set_default_backend(sgl.RuntimeEndpoint(self.base_url)) - try: - accuracy, _ = test_hellaswag_select() - finally: - sgl.set_default_backend(None) - self.assertGreater( - accuracy, - self.hellaswag_accuracy_threshold, - f"hellaswag accuracy floor breached: {accuracy:.3f}", - ) diff --git a/python/sglang/test/test_programs.py b/python/sglang/test/test_programs.py index 01755192b..d9cd71a9b 100644 --- a/python/sglang/test/test_programs.py +++ b/python/sglang/test/test_programs.py @@ -3,13 +3,9 @@ import asyncio import json import re -import time - -import numpy as np import sglang as sgl from sglang.srt.utils import is_hip -from sglang.utils import download_and_cache_file, read_jsonl _is_hip = is_hip() @@ -500,95 +496,6 @@ def test_chat_completion_speculative(): gen_character_spec().sync() -def test_hellaswag_select(): - """Benchmark the accuracy of sgl.select on the HellaSwag dataset.""" - - def get_one_example(lines, i, include_answer): - ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " - if include_answer: - ret += lines[i]["endings"][lines[i]["label"]] - return ret - - def get_few_shot_examples(lines, k): - ret = "" - for i in range(k): - ret += get_one_example(lines, i, True) + "\n\n" - return ret - - # Read data - url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" - filename = download_and_cache_file(url) - lines = list(read_jsonl(filename)) - - # Construct prompts - num_questions = 200 - num_shots = 20 - few_shot_examples = get_few_shot_examples(lines, num_shots) - - questions = [] - choices = [] - labels = [] - for i in range(len(lines[:num_questions])): - questions.append(get_one_example(lines, i, False)) - choices.append(lines[i]["endings"]) - labels.append(lines[i]["label"]) - arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] - - ##################################### - ######### SGL Program Begin ######### - ##################################### - - import sglang as sgl - - @sgl.function - def few_shot_hellaswag(s, question, choices): - s += few_shot_examples + question - s += sgl.select("answer", choices=choices) - - ##################################### - ########## SGL Program End ########## - ##################################### - - # Run requests - tic = time.perf_counter() - rets = few_shot_hellaswag.run_batch( - arguments, - temperature=0, - num_threads=64, - progress_bar=True, - generator_style=False, - ) - preds = [] - for i, ret in enumerate(rets): - preds.append(choices[i].index(ret["answer"])) - latency = time.perf_counter() - tic - - # Compute accuracy - accuracy = np.mean(np.array(preds) == np.array(labels)) - - # Test generator style of run_batch - tic = time.perf_counter() - rets = few_shot_hellaswag.run_batch( - arguments, - temperature=0, - num_threads=64, - progress_bar=True, - generator_style=True, - ) - preds_gen = [] - for i, ret in enumerate(rets): - preds_gen.append(choices[i].index(ret["answer"])) - latency_gen = time.perf_counter() - tic - - # Compute accuracy - accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels)) - print(f"{accuracy=}, {accuracy_gen=} {latency=:.2f}s {latency_gen=:.2f}s") - assert np.abs(accuracy_gen - accuracy) < 0.1 - # No latency assert: the 2nd run hits the radix cache the 1st filled. - - return accuracy, latency - - def test_gen_min_new_tokens(): """ Validate sgl.gen(min_tokens) functionality. diff --git a/scripts/lint/check_registered_tests.py b/scripts/lint/check_registered_tests.py index 507609bb1..2cb489a97 100755 --- a/scripts/lint/check_registered_tests.py +++ b/scripts/lint/check_registered_tests.py @@ -142,9 +142,13 @@ def taxonomy_errors(path: str, registries: list, tree: ast.AST) -> list[str]: kind = relative_parts[0] errors = [] if kind == "unit": - non_cpu = [r for r in registries if r.backend.name != "CPU"] - if non_cpu: - errors.append(f"{path}: unit tests may register only CPU suites") + invalid = [ + r + for r in registries + if r.backend.name != "CPU" and "-unit-" not in (r.effective_suite or "") + ] + if invalid: + errors.append(f"{path}: unit tests must use CPU or dedicated unit suites") if any(r.est_time > 60 for r in registries): errors.append(f"{path}: unit test est_time must be <= 60 seconds") if _contains_call(tree, "popen_launch_server"): diff --git a/test/registered/core/test_basic_sanity.py b/test/registered/core/test_basic_sanity.py index 893dab228..9e11feeae 100644 --- a/test/registered/core/test_basic_sanity.py +++ b/test/registered/core/test_basic_sanity.py @@ -1,7 +1,7 @@ """Stage-a basic sanity: small-but-broad server coverage that downstream stages depend on. Multiple sanity-kit mixins driving one shared server, covering protocol, decode correctness, scheduler stress, occupancy, and -hellaswag accuracy.""" +MMLU accuracy.""" import unittest @@ -10,8 +10,8 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin +from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin -from sglang.test.kits.hellaswag_kit import HellaswagMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -29,7 +29,6 @@ class TestBasicSanity( BasicDecodeCorrectnessMixin, BasicSchedulerStressMixin, FwdOccupancyMixin, - HellaswagMixin, CustomTestCase, ): served_model_name = DEFAULT_MODEL_NAME_FOR_TEST @@ -57,6 +56,18 @@ class TestBasicSanity( env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"}, ) + def test_accuracy_floor(self): + _run_sgl_eval( + self, + eval_name="mmlu", + score_threshold=0.60, + num_examples=200, + num_threads=64, + thinking=False, + max_tokens=256, + temperature=0, + ) + @classmethod def tearDownClass(cls): kill_process_tree(cls.process.pid) diff --git a/test/registered/unit/models/test_flux2_fp8_norm_quant_gate.py b/test/registered/unit/diffusion/test_flux2_fp8_norm_quant_gate.py similarity index 97% rename from test/registered/unit/models/test_flux2_fp8_norm_quant_gate.py rename to test/registered/unit/diffusion/test_flux2_fp8_norm_quant_gate.py index cfc11588b..e4cd39539 100644 --- a/test/registered/unit/models/test_flux2_fp8_norm_quant_gate.py +++ b/test/registered/unit/diffusion/test_flux2_fp8_norm_quant_gate.py @@ -16,7 +16,7 @@ from sglang.multimodal_gen.runtime.models.dits.flux_2 import ( from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=13, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=13, stage="base-b", runner_config="diffusion-unit-1-gpu-h100") def _fp8_linear(input_scale: float) -> nn.Module: diff --git a/test/registered/unit/models/test_qwen_image_fp8_norm_quant.py b/test/registered/unit/diffusion/test_qwen_image_fp8_norm_quant.py similarity index 97% rename from test/registered/unit/models/test_qwen_image_fp8_norm_quant.py rename to test/registered/unit/diffusion/test_qwen_image_fp8_norm_quant.py index 08db0c0e0..16bc61949 100644 --- a/test/registered/unit/models/test_qwen_image_fp8_norm_quant.py +++ b/test/registered/unit/diffusion/test_qwen_image_fp8_norm_quant.py @@ -16,7 +16,7 @@ from sglang.multimodal_gen.runtime.models.dits.qwen_image import ( from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=12, stage="base-b", runner_config="diffusion-unit-1-gpu-h100") def _fp8_linear(input_scale: float) -> nn.Module: