[CI] Fix sanity evaluation and diffusion test suite blockers (#39892)

.github/workflows/_pr-test-check-changes.yml

@@ -113,6 +113,7 @@ jobs:
               - "test/registered/kernels/ops/diffusion/**"
               - "test/registered/kernels/benchmark/diffusion/**"
               - "test/registered/kernel/diffusion/**"
+              - "test/registered/unit/diffusion/**"
               - "python/sglang/cli/**"
             jit_kernel:
               - ".github/workflows/pr-test.yml"

python/sglang/test/kits/hellaswag_kit.py

@@ -1,29 +0,0 @@
-"""Hellaswag sanity kit.
-
-Runs hellaswag via the sgl frontend DSL bound to ``self.base_url`` and
-asserts accuracy above a threshold. Catches systematic regressions that
-pass every cheap single-prompt probe but tank multi-choice reasoning.
-
-Mix into a ``CustomTestCase`` subclass exposing ``self.base_url``.
-"""
-
-
-class HellaswagMixin:
-    """Assert hellaswag accuracy > threshold."""
-
-    hellaswag_accuracy_threshold: float = 0.60
-
-    def test_accuracy_floor(self):
-        import sglang as sgl
-        from sglang.test.test_programs import test_hellaswag_select
-
-        sgl.set_default_backend(sgl.RuntimeEndpoint(self.base_url))
-        try:
-            accuracy, _ = test_hellaswag_select()
-        finally:
-            sgl.set_default_backend(None)
-        self.assertGreater(
-            accuracy,
-            self.hellaswag_accuracy_threshold,
-            f"hellaswag accuracy floor breached: {accuracy:.3f}",
-        )

python/sglang/test/test_programs.py

@@ -3,13 +3,9 @@
 import asyncio
 import json
 import re
-import time
-
-import numpy as np
 
 import sglang as sgl
 from sglang.srt.utils import is_hip
-from sglang.utils import download_and_cache_file, read_jsonl
 
 _is_hip = is_hip()
 
@@ -500,95 +496,6 @@ def test_chat_completion_speculative():
     gen_character_spec().sync()
 
 
-def test_hellaswag_select():
-    """Benchmark the accuracy of sgl.select on the HellaSwag dataset."""
-
-    def get_one_example(lines, i, include_answer):
-        ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " "
-        if include_answer:
-            ret += lines[i]["endings"][lines[i]["label"]]
-        return ret
-
-    def get_few_shot_examples(lines, k):
-        ret = ""
-        for i in range(k):
-            ret += get_one_example(lines, i, True) + "\n\n"
-        return ret
-
-    # Read data
-    url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl"
-    filename = download_and_cache_file(url)
-    lines = list(read_jsonl(filename))
-
-    # Construct prompts
-    num_questions = 200
-    num_shots = 20
-    few_shot_examples = get_few_shot_examples(lines, num_shots)
-
-    questions = []
-    choices = []
-    labels = []
-    for i in range(len(lines[:num_questions])):
-        questions.append(get_one_example(lines, i, False))
-        choices.append(lines[i]["endings"])
-        labels.append(lines[i]["label"])
-    arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)]
-
-    #####################################
-    ######### SGL Program Begin #########
-    #####################################
-
-    import sglang as sgl
-
-    @sgl.function
-    def few_shot_hellaswag(s, question, choices):
-        s += few_shot_examples + question
-        s += sgl.select("answer", choices=choices)
-
-    #####################################
-    ########## SGL Program End ##########
-    #####################################
-
-    # Run requests
-    tic = time.perf_counter()
-    rets = few_shot_hellaswag.run_batch(
-        arguments,
-        temperature=0,
-        num_threads=64,
-        progress_bar=True,
-        generator_style=False,
-    )
-    preds = []
-    for i, ret in enumerate(rets):
-        preds.append(choices[i].index(ret["answer"]))
-    latency = time.perf_counter() - tic
-
-    # Compute accuracy
-    accuracy = np.mean(np.array(preds) == np.array(labels))
-
-    # Test generator style of run_batch
-    tic = time.perf_counter()
-    rets = few_shot_hellaswag.run_batch(
-        arguments,
-        temperature=0,
-        num_threads=64,
-        progress_bar=True,
-        generator_style=True,
-    )
-    preds_gen = []
-    for i, ret in enumerate(rets):
-        preds_gen.append(choices[i].index(ret["answer"]))
-    latency_gen = time.perf_counter() - tic
-
-    # Compute accuracy
-    accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels))
-    print(f"{accuracy=}, {accuracy_gen=} {latency=:.2f}s {latency_gen=:.2f}s")
-    assert np.abs(accuracy_gen - accuracy) < 0.1
-    # No latency assert: the 2nd run hits the radix cache the 1st filled.
-
-    return accuracy, latency
-
-
 def test_gen_min_new_tokens():
     """
     Validate sgl.gen(min_tokens) functionality.

scripts/lint/check_registered_tests.py

@@ -142,9 +142,13 @@ def taxonomy_errors(path: str, registries: list, tree: ast.AST) -> list[str]:
     kind = relative_parts[0]
     errors = []
     if kind == "unit":
-        non_cpu = [r for r in registries if r.backend.name != "CPU"]
+        invalid = [
-        if non_cpu:
+            r
-            errors.append(f"{path}: unit tests may register only CPU suites")
+            for r in registries
+            if r.backend.name != "CPU" and "-unit-" not in (r.effective_suite or "")
+        ]
+        if invalid:
+            errors.append(f"{path}: unit tests must use CPU or dedicated unit suites")
         if any(r.est_time > 60 for r in registries):
             errors.append(f"{path}: unit test est_time must be <= 60 seconds")
         if _contains_call(tree, "popen_launch_server"):

test/registered/core/test_basic_sanity.py

@@ -1,7 +1,7 @@
 """Stage-a basic sanity: small-but-broad server coverage that downstream
 stages depend on. Multiple sanity-kit mixins driving one shared server,
 covering protocol, decode correctness, scheduler stress, occupancy, and
-hellaswag accuracy."""
+MMLU accuracy."""
 
 import unittest
 
@@ -10,8 +10,8 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin
 from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin
 from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin
+from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval
 from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin
-from sglang.test.kits.hellaswag_kit import HellaswagMixin
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -29,7 +29,6 @@ class TestBasicSanity(
     BasicDecodeCorrectnessMixin,
     BasicSchedulerStressMixin,
     FwdOccupancyMixin,
-    HellaswagMixin,
     CustomTestCase,
 ):
     served_model_name = DEFAULT_MODEL_NAME_FOR_TEST
@@ -57,6 +56,18 @@ class TestBasicSanity(
             env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"},
         )
 
+    def test_accuracy_floor(self):
+        _run_sgl_eval(
+            self,
+            eval_name="mmlu",
+            score_threshold=0.60,
+            num_examples=200,
+            num_threads=64,
+            thinking=False,
+            max_tokens=256,
+            temperature=0,
+        )
+
     @classmethod
     def tearDownClass(cls):
         kill_process_tree(cls.process.pid)

test/registered/unit/diffusion/test_flux2_fp8_norm_quant_gate.py

@@ -16,7 +16,7 @@ from sglang.multimodal_gen.runtime.models.dits.flux_2 import (
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=13, stage="base-b", runner_config="1-gpu-small")
+register_cuda_ci(est_time=13, stage="base-b", runner_config="diffusion-unit-1-gpu-h100")
 
 
 def _fp8_linear(input_scale: float) -> nn.Module:

test/registered/unit/diffusion/test_qwen_image_fp8_norm_quant.py

@@ -16,7 +16,7 @@ from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-small")
+register_cuda_ci(est_time=12, stage="base-b", runner_config="diffusion-unit-1-gpu-h100")
 
 
 def _fp8_linear(input_scale: float) -> nn.Module: