minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Fix sanity evaluation and diffusion test suite blockers (#39892)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-09-16 21:47:41 -07:00
committed by GitHub
parent 25ce8063f7
commit 923e4a56d4
7 changed files with 24 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_pr-test-check-changes.yml
+1
View File
@@ -113,6 +113,7 @@ jobs:
- "test/registered/kernels/ops/diffusion/**"
- "test/registered/kernels/benchmark/diffusion/**"
- "test/registered/kernel/diffusion/**"
- "test/registered/unit/diffusion/**"
- "python/sglang/cli/**"
jit_kernel:
- ".github/workflows/pr-test.yml"
python/sglang/test/kits/hellaswag_kit.py
-29
View File
@@ -1,29 +0,0 @@
"""Hellaswag sanity kit.
Runs hellaswag via the sgl frontend DSL bound to ``self.base_url`` and
asserts accuracy above a threshold. Catches systematic regressions that
pass every cheap single-prompt probe but tank multi-choice reasoning.
Mix into a ``CustomTestCase`` subclass exposing ``self.base_url``.
"""
class HellaswagMixin:
"""Assert hellaswag accuracy > threshold."""
hellaswag_accuracy_threshold: float = 0.60
def test_accuracy_floor(self):
import sglang as sgl
from sglang.test.test_programs import test_hellaswag_select
sgl.set_default_backend(sgl.RuntimeEndpoint(self.base_url))
try:
accuracy, _ = test_hellaswag_select()
finally:
sgl.set_default_backend(None)
self.assertGreater(
accuracy,
self.hellaswag_accuracy_threshold,
f"hellaswag accuracy floor breached: {accuracy:.3f}",
)
python/sglang/test/test_programs.py
-93
View File
@@ -3,13 +3,9 @@
import asyncio
import json
import re
import time
import numpy as np
import sglang as sgl
from sglang.srt.utils import is_hip
from sglang.utils import download_and_cache_file, read_jsonl
_is_hip = is_hip()
@@ -500,95 +496,6 @@ def test_chat_completion_speculative():
gen_character_spec().sync()
def test_hellaswag_select():
"""Benchmark the accuracy of sgl.select on the HellaSwag dataset."""
def get_one_example(lines, i, include_answer):
ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " "
if include_answer:
ret += lines[i]["endings"][lines[i]["label"]]
return ret
def get_few_shot_examples(lines, k):
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
# Read data
url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl"
filename = download_and_cache_file(url)
lines = list(read_jsonl(filename))
# Construct prompts
num_questions = 200
num_shots = 20
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
choices = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
choices.append(lines[i]["endings"])
labels.append(lines[i]["label"])
arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)]
#####################################
######### SGL Program Begin #########
#####################################
import sglang as sgl
@sgl.function
def few_shot_hellaswag(s, question, choices):
s += few_shot_examples + question
s += sgl.select("answer", choices=choices)
#####################################
########## SGL Program End ##########
#####################################
# Run requests
tic = time.perf_counter()
rets = few_shot_hellaswag.run_batch(
arguments,
temperature=0,
num_threads=64,
progress_bar=True,
generator_style=False,
)
preds = []
for i, ret in enumerate(rets):
preds.append(choices[i].index(ret["answer"]))
latency = time.perf_counter() - tic
# Compute accuracy
accuracy = np.mean(np.array(preds) == np.array(labels))
# Test generator style of run_batch
tic = time.perf_counter()
rets = few_shot_hellaswag.run_batch(
arguments,
temperature=0,
num_threads=64,
progress_bar=True,
generator_style=True,
)
preds_gen = []
for i, ret in enumerate(rets):
preds_gen.append(choices[i].index(ret["answer"]))
latency_gen = time.perf_counter() - tic
# Compute accuracy
accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels))
print(f"{accuracy=}, {accuracy_gen=} {latency=:.2f}s {latency_gen=:.2f}s")
assert np.abs(accuracy_gen - accuracy) < 0.1
# No latency assert: the 2nd run hits the radix cache the 1st filled.
return accuracy, latency
def test_gen_min_new_tokens():
"""
Validate sgl.gen(min_tokens) functionality.
scripts/lint/check_registered_tests.py
+7 -3
View File
@@ -142,9 +142,13 @@ def taxonomy_errors(path: str, registries: list, tree: ast.AST) -> list[str]:
kind = relative_parts[0]
errors = []
if kind == "unit":
non_cpu = [r for r in registries if r.backend.name != "CPU"]
if non_cpu:
errors.append(f"{path}: unit tests may register only CPU suites")
invalid = [
r
for r in registries
if r.backend.name != "CPU" and "-unit-" not in (r.effective_suite or "")
]
if invalid:
errors.append(f"{path}: unit tests must use CPU or dedicated unit suites")
if any(r.est_time > 60 for r in registries):
errors.append(f"{path}: unit test est_time must be <= 60 seconds")
if _contains_call(tree, "popen_launch_server"):
test/registered/core/test_basic_sanity.py
+14 -3
View File
@@ -1,7 +1,7 @@
"""Stage-a basic sanity: small-but-broad server coverage that downstream
stages depend on. Multiple sanity-kit mixins driving one shared server,
covering protocol, decode correctness, scheduler stress, occupancy, and
hellaswag accuracy."""
MMLU accuracy."""
import unittest
@@ -10,8 +10,8 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin
from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin
from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin
from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval
from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin
from sglang.test.kits.hellaswag_kit import HellaswagMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -29,7 +29,6 @@ class TestBasicSanity(
BasicDecodeCorrectnessMixin,
BasicSchedulerStressMixin,
FwdOccupancyMixin,
HellaswagMixin,
CustomTestCase,
):
served_model_name = DEFAULT_MODEL_NAME_FOR_TEST
@@ -57,6 +56,18 @@ class TestBasicSanity(
env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"},
)
def test_accuracy_floor(self):
_run_sgl_eval(
self,
eval_name="mmlu",
score_threshold=0.60,
num_examples=200,
num_threads=64,
thinking=False,
max_tokens=256,
temperature=0,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
test/registered/unit/models/test_flux2_fp8_norm_quant_gate.py → test/registered/unit/diffusion/test_flux2_fp8_norm_quant_gate.py
+1 -1
View File
@@ -16,7 +16,7 @@ from sglang.multimodal_gen.runtime.models.dits.flux_2 import (
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=13, stage="base-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=13, stage="base-b", runner_config="diffusion-unit-1-gpu-h100")
def _fp8_linear(input_scale: float) -> nn.Module:
test/registered/unit/models/test_qwen_image_fp8_norm_quant.py → test/registered/unit/diffusion/test_qwen_image_fp8_norm_quant.py
+1 -1
View File
@@ -16,7 +16,7 @@ from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=12, stage="base-b", runner_config="diffusion-unit-1-gpu-h100")
def _fp8_linear(input_scale: float) -> nn.Module: