minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Migrate FP8/TorchAO tests to test/registered/quant/ (#16453)

Browse Source
This commit is contained in:
Alison Shao
2026-01-06 18:27:43 -08:00
committed by GitHub
parent badcd02896
commit 90eac38a12
13 changed files with 10 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/srt/run_suite.py
-4
View File
@@ -10,10 +10,8 @@ from sglang.test.ci.ci_utils import TestFile, run_unittest_files
suites = {
"per-commit-1-gpu": [
TestFile("test_deterministic.py", 228),
TestFile("test_eval_fp8_accuracy.py", 250),
TestFile("test_evs.py", 20),
TestFile("test_external_models.py", 30),
TestFile("test_fp8_utils.py", 9),
TestFile("test_gpt_oss_1gpu.py", 402),
TestFile("test_hidden_states.py", 55),
TestFile("test_input_embeddings.py", 38),
@@ -35,7 +33,6 @@ suites = {
TestFile("test_profile_merger_http_api.py", 9),
TestFile("test_swa_unittest.py", 8),
TestFile("test_torch_compile.py", 190),
TestFile("test_torchao.py", 103),
TestFile("test_utils_update_weights.py", 29),
TestFile("test_video_utils.py", 5),
TestFile("test_modelopt_export.py", 9),
@@ -128,7 +125,6 @@ suite_amd = {
# TestFile("lora/test_lora_cuda_graph.py", 250), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
# TestFile("lora/test_lora_qwen3.py", 97), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107
TestFile("test_bench_typebaseddispatcher.py", 10),
TestFile("test_eval_fp8_accuracy.py", 303),
TestFile("test_external_models.py", 45),
TestFile("test_input_embeddings.py", 38),
TestFile("test_io_struct.py", 8),
test/srt/test_eval_fp8_accuracy.py
-114
View File
@@ -1,114 +0,0 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8,
DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
class TestEvalFP8Accuracy(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
if is_hip():
# Another threshold for AMD because fp8 dtype is difference
self.assertGreaterEqual(metrics["score"], 0.60)
else:
self.assertGreaterEqual(metrics["score"], 0.60)
class TestEvalFP8DynamicQuantAccuracy(CustomTestCase):
def _run_test(self, model, other_args, expected_score):
base_url = DEFAULT_URL_FOR_TEST
other_args = other_args or []
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
try:
args = SimpleNamespace(
base_url=base_url,
model=model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], expected_score)
finally:
kill_process_tree(process.pid)
def test_mmlu_offline_only(self):
"""Test with offline quantization only."""
self._run_test(
model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8,
other_args=[],
expected_score=0.64,
)
def test_mmlu_offline_and_online_override(self):
"""Test with both offline and online quantization."""
self._run_test(
model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8,
other_args=["--quantization", "w8a8_fp8"],
# inference will use sgl kernel w/ online quant override
# we observed that the accuracy is higher then offline only
expected_score=0.64,
)
def test_mmlu_online_only(self):
"""Test with online quantization only."""
self._run_test(
model=DEFAULT_MODEL_NAME_FOR_TEST,
# inference will use sgl kernel w/ online quantization only
# we observed that the accuracy is higher then offline only
other_args=["--quantization", "w8a8_fp8"],
expected_score=0.64,
)
def test_mmlu_fp16_baseline(self):
"""Test with unquantized fp16 baseline."""
self._run_test(
model=DEFAULT_MODEL_NAME_FOR_TEST,
other_args=[],
expected_score=0.64,
)
if __name__ == "__main__":
unittest.main()
test/srt/test_fp8_utils.py
-44
View File
@@ -1,44 +0,0 @@
import unittest
import torch
from sglang.srt.layers.quantization.fp8_utils import (
inverse_transform_scale_ue8m0,
quant_weight_ue8m0,
transform_scale_ue8m0,
)
from sglang.test.test_utils import CustomTestCase
class TestInverseTransformScaleUe8m0(CustomTestCase):
def test_round_trip(self):
for _ in range(100):
weight_bf16 = torch.randn(
# DeepSeek V3 kv_b_proj
(32768, 512),
dtype=torch.bfloat16,
device="cuda",
)
weight_block_size = [128, 128]
qweight, sf_fp32_original = quant_weight_ue8m0(
weight_bf16, weight_block_size=weight_block_size
)
mn = qweight.shape[-2]
sf_packed_original = transform_scale_ue8m0(sf_fp32_original, mn=mn)
sf_fp32_recreated = inverse_transform_scale_ue8m0(sf_packed_original, mn=mn)
sf_packed_recreated = transform_scale_ue8m0(sf_fp32_recreated, mn=mn)
assert torch.all(
sf_packed_original == sf_packed_recreated
), f"{sf_packed_original=} {sf_packed_recreated}"
assert torch.all(
sf_fp32_original == sf_fp32_recreated
), f"{sf_fp32_original=} {sf_fp32_recreated}"
if __name__ == "__main__":
unittest.main()
test/srt/test_torchao.py
-95
View File
@@ -1,95 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang import Engine
from sglang.lang.chat_template import get_chat_template_by_model_path
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_IMAGE_URL,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
class TestTorchAO(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--torchao-config", "int4wo-128"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
assert metrics["score"] >= 0.60
def run_decode(self, max_new_tokens):
response = requests.post(
self.base_url + "/generate",
json={
"text": "The capital of France is",
"sampling_params": {
"temperature": 0,
"max_new_tokens": max_new_tokens,
},
"ignore_eos": True,
},
)
return response.json()
def test_throughput(self):
import time
max_tokens = 256
tic = time.perf_counter()
res = self.run_decode(max_tokens)
tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s")
assert throughput >= 210
class TestTorchAOForVLM(CustomTestCase):
def test_vlm_generate(self):
model_path = DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST
chat_template = get_chat_template_by_model_path(model_path)
text = f"{chat_template.image_token}What is in this picture? Answer: "
engine = Engine(
model_path=model_path,
max_total_tokens=512,
enable_multimodal=True,
torchao_config="fp8wo",
)
out = engine.generate([text], image_data=[DEFAULT_IMAGE_URL])
engine.shutdown()
self.assertGreater(len(out), 0)
if __name__ == "__main__":
unittest.main()