minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Test] Replace GEMM backend e2e matrices with layer-level unit tests (#33596)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-04 15:50:41 -07:00
committed by GitHub
parent b0fd31ba07
commit a0b3f1dde6
6 changed files with 421 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/quant/test_fp8_blockwise_gemm.py
-145
View File
@@ -1,145 +0,0 @@
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
popen_launch_server,
try_cached_model,
)
register_cuda_ci(est_time=430, stage="extra-b", runner_config="4-gpu-b200")
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
class FP8BlockwiseGemmBase:
backend = None
@classmethod
def setUpClass(cls):
if cls.backend is None:
raise NotImplementedError("Subclass must set 'backend' attribute")
cls.model = try_cached_model(MODEL_PATH)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--fp8-gemm-backend",
cls.backend,
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=200,
num_shots=8,
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["score"], 0.8)
class MXFP8GemmBase:
backend = None
@classmethod
def setUpClass(cls):
if cls.backend is None:
raise NotImplementedError("Subclass must set 'backend' attribute")
cls.model = try_cached_model(MXFP8_MODEL_PATH)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--fp8-gemm-backend",
cls.backend,
# TODO: pin tc_piecewise — default `breakable` prefill runs MXFP8 RMSNorm in bf16, hurting accuracy; unrelated to BCG-default change.
"--cuda-graph-backend-prefill",
"tc_piecewise",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=200,
num_shots=8,
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["score"], 0.8)
class TestFP8BlockwiseGemmTriton(FP8BlockwiseGemmBase, unittest.TestCase):
backend = "triton"
class TestFP8BlockwiseGemmDeepGemm(FP8BlockwiseGemmBase, unittest.TestCase):
backend = "deep_gemm"
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP8BlockwiseGemmFlashinferTrtllm(FP8BlockwiseGemmBase, unittest.TestCase):
backend = "flashinfer_trtllm"
@unittest.skipIf(get_device_sm() != 90, "Test requires CUDA SM 90")
class TestFP8BlockwiseGemmFlashinferDeepGemm(FP8BlockwiseGemmBase, unittest.TestCase):
backend = "flashinfer_deepgemm"
@unittest.skip("Currently PCG capture takes too long to complete, disable until fixed")
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestMXFP8GemmTriton(MXFP8GemmBase, unittest.TestCase):
backend = "triton"
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestMXFP8GemmFlashinferTrtllm(MXFP8GemmBase, unittest.TestCase):
backend = "flashinfer_trtllm"
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestMXFP8GemmFlashinferCutlass(MXFP8GemmBase, unittest.TestCase):
backend = "flashinfer_cutlass"
if __name__ == "__main__":
unittest.main()
test/registered/quant/test_fp8_gemm_sm120.py
-86
View File
@@ -1,86 +0,0 @@
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
popen_launch_server,
try_cached_model,
)
register_cuda_ci(est_time=146, stage="extra-a", runner_config="1-gpu-small")
PERTENSOR_MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-FP8"
BLOCKWISE_MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
class FP8GemmSM120Base:
model_path = None
backend = None
quantization = None
@classmethod
def setUpClass(cls):
if cls.backend is None:
raise NotImplementedError("Subclass must set 'backend' attribute")
cls.model = try_cached_model(cls.model_path)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--fp8-gemm-backend",
cls.backend,
"--cuda-graph-backend-prefill=disabled",
]
if cls.quantization:
other_args += ["--quantization", cls.quantization]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
if hasattr(cls, "process"):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
num_shots=self.num_shots,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=200,
host=parsed_url.hostname,
port=parsed_url.port,
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["accuracy"], self.accuracy_threshold)
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP8PerTensorGemmSM120Auto(FP8GemmSM120Base, unittest.TestCase):
model_path = PERTENSOR_MODEL_PATH
backend = "auto"
quantization = "modelopt_fp8"
num_shots = 5
accuracy_threshold = 0.73
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP8BlockwiseGemmSM120Auto(FP8GemmSM120Base, unittest.TestCase):
model_path = BLOCKWISE_MODEL_PATH
backend = "auto"
num_shots = 8
accuracy_threshold = 0.87
if __name__ == "__main__":
unittest.main()
test/registered/quant/test_nvfp4_gemm.py
-86
View File
@@ -1,86 +0,0 @@
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
popen_launch_server,
try_cached_model,
)
register_cuda_ci(est_time=350, stage="base-c", runner_config="4-gpu-b200")
MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
class FP4GemmBase:
backend = None
@classmethod
def setUpClass(cls):
if cls.backend is None:
raise NotImplementedError("Subclass must set 'backend' attribute")
cls.model = try_cached_model(MODEL_PATH)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--quantization",
"modelopt_fp4",
"--fp4-gemm-backend",
cls.backend,
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=200,
)
metrics = run_eval(args)
print(metrics)
# TODO: restore 0.64 once the BCG-prefill RMSNorm fp32 fix lands.
self.assertGreater(metrics["score"], 0.63)
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP4GemmFlashinferCutlass(FP4GemmBase, unittest.TestCase):
backend = "flashinfer_cutlass"
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP4GemmFlashinferCudnn(FP4GemmBase, unittest.TestCase):
backend = "flashinfer_cudnn"
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP4GemmFlashinferTrtllm(FP4GemmBase, unittest.TestCase):
backend = "flashinfer_trtllm"
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
class TestFP4GemmFlashinferCutedsl(FP4GemmBase, unittest.TestCase):
backend = "flashinfer_cutedsl"
if __name__ == "__main__":
unittest.main()