minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Merge tokenizer worker tests and drop redundant triton attention e2e (#33641)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-05 11:55:11 -07:00
committed by GitHub
parent 5f79cf3511
commit 1a045669e4
8 changed files with 133 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/attention/test_triton_attention_backend.py
-71
View File
@@ -1,71 +0,0 @@
"""
Usage:
python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu
"""
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
run_bench_offline_throughput,
)
# Triton attention backend integration test with latency benchmark and MMLU eval
register_cuda_ci(est_time=177, stage="base-b", runner_config="1-gpu-large")
register_amd_ci(est_time=1400, suite="stage-b-test-1-gpu-small-amd")
class TestTritonAttnBackend(CustomTestCase):
def test_latency(self):
output_throughput = run_bench_offline_throughput(
DEFAULT_MODEL_NAME_FOR_TEST,
[
"--attention-backend",
"triton",
"--enable-torch-compile",
"--cuda-graph-max-bs-decode",
4,
],
)
print(f"{output_throughput=}")
if is_in_ci():
self.assertGreater(output_throughput, 153)
def test_mmlu(self):
model = DEFAULT_MODEL_NAME_FOR_TEST
base_url = DEFAULT_URL_FOR_TEST
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--attention-backend", "triton"],
)
try:
args = SimpleNamespace(
base_url=base_url,
model=model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
finally:
kill_process_tree(process.pid)
if __name__ == "__main__":
unittest.main()
test/registered/mla/test_flashmla.py
-93
View File
@@ -1,93 +0,0 @@
"""
Usage:
python3 test/registered/mla/test_flashmla.py
"""
import unittest
from types import SimpleNamespace
import requests
import torch
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
# FlashMLA attention backend tests with MTP speculative decoding
register_cuda_ci(est_time=160, stage="base-b", runner_config="1-gpu-large")
class TestFlashMLAMTP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "lmsys/sglang-ci-dsv3-test"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code"]
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs-decode",
"4",
"--disable-radix",
"--enable-torch-compile",
"--torch-compile-max-bs",
"1",
"--speculative-algorithm",
"EAGLE",
"--speculative-draft-model-path",
"lmsys/sglang-ci-dsv3-test-NextN",
"--speculative-num-steps",
"2",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"3",
"--attention-backend",
"flashmla",
]
)
# Use longer timeout for DeepGEMM JIT compilation which can take 10-20 minutes
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.60)
server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 2.4)
if __name__ == "__main__":
unittest.main()
test/registered/mla/test_mla_flashinfer.py
-84
View File
@@ -1,84 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
import torch
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
# FlashInfer MLA backend tests with MTP speculative decoding
register_cuda_ci(est_time=130, stage="base-b", runner_config="1-gpu-large")
class TestFlashinferMLAMTP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "lmsys/sglang-ci-dsv3-test"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code"]
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs-decode",
"4",
"--enable-torch-compile",
"--torch-compile-max-bs",
"1",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--attention-backend",
"flashinfer",
]
)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.60)
server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 2.5)
if __name__ == "__main__":
unittest.main()
test/registered/mla/test_mla_int8_deepseek_v3.py
+2 -63
View File
@@ -11,11 +11,11 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
# DeepSeek-V3 INT8 quantization tests (channel and block INT8)
# DeepSeek-V3 channel-INT8 + MTP smoke; int8 GEMM numerics live in
# unit/layers/quantization/test_int8_linear_methods.py
register_cuda_ci(est_time=160, stage="base-b", runner_config="1-gpu-large")
@@ -81,66 +81,5 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase):
self.assertGreater(avg_spec_accept_length, 2.5)
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestDeepseekV3MTPBlockInt8(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "lmsys/sglang-ci-dsv3-block-int8-test"
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code"]
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
"2",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"2",
"--speculative-eagle-topk",
"4",
"--speculative-num-draft-tokens",
"4",
]
)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["score"], 0.60)
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 2.5)
if __name__ == "__main__":
unittest.main()
test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py
-22
View File
@@ -1,22 +0,0 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
class TestGptOss4GpuBf16(BaseTestGptOss):
def test_bf16_120b(self):
self.run_test(
model_variant="120b",
quantization="bf16",
expected_score_of_reasoning_effort={
"low": 0.58,
},
other_args=["--tp", "4", "--cuda-graph-max-bs-decode", "200"],
)
if __name__ == "__main__":
unittest.main()
test/registered/tokenizer/test_multi_detokenizer.py
-80
View File
@@ -1,80 +0,0 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
auto_config_device,
get_benchmark_args,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
run_benchmark,
write_github_step_summary,
)
register_cuda_ci(est_time=211, stage="base-b", runner_config="1-gpu-large")
register_amd_ci(est_time=345, suite="stage-b-test-1-gpu-small-amd")
class TestMultiDetokenizer(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tokenizer-worker-num",
8,
"--detokenizer-worker-num",
4,
"--mem-fraction-static",
0.7,
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_multi_detokenizer_ttft(self):
args = get_benchmark_args(
base_url=self.base_url,
dataset_name="random",
dataset_path="",
tokenizer=None,
num_prompts=100,
random_input_len=4096,
random_output_len=2048,
sharegpt_context_len=None,
request_rate=1,
disable_stream=False,
disable_ignore_eos=False,
seed=0,
device=auto_config_device(),
lora_name=None,
)
res = run_benchmark(args)
if is_in_ci():
write_github_step_summary(
f"### test_multi_detokenizer_ttft\n"
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
)
self.assertLess(res["median_e2e_latency_ms"], 11000)
self.assertLess(res["median_ttft_ms"], 130 if is_in_amd_ci() else 86)
self.assertLess(res["median_itl_ms"], 10)
if __name__ == "__main__":
unittest.main()
test/registered/tokenizer/test_multi_tokenizer.py
+5
View File
@@ -24,6 +24,9 @@ register_amd_ci(est_time=355, suite="stage-b-test-1-gpu-small-amd")
class TestMultiTokenizer(CustomTestCase, MMLUMixin):
"""One server covering both worker pools: multi-tokenizer and
multi-detokenizer (the flags are orthogonal)."""
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@@ -39,6 +42,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
other_args=[
"--tokenizer-worker-num",
8,
"--detokenizer-worker-num",
4,
"--mem-fraction-static",
0.7,
],
test/registered/unit/layers/quantization/test_int8_linear_methods.py
+126
View File
@@ -0,0 +1,126 @@
"""Numerics for the INT8 dense-linear methods.
Real layer path vs a dequantized-reference matmul, in two formats:
channel W8A8 (W8A8Int8LinearMethod, per-channel weight scale + dynamic
per-token int8 activations) and blockwise (BlockInt8LinearMethod,
(128, 128) block weight scale).
"""
import unittest
import torch
from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config
from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config
from sglang.srt.utils import get_device_sm
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.layer_ut_utils import (
assert_output_close,
init_single_process_dist,
load_linear_weights,
make_tp1_column_parallel_linear,
)
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-large")
INT8_MAX = 127.0
# (M, N, K); channel int8 has no block-alignment constraints.
CHANNEL_SHAPES = [
(64, 512, 512),
(5, 160, 336),
(128, 1024, 1024),
]
# (M, N, K), N and K multiples of the (128, 128) weight block.
BLOCK_SHAPES = [
(64, 512, 512),
(5, 384, 896),
(128, 1024, 1024),
]
def _quantize_int8_channel(w: torch.Tensor):
"""Per-output-channel symmetric int8; returns checkpoint-format
(w_int8 [N, K], scale fp32 [N, 1]) and the dequant reference."""
amax = w.float().abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
scale = amax / INT8_MAX
w_int8 = torch.round(w.float() / scale).clamp(-INT8_MAX, INT8_MAX).to(torch.int8)
w_dequant = w_int8.float() * scale
return w_int8, scale, w_dequant
def _quantize_int8_block(w: torch.Tensor, block: int = 128):
"""Per (block, block) tile symmetric int8; returns checkpoint-format
(w_int8 [N, K], scale_inv fp32 [N/block, K/block]) and the dequant reference."""
n, k = w.shape
tiles = w.float().reshape(n // block, block, k // block, block)
amax = tiles.abs().amax(dim=(1, 3)).clamp(min=1e-12)
scale = amax / INT8_MAX
w_int8 = (
torch.round(tiles / scale[:, None, :, None])
.clamp(-INT8_MAX, INT8_MAX)
.to(torch.int8)
)
w_dequant = (w_int8.float() * scale[:, None, :, None]).reshape(n, k)
return w_int8.reshape(n, k), scale, w_dequant
class _Int8LinearCheck(CustomTestCase):
@classmethod
def setUpClass(cls):
init_single_process_dist()
def _check(self, shapes, build_layer):
torch.manual_seed(7)
for m, n, k in shapes:
with self.subTest(shape=(m, n, k)):
layer, w_dequant = build_layer(n, k)
layer.quant_method.process_weights_after_loading(layer)
x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
out, _ = layer(x)
ref = x.float() @ w_dequant.T
# atol absorbs the dynamic per-token int8 activation quant,
# which the reference does not mirror.
assert_output_close(self, out, ref, rtol=5e-2, atol=1e-1)
@unittest.skipIf(
get_device_sm() >= 100, "sgl-kernel int8_scaled_mm has no SM100+ kernel"
)
class TestW8A8Int8Linear(_Int8LinearCheck):
@staticmethod
def _build_layer(n: int, k: int):
layer = make_tp1_column_parallel_linear(W8A8Int8Config({}), n, k)
w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10
w_int8, scale, w_dequant = _quantize_int8_channel(w)
load_linear_weights(layer, weight=w_int8, weight_scale=scale)
return layer, w_dequant
def test_channel(self):
self._check(CHANNEL_SHAPES, self._build_layer)
class TestBlockInt8Linear(_Int8LinearCheck):
@staticmethod
def _build_layer(n: int, k: int):
quant_config = BlockInt8Config(
is_checkpoint_int8_serialized=True,
activation_scheme="dynamic",
weight_block_size=[128, 128],
)
layer = make_tp1_column_parallel_linear(quant_config, n, k)
w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10
w_int8, scale_inv, w_dequant = _quantize_int8_block(w)
load_linear_weights(layer, weight=w_int8, weight_scale_inv=scale_inv)
return layer, w_dequant
def test_block(self):
self._check(BLOCK_SHAPES, self._build_layer)
if __name__ == "__main__":
unittest.main()