From 1a045669e497c72868b64eba29a36af463686b6f Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 5 Aug 2026 11:55:11 -0700 Subject: [PATCH] [CI] Merge tokenizer worker tests and drop redundant triton attention e2e (#33641) --- .../test_triton_attention_backend.py | 71 ---------- test/registered/mla/test_flashmla.py | 93 ------------- test/registered/mla/test_mla_flashinfer.py | 84 ------------ .../mla/test_mla_int8_deepseek_v3.py | 65 +-------- .../models_e2e/test_gpt_oss_4gpu_bf16.py | 22 --- .../tokenizer/test_multi_detokenizer.py | 80 ----------- .../tokenizer/test_multi_tokenizer.py | 5 + .../quantization/test_int8_linear_methods.py | 126 ++++++++++++++++++ 8 files changed, 133 insertions(+), 413 deletions(-) delete mode 100644 test/registered/attention/test_triton_attention_backend.py delete mode 100644 test/registered/mla/test_flashmla.py delete mode 100644 test/registered/mla/test_mla_flashinfer.py delete mode 100644 test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py delete mode 100644 test/registered/tokenizer/test_multi_detokenizer.py create mode 100644 test/registered/unit/layers/quantization/test_int8_linear_methods.py diff --git a/test/registered/attention/test_triton_attention_backend.py b/test/registered/attention/test_triton_attention_backend.py deleted file mode 100644 index f5371ee68..000000000 --- a/test/registered/attention/test_triton_attention_backend.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Usage: -python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu -""" - -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - run_bench_offline_throughput, -) - -# Triton attention backend integration test with latency benchmark and MMLU eval -register_cuda_ci(est_time=177, stage="base-b", runner_config="1-gpu-large") -register_amd_ci(est_time=1400, suite="stage-b-test-1-gpu-small-amd") - - -class TestTritonAttnBackend(CustomTestCase): - def test_latency(self): - output_throughput = run_bench_offline_throughput( - DEFAULT_MODEL_NAME_FOR_TEST, - [ - "--attention-backend", - "triton", - "--enable-torch-compile", - "--cuda-graph-max-bs-decode", - 4, - ], - ) - - print(f"{output_throughput=}") - - if is_in_ci(): - self.assertGreater(output_throughput, 153) - - def test_mmlu(self): - model = DEFAULT_MODEL_NAME_FOR_TEST - base_url = DEFAULT_URL_FOR_TEST - process = popen_launch_server( - model, - base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--attention-backend", "triton"], - ) - - try: - args = SimpleNamespace( - base_url=base_url, - model=model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - finally: - kill_process_tree(process.pid) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/mla/test_flashmla.py b/test/registered/mla/test_flashmla.py deleted file mode 100644 index 955c859cf..000000000 --- a/test/registered/mla/test_flashmla.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Usage: -python3 test/registered/mla/test_flashmla.py -""" - -import unittest -from types import SimpleNamespace - -import requests -import torch - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -# FlashMLA attention backend tests with MTP speculative decoding -register_cuda_ci(est_time=160, stage="base-b", runner_config="1-gpu-large") - - -class TestFlashMLAMTP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "lmsys/sglang-ci-dsv3-test" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--cuda-graph-max-bs-decode", - "4", - "--disable-radix", - "--enable-torch-compile", - "--torch-compile-max-bs", - "1", - "--speculative-algorithm", - "EAGLE", - "--speculative-draft-model-path", - "lmsys/sglang-ci-dsv3-test-NextN", - "--speculative-num-steps", - "2", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "3", - "--attention-backend", - "flashmla", - ] - ) - # Use longer timeout for DeepGEMM JIT compilation which can take 10-20 minutes - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.60) - - server_info = requests.get(self.base_url + "/server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 2.4) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/mla/test_mla_flashinfer.py b/test/registered/mla/test_mla_flashinfer.py deleted file mode 100644 index 4dd4e8637..000000000 --- a/test/registered/mla/test_mla_flashinfer.py +++ /dev/null @@ -1,84 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests -import torch - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -# FlashInfer MLA backend tests with MTP speculative decoding -register_cuda_ci(est_time=130, stage="base-b", runner_config="1-gpu-large") - - -class TestFlashinferMLAMTP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "lmsys/sglang-ci-dsv3-test" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--cuda-graph-max-bs-decode", - "4", - "--enable-torch-compile", - "--torch-compile-max-bs", - "1", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--attention-backend", - "flashinfer", - ] - ) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.60) - - server_info = requests.get(self.base_url + "/server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 2.5) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/mla/test_mla_int8_deepseek_v3.py b/test/registered/mla/test_mla_int8_deepseek_v3.py index 1add20c65..a0e5ea8d0 100644 --- a/test/registered/mla/test_mla_int8_deepseek_v3.py +++ b/test/registered/mla/test_mla_int8_deepseek_v3.py @@ -11,11 +11,11 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, ) -# DeepSeek-V3 INT8 quantization tests (channel and block INT8) +# DeepSeek-V3 channel-INT8 + MTP smoke; int8 GEMM numerics live in +# unit/layers/quantization/test_int8_linear_methods.py register_cuda_ci(est_time=160, stage="base-b", runner_config="1-gpu-large") @@ -81,66 +81,5 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase): self.assertGreater(avg_spec_accept_length, 2.5) -@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") -class TestDeepseekV3MTPBlockInt8(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "lmsys/sglang-ci-dsv3-block-int8-test" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--cuda-graph-max-bs-decode", - "16", - "--enable-torch-compile", - "--torch-compile-max-bs", - "2", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "2", - "--speculative-eagle-topk", - "4", - "--speculative-num-draft-tokens", - "4", - ] - ) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.60) - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 2.5) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py b/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py deleted file mode 100644 index aa60132db..000000000 --- a/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest - -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.gpt_oss_common import BaseTestGptOss - -register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100") - - -class TestGptOss4GpuBf16(BaseTestGptOss): - def test_bf16_120b(self): - self.run_test( - model_variant="120b", - quantization="bf16", - expected_score_of_reasoning_effort={ - "low": 0.58, - }, - other_args=["--tp", "4", "--cuda-graph-max-bs-decode", "200"], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/tokenizer/test_multi_detokenizer.py b/test/registered/tokenizer/test_multi_detokenizer.py deleted file mode 100644 index 2129961a6..000000000 --- a/test/registered/tokenizer/test_multi_detokenizer.py +++ /dev/null @@ -1,80 +0,0 @@ -import unittest - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.kits.eval_accuracy_kit import MMLUMixin -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - auto_config_device, - get_benchmark_args, - is_in_amd_ci, - is_in_ci, - popen_launch_server, - run_benchmark, - write_github_step_summary, -) - -register_cuda_ci(est_time=211, stage="base-b", runner_config="1-gpu-large") -register_amd_ci(est_time=345, suite="stage-b-test-1-gpu-small-amd") - - -class TestMultiDetokenizer(CustomTestCase, MMLUMixin): - mmlu_score_threshold = 0.65 - mmlu_num_examples = 64 - mmlu_num_threads = 32 - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tokenizer-worker-num", - 8, - "--detokenizer-worker-num", - 4, - "--mem-fraction-static", - 0.7, - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_multi_detokenizer_ttft(self): - args = get_benchmark_args( - base_url=self.base_url, - dataset_name="random", - dataset_path="", - tokenizer=None, - num_prompts=100, - random_input_len=4096, - random_output_len=2048, - sharegpt_context_len=None, - request_rate=1, - disable_stream=False, - disable_ignore_eos=False, - seed=0, - device=auto_config_device(), - lora_name=None, - ) - res = run_benchmark(args) - if is_in_ci(): - write_github_step_summary( - f"### test_multi_detokenizer_ttft\n" - f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" - ) - self.assertLess(res["median_e2e_latency_ms"], 11000) - self.assertLess(res["median_ttft_ms"], 130 if is_in_amd_ci() else 86) - self.assertLess(res["median_itl_ms"], 10) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/tokenizer/test_multi_tokenizer.py b/test/registered/tokenizer/test_multi_tokenizer.py index a49cf57d0..72466abb6 100644 --- a/test/registered/tokenizer/test_multi_tokenizer.py +++ b/test/registered/tokenizer/test_multi_tokenizer.py @@ -24,6 +24,9 @@ register_amd_ci(est_time=355, suite="stage-b-test-1-gpu-small-amd") class TestMultiTokenizer(CustomTestCase, MMLUMixin): + """One server covering both worker pools: multi-tokenizer and + multi-detokenizer (the flags are orthogonal).""" + mmlu_score_threshold = 0.65 mmlu_num_examples = 64 mmlu_num_threads = 32 @@ -39,6 +42,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin): other_args=[ "--tokenizer-worker-num", 8, + "--detokenizer-worker-num", + 4, "--mem-fraction-static", 0.7, ], diff --git a/test/registered/unit/layers/quantization/test_int8_linear_methods.py b/test/registered/unit/layers/quantization/test_int8_linear_methods.py new file mode 100644 index 000000000..57dbe8377 --- /dev/null +++ b/test/registered/unit/layers/quantization/test_int8_linear_methods.py @@ -0,0 +1,126 @@ +"""Numerics for the INT8 dense-linear methods. + +Real layer path vs a dequantized-reference matmul, in two formats: +channel W8A8 (W8A8Int8LinearMethod, per-channel weight scale + dynamic +per-token int8 activations) and blockwise (BlockInt8LinearMethod, +(128, 128) block weight scale). +""" + +import unittest + +import torch + +from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config +from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config +from sglang.srt.utils import get_device_sm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.layer_ut_utils import ( + assert_output_close, + init_single_process_dist, + load_linear_weights, + make_tp1_column_parallel_linear, +) +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-large") + +INT8_MAX = 127.0 + +# (M, N, K); channel int8 has no block-alignment constraints. +CHANNEL_SHAPES = [ + (64, 512, 512), + (5, 160, 336), + (128, 1024, 1024), +] + +# (M, N, K), N and K multiples of the (128, 128) weight block. +BLOCK_SHAPES = [ + (64, 512, 512), + (5, 384, 896), + (128, 1024, 1024), +] + + +def _quantize_int8_channel(w: torch.Tensor): + """Per-output-channel symmetric int8; returns checkpoint-format + (w_int8 [N, K], scale fp32 [N, 1]) and the dequant reference.""" + amax = w.float().abs().amax(dim=1, keepdim=True).clamp(min=1e-12) + scale = amax / INT8_MAX + w_int8 = torch.round(w.float() / scale).clamp(-INT8_MAX, INT8_MAX).to(torch.int8) + w_dequant = w_int8.float() * scale + return w_int8, scale, w_dequant + + +def _quantize_int8_block(w: torch.Tensor, block: int = 128): + """Per (block, block) tile symmetric int8; returns checkpoint-format + (w_int8 [N, K], scale_inv fp32 [N/block, K/block]) and the dequant reference.""" + n, k = w.shape + tiles = w.float().reshape(n // block, block, k // block, block) + amax = tiles.abs().amax(dim=(1, 3)).clamp(min=1e-12) + scale = amax / INT8_MAX + w_int8 = ( + torch.round(tiles / scale[:, None, :, None]) + .clamp(-INT8_MAX, INT8_MAX) + .to(torch.int8) + ) + w_dequant = (w_int8.float() * scale[:, None, :, None]).reshape(n, k) + return w_int8.reshape(n, k), scale, w_dequant + + +class _Int8LinearCheck(CustomTestCase): + @classmethod + def setUpClass(cls): + init_single_process_dist() + + def _check(self, shapes, build_layer): + torch.manual_seed(7) + for m, n, k in shapes: + with self.subTest(shape=(m, n, k)): + layer, w_dequant = build_layer(n, k) + layer.quant_method.process_weights_after_loading(layer) + + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10 + out, _ = layer(x) + + ref = x.float() @ w_dequant.T + # atol absorbs the dynamic per-token int8 activation quant, + # which the reference does not mirror. + assert_output_close(self, out, ref, rtol=5e-2, atol=1e-1) + + +@unittest.skipIf( + get_device_sm() >= 100, "sgl-kernel int8_scaled_mm has no SM100+ kernel" +) +class TestW8A8Int8Linear(_Int8LinearCheck): + @staticmethod + def _build_layer(n: int, k: int): + layer = make_tp1_column_parallel_linear(W8A8Int8Config({}), n, k) + w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10 + w_int8, scale, w_dequant = _quantize_int8_channel(w) + load_linear_weights(layer, weight=w_int8, weight_scale=scale) + return layer, w_dequant + + def test_channel(self): + self._check(CHANNEL_SHAPES, self._build_layer) + + +class TestBlockInt8Linear(_Int8LinearCheck): + @staticmethod + def _build_layer(n: int, k: int): + quant_config = BlockInt8Config( + is_checkpoint_int8_serialized=True, + activation_scheme="dynamic", + weight_block_size=[128, 128], + ) + layer = make_tp1_column_parallel_linear(quant_config, n, k) + w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10 + w_int8, scale_inv, w_dequant = _quantize_int8_block(w) + load_linear_weights(layer, weight=w_int8, weight_scale_inv=scale_inv) + return layer, w_dequant + + def test_block(self): + self._check(BLOCK_SHAPES, self._build_layer) + + +if __name__ == "__main__": + unittest.main()