[CI] Merge tokenizer worker tests and drop redundant triton attention e2e (#33641)

test/registered/attention/test_triton_attention_backend.py

@@ -1,71 +0,0 @@
-"""
-Usage:
-python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu
-"""
-
-import unittest
-from types import SimpleNamespace
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_MODEL_NAME_FOR_TEST,
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    is_in_ci,
-    popen_launch_server,
-    run_bench_offline_throughput,
-)
-
-# Triton attention backend integration test with latency benchmark and MMLU eval
-register_cuda_ci(est_time=177, stage="base-b", runner_config="1-gpu-large")
-register_amd_ci(est_time=1400, suite="stage-b-test-1-gpu-small-amd")
-
-
-class TestTritonAttnBackend(CustomTestCase):
-    def test_latency(self):
-        output_throughput = run_bench_offline_throughput(
-            DEFAULT_MODEL_NAME_FOR_TEST,
-            [
-                "--attention-backend",
-                "triton",
-                "--enable-torch-compile",
-                "--cuda-graph-max-bs-decode",
-                4,
-            ],
-        )
-
-        print(f"{output_throughput=}")
-
-        if is_in_ci():
-            self.assertGreater(output_throughput, 153)
-
-    def test_mmlu(self):
-        model = DEFAULT_MODEL_NAME_FOR_TEST
-        base_url = DEFAULT_URL_FOR_TEST
-        process = popen_launch_server(
-            model,
-            base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=["--attention-backend", "triton"],
-        )
-
-        try:
-            args = SimpleNamespace(
-                base_url=base_url,
-                model=model,
-                eval_name="mmlu",
-                num_examples=64,
-                num_threads=32,
-            )
-
-            metrics = run_eval(args)
-            self.assertGreaterEqual(metrics["score"], 0.65)
-        finally:
-            kill_process_tree(process.pid)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/mla/test_flashmla.py

@@ -1,93 +0,0 @@
-"""
-Usage:
-python3 test/registered/mla/test_flashmla.py
-"""
-
-import unittest
-from types import SimpleNamespace
-
-import requests
-import torch
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-# FlashMLA attention backend tests with MTP speculative decoding
-register_cuda_ci(est_time=160, stage="base-b", runner_config="1-gpu-large")
-
-
-class TestFlashMLAMTP(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = "lmsys/sglang-ci-dsv3-test"
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = ["--trust-remote-code"]
-        if torch.cuda.is_available() and torch.version.cuda:
-            other_args.extend(
-                [
-                    "--cuda-graph-max-bs-decode",
-                    "4",
-                    "--disable-radix",
-                    "--enable-torch-compile",
-                    "--torch-compile-max-bs",
-                    "1",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-draft-model-path",
-                    "lmsys/sglang-ci-dsv3-test-NextN",
-                    "--speculative-num-steps",
-                    "2",
-                    "--speculative-eagle-topk",
-                    "1",
-                    "--speculative-num-draft-tokens",
-                    "3",
-                    "--attention-backend",
-                    "flashmla",
-                ]
-            )
-        # Use longer timeout for DeepGEMM JIT compilation which can take 10-20 minutes
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_gsm8k(self):
-        requests.get(self.base_url + "/flush_cache")
-
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            model=self.model,
-            eval_name="gsm8k",
-            api="completion",
-            max_tokens=512,
-            num_examples=200,
-            num_threads=128,
-        )
-        metrics = run_eval(args)
-        print(metrics)
-
-        self.assertGreater(metrics["score"], 0.60)
-
-        server_info = requests.get(self.base_url + "/server_info").json()
-        avg_spec_accept_length = server_info["internal_states"][0][
-            "avg_spec_accept_length"
-        ]
-        print(f"{avg_spec_accept_length=}")
-        self.assertGreater(avg_spec_accept_length, 2.4)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/mla/test_mla_flashinfer.py

@@ -1,84 +0,0 @@
-import unittest
-from types import SimpleNamespace
-
-import requests
-import torch
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-# FlashInfer MLA backend tests with MTP speculative decoding
-register_cuda_ci(est_time=130, stage="base-b", runner_config="1-gpu-large")
-
-
-class TestFlashinferMLAMTP(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = "lmsys/sglang-ci-dsv3-test"
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = ["--trust-remote-code"]
-        if torch.cuda.is_available() and torch.version.cuda:
-            other_args.extend(
-                [
-                    "--cuda-graph-max-bs-decode",
-                    "4",
-                    "--enable-torch-compile",
-                    "--torch-compile-max-bs",
-                    "1",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-num-steps",
-                    "3",
-                    "--speculative-eagle-topk",
-                    "1",
-                    "--speculative-num-draft-tokens",
-                    "4",
-                    "--attention-backend",
-                    "flashinfer",
-                ]
-            )
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_gsm8k(self):
-        requests.get(self.base_url + "/flush_cache")
-
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            model=self.model,
-            eval_name="gsm8k",
-            api="completion",
-            max_tokens=512,
-            num_examples=200,
-            num_threads=128,
-        )
-        metrics = run_eval(args)
-        print(metrics)
-
-        self.assertGreater(metrics["score"], 0.60)
-
-        server_info = requests.get(self.base_url + "/server_info").json()
-        avg_spec_accept_length = server_info["internal_states"][0][
-            "avg_spec_accept_length"
-        ]
-        print(f"{avg_spec_accept_length=}")
-        self.assertGreater(avg_spec_accept_length, 2.5)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/mla/test_mla_int8_deepseek_v3.py

@@ -11,11 +11,11 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
-    is_in_ci,
     popen_launch_server,
 )
 
-# DeepSeek-V3 INT8 quantization tests (channel and block INT8)
+# DeepSeek-V3 channel-INT8 + MTP smoke; int8 GEMM numerics live in
+# unit/layers/quantization/test_int8_linear_methods.py
 register_cuda_ci(est_time=160, stage="base-b", runner_config="1-gpu-large")
 
 
@@ -81,66 +81,5 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase):
         self.assertGreater(avg_spec_accept_length, 2.5)
 
 
-@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
-class TestDeepseekV3MTPBlockInt8(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = "lmsys/sglang-ci-dsv3-block-int8-test"
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = ["--trust-remote-code"]
-        if torch.cuda.is_available() and torch.version.cuda:
-            other_args.extend(
-                [
-                    "--cuda-graph-max-bs-decode",
-                    "16",
-                    "--enable-torch-compile",
-                    "--torch-compile-max-bs",
-                    "2",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-num-steps",
-                    "2",
-                    "--speculative-eagle-topk",
-                    "4",
-                    "--speculative-num-draft-tokens",
-                    "4",
-                ]
-            )
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_gsm8k(self):
-        requests.get(self.base_url + "/flush_cache")
-
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            model=self.model,
-            eval_name="gsm8k",
-            api="completion",
-            max_tokens=512,
-            num_examples=200,
-            num_threads=128,
-        )
-        metrics = run_eval(args)
-        print(metrics)
-
-        self.assertGreater(metrics["score"], 0.60)
-
-        server_info = requests.get(self.base_url + "/server_info")
-        avg_spec_accept_length = server_info.json()["internal_states"][0][
-            "avg_spec_accept_length"
-        ]
-        print(f"{avg_spec_accept_length=}")
-        self.assertGreater(avg_spec_accept_length, 2.5)
-
-
 if __name__ == "__main__":
     unittest.main()

test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py

@@ -1,22 +0,0 @@
-import unittest
-
-from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.gpt_oss_common import BaseTestGptOss
-
-register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100")
-
-
-class TestGptOss4GpuBf16(BaseTestGptOss):
-    def test_bf16_120b(self):
-        self.run_test(
-            model_variant="120b",
-            quantization="bf16",
-            expected_score_of_reasoning_effort={
-                "low": 0.58,
-            },
-            other_args=["--tp", "4", "--cuda-graph-max-bs-decode", "200"],
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/tokenizer/test_multi_detokenizer.py

@@ -1,80 +0,0 @@
-import unittest
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.kits.eval_accuracy_kit import MMLUMixin
-from sglang.test.test_utils import (
-    DEFAULT_MODEL_NAME_FOR_TEST,
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    auto_config_device,
-    get_benchmark_args,
-    is_in_amd_ci,
-    is_in_ci,
-    popen_launch_server,
-    run_benchmark,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=211, stage="base-b", runner_config="1-gpu-large")
-register_amd_ci(est_time=345, suite="stage-b-test-1-gpu-small-amd")
-
-
-class TestMultiDetokenizer(CustomTestCase, MMLUMixin):
-    mmlu_score_threshold = 0.65
-    mmlu_num_examples = 64
-    mmlu_num_threads = 32
-
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEFAULT_MODEL_NAME_FOR_TEST
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=[
-                "--tokenizer-worker-num",
-                8,
-                "--detokenizer-worker-num",
-                4,
-                "--mem-fraction-static",
-                0.7,
-            ],
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_multi_detokenizer_ttft(self):
-        args = get_benchmark_args(
-            base_url=self.base_url,
-            dataset_name="random",
-            dataset_path="",
-            tokenizer=None,
-            num_prompts=100,
-            random_input_len=4096,
-            random_output_len=2048,
-            sharegpt_context_len=None,
-            request_rate=1,
-            disable_stream=False,
-            disable_ignore_eos=False,
-            seed=0,
-            device=auto_config_device(),
-            lora_name=None,
-        )
-        res = run_benchmark(args)
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_multi_detokenizer_ttft\n"
-                f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
-            )
-            self.assertLess(res["median_e2e_latency_ms"], 11000)
-            self.assertLess(res["median_ttft_ms"], 130 if is_in_amd_ci() else 86)
-            self.assertLess(res["median_itl_ms"], 10)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/tokenizer/test_multi_tokenizer.py

@@ -24,6 +24,9 @@ register_amd_ci(est_time=355, suite="stage-b-test-1-gpu-small-amd")
 
 
 class TestMultiTokenizer(CustomTestCase, MMLUMixin):
+    """One server covering both worker pools: multi-tokenizer and
+    multi-detokenizer (the flags are orthogonal)."""
+
     mmlu_score_threshold = 0.65
     mmlu_num_examples = 64
     mmlu_num_threads = 32
@@ -39,6 +42,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
             other_args=[
                 "--tokenizer-worker-num",
                 8,
+                "--detokenizer-worker-num",
+                4,
                 "--mem-fraction-static",
                 0.7,
             ],

test/registered/unit/layers/quantization/test_int8_linear_methods.py

@@ -0,0 +1,126 @@
+"""Numerics for the INT8 dense-linear methods.
+
+Real layer path vs a dequantized-reference matmul, in two formats:
+channel W8A8 (W8A8Int8LinearMethod, per-channel weight scale + dynamic
+per-token int8 activations) and blockwise (BlockInt8LinearMethod,
+(128, 128) block weight scale).
+"""
+
+import unittest
+
+import torch
+
+from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config
+from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config
+from sglang.srt.utils import get_device_sm
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.layer_ut_utils import (
+    assert_output_close,
+    init_single_process_dist,
+    load_linear_weights,
+    make_tp1_column_parallel_linear,
+)
+from sglang.test.test_utils import CustomTestCase
+
+register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-large")
+
+INT8_MAX = 127.0
+
+# (M, N, K); channel int8 has no block-alignment constraints.
+CHANNEL_SHAPES = [
+    (64, 512, 512),
+    (5, 160, 336),
+    (128, 1024, 1024),
+]
+
+# (M, N, K), N and K multiples of the (128, 128) weight block.
+BLOCK_SHAPES = [
+    (64, 512, 512),
+    (5, 384, 896),
+    (128, 1024, 1024),
+]
+
+
+def _quantize_int8_channel(w: torch.Tensor):
+    """Per-output-channel symmetric int8; returns checkpoint-format
+    (w_int8 [N, K], scale fp32 [N, 1]) and the dequant reference."""
+    amax = w.float().abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
+    scale = amax / INT8_MAX
+    w_int8 = torch.round(w.float() / scale).clamp(-INT8_MAX, INT8_MAX).to(torch.int8)
+    w_dequant = w_int8.float() * scale
+    return w_int8, scale, w_dequant
+
+
+def _quantize_int8_block(w: torch.Tensor, block: int = 128):
+    """Per (block, block) tile symmetric int8; returns checkpoint-format
+    (w_int8 [N, K], scale_inv fp32 [N/block, K/block]) and the dequant reference."""
+    n, k = w.shape
+    tiles = w.float().reshape(n // block, block, k // block, block)
+    amax = tiles.abs().amax(dim=(1, 3)).clamp(min=1e-12)
+    scale = amax / INT8_MAX
+    w_int8 = (
+        torch.round(tiles / scale[:, None, :, None])
+        .clamp(-INT8_MAX, INT8_MAX)
+        .to(torch.int8)
+    )
+    w_dequant = (w_int8.float() * scale[:, None, :, None]).reshape(n, k)
+    return w_int8.reshape(n, k), scale, w_dequant
+
+
+class _Int8LinearCheck(CustomTestCase):
+    @classmethod
+    def setUpClass(cls):
+        init_single_process_dist()
+
+    def _check(self, shapes, build_layer):
+        torch.manual_seed(7)
+        for m, n, k in shapes:
+            with self.subTest(shape=(m, n, k)):
+                layer, w_dequant = build_layer(n, k)
+                layer.quant_method.process_weights_after_loading(layer)
+
+                x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
+                out, _ = layer(x)
+
+                ref = x.float() @ w_dequant.T
+                # atol absorbs the dynamic per-token int8 activation quant,
+                # which the reference does not mirror.
+                assert_output_close(self, out, ref, rtol=5e-2, atol=1e-1)
+
+
+@unittest.skipIf(
+    get_device_sm() >= 100, "sgl-kernel int8_scaled_mm has no SM100+ kernel"
+)
+class TestW8A8Int8Linear(_Int8LinearCheck):
+    @staticmethod
+    def _build_layer(n: int, k: int):
+        layer = make_tp1_column_parallel_linear(W8A8Int8Config({}), n, k)
+        w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10
+        w_int8, scale, w_dequant = _quantize_int8_channel(w)
+        load_linear_weights(layer, weight=w_int8, weight_scale=scale)
+        return layer, w_dequant
+
+    def test_channel(self):
+        self._check(CHANNEL_SHAPES, self._build_layer)
+
+
+class TestBlockInt8Linear(_Int8LinearCheck):
+    @staticmethod
+    def _build_layer(n: int, k: int):
+        quant_config = BlockInt8Config(
+            is_checkpoint_int8_serialized=True,
+            activation_scheme="dynamic",
+            weight_block_size=[128, 128],
+        )
+        layer = make_tp1_column_parallel_linear(quant_config, n, k)
+        w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10
+        w_int8, scale_inv, w_dequant = _quantize_int8_block(w)
+        load_linear_weights(layer, weight=w_int8, weight_scale_inv=scale_inv)
+        return layer, w_dequant
+
+    def test_block(self):
+        self._check(BLOCK_SHAPES, self._build_layer)
+
+
+if __name__ == "__main__":
+    unittest.main()