From 5f017ffabb6ab8d214f6a4616ee8bd98a376034a Mon Sep 17 00:00:00 2001 From: ZY Y <99327288+YZY00Raiser@users.noreply.github.com> Date: Sun, 20 Sep 2026 22:42:00 +0800 Subject: [PATCH] Update test cases and performance testing framework (#40392) --- .../ascend/e2e/test_npu_performance_utils.py | 64 +++++++++++++++++ ...test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py | 56 ++++++++++++--- ...epseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py | 70 ++++++++++++++----- 3 files changed, 162 insertions(+), 28 deletions(-) diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index 0f3d07df5..e263508cc 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -228,6 +228,20 @@ TTFT_TOLERANCE = 1.02 # +2% E2E_TOLERANCE = 1.02 # +2% OUTPUT_TOKEN_THROUGHPUT_TOLERANCE = 0.98 # -2% + +def _get_spec_num_draft_tokens(other_args): + """Extract the value of --speculative-num-draft-tokens from server args.""" + if not other_args: + return None + args = [str(arg) for arg in other_args] + if "--speculative-num-draft-tokens" not in args: + return None + idx = args.index("--speculative-num-draft-tokens") + if idx + 1 >= len(args): + return None + return int(args[idx + 1]) + + # Package filtering keywords PACKAGE_FILTER_KEYWORDS = [ "sglang", @@ -570,6 +584,11 @@ def run_bench_serving( parts = stripped_line.split() if len(parts) >= 5: metrics["mean_e2e_latency"] = parts[4] + elif "Accept length" in stripped_line: + # Format: "Accept length: 4.35" + parts = stripped_line.split() + if len(parts) >= 3: + metrics["accept_length"] = parts[2] reader_done.set() process.wait() if process.returncode != 0: @@ -919,6 +938,29 @@ def assert_metrics(self, metrics): labels={"test_case": tc_name, "type": "perf"}, ) + spec_num_draft_tokens = _get_spec_num_draft_tokens( + getattr(self, "other_args", None) + ) + # accept_rate = accept_length / --speculative-num-draft-tokens. + # Dump only when all inputs are present; the mandatory checks below + # decide pass/fail when the baseline is set. + if ( + getattr(self, "accept_rate", None) + and metrics.get("accept_length") + and spec_num_draft_tokens + ): + accept_rate = float(metrics["accept_length"]) / spec_num_draft_tokens + dump_metric( + "accept_rate", + accept_rate, + labels={"test_case": tc_name, "type": "perf"}, + ) + dump_metric( + "accept_rate_baseline", + float(self.accept_rate), + labels={"test_case": tc_name, "type": "perf"}, + ) + if self.tpot: if self.tpot < TPOT_THRESHOLD: self.assertLessEqual( @@ -945,6 +987,25 @@ def assert_metrics(self, metrics): float(metrics["mean_e2e_latency"]), self.mean_e2e_latency * E2E_TOLERANCE, ) + # Once an accept_rate baseline is set, a missing "Accept length" line + # (e.g. server_info request failed or spec decoding inactive) or a + # missing --speculative-num-draft-tokens arg must fail the test + # instead of being silently skipped. + if getattr(self, "accept_rate", None): + self.assertIsNotNone( + metrics.get("accept_length"), + "accept_length not found in bench_serving output " + "while accept_rate baseline is set", + ) + self.assertIsNotNone( + spec_num_draft_tokens, + "--speculative-num-draft-tokens not found in other_args " + "while accept_rate baseline is set", + ) + self.assertGreaterEqual( + float(metrics["accept_length"]) / spec_num_draft_tokens, + self.accept_rate, + ) class TestNpuPerformanceTestCaseBase(CustomTestCase): @@ -975,6 +1036,9 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): tpot = None mean_e2e_latency = None output_token_throughput = None + # Baseline for accept_length / speculative-num-draft-tokens; None disables + # the assertion. + accept_rate = None dp = None generation_kwargs = None diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py index eabddae0a..bf930ab80 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -1,3 +1,5 @@ +import os +import subprocess import unittest from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( @@ -8,6 +10,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, ) from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST +from sglang.utils import wait_for_server register_npu_ci( est_time=3600, @@ -29,6 +33,7 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = { "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_NPU_USE_MULTI_STREAM": "1", # skip gpu branch "SGLANG_OPT_FP8_WO_A_GEMM": "0", "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", @@ -43,10 +48,13 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = { # DSPARK "SGLANG_RAGGED_VERIFY_MODE": "static", "SGLANG_DSPARK_FAST_KERNEL": "0", + # mtp + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", # deepep "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "DEEPEP_HCCL_BUFFSIZE": "2048", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", + "DEEPEP_HCCL_BUFFSIZE": "2500", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96", "DEEPEP_HYBRID_DEPLOYMENT": "1", # war barrier "SGLANG_ENABLE_WAR_BARRIER": "1", @@ -66,15 +74,15 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ "--watchdog-timeout", 9000, "--mem-fraction-static", - 0.62, + 0.68, "--prefill-max-requests", - 32, + 192, "--max-prefill-tokens", - 131072, + 80000, "--chunked-prefill-size", 131072, "--max-running-requests", - 96, + 192, "--dp-size", 16, "--enable-dp-attention", @@ -87,6 +95,8 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ "--enable-dp-lm-head", "--kv-cache-dtype", "bfloat16", + "--load-balance-method", + "round_robin", "--speculative-algorithm", "DSPARK", "--speculative-draft-model-path", @@ -96,16 +106,17 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ "--speculative-draft-attention-backend", "ascend", "--speculative-num-draft-tokens", - 6, + 7, "--speculative-dspark-block-size", - 5, - "--skip-server-warmup", + 6, "--cuda-graph-bs-decode", 1, 2, 4, - 5, 6, + 8, + 10, + "--disable-radix-cache", ] @@ -136,6 +147,31 @@ class TestNPUDeepSeekV4FlashW8A88PGPQA(TestNpuAccuracyTestCaseBase): timeout = 6000 seed = 1 + @classmethod + def setUpClass(cls): + """Launch server via `python3 -m sglang.launch_server` instead of `sglang serve`.""" + cls._setup_per_case_output() + cls.base_url = DEFAULT_URL_FOR_TEST + env = os.environ.copy() + if cls.envs: + env.update(cls.envs) + + _, host, port = cls.base_url.split(":") + command = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + cls.model, + *[str(x) for x in cls.other_args], + "--host", + host[2:], + "--port", + port, + ] + cls.process = subprocess.Popen(command, env=env) + wait_for_server(cls.base_url, timeout=cls.server_timeout, process=cls.process) + def test_npu_deepseek_v4_flash_w8a8_8p_gpqa(self): """Run NPU accuracy test for DeepSeek-V4-Flash W8A8 8p DSPARK GPQA.""" self.run_accuracy() diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 2de42b09b..228985014 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -1,3 +1,5 @@ +import os +import subprocess import unittest from sglang.test.ascend.e2e.test_npu_performance_utils import ( @@ -7,6 +9,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST +from sglang.utils import wait_for_server register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3", nightly=True) register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3-cann910", nightly=True) @@ -23,9 +27,11 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { "SGLANG_NPU_USE_MULTI_STREAM": "1", # deepep "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "DEEPEP_HCCL_BUFFSIZE": "2048", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "35", + "DEEPEP_HCCL_BUFFSIZE": "2500", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96", "DEEPEP_HYBRID_DEPLOYMENT": "1", + "SGLANG_RAGGED_VERIFY_MODE": "static", + "SGLANG_DSPARK_FAST_KERNEL": "0", # war barrier "SGLANG_ENABLE_WAR_BARRIER": "1", "SGLANG_FORCE_COARSE_WAR_BARRIER": "1", @@ -55,7 +61,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "--device", "npu", "--prefill-max-requests", - 160, + 192, "--max-prefill-tokens", 80000, "--attention-backend", @@ -67,7 +73,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "--chunked-prefill-size", 131072, "--max-running-requests", - 160, + 192, "--dp-size", 16, "--enable-dp-attention", @@ -80,24 +86,28 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "--enable-dp-lm-head", "--kv-cache-dtype", "bfloat16", - "--skip-server-warmup", + "--load-balance-method", + "round_robin", "--cuda-graph-bs-decode", 1, 2, 4, + 6, 8, 10, - # MTP (EAGLE) configuration. + # DSPARK configuration. "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - 2, - "--speculative-eagle-topk", - 1, + "DSPARK", + "--speculative-draft-model-path", + DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, + "--speculative-draft-model-quantization", + "modelslim", + "--speculative-draft-attention-backend", + "ascend", "--speculative-num-draft-tokens", - 3, - "--ep-size", - 16, + 7, + "--speculative-dspark-block-size", + 6, "--disable-radix-cache", ] @@ -111,18 +121,42 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase): other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS dataset_name = "random" - dataset_path = "/root/.cache/modelscope/hub/datasets/gsm8k_deepseekv4/cache0_8000/formal_run1_160_8000_cache0.json" input_len = 8000 output_len = 1000 num_prompts = 160 max_concurrency = 160 random_range_ratio = 1 - warmup_requests = 16 + warmup_requests = 32 request_rate = float("inf") - seed = 1 tpot = 50 max_attempts = 3 - output_token_throughput = 2825 + output_token_throughput = 3100 + accept_rate = 0.5 + + @classmethod + def setUpClass(cls): + """Launch server via `python3 -m sglang.launch_server` instead of `sglang serve`.""" + cls._setup_per_case_output() + cls.base_url = DEFAULT_URL_FOR_TEST + env = os.environ.copy() + if cls.envs: + env.update(cls.envs) + + _, host, port = cls.base_url.split(":") + command = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + cls.model, + *[str(x) for x in cls.other_args], + "--host", + host[2:], + "--port", + port, + ] + cls.process = subprocess.Popen(command, env=env) + wait_for_server(cls.base_url, timeout=cls.timeout, process=cls.process) def test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms(self): """Run NPU performance test for DeepSeek-V4-Flash W8A8 8p in8k out1k."""