minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[NPU] Add GitHub test summary and deduplicate test code. Part 2 (#24689)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Elizaveta Martirosian <elizaveta.martirosian@gmail.com>
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
Elizaveta Martirosian
2026-06-08 18:08:45 +03:00
committed by GitHub
co-authored by github-actions[bot] Elizaveta Martirosian ronnie_zheng
parent 62c505a196
commit 40030d8af8
12 changed files with 384 additions and 682 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/CODEOWNERS
+2 -1
View File
@@ -83,12 +83,13 @@
/sgl-model-gateway/examples/wasm @slin1237 /sgl-model-gateway/examples/wasm @slin1237
/test/registered/prefill_only @sundar24295s @chanh @fortunecookiee /test/registered/prefill_only @sundar24295s @chanh @fortunecookiee
/benchmark/prefill_only/bench_score.py @sundar24295s @chanh @fortunecookiee /benchmark/prefill_only/bench_score.py @sundar24295s @chanh @fortunecookiee
/test/srt/ascend @ping1jing2 @iforgetmyname /test/registered/ascend @ping1jing2 @ssshinigami @e-martirosian
/test/srt/test_modelopt* @Edwardf0t1 /test/srt/test_modelopt* @Edwardf0t1
/python/sglang/srt/layers/gemma4_fused_ops.py @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 @kpham-sgl @pyc96 /python/sglang/srt/layers/gemma4_fused_ops.py @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 @kpham-sgl @pyc96
/python/sglang/srt/function_call/gemma4_detector.py @CatherineSue @JustinTong0323 @kpham-sgl @pyc96 /python/sglang/srt/function_call/gemma4_detector.py @CatherineSue @JustinTong0323 @kpham-sgl @pyc96
/python/sglang/srt/models/gemma4_*.py @kpham-sgl @pyc96 /python/sglang/srt/models/gemma4_*.py @kpham-sgl @pyc96
/python/sglang/srt/multimodal/processors/gemma4.py @mickqian @JustinTong0323 @yhyang201 @yuan-luo @kpham-sgl @pyc96 /python/sglang/srt/multimodal/processors/gemma4.py @mickqian @JustinTong0323 @yhyang201 @yuan-luo @kpham-sgl @pyc96
/python/sglang/test/ascend @ping1jing2 @ssshinigami @e-martirosian
/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96 /docs_new/cookbook/autoregressive/Google/Gemma4.mdx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96
/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96 /docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96
/python/sglang/srt/speculative/ngram_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl /python/sglang/srt/speculative/ngram_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl
python/sglang/test/ascend/gsm8k_ascend_mixin.py
+3 -1
View File
@@ -29,6 +29,7 @@ class GSM8KAscendMixin(ABC):
server_cmd = "" server_cmd = ""
gsm8k_num_shots = 5 gsm8k_num_shots = 5
num_questions = 200 num_questions = 200
gsm8k_parallel = 128
env = { env = {
**os.environ, **os.environ,
@@ -82,13 +83,14 @@ class GSM8KAscendMixin(ABC):
data_path=None, data_path=None,
num_questions=self.num_questions, num_questions=self.num_questions,
max_new_tokens=512, max_new_tokens=512,
parallel=128, parallel=self.gsm8k_parallel,
host="http://127.0.0.1", host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]), port=int(self.base_url.split(":")[-1]),
) )
metrics = run_eval(args) metrics = run_eval(args)
model_metrics["accuracy"] = metrics["accuracy"] model_metrics["accuracy"] = metrics["accuracy"]
model_metrics["output_throughput"] = metrics["output_throughput"] model_metrics["output_throughput"] = metrics["output_throughput"]
model_metrics["latency"] = metrics["latency"]
self.assertGreaterEqual( self.assertGreaterEqual(
metrics["accuracy"], metrics["accuracy"],
accuracy_threshold, accuracy_threshold,
python/sglang/test/ascend/test_ascend_utils.py
+6 -1
View File
@@ -58,6 +58,9 @@ DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join(
DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join( DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base" MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base"
) )
ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS"
)
ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join( ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT" MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT"
) )
@@ -154,6 +157,9 @@ QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE" MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE"
) )
QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8") QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8")
REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8"
)
SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B") SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B")
STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join( STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b" MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b"
@@ -565,7 +571,6 @@ def run_bench_serving(
HEADER = """ HEADER = """
### Models
| Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status | | Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status |
| ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ | | ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ |
""" """
python/sglang/test/ascend/test_mmlu.py
+5 -1
View File
@@ -7,6 +7,8 @@ from sglang.test.run_eval import run_eval
class TestMMLU: class TestMMLU:
mmlu_num_examples = 128
def test_mmlu(self): def test_mmlu(self):
accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00) accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00)
@@ -23,12 +25,14 @@ class TestMMLU:
base_url=self.base_url, base_url=self.base_url,
model=self.model, model=self.model,
eval_name="mmlu", eval_name="mmlu",
num_examples=128, num_examples=self.mmlu_num_examples,
num_threads=32, num_threads=32,
) )
print("Starting mmlu test...") print("Starting mmlu test...")
metrics = run_eval(args) metrics = run_eval(args)
model_metrics["accuracy"] = metrics["score"] model_metrics["accuracy"] = metrics["score"]
model_metrics["latency"] = metrics.get("latency", "-")
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
self.assertGreater(metrics["score"], accuracy_mmlu_threshold) self.assertGreater(metrics["score"], accuracy_mmlu_threshold)
except Exception as e: except Exception as e:
model_metrics["error"] = e model_metrics["error"] = e
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py
+60 -104
View File
@@ -1,25 +1,18 @@
import os import os
import unittest import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH, QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
) )
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k from sglang.test.test_utils import CustomTestCase
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpQwen(CustomTestCase): class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
""" """
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's auto mode enabled, Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's auto mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled. and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
@@ -28,101 +21,64 @@ class TestDeepEpQwen(CustomTestCase):
[Test Target] --moe-a2a-backend, --deepep-mode [Test Target] --moe-a2a-backend, --deepep-mode
""" """
@classmethod model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
def setUpClass(cls): other_args = [
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH "--trust-remote-code",
cls.base_url = DEFAULT_URL_FOR_TEST "--nnodes",
cls.process = popen_launch_server( "1",
cls.model, "--node-rank",
cls.base_url, "0",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, "--attention-backend",
other_args=[ "ascend",
"--trust-remote-code", "--device",
"--nnodes", "npu",
"1", "--quantization",
"--node-rank", "modelslim",
"0", "--max-running-requests",
"--attention-backend", 96,
"ascend", "--context-length",
"--device", 8192,
"npu", "--dtype",
"--quantization", "bfloat16",
"modelslim", "--chunked-prefill-size",
"--max-running-requests", 28672,
96, "--max-prefill-tokens",
"--context-length", 458880,
8192, "--disable-radix-cache",
"--dtype", "--moe-a2a-backend",
"bfloat16", "deepep",
"--chunked-prefill-size", "--deepep-mode",
28672, "auto",
"--max-prefill-tokens", "--tp-size",
458880, 16,
"--disable-radix-cache", "--dp-size",
"--moe-a2a-backend", 4,
"deepep", "--enable-dp-attention",
"--deepep-mode", "--enable-dp-lm-head",
"auto", "--mem-fraction-static",
"--tp-size", 0.7,
16, "--cuda-graph-bs",
"--dp-size", 16,
4, 20,
"--enable-dp-attention", 24,
"--enable-dp-lm-head", ]
"--mem-fraction-static", env = {
0.7, "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"--cuda-graph-bs", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
16, "HCCL_BUFFSIZE": "2100",
20, "HCCL_OP_EXPANSION_MODE": "AIV",
24, "TRANSFORMERS_VERBOSITY": "error",
], **os.environ,
env={ }
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
},
)
@classmethod # MMLU Configs
def tearDownClass(cls): mmlu_num_examples = 8
kill_process_tree(cls.process.pid) accuracy_mmlu_threshold = 0.61 # MMLU accuracy ≥0.61
def test_mmlu(self): # GSM8K Configs
expect_score = 0.61 accuracy = 0.91 # GSM8K accuracy ≥0.91
num_questions = 200
args = SimpleNamespace( gsm8k_num_shots = 8
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
host = "http://127.0.0.1"
port = int(self.base_url.split(":")[-1])
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=host,
port=port,
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__": if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py
+56 -96
View File
@@ -1,20 +1,13 @@
import os import os
import unittest import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH, QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
) )
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k from sglang.test.test_utils import CustomTestCase
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci( register_npu_ci(
est_time=200, est_time=200,
@@ -23,7 +16,7 @@ register_npu_ci(
) )
class TestQwen3Next(CustomTestCase): class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase):
""" """
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's auto mode enabled, and verify that there is Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's auto mode enabled, and verify that there is
no drop in accuracy compared to when DeepEP is not enabled. no drop in accuracy compared to when DeepEP is not enabled.
@@ -32,93 +25,60 @@ class TestQwen3Next(CustomTestCase):
[Test Target] --moe-a2a-backend deepep, --deepep-mode auto [Test Target] --moe-a2a-backend deepep, --deepep-mode auto
""" """
@classmethod model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
def setUpClass(cls): other_args = [
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH "--trust-remote-code",
cls.base_url = DEFAULT_URL_FOR_TEST "--attention-backend",
cls.process = popen_launch_server( "ascend",
cls.model, "--device",
cls.base_url, "npu",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, "--tp-size",
other_args=[ 8,
"--trust-remote-code", "--mem-fraction-static",
"--attention-backend", 0.8,
"ascend", "--max-running-requests",
"--device", 80,
"npu", "--watchdog-timeout",
"--tp-size", 9000,
8, "--disable-radix-cache",
"--mem-fraction-static", "--cuda-graph-bs",
0.8, 2,
"--max-running-requests", 4,
80, 6,
"--watchdog-timeout", 8,
9000, "--max-prefill-tokens",
"--disable-radix-cache", 28672,
"--cuda-graph-bs", "--max-total-tokens",
2, 450560,
4, "--moe-a2a-backend",
6, "deepep",
8, "--deepep-mode",
"--max-prefill-tokens", "auto",
28672, "--chunked-prefill-size",
"--max-total-tokens", -1,
450560, ]
"--moe-a2a-backend", env = {
"deepep", # In NPU scenarios, operators only support BF16 precision.
"--deepep-mode", # This environment variable needs to be set for quantizing weights.
"auto", "SGLANG_DEEPEP_BF16_DISPATCH": "1",
"--chunked-prefill-size", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
-1, "STREAMS_PER_DEVICE": "32",
], "HCCL_OP_EXPANSION_MODE": "AIV",
env={ "HCCL_ALGO": "level0:NA;level1:ring",
# In NPU scenarios, operators only support BF16 precision. "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
# This environment variable needs to be set for quantizing weights. "HCCL_BUFFSIZE": "2000",
"SGLANG_DEEPEP_BF16_DISPATCH": "1", "GDN_ATTN_BACKEND_TRITON": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", **os.environ,
"STREAMS_PER_DEVICE": "32", }
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2000",
"GDN_ATTN_BACKEND_TRITON": "1",
**os.environ,
},
)
@classmethod # MMLU Configs
def tearDownClass(cls): mmlu_num_examples = 8
kill_process_tree(cls.process.pid) accuracy_mmlu_threshold = 0.56 # MMLU accuracy ≥0.56
def test_mmlu(self): # GSM8K Configs
expect_score = 0.56 accuracy = 0.9 # GSM8K accuracy ≥0.9
args = SimpleNamespace( num_questions = 200
base_url=self.base_url, gsm8k_num_shots = 5
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__": if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py
+46 -87
View File
@@ -1,22 +1,16 @@
import os import os
import unittest import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k from sglang.test.test_utils import CustomTestCase
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpDeepseekV32(CustomTestCase): class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"""Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario, """Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario,
its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode low_latency is configured. its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode low_latency is configured.
@@ -25,85 +19,50 @@ class TestDeepEpDeepseekV32(CustomTestCase):
[Test Suggestions] Mixing deployment + low_latency mode is not recommended. [Test Suggestions] Mixing deployment + low_latency mode is not recommended.
""" """
@classmethod model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
def setUpClass(cls): timeout_for_server_launch = 6000
cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH other_args = [
cls.base_url = DEFAULT_URL_FOR_TEST "--trust-remote-code",
cls.process = popen_launch_server( "--tp-size",
cls.model, "16",
cls.base_url, "--quantization",
timeout=6000, "modelslim",
other_args=[ "--moe-a2a-backend",
"--trust-remote-code", "deepep",
"--tp-size", "--deepep-mode",
"16", "low_latency",
"--quantization", "--mem-fraction-static",
"modelslim", 0.82,
"--moe-a2a-backend", "--disable-cuda-graph",
"deepep", "--disable-radix-cache",
"--deepep-mode", "--context-length",
"low_latency", 40960,
"--mem-fraction-static", "--max-prefill-tokens",
0.82, 128,
"--disable-cuda-graph", "--max-total-tokens",
"--disable-radix-cache", 40960,
"--context-length", "--watchdog-timeout",
40960, 900,
"--max-prefill-tokens", ]
128, env = {
"--max-total-tokens", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
40960, "STREAMS_PER_DEVICE": "32",
"--watchdog-timeout", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
900, "HCCL_BUFFSIZE": "2048",
], "HCCL_OP_EXPANSION_MODE": "AIV",
env={ "TASK_QUEUE_ENABLE": "0",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "TRANSFORMERS_VERBOSITY": "error",
"STREAMS_PER_DEVICE": "32", **os.environ,
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", }
"HCCL_BUFFSIZE": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "0",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
},
)
@classmethod # MMLU Configs
def tearDownClass(cls): mmlu_num_examples = 128
kill_process_tree(cls.process.pid) accuracy_mmlu_threshold = 0.85 # MMLU accuracy ≥0.85
def test_mmlu(self): # GSM8K Configs
expect_score = 0.85 accuracy = 0.95 # GSM8K accuracy ≥0.95
args = SimpleNamespace( num_questions = 200
base_url=self.base_url, gsm8k_num_shots = 8
model=self.model,
eval_name="mmlu",
num_examples=128,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.95
args = SimpleNamespace(
num_shots=8,
data_path=None,
timeout=60000,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
print("Starting gsm8k test...")
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__": if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py
+60 -100
View File
@@ -1,25 +1,18 @@
import os import os
import unittest import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH, QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
) )
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k from sglang.test.test_utils import CustomTestCase
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpQwen(CustomTestCase): class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
""" """
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's low_latency mode enabled, Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's low_latency mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled. and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
@@ -29,97 +22,64 @@ class TestDeepEpQwen(CustomTestCase):
[Test Suggestions] Mixing deployment + low_latency mode is not recommended. [Test Suggestions] Mixing deployment + low_latency mode is not recommended.
""" """
@classmethod model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
def setUpClass(cls): other_args = [
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH "--trust-remote-code",
cls.base_url = DEFAULT_URL_FOR_TEST "--nnodes",
cls.process = popen_launch_server( "1",
cls.model, "--node-rank",
cls.base_url, "0",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, "--attention-backend",
other_args=[ "ascend",
"--trust-remote-code", "--device",
"--nnodes", "npu",
"1", "--quantization",
"--node-rank", "modelslim",
"0", "--max-running-requests",
"--attention-backend", 96,
"ascend", "--context-length",
"--device", 8192,
"npu", "--dtype",
"--quantization", "bfloat16",
"modelslim", "--chunked-prefill-size",
"--max-running-requests", 1024,
96, "--max-prefill-tokens",
"--context-length", 458880,
8192, "--disable-radix-cache",
"--dtype", "--moe-a2a-backend",
"bfloat16", "deepep",
"--chunked-prefill-size", "--deepep-mode",
1024, "low_latency",
"--max-prefill-tokens", "--tp-size",
458880, 16,
"--disable-radix-cache", "--dp-size",
"--moe-a2a-backend", 4,
"deepep", "--enable-dp-attention",
"--deepep-mode", "--enable-dp-lm-head",
"low_latency", "--mem-fraction-static",
"--tp-size", 0.7,
16, "--cuda-graph-bs",
"--dp-size", 16,
4, 20,
"--enable-dp-attention", 24,
"--enable-dp-lm-head", ]
"--mem-fraction-static", env = {
0.7, "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"--cuda-graph-bs", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
16, "HCCL_BUFFSIZE": "2100",
20, "HCCL_OP_EXPANSION_MODE": "AIV",
24, "TRANSFORMERS_VERBOSITY": "error",
], **os.environ,
env={ }
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
},
)
@classmethod # MMLU Configs
def tearDownClass(cls): mmlu_num_examples = 8
kill_process_tree(cls.process.pid) accuracy_mmlu_threshold = 0.61 # MMLU accuracy ≥0.61
def test_mmlu(self): # GSM8K Configs
expect_score = 0.61 accuracy = 0.91 # GSM8K accuracy ≥0.91
args = SimpleNamespace( num_questions = 200
base_url=self.base_url, gsm8k_num_shots = 8
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__": if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py
+60 -100
View File
@@ -1,20 +1,13 @@
import os import os
import unittest import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH, QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
) )
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k from sglang.test.test_utils import CustomTestCase
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci( register_npu_ci(
est_time=200, est_time=200,
@@ -23,7 +16,7 @@ register_npu_ci(
) )
class TestQwen3Next(CustomTestCase): class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase):
""" """
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's low_latency mode enabled, and verify that Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's low_latency mode enabled, and verify that
there is no drop in accuracy compared to when DeepEP is not enabled. there is no drop in accuracy compared to when DeepEP is not enabled.
@@ -32,97 +25,64 @@ class TestQwen3Next(CustomTestCase):
[Test Target] --moe-a2a-backend deepep, --deepep-mode low_latency [Test Target] --moe-a2a-backend deepep, --deepep-mode low_latency
""" """
@classmethod model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
def setUpClass(cls): other_args = [
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH "--trust-remote-code",
cls.base_url = DEFAULT_URL_FOR_TEST "--attention-backend",
cls.process = popen_launch_server( "ascend",
cls.model, "--device",
cls.base_url, "npu",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, "--tp-size",
other_args=[ 8,
"--trust-remote-code", "--mem-fraction-static",
"--attention-backend", 0.8,
"ascend", "--max-running-requests",
"--device", 80,
"npu", "--watchdog-timeout",
"--tp-size", 9000,
8, "--disable-radix-cache",
"--mem-fraction-static", "--cuda-graph-bs",
0.8, 2,
"--max-running-requests", 4,
80, 6,
"--watchdog-timeout", 8,
9000, "--chunked-prefill-size",
"--disable-radix-cache", 1024,
"--cuda-graph-bs", "--max-prefill-tokens",
2, 28672,
4, "--max-total-tokens",
6, 450560,
8, "--moe-a2a-backend",
"--chunked-prefill-size", "deepep",
1024, "--deepep-mode",
"--max-prefill-tokens", "low_latency",
28672, ]
"--max-total-tokens", env = {
450560, # The product of the following two environment variables must be greater than --max-prefill-tokens
"--moe-a2a-backend", # divide by dp size
"deepep", "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000",
"--deepep-mode", "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
"low_latency", # In NPU scenarios, operators only support BF16 precision.
], # This environment variable needs to be set for quantizing weights.
env={ "SGLANG_DEEPEP_BF16_DISPATCH": "1",
# The product of the following two environment variables must be greater than --max-prefill-tokens "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
# divide by dp size "STREAMS_PER_DEVICE": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000", "HCCL_OP_EXPANSION_MODE": "AIV",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "10", "HCCL_ALGO": "level0:NA;level1:ring",
# In NPU scenarios, operators only support BF16 precision. "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160",
# This environment variable needs to be set for quantizing weights. "HCCL_BUFFSIZE": "2048",
"SGLANG_DEEPEP_BF16_DISPATCH": "1", "GDN_ATTN_BACKEND_TRITON": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", **os.environ,
"STREAMS_PER_DEVICE": "32", }
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160",
"HCCL_BUFFSIZE": "2048",
"GDN_ATTN_BACKEND_TRITON": "1",
**os.environ,
},
)
@classmethod # MMLU Configs
def tearDownClass(cls): mmlu_num_examples = 8
kill_process_tree(cls.process.pid) accuracy_mmlu_threshold = 0.56 # MMLU accuracy ≥0.56
def test_mmlu(self): # GSM8K Configs
expect_score = 0.56 accuracy = 0.9 # GSM8K accuracy ≥0.9
args = SimpleNamespace( num_questions = 200
base_url=self.base_url, gsm8k_num_shots = 5
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__": if __name__ == "__main__":
test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py
+32 -63
View File
@@ -6,79 +6,47 @@ python3 -m unittest test_ascend_w4a4_quantization.TestAscendW4A4.test_gsm8k
import os import os
import time import time
import unittest import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
import requests import requests
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ascend.test_ascend_utils import (
from sglang.test.few_shot_gsm8k import run_eval ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH,
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
) )
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase, is_in_ci, write_github_step_summary
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ:
os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1,2,3"
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
class TestAscendW4A4(GSM8KAscendMixin, CustomTestCase):
class TestAscendW4A4(CustomTestCase): model = ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH
@classmethod other_args = [
def setUpClass(cls): "--trust-remote-code",
cls.model = "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3-32B-w4a4-LAOS" "--device",
cls.base_url = DEFAULT_URL_FOR_TEST "npu",
cls.process = popen_launch_server( "--attention-backend",
cls.model, "ascend",
cls.base_url, "--tp-size",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, "4",
other_args=[ "--mem-fraction-static",
"--trust-remote-code", "0.8",
"--device", "--cuda-graph-bs",
"npu", "64",
"--attention-backend", "--disable-radix-cache",
"ascend", ]
"--tp-size",
"4",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"64",
"--disable-radix-cache",
],
)
@classmethod env = {
def tearDownClass(cls): **os.environ,
kill_process_tree(cls.process.pid) }
def test_gsm8k(self): # GSM8K Configs
base_url = DEFAULT_URL_FOR_TEST accuracy = 0.80 # GSM8K accuracy ≥0.80
url = urlparse(base_url) num_questions = 1319
args = SimpleNamespace( gsm8k_num_shots = 5
num_shots=5, output_throughput = 1000 # GSM8K output throughput ≥1000 tokens/s
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=64,
host=f"http://{url.hostname}",
port=int(url.port),
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.80)
self.assertGreaterEqual(metrics["output_throughput"], 1000)
def run_decode(self, max_new_tokens): def run_decode(self, max_new_tokens):
response = requests.post( response = requests.post(
@@ -100,11 +68,12 @@ class TestAscendW4A4(CustomTestCase):
tic = time.perf_counter() tic = time.perf_counter()
res = self.run_decode(max_tokens) res = self.run_decode(max_tokens)
tok = time.perf_counter() tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic) throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s") summary = res["text"] + f"\nThroughput: {throughput} tokens/s"
print(summary)
if is_in_ci(): if is_in_ci():
write_github_step_summary(summary + "\nThroughput threshold: 35 tokens/s")
self.assertGreaterEqual(throughput, 35) self.assertGreaterEqual(throughput, 35)
test/registered/ascend/basic_function/quant/test_npu_w8a8_quantization.py
+26 -60
View File
@@ -6,74 +6,39 @@ python3 -m unittest test_ascend_w8a8_quantization.TestAscendW8A8.test_gsm8k
import os import os
import time import time
import unittest import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
import requests import requests
from sglang.srt.utils import kill_process_tree from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ascend.test_ascend_utils import (
from sglang.test.few_shot_gsm8k import run_eval REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH,
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
) )
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase, is_in_ci, write_github_step_summary
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ:
os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1"
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
class TestAscendW8A8CompressedTensors(GSM8KAscendMixin, CustomTestCase):
model = REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--disable-cuda-graph",
"--device",
"npu",
"--attention-backend",
"ascend",
]
env = {
**os.environ,
}
class TestAscendW8A8CompressedTensors(CustomTestCase): # GSM8K Configs
@classmethod accuracy = 0.3 # GSM8K accuracy ≥0.3
def setUpClass(cls): num_questions = 200
# TODO: Move model to CI or Modelscope gsm8k_num_shots = 5
cls.model = "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8" output_throughput = 700 # GSM8K output throughput >=700 tokens/s
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--disable-cuda-graph",
"--device",
"npu",
"--attention-backend",
"ascend",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
base_url = DEFAULT_URL_FOR_TEST
url = urlparse(base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=f"http://{url.hostname}",
port=int(url.port),
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.3)
self.assertGreaterEqual(metrics["output_throughput"], 700)
def run_decode(self, max_new_tokens): def run_decode(self, max_new_tokens):
response = requests.post( response = requests.post(
@@ -95,11 +60,12 @@ class TestAscendW8A8CompressedTensors(CustomTestCase):
tic = time.perf_counter() tic = time.perf_counter()
res = self.run_decode(max_tokens) res = self.run_decode(max_tokens)
tok = time.perf_counter() tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic) throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s") summary = f"\nThroughput: {throughput} tokens/s"
print(res["text"] + summary)
if is_in_ci(): if is_in_ci():
write_github_step_summary(summary + "\nThroughput threshold: 25 tokens/s")
self.assertGreaterEqual(throughput, 25) self.assertGreaterEqual(throughput, 25)
test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py
+28 -68
View File
@@ -1,30 +1,15 @@
import os import os
import unittest import unittest
from types import SimpleNamespace
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import CustomTestCase
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
kill_process_tree,
popen_launch_server,
)
register_npu_ci(est_time=500, suite="nightly-4-npu-a3", nightly=True) register_npu_ci(est_time=500, suite="nightly-4-npu-a3", nightly=True)
QWEN3_30B_MODEL = QWEN3_30B_A3B_WEIGHTS_PATH
GSM8K_MIN_ACCURACY = 0.92
GSM8K_NUM_QUESTIONS = 100
_NPU_ENV_VARS = { class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase):
"ASCEND_USE_FIA": "1",
}
class TestQwen330BAttnCP(CustomTestCase):
"""GSM8K accuracy test for Qwen3-30B-A3B mixed deployment on 4 NPUs. """GSM8K accuracy test for Qwen3-30B-A3B mixed deployment on 4 NPUs.
The test uses: The test uses:
@@ -37,58 +22,33 @@ class TestQwen330BAttnCP(CustomTestCase):
environment variables from the PD GSM8K test. environment variables from the PD GSM8K test.
""" """
@classmethod model = QWEN3_30B_A3B_WEIGHTS_PATH
def setUpClass(cls): other_args = [
cls.model = QWEN3_30B_MODEL "--trust-remote-code",
cls.base_url = DEFAULT_URL_FOR_TEST "--mem-fraction-static",
cls.npu_env = {**os.environ, **_NPU_ENV_VARS} "0.7",
cls.process = popen_launch_server( "--max-running-requests",
cls.model, "32",
cls.base_url, "--attention-backend",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, "ascend",
other_args=[ "--tp-size",
"--trust-remote-code", "4",
"--mem-fraction-static", "--moe-dp-size",
"0.7", "2",
"--max-running-requests", "--attn-cp-size",
"32", "2",
"--attention-backend", "--cuda-graph-max-bs",
"ascend", "32",
"--tp-size", "--enable-prefill-context-parallel",
"4", ]
"--moe-dp-size",
"2",
"--attn-cp-size",
"2",
"--cuda-graph-max-bs",
"32",
"--enable-prefill-context-parallel",
],
env=cls.npu_env,
)
@classmethod env = {**os.environ, "ASCEND_USE_FIA": "1"}
def tearDownClass(cls):
if hasattr(cls, "process") and cls.process is not None:
kill_process_tree(cls.process.pid)
def test_gsm8k_accuracy(self): # GSM8K Configs
args = SimpleNamespace( accuracy = 0.92 # GSM8K accuracy ≥0.92
num_shots=5, gsm8k_parallel = 32
data_path=None, num_questions = 100
num_questions=GSM8K_NUM_QUESTIONS, gsm8k_num_shots = 5
max_new_tokens=512,
parallel=32,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(
"GSM8K accuracy "
f"(mixed TP=4 MOE_DP=2 ATTN_CP=2, {GSM8K_NUM_QUESTIONS} samples): "
f"{metrics['accuracy']:.3f}"
)
self.assertGreaterEqual(metrics["accuracy"], GSM8K_MIN_ACCURACY)
if __name__ == "__main__": if __name__ == "__main__":