minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[NPU] Add GitHub test summary and deduplicate test code. Part 2 (#24689)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Elizaveta Martirosian <elizaveta.martirosian@gmail.com>
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
Elizaveta Martirosian
2026-06-08 18:08:45 +03:00
committed by GitHub
co-authored by github-actions[bot] Elizaveta Martirosian ronnie_zheng
parent 62c505a196
commit 40030d8af8
12 changed files with 384 additions and 682 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/CODEOWNERS
+2 -1
View File
@@ -83,12 +83,13 @@
/sgl-model-gateway/examples/wasm @slin1237
/test/registered/prefill_only @sundar24295s @chanh @fortunecookiee
/benchmark/prefill_only/bench_score.py @sundar24295s @chanh @fortunecookiee
/test/srt/ascend @ping1jing2 @iforgetmyname
/test/registered/ascend @ping1jing2 @ssshinigami @e-martirosian
/test/srt/test_modelopt* @Edwardf0t1
/python/sglang/srt/layers/gemma4_fused_ops.py @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 @kpham-sgl @pyc96
/python/sglang/srt/function_call/gemma4_detector.py @CatherineSue @JustinTong0323 @kpham-sgl @pyc96
/python/sglang/srt/models/gemma4_*.py @kpham-sgl @pyc96
/python/sglang/srt/multimodal/processors/gemma4.py @mickqian @JustinTong0323 @yhyang201 @yuan-luo @kpham-sgl @pyc96
/python/sglang/test/ascend @ping1jing2 @ssshinigami @e-martirosian
/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96
/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96
/python/sglang/srt/speculative/ngram_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl
python/sglang/test/ascend/gsm8k_ascend_mixin.py
+3 -1
View File
@@ -29,6 +29,7 @@ class GSM8KAscendMixin(ABC):
server_cmd = ""
gsm8k_num_shots = 5
num_questions = 200
gsm8k_parallel = 128
env = {
**os.environ,
@@ -82,13 +83,14 @@ class GSM8KAscendMixin(ABC):
data_path=None,
num_questions=self.num_questions,
max_new_tokens=512,
parallel=128,
parallel=self.gsm8k_parallel,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
model_metrics["accuracy"] = metrics["accuracy"]
model_metrics["output_throughput"] = metrics["output_throughput"]
model_metrics["latency"] = metrics["latency"]
self.assertGreaterEqual(
metrics["accuracy"],
accuracy_threshold,
python/sglang/test/ascend/test_ascend_utils.py
+6 -1
View File
@@ -58,6 +58,9 @@ DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join(
DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base"
)
ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS"
)
ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT"
)
@@ -154,6 +157,9 @@ QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE"
)
QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8")
REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8"
)
SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B")
STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b"
@@ -565,7 +571,6 @@ def run_bench_serving(
HEADER = """
### Models
| Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status |
| ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ |
"""
python/sglang/test/ascend/test_mmlu.py
+5 -1
View File
@@ -7,6 +7,8 @@ from sglang.test.run_eval import run_eval
class TestMMLU:
mmlu_num_examples = 128
def test_mmlu(self):
accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00)
@@ -23,12 +25,14 @@ class TestMMLU:
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_examples=self.mmlu_num_examples,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
model_metrics["accuracy"] = metrics["score"]
model_metrics["latency"] = metrics.get("latency", "-")
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
self.assertGreater(metrics["score"], accuracy_mmlu_threshold)
except Exception as e:
model_metrics["error"] = e
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py
+60 -104
View File
@@ -1,25 +1,18 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
)
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpQwen(CustomTestCase):
class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"""
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's auto mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
@@ -28,101 +21,64 @@ class TestDeepEpQwen(CustomTestCase):
[Test Target] --moe-a2a-backend, --deepep-mode
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
28672,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
},
)
model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
28672,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
]
env = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
}
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
# MMLU Configs
mmlu_num_examples = 8
accuracy_mmlu_threshold = 0.61 # MMLU accuracy ≥0.61
def test_mmlu(self):
expect_score = 0.61
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
host = "http://127.0.0.1"
port = int(self.base_url.split(":")[-1])
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=host,
port=port,
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
# GSM8K Configs
accuracy = 0.91 # GSM8K accuracy ≥0.91
num_questions = 200
gsm8k_num_shots = 8
if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py
+56 -96
View File
@@ -1,20 +1,13 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=200,
@@ -23,7 +16,7 @@ register_npu_ci(
)
class TestQwen3Next(CustomTestCase):
class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"""
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's auto mode enabled, and verify that there is
no drop in accuracy compared to when DeepEP is not enabled.
@@ -32,93 +25,60 @@ class TestQwen3Next(CustomTestCase):
[Test Target] --moe-a2a-backend deepep, --deepep-mode auto
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--cuda-graph-bs",
2,
4,
6,
8,
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--chunked-prefill-size",
-1,
],
env={
# In NPU scenarios, operators only support BF16 precision.
# This environment variable needs to be set for quantizing weights.
"SGLANG_DEEPEP_BF16_DISPATCH": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2000",
"GDN_ATTN_BACKEND_TRITON": "1",
**os.environ,
},
)
model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--cuda-graph-bs",
2,
4,
6,
8,
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--chunked-prefill-size",
-1,
]
env = {
# In NPU scenarios, operators only support BF16 precision.
# This environment variable needs to be set for quantizing weights.
"SGLANG_DEEPEP_BF16_DISPATCH": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2000",
"GDN_ATTN_BACKEND_TRITON": "1",
**os.environ,
}
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
# MMLU Configs
mmlu_num_examples = 8
accuracy_mmlu_threshold = 0.56 # MMLU accuracy ≥0.56
def test_mmlu(self):
expect_score = 0.56
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
# GSM8K Configs
accuracy = 0.9 # GSM8K accuracy ≥0.9
num_questions = 200
gsm8k_num_shots = 5
if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py
+46 -87
View File
@@ -1,22 +1,16 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpDeepseekV32(CustomTestCase):
class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"""Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario,
its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode low_latency is configured.
@@ -25,85 +19,50 @@ class TestDeepEpDeepseekV32(CustomTestCase):
[Test Suggestions] Mixing deployment + low_latency mode is not recommended.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=6000,
other_args=[
"--trust-remote-code",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--mem-fraction-static",
0.82,
"--disable-cuda-graph",
"--disable-radix-cache",
"--context-length",
40960,
"--max-prefill-tokens",
128,
"--max-total-tokens",
40960,
"--watchdog-timeout",
900,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"HCCL_BUFFSIZE": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "0",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
},
)
model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
timeout_for_server_launch = 6000
other_args = [
"--trust-remote-code",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--mem-fraction-static",
0.82,
"--disable-cuda-graph",
"--disable-radix-cache",
"--context-length",
40960,
"--max-prefill-tokens",
128,
"--max-total-tokens",
40960,
"--watchdog-timeout",
900,
]
env = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"HCCL_BUFFSIZE": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "0",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
}
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
# MMLU Configs
mmlu_num_examples = 128
accuracy_mmlu_threshold = 0.85 # MMLU accuracy ≥0.85
def test_mmlu(self):
expect_score = 0.85
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.95
args = SimpleNamespace(
num_shots=8,
data_path=None,
timeout=60000,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
print("Starting gsm8k test...")
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
# GSM8K Configs
accuracy = 0.95 # GSM8K accuracy ≥0.95
num_questions = 200
gsm8k_num_shots = 8
if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py
+60 -100
View File
@@ -1,25 +1,18 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
)
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpQwen(CustomTestCase):
class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"""
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's low_latency mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
@@ -29,97 +22,64 @@ class TestDeepEpQwen(CustomTestCase):
[Test Suggestions] Mixing deployment + low_latency mode is not recommended.
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
},
)
model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
]
env = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TRANSFORMERS_VERBOSITY": "error",
**os.environ,
}
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
# MMLU Configs
mmlu_num_examples = 8
accuracy_mmlu_threshold = 0.61 # MMLU accuracy ≥0.61
def test_mmlu(self):
expect_score = 0.61
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
# GSM8K Configs
accuracy = 0.91 # GSM8K accuracy ≥0.91
num_questions = 200
gsm8k_num_shots = 8
if __name__ == "__main__":
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py
+60 -100
View File
@@ -1,20 +1,13 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ascend.test_mmlu import TestMMLU
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=200,
@@ -23,7 +16,7 @@ register_npu_ci(
)
class TestQwen3Next(CustomTestCase):
class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"""
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's low_latency mode enabled, and verify that
there is no drop in accuracy compared to when DeepEP is not enabled.
@@ -32,97 +25,64 @@ class TestQwen3Next(CustomTestCase):
[Test Target] --moe-a2a-backend deepep, --deepep-mode low_latency
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--cuda-graph-bs",
2,
4,
6,
8,
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
],
env={
# The product of the following two environment variables must be greater than --max-prefill-tokens
# divide by dp size
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
# In NPU scenarios, operators only support BF16 precision.
# This environment variable needs to be set for quantizing weights.
"SGLANG_DEEPEP_BF16_DISPATCH": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160",
"HCCL_BUFFSIZE": "2048",
"GDN_ATTN_BACKEND_TRITON": "1",
**os.environ,
},
)
model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--cuda-graph-bs",
2,
4,
6,
8,
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
]
env = {
# The product of the following two environment variables must be greater than --max-prefill-tokens
# divide by dp size
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
# In NPU scenarios, operators only support BF16 precision.
# This environment variable needs to be set for quantizing weights.
"SGLANG_DEEPEP_BF16_DISPATCH": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160",
"HCCL_BUFFSIZE": "2048",
"GDN_ATTN_BACKEND_TRITON": "1",
**os.environ,
}
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
# MMLU Configs
mmlu_num_examples = 8
accuracy_mmlu_threshold = 0.56 # MMLU accuracy ≥0.56
def test_mmlu(self):
expect_score = 0.56
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
# GSM8K Configs
accuracy = 0.9 # GSM8K accuracy ≥0.9
num_questions = 200
gsm8k_num_shots = 5
if __name__ == "__main__":
test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py
+32 -63
View File
@@ -6,79 +6,47 @@ python3 -m unittest test_ascend_w4a4_quantization.TestAscendW4A4.test_gsm8k
import os
import time
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase, is_in_ci, write_github_step_summary
register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ:
os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1,2,3"
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
class TestAscendW4A4(GSM8KAscendMixin, CustomTestCase):
class TestAscendW4A4(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3-32B-w4a4-LAOS"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--device",
"npu",
"--attention-backend",
"ascend",
"--tp-size",
"4",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"64",
"--disable-radix-cache",
],
)
model = ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--device",
"npu",
"--attention-backend",
"ascend",
"--tp-size",
"4",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"64",
"--disable-radix-cache",
]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
env = {
**os.environ,
}
def test_gsm8k(self):
base_url = DEFAULT_URL_FOR_TEST
url = urlparse(base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=64,
host=f"http://{url.hostname}",
port=int(url.port),
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.80)
self.assertGreaterEqual(metrics["output_throughput"], 1000)
# GSM8K Configs
accuracy = 0.80 # GSM8K accuracy ≥0.80
num_questions = 1319
gsm8k_num_shots = 5
output_throughput = 1000 # GSM8K output throughput ≥1000 tokens/s
def run_decode(self, max_new_tokens):
response = requests.post(
@@ -100,11 +68,12 @@ class TestAscendW4A4(CustomTestCase):
tic = time.perf_counter()
res = self.run_decode(max_tokens)
tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s")
summary = res["text"] + f"\nThroughput: {throughput} tokens/s"
print(summary)
if is_in_ci():
write_github_step_summary(summary + "\nThroughput threshold: 35 tokens/s")
self.assertGreaterEqual(throughput, 35)
test/registered/ascend/basic_function/quant/test_npu_w8a8_quantization.py
+26 -60
View File
@@ -6,74 +6,39 @@ python3 -m unittest test_ascend_w8a8_quantization.TestAscendW8A8.test_gsm8k
import os
import time
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase, is_in_ci, write_github_step_summary
register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ:
os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1"
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
class TestAscendW8A8CompressedTensors(GSM8KAscendMixin, CustomTestCase):
model = REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--disable-cuda-graph",
"--device",
"npu",
"--attention-backend",
"ascend",
]
env = {
**os.environ,
}
class TestAscendW8A8CompressedTensors(CustomTestCase):
@classmethod
def setUpClass(cls):
# TODO: Move model to CI or Modelscope
cls.model = "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--disable-cuda-graph",
"--device",
"npu",
"--attention-backend",
"ascend",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
base_url = DEFAULT_URL_FOR_TEST
url = urlparse(base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=f"http://{url.hostname}",
port=int(url.port),
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.3)
self.assertGreaterEqual(metrics["output_throughput"], 700)
# GSM8K Configs
accuracy = 0.3 # GSM8K accuracy ≥0.3
num_questions = 200
gsm8k_num_shots = 5
output_throughput = 700 # GSM8K output throughput >=700 tokens/s
def run_decode(self, max_new_tokens):
response = requests.post(
@@ -95,11 +60,12 @@ class TestAscendW8A8CompressedTensors(CustomTestCase):
tic = time.perf_counter()
res = self.run_decode(max_tokens)
tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s")
summary = f"\nThroughput: {throughput} tokens/s"
print(res["text"] + summary)
if is_in_ci():
write_github_step_summary(summary + "\nThroughput threshold: 25 tokens/s")
self.assertGreaterEqual(throughput, 25)
test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py
+28 -68
View File
@@ -1,30 +1,15 @@
import os
import unittest
from types import SimpleNamespace
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
kill_process_tree,
popen_launch_server,
)
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=500, suite="nightly-4-npu-a3", nightly=True)
QWEN3_30B_MODEL = QWEN3_30B_A3B_WEIGHTS_PATH
GSM8K_MIN_ACCURACY = 0.92
GSM8K_NUM_QUESTIONS = 100
_NPU_ENV_VARS = {
"ASCEND_USE_FIA": "1",
}
class TestQwen330BAttnCP(CustomTestCase):
class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase):
"""GSM8K accuracy test for Qwen3-30B-A3B mixed deployment on 4 NPUs.
The test uses:
@@ -37,58 +22,33 @@ class TestQwen330BAttnCP(CustomTestCase):
environment variables from the PD GSM8K test.
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_30B_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.npu_env = {**os.environ, **_NPU_ENV_VARS}
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--mem-fraction-static",
"0.7",
"--max-running-requests",
"32",
"--attention-backend",
"ascend",
"--tp-size",
"4",
"--moe-dp-size",
"2",
"--attn-cp-size",
"2",
"--cuda-graph-max-bs",
"32",
"--enable-prefill-context-parallel",
],
env=cls.npu_env,
)
model = QWEN3_30B_A3B_WEIGHTS_PATH
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.7",
"--max-running-requests",
"32",
"--attention-backend",
"ascend",
"--tp-size",
"4",
"--moe-dp-size",
"2",
"--attn-cp-size",
"2",
"--cuda-graph-max-bs",
"32",
"--enable-prefill-context-parallel",
]
@classmethod
def tearDownClass(cls):
if hasattr(cls, "process") and cls.process is not None:
kill_process_tree(cls.process.pid)
env = {**os.environ, "ASCEND_USE_FIA": "1"}
def test_gsm8k_accuracy(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=GSM8K_NUM_QUESTIONS,
max_new_tokens=512,
parallel=32,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(
"GSM8K accuracy "
f"(mixed TP=4 MOE_DP=2 ATTN_CP=2, {GSM8K_NUM_QUESTIONS} samples): "
f"{metrics['accuracy']:.3f}"
)
self.assertGreaterEqual(metrics["accuracy"], GSM8K_MIN_ACCURACY)
# GSM8K Configs
accuracy = 0.92 # GSM8K accuracy ≥0.92
gsm8k_parallel = 32
num_questions = 100
gsm8k_num_shots = 5
if __name__ == "__main__":