minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

test(npu): remove obsolete npu pr nightly cases, move accuracy cases to full (#37990)

Browse Source 
Co-authored-by: Sugar920 <Sugar920@users.noreply.github.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
Sugar920
2026-09-05 20:58:52 +08:00
committed by GitHub
co-authored by Sugar920 Claude Code
parent a18106bbc3
commit 4b802c052b
60 changed files with 283 additions and 3134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=6500,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py
+1 -1
View File
@@ -6,7 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=6500, suite="nightly-acc-2-npu-a3", nightly=True)
register_npu_ci(est_time=6500, suite="full-acc-2-npu-a3", nightly=True)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=7200,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3700,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3700,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=2800,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-16-npu-a3",
suite="full-acc-16-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-4-npu-a3",
suite="full-acc-4-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-1-npu-a2",
suite="full-4-npu-a2",
nightly=True,
)
test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=2800, suite="nightly-acc-2-npu-a3", nightly=True)
register_npu_ci(est_time=2800, suite="full-acc-2-npu-a3", nightly=True)
QWEN3_5_9B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=4800,
suite="nightly-acc-4-npu-a3",
suite="full-acc-4-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True)
register_npu_ci(est_time=4800, suite="full-acc-4-npu-a3", nightly=True)
QWEN3_VL_30B_A3B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
+1 -1
View File
@@ -10,7 +10,7 @@ from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=8400,
suite="nightly-acc-2-npu-a3",
suite="full-acc-2-npu-a3",
nightly=True,
)
test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=4800, suite="nightly-acc-4-npu-a3", nightly=True)
register_npu_ci(est_time=4800, suite="full-acc-4-npu-a3", nightly=True)
QWEN3_VL_8B_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
+1 -1
View File
@@ -9,7 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=12000, suite="nightly-acc-2-npu-a3", nightly=True)
register_npu_ci(est_time=12000, suite="full-acc-2-npu-a3", nightly=True)
_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request"
test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py
-111
View File
@@ -1,111 +0,0 @@
import multiprocessing as mp
import unittest
from typing import Optional
import torch
from transformers import AutoConfig, AutoTokenizer
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase, get_similarities
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
disabled="embeddings are not all close",
)
DEFAULT_PROMPTS = [
"The capital of the United Kingdom is",
"Today is a sunny day and I like",
"AI is a field of computer science focused on",
]
MODELS = [
("/root/.cache/modelscope/hub/models/bge-large-en-v1.5", 1, 1e-5),
]
TORCH_DTYPES = [torch.float16]
class TestEmbeddingModels(CustomTestCase):
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def _truncate_prompts(self, prompts, model_path):
config = AutoConfig.from_pretrained(model_path)
max_length = getattr(config, "max_position_embeddings", 2048)
tokenizer = AutoTokenizer.from_pretrained(model_path)
truncated_prompts = []
for prompt in prompts:
tokens = tokenizer(prompt, return_tensors="pt", truncation=False)
if len(tokens.input_ids[0]) > max_length:
truncated_text = tokenizer.decode(
tokens.input_ids[0][: max_length - 1], skip_special_tokens=True
)
truncated_prompts.append(truncated_text)
else:
truncated_prompts.append(prompt)
return truncated_prompts
def assert_close_prefill_logits(
self,
prompts,
model_path,
tp_size,
torch_dtype,
prefill_tolerance,
matryoshka_dim: Optional[int] = None,
) -> None:
truncated_prompts = self._truncate_prompts(prompts, model_path)
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="embedding",
matryoshka_dim=matryoshka_dim,
) as hf_runner:
hf_outputs = hf_runner.forward(truncated_prompts)
attention_backend = "ascend"
with SRTRunner(
model_path,
tp_size=tp_size,
torch_dtype=torch_dtype,
model_type="embedding",
attention_backend=attention_backend,
json_model_override_args=(
{"matryoshka_dimensions": [matryoshka_dim]} if matryoshka_dim else None
),
) as srt_runner:
srt_outputs = srt_runner.forward(
truncated_prompts, dimensions=matryoshka_dim
)
for i in range(len(prompts)):
hf_logits = torch.Tensor(hf_outputs.embed_logits[i])
srt_logits = torch.Tensor(srt_outputs.embed_logits[i])
similarity = torch.tensor(get_similarities(hf_logits, srt_logits))
print("similarity diff", abs(similarity - 1))
if len(prompts[i]) <= 1000:
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
def test_prefill_logits(self):
models_to_test = MODELS
for model, tp_size, prefill_tolerance in models_to_test:
for torch_dtype in TORCH_DTYPES:
self.assert_close_prefill_logits(
DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance
)
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_ascend_dbrx_instruct.py
-26
View File
@@ -1,26 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-8-npu-a3", nightly=True)
class TestDbrx(GSM8KAscendMixin, CustomTestCase):
model = "/root/.cache/modelscope/hub/models/AI-ModelScope/dbrx-instruct"
accuracy = 0.735
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--tp-size",
"8",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py
-37
View File
@@ -1,37 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import BAICHUAN2_13B_CHAT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestBaichuan(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the baichuan-inc/Baichuan2-13B-Chat model on the GSM8K dataset is no less than 0.48.
[Test Category] Model
[Test Target] baichuan-inc/Baichuan2-13B-Chat
"""
model = BAICHUAN2_13B_CHAT_WEIGHTS_PATH
accuracy = 0.48
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--max-running-requests",
"128",
"--disable-radix-cache",
"--chunked-prefill-size",
"-1",
]
gsm8k_num_shots = 1
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py
-40
View File
@@ -1,40 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH,
C4AI_COMMAND_R_V01_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)
class TestC4AI(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the CohereForAI/c4ai-command-r-v01 model on the GSM8K dataset is no less than 0.55.
[Test Category] Model
[Test Target] CohereForAI/c4ai-command-r-v01
"""
model = C4AI_COMMAND_R_V01_WEIGHTS_PATH
accuracy = 0.55
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--chat-template",
C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH,
"--tp-size",
"2",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_chatglm2_6b.py
-33
View File
@@ -1,33 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import CHATGLM2_6B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestChatGlm2(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ZhipuAI/chatglm2-6b model on the GSM8K dataset is no less than 0.25.
[Test Category] Model
[Test Target] ZhipuAI/chatglm2-6b
"""
model = CHATGLM2_6B_WEIGHTS_PATH
accuracy = 0.25
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_exaone_3.py
-34
View File
@@ -1,34 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestEXAONE(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct model on the GSM8K dataset is no less than 0.8.
[Test Category] Model
[Test Target] LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct
"""
model = EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH
# Allow 1% tolerance for the accuracy threshold
accuracy = round(0.8 * 0.99, 3)
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_glm4_9b_chat.py
-27
View File
@@ -1,27 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import GLM_4_9B_CHAT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
class TestGLM49BChat(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ZhipuAI/glm-4-9b-chat model on the GSM8K dataset is no less than 0.77.
[Test Category] Model
[Test Target] ZhipuAI/glm-4-9b-chat
"""
model = GLM_4_9B_CHAT_WEIGHTS_PATH
accuracy = 0.77
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py
-25
View File
@@ -1,25 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import (
GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestGranite(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ibm-granite/granite-3.0-3b-a800m-instruct model on the GSM8K dataset is no less than 0.38.
[Test Category] Model
[Test Target] ibm-granite/granite-3.0-3b-a800m-instruct
"""
model = GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH
accuracy = 0.38
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_granite_3_1_8b.py
-23
View File
@@ -1,23 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestGranite(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the ibm-granite/granite-3.1-8b-instruct model on the GSM8K dataset is no less than 0.695.
[Test Category] Model
[Test Target] ibm-granite/granite-3.1-8b-instruct
"""
model = GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH
accuracy = 0.695
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_grok_2.py
-33
View File
@@ -1,33 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-16-npu-a3",
nightly=True,
)
class TestGrok2(GSM8KAscendMixin, CustomTestCase):
model = "/root/.cache/modelscope/hub/models/huihui-ai/grok-2"
accuracy = 0.91
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-radix-cache",
"--disable-cuda-graph",
"--tokenizer-path",
"/root/.cache/modelscope/hub/models/huihui-ai/grok-2/tokenizer.tok.json",
"--tp-size",
"16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_internlm2_7b.py
-29
View File
@@ -1,29 +0,0 @@
import os
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import INTERNLM2_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
class TestInternlm2(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the Shanghai_AI_Laboratory/internlm2-7b model on the GSM8K dataset is no less than 0.585.
[Test Category] Model
[Test Target] Shanghai_AI_Laboratory/internlm2-7b
"""
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
model = INTERNLM2_7B_WEIGHTS_PATH
accuracy = 0.585
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_llama_2_7b.py
-23
View File
@@ -1,23 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestLlama(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the LLM-Research/Llama-2-7B model on the GSM8K dataset is no less than 0.18.
[Test Category] Model
[Test Target] LLM-Research/Llama-2-7B
"""
model = LLAMA_2_7B_WEIGHTS_PATH
accuracy = 0.18
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py
-37
View File
@@ -1,37 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import LLAMA_2_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True)
class TestLlama(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the LLM-Research/Llama-2-7B model on the GSM8K dataset with tp communications quantization is no less than 0.18.
[Test Category] Model
[Test Target] LLM-Research/Llama-2-7B
"""
model = LLAMA_2_7B_WEIGHTS_PATH
accuracy = 0.18
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
0.8,
"--max-running-requests",
32,
"--attention-backend",
"ascend",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
2,
"--enable-quant-communications",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_minicpm3_4b.py
-37
View File
@@ -1,37 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import MINICPM3_4B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestMiniCPM3(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the OpenBMB/MiniCPM3-4B model on the GSM8K dataset is no less than 0.69.
[Test Category] Model
[Test Target] OpenBMB/MiniCPM3-4B
"""
model = MINICPM3_4B_WEIGHTS_PATH
accuracy = 0.69
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--disable-radix-cache",
"--disable-overlap-schedule",
"--max-running-requests",
"128",
"--chunked-prefill-size",
"-1",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_mistral_7b.py
-23
View File
@@ -1,23 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the mistralai/Mistral-7B-Instruct-v0.2 model on the GSM8K dataset is no less than 0.375.
[Test Category] Model
[Test Target] mistralai/Mistral-7B-Instruct-v0.2
"""
model = MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH
accuracy = 0.375
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py
-29
View File
@@ -1,29 +0,0 @@
import os
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import PERSIMMON_8B_CHAT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
class TestPersimmon8BChat(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the Howeee/persimmon-8b-chat model on the GSM8K dataset is no less than 0.17.
[Test Category] Model
[Test Target] Howeee/persimmon-8b-chat
"""
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
model = PERSIMMON_8B_CHAT_WEIGHTS_PATH
accuracy = 0.17
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_smollm_1_7b.py
-33
View File
@@ -1,33 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import SMOLLM_1_7B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestSmolLM(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the HuggingFaceTB/SmolLM-1.7B model on the GSM8K dataset is no less than 0.05.
[Test Category] Model
[Test Target] HuggingFaceTB/SmolLM-1.7B
"""
model = SMOLLM_1_7B_WEIGHTS_PATH
accuracy = 0.05
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py
-34
View File
@@ -1,34 +0,0 @@
import unittest
from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
from sglang.test.ascend.test_ascend_utils import STABLELM_2_1_6B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestStablelm(GSM8KAscendMixin, CustomTestCase):
"""Testcase: Verify that the inference accuracy of the stabilityai/stablelm-2-1_6b model on the GSM8K dataset is no less than 0.195.
[Test Category] Model
[Test Target] stabilityai/stablelm-2-1_6b
"""
model = STABLELM_2_1_6B_WEIGHTS_PATH
accuracy = 0.195
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--tp-size",
1,
"--enable-torch-compile",
]
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py
-113
View File
@@ -1,113 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
KIMI_K2_6_EAGLE3_MODEL_PATH,
KIMI_K2_6_W4A8_MODEL_PATH,
TestNpuPerfMultiNodePdMixTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=1800,
suite="full-8-npu-a3",
nightly=True,
disabled="Currently it is executed by the npu performance workflow.",
)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
"DEEPEP_HCCL_BUFFSIZE": "4400",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--tp-size",
32,
"--nnodes",
2,
"--mem-fraction-static",
0.68,
"--max-running-requests",
32,
"--chunked-prefill-size",
32768,
"--context-length",
75000,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--sampling-backend",
"ascend",
"--enable-dp-attention",
"--dp-size",
32,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs",
1,
"--disable-radix-cache",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
KIMI_K2_6_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--reasoning-parser",
"kimi_k2",
"--tool-call-parser",
"kimi_k2",
]
MODEL_CONFIG = {
"model_path": KIMI_K2_6_W4A8_MODEL_PATH,
"other_args": OTHER_ARGS,
"node_envs": ENVS,
}
class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model_config = MODEL_CONFIG
dataset_name = "random"
max_concurrency = 32
num_prompts = 32
input_len = 64000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 100
output_token_throughput = 160
def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py
-126
View File
@@ -1,126 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
KIMI_K2_6_EAGLE3_MODEL_PATH,
KIMI_K2_6_W4A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True)
KIMI_K2_6_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
"STREAMS_PER_DEVICE": "32",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96",
"DEEPEP_HCCL_BUFFSIZE": "1200",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_NPU_USE_MLAPO": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "1",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64",
}
KIMI_K2_6_OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--tp-size",
16,
"--mem-fraction-static",
0.865,
"--max-running-requests",
80,
"--chunked-prefill-size",
32768,
"--context-length",
6144,
"--max-prefill-tokens",
65536,
"--max-total-tokens",
32256,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--sampling-backend",
"ascend",
"--enable-dp-attention",
"--dp-size",
16,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs-decode",
1,
2,
3,
4,
5,
"--disable-radix-cache",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
KIMI_K2_6_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--speculative-draft-model-quantization",
"unquant",
"--prefill-delayer-max-delay-passes",
200,
"--enable-prefill-delayer",
"--reasoning-parser",
"kimi_k2",
"--tool-call-parser",
"kimi_k2",
]
class TestKimiK25W4A8(
TestNpuPerformanceTestCaseBase,
):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
max_attempts = 5
model = KIMI_K2_6_W4A8_MODEL_PATH
other_args = KIMI_K2_6_OTHER_ARGS
envs = KIMI_K2_6_ENVS
backend = "sglang"
dataset_name = "random"
max_concurrency = 64
num_prompts = 256
input_len = 3500
output_len = 1500
random_range_ratio = 1
warmup_requests = 0
seed = 1
tpot = 20
output_token_throughput = 1900
def test_kimi_k2_6_w4a8(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py
-191
View File
@@ -1,191 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MIMO_V2_FLASH_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
PREFILL_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"ASCEND_USE_FIA": "1",
}
DECODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"DEEPEP_HCCL_BUFFSIZE": "800",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"SGLANG_PROFILE_WITH_STACK": "True",
"ASCEND_USE_FIA": "1",
}
PREFILL_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--nnodes",
"1",
"--node-rank",
"0",
"--chunked-prefill-size",
8192,
"--trust-remote-code",
"--max-running-requests",
64,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--disaggregation-mode",
"prefill",
"--disaggregation-transfer-backend",
"ascend",
"--disable-radix-cache",
"--disable-cuda-graph",
"--disable-piecewise-cuda-graph",
"--dp-size",
"2",
]
DECODE_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
16,
"--nnodes",
"1",
"--node-rank",
"0",
"--trust-remote-code",
"--max-running-requests",
64,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
20,
24,
28,
32,
"--disaggregation-mode",
"decode",
"--disaggregation-transfer-backend",
"ascend",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--enable-multi-layer-eagle",
"--disable-radix-cache",
"--dp-size",
"2",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
]
ROUTER_ARGS = [
"--health-check-interval-secs",
"3600",
"--mini-lb",
]
MODEL_CONFIG = {
"model_path": MIMO_V2_FLASH_MODEL_PATH,
"prefill_args": PREFILL_ARGS,
"decode_args": DECODE_ARGS,
"prefill_envs": PREFILL_ENVS,
"decode_envs": DECODE_ENVS,
"router_args": ROUTER_ARGS,
"router_envs": {},
}
class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1_TTFT_5s(
TestNpuPerfMultiNodePdSepTestCaseBase
):
"""Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1, 0 cache, TTFT=5s"""
model_config = MODEL_CONFIG
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_name = "random"
max_concurrency = 64
num_prompts = 128
request_rate = 0.4
input_len = 16000
output_len = 1
random_range_ratio = 1
seed = 1
ttft = 5000
def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1_ttft_5s(self):
"""Run NPU performance test for 1P+1D 16p with 16k input, 1 output, 0 cache, TTFT=5s"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py
-187
View File
@@ -1,187 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MIMO_V2_FLASH_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
PREFILL_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"ASCEND_USE_FIA": "1",
}
DECODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"DEEPEP_HCCL_BUFFSIZE": "800",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "32",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_BF16_DISPATCH": "0",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_CONNECT_TIMEOUT": "1800",
"SGLANG_PROFILE_WITH_STACK": "True",
"ASCEND_USE_FIA": "1",
}
PREFILL_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--nnodes",
"1",
"--node-rank",
"0",
"--chunked-prefill-size",
8192,
"--trust-remote-code",
"--max-running-requests",
64,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--disaggregation-mode",
"prefill",
"--disaggregation-transfer-backend",
"ascend",
"--disable-radix-cache",
"--disable-cuda-graph",
"--disable-piecewise-cuda-graph",
"--dp-size",
"2",
]
DECODE_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
16,
"--nnodes",
"1",
"--node-rank",
"0",
"--trust-remote-code",
"--max-running-requests",
32,
"--mem-fraction-static",
0.8,
"--swa-full-tokens-ratio",
0.3,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
"--disaggregation-mode",
"decode",
"--disaggregation-transfer-backend",
"ascend",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--enable-multi-layer-eagle",
"--disable-radix-cache",
"--dp-size",
"2",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
]
ROUTER_ARGS = [
"--health-check-interval-secs",
"3600",
"--mini-lb",
]
MODEL_CONFIG = {
"model_path": MIMO_V2_FLASH_MODEL_PATH,
"prefill_args": PREFILL_ARGS,
"decode_args": DECODE_ARGS,
"prefill_envs": PREFILL_ENVS,
"decode_envs": DECODE_ENVS,
"router_args": ROUTER_ARGS,
"router_envs": {},
}
class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1k_TTFT_5s(
TestNpuPerfMultiNodePdSepTestCaseBase
):
"""Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1000, 0 cache, TPOT=20ms"""
model_config = MODEL_CONFIG
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_name = "random"
max_concurrency = 32
num_prompts = 128
request_rate = float("inf")
input_len = 16000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 20
def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1k_tpot_20ms(self):
"""Run NPU performance test for 1P+1D 16p with 16k input, 1k output, 0 cache, TPOT=20ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
-118
View File
@@ -1,118 +0,0 @@
import os
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
BENCHMARK_TOOL_DEFAULT,
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
MINIMAX_M2_5_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-8-npu-a3", nightly=True)
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "999999999",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
}
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
"--tp-size",
8,
"--mem-fraction-static",
0.63,
"--max-running-requests",
24,
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
"--enable-prefill-delayer",
"--prefill-delayer-max-delay-passes",
30,
"--prefill-max-requests",
10,
"--chunked-prefill-size",
67072,
"--max-prefill-token",
67000,
"--cuda-graph-bs",
2,
4,
8,
12,
16,
18,
20,
22,
24,
"--moe-a2a-backend",
"ascend_fuseep",
"--fuseep-mode",
2,
"--deepep-mode",
"auto",
"--quantization",
"modelslim",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--dtype",
"bfloat16",
"--trust-remote-code",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
]
class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms(
TestNpuPerformanceTestCaseBase
):
"""MiniMax-M2.5-w8a8 4p (4 cards) 64k input 1k output with 90% prefix cache performance test"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS
envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 24
num_prompts = 96
input_len = 65536
output_len = 1024
random_range_ratio = 1
seed = 1
repeat_rate = 0.9
tpot = 50
output_token_throughput = 390.5859
request_rate = float("inf")
def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self):
"""Run MiniMax-M2.5-w8a8 4p 64k/1k prefix90 performance test"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py
-117
View File
@@ -1,117 +0,0 @@
import os
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
MINIMAX_M2_5_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-perf-16-npu-a3",
nightly=True,
)
MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"TASK_QUEUE_ENABLE": "1",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
}
MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [
"--tp-size",
16,
"--enable-dp-attention",
"--dp-size",
16,
"--mem-fraction-static",
0.75,
"--max-running-requests",
320,
"--disable-radix-cache",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
"--prefill-delayer-max-delay-passes",
500,
"--enable-prefill-delayer",
"--chunked-prefill-size",
196608,
"--max-prefill-token",
8192,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
20,
"--moe-a2a-backend",
"ascend_fuseep",
"--fuseep-mode",
2,
"--quantization",
"modelslim",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--dtype",
"bfloat16",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
]
class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_HighThroughput(
TestNpuPerformanceTestCaseBase
):
"""Test NPU performance for MiniMax-M2.5-w8a8 8p single node high throughput in3k5 out1k5"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS
envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS
dataset_name = "random"
max_concurrency = 320
num_prompts = 1280
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 5717.58
def test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_high_throughput(self):
"""Run NPU performance test for MiniMax-M2.5-w8a8 high throughput"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
-105
View File
@@ -1,105 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_8B_EAGLE_MODEL_PATH,
QWEN3_8B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True)
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
}
QWEN3_8B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
70,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
16384,
"--tp-size",
1,
"--mem-fraction-static",
0.85,
"--cuda-graph-bs",
8,
12,
24,
36,
48,
51,
55,
60,
63,
64,
66,
68,
70,
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen8B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_8B_W8A8_MODEL_PATH
other_args = QWEN3_8B_OTHER_ARGS
envs = QWEN3_8B_ENVS
dataset_name = "random"
max_concurrency = 64
num_prompts = 256
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 37
output_token_throughput = 1586
def test_qwen3_8b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py
-93
View File
@@ -1,93 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_8B_EAGLE_MODEL_PATH,
QWEN3_8B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-perf-2-npu-a3",
nightly=True,
)
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
}
QWEN3_8B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
16,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
-1,
"--tp-size",
2,
"--mem-fraction-static",
0.894,
"--cuda-graph-bs",
1,
5,
15,
16,
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen8B(TestNpuPerformanceTestCaseBase):
max_attempts = 5
model = QWEN3_8B_W8A8_MODEL_PATH
other_args = QWEN3_8B_OTHER_ARGS
envs = QWEN3_8B_ENVS
dataset_name = "random"
max_concurrency = 16
num_prompts = 16
input_len = 6144
output_len = 1500
random_range_ratio = 1
tpot = 11.79
output_token_throughput = 1040.96
def test_qwen3_8b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
-122
View File
@@ -1,122 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_235B_A22B_EAGLE_MODEL_PATH,
QWEN3_235B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-16-npu-a3", nightly=True)
QWEN3_235B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"DEEPEP_HCCL_BUFFSIZE": "570",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "16",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_BS": "27",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416",
}
QWEN3_235B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
432,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
94208,
"--max-prefill-tokens",
458880,
"--sampling-backend",
"ascend",
"--ep-dispatch-algorithm",
"static",
"--init-expert-location",
"/root/.cache/modelscope/hub/models/hot_map/235B_3_5k_decode.pt",
"--disable-radix-cache",
"--moe-a2a-backend",
"ascend_fuseep",
"--fuseep-mode",
2,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_235B_A22B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--speculative-draft-model-quantization",
"unquant",
"--tp",
"16",
"--dp-size",
"16",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"1",
"2",
"4",
"8",
"16",
"20",
"24",
"26",
"27",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen25",
]
class TestQwen235B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_235B_W8A8_MODEL_PATH
other_args = QWEN3_235B_OTHER_ARGS
envs = QWEN3_235B_ENVS
dataset_name = "random"
max_concurrency = 432
num_prompts = int(max_concurrency) * 4
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50.1
output_token_throughput = 6189
def test_qwen3_235b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH,
QWEN3_A3B_EAGLE_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-2-npu-a3", nightly=True)
QWEN3_30B_A3B_ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"DEEPEP_HCCL_BUFFSIZE": "400",
}
QWEN3_30B_A3B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
162,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_A3B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
1,
5,
15,
40,
70,
100,
120,
130,
140,
146,
150,
154,
156,
158,
160,
162,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen30B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH
other_args = QWEN3_30B_A3B_OTHER_ARGS
envs = QWEN3_30B_A3B_ENVS
dataset_name = "random"
max_concurrency = 160
num_prompts = int(max_concurrency) * 4
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 3200
def test_qwen3_30b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
-94
View File
@@ -1,94 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-perf-16-npu-a3",
nightly=True,
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
1,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
65536,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--tp-size",
16,
"--mem-fraction-static",
0.72,
"--cuda-graph-bs",
1,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 1
num_prompts = 1
input_len = 18000
output_len = 4000
random_range_ratio = 1
seed = 1
tpot = 6
output_token_throughput = 171
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
-107
View File
@@ -1,107 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
101,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
16,
32,
64,
72,
88,
90,
92,
94,
96,
97,
98,
99,
100,
101,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 100
num_prompts = 400
input_len = 3584
output_len = 1536
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 1600
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
DEFAULT_URL_FOR_TEST,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-1-npu-a2",
nightly=True,
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
101,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
16,
32,
64,
72,
88,
90,
92,
94,
96,
97,
98,
99,
100,
101,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
base_url = DEFAULT_URL_FOR_TEST
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 100
num_prompts = 400
input_len = 3584
output_len = 1536
random_range_ratio = 1
seed = 1
tpot = 55
output_token_throughput = 1500
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py
-100
View File
@@ -1,100 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_64K_1K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_64K_1K_OTHER_ARGS = [
"--tp-size",
4,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
50000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
28,
"--max-mamba-cache-size",
50,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
2,
4,
6,
"--enable-multimodal",
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS
envs = QWEN3_6_27B_64K_1K_ENVS
dataset_name = "random"
max_concurrency = 6
num_prompts = 24
input_len = 64000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 122.6
def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py
-110
View File
@@ -1,110 +0,0 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_NEXT_80B_A3B_MODEL_PATH,
QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-4-npu-a3", nightly=True)
QWEN3_NEXT_80B_A3B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "400",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "0",
"SGLANG_WARMUP_TIMEOUT": "3600",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "2000",
"ZBCCL_LOCAL_MEM_SIZE": "60416",
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
"ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669",
"ZBCCL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True",
"ZBCCL_ENABLE_GRAPH": "1",
}
QWEN3_NEXT_80B_A3B_OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--page-size",
128,
"--tp-size",
4,
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.85,
"--disable-radix-cache",
"--max-prefill-tokens",
28672,
"--context-length",
81920,
"--max-total-tokens",
122304,
"--dp-size",
2,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-running-requests",
16,
"--cuda-graph-bs",
2,
4,
8,
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-draft-model-path",
QWEN3_NEXT_80B_A3B_MODEL_PATH,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase):
max_attempts = 5
model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH
other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS
envs = QWEN3_NEXT_80B_A3B_ENVS
dataset_name = "random"
max_concurrency = 16
num_prompts = 16
input_len = 6144
output_len = 1500
random_range_ratio = 1
tpot = 15.62
def test_qwen3_next_80b_a3b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py
-98
View File
@@ -1,98 +0,0 @@
import multiprocessing as mp
import unittest
import torch
from sglang.test.ascend.test_ascend_utils import BGE_RERANKER_V2_M3_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-1-npu-a3",
nightly=True,
)
MODELS = [
(BGE_RERANKER_V2_M3_WEIGHTS_PATH, 1, 1e-2),
]
ATTENTION_BACKEND = ["ascend"]
TORCH_DTYPES = [torch.bfloat16]
class TestBgeReranker(CustomTestCase):
"""Testcase: This test case validates that the cross-encoder scores from the BAAI/bge-reranker-v2-m3 model in the
SGLang framework are less than 1e-2 different from the Hugging Face implementation.
[Test Category] Model
[Test Target] BAAI/bge-reranker-v2-m3
"""
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def assert_close_prefill_logits(
self,
prompts,
model_path,
tp_size,
torch_dtype,
score_tolerance,
attention_backend,
) -> None:
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="cross_encoder",
) as hf_runner:
hf_scores = hf_runner.forward(prompts).scores
with SRTRunner(
model_path,
tp_size=tp_size,
torch_dtype=torch_dtype,
model_type="cross_encoder",
attention_backend=attention_backend,
chunked_prefill_size=-1,
disable_radix_cache=True,
) as srt_runner:
srt_scores = srt_runner.forward(prompts).scores
for i in range(len(srt_scores)):
score_difference = abs(hf_scores[i] - srt_scores[i])
assert score_difference < score_tolerance, (
"cross encoder scores are not all close"
)
def preprocess_prompts(self, prompt):
processed_prompts = []
query = prompt["query"]
documents = prompt["documents"]
for document in documents:
processed_prompts.append([query, document])
return processed_prompts
def test_prefill_logits(self):
models_to_test = MODELS
for model, tp_size, prefill_tolerance in models_to_test:
for attention_backend in ATTENTION_BACKEND:
for queryDocs in TEST_RERANK_QUERY_DOCS:
prompts = self.preprocess_prompts(queryDocs)
for torch_dtype in TORCH_DTYPES:
self.assert_close_prefill_logits(
prompts,
model,
tp_size,
torch_dtype,
prefill_tolerance,
attention_backend,
)
if __name__ == "__main__":
unittest.main()
test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py
-65
View File
@@ -1,65 +0,0 @@
import os
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
import multiprocessing as mp
import unittest
import torch
from sglang.test.ascend.test_ascend_utils import INTERNLM2_7B_REWARD_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import SRTRunner
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="full-4-npu-a3",
nightly=True,
)
PROMPT = (
"What is the range of the numeric output of a sigmoid node in a neural network?"
)
RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1."
RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1."
CONVS = [
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}],
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}],
]
class TestInternlm2(CustomTestCase):
"""Testcase: This test case verifies that the Shanghai_AI_Laboratory/internlm2-7b-reward model can successfully generate reward
scores for different conversational responses using the SGLang framework, without comparing to a reference implementation.
[Test Category] Model
[Test Target] Shanghai_AI_Laboratory/internlm2-7b-reward
"""
model_path = INTERNLM2_7B_REWARD_WEIGHTS_PATH
torch_dtype = torch.float16
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def test_assert_close_reward_scores(self):
with SRTRunner(
self.model_path,
torch_dtype=self.torch_dtype,
model_type="reward",
trust_remote_code=True,
disable_cuda_graph=True,
tp_size=4,
mem_fraction_static=0.8,
) as srt_runner:
prompts = srt_runner.tokenizer.apply_chat_template(CONVS, tokenize=False)
srt_outputs = srt_runner.forward(prompts)
srt_scores = torch.tensor(srt_outputs.scores)
print(f"accuracy: {srt_scores}")
self.assertIsInstance(srt_scores, torch.Tensor)
if __name__ == "__main__":
unittest.main()
test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py
-86
View File
@@ -1,86 +0,0 @@
import multiprocessing as mp
import unittest
import torch
from sglang.test.ascend.test_ascend_utils import (
SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.runners import HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
MODELS = [
(SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH, 1, 4e-2),
]
TORCH_DTYPES = [torch.float16]
PROMPT = (
"What is the range of the numeric output of a sigmoid node in a neural network?"
)
RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1."
RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1."
CONVS = [
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}],
[{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}],
]
class TestLlama(CustomTestCase):
"""Testcase: This test case validates that the reward scores from the Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 model
in the SGLang framework are less than 4e-2 different from the Hugging Face implementation.
[Test Category] Model
[Test Target] Skywork/Skywork-Reward-Llama-3.1-8B-v0.2
"""
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def assert_close_reward_scores(
self,
convs,
model_path,
tp_size,
torch_dtype,
tolerance,
) -> None:
with HFRunner(
model_path,
torch_dtype=torch_dtype,
model_type="reward",
) as hf_runner:
hf_outputs = hf_runner.forward(convs)
with SRTRunner(
model_path,
tp_size=tp_size,
torch_dtype=torch_dtype,
model_type="reward",
) as srt_runner:
prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False)
srt_outputs = srt_runner.forward(prompts)
hf_scores = torch.tensor(hf_outputs.scores)
srt_scores = torch.tensor(srt_outputs.scores)
print(f"{hf_scores=}")
print(f"{srt_scores=}")
assert torch.all(abs(hf_scores - srt_scores) < tolerance), (
"reward scores are not all close"
)
def test_reward_scores(self):
for model, tp_size, tolerance in MODELS:
for torch_dtype in TORCH_DTYPES:
self.assert_close_reward_scores(
CONVS, model, tp_size, torch_dtype, tolerance
)
if __name__ == "__main__":
unittest.main()
test/registered/npu/vlm_models/test_npu_deepseek_vl2.py
-29
View File
@@ -1,29 +0,0 @@
import unittest
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_VL2_WEIGHTS_PATH
from sglang.test.ascend.vlm_utils import TestVLMModels
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=400,
suite="full-4-npu-a3",
nightly=True,
)
class TestDeepseekVl2(TestVLMModels):
"""Testcase: Verify that the inference accuracy of the deepseek-ai/deepseek-vl2 model on the MMMU dataset is no less than 0.2.
[Test Category] Model
[Test Target] deepseek-ai/deepseek-vl2
"""
model = DEEPSEEK_VL2_WEIGHTS_PATH
mmmu_accuracy = 0.2
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test()
if __name__ == "__main__":
unittest.main()
test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py
-36
View File
@@ -1,36 +0,0 @@
import unittest
from sglang.test.ascend.test_ascend_utils import (
LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ascend.vlm_utils import TestVLMModels
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestLlama3211BVisionInstruct(TestVLMModels):
"""Testcase: Verify that the inference accuracy of the LLM-Research/Llama-3.2-11B-Vision-Instruct model on the MMMU dataset is no less than 0.2.
[Test Category] Model
[Test Target] LLM-Research/Llama-3.2-11B-Vision-Instruct
"""
model = LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH
mmmu_accuracy = 0.2
other_args = [
"--trust-remote-code",
"--mem-fraction-static",
"0.8",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--disable-radix-cache",
]
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test()
if __name__ == "__main__":
unittest.main()
test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py
-25
View File
@@ -1,25 +0,0 @@
import unittest
from sglang.test.ascend.test_ascend_utils import MINICPM_V_2_6_WEIGHTS_PATH
from sglang.test.ascend.vlm_utils import TestVLMModels
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=400, suite="full-4-npu-a3", nightly=True)
class TestMiniCPMModelsV(TestVLMModels):
"""Testcase: Verify that the inference accuracy of the openbmb/MiniCPM-V-2_6 model on the MMMU dataset is no less than 0.2.
[Test Category] Model
[Test Target] openbmb/MiniCPM-V-2_6
"""
model = MINICPM_V_2_6_WEIGHTS_PATH
mmmu_accuracy = 0.2
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test()
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+8 -3
View File
@@ -153,25 +153,30 @@ NIGHTLY_SUITES = {
],
HWBackend.CPU: [],
HWBackend.NPU: [
"nightly-1-npu-a2",
"nightly-1-npu-a3",
"nightly-2-npu-a3",
"nightly-4-npu-a3",
"nightly-8-npu-a3",
"nightly-16-npu-a3",
"nightly-acc-2-npu-a3",
"nightly-acc-4-npu-a3",
"nightly-acc-8-npu-a3",
"nightly-acc-16-npu-a3",
"nightly-perf-2-npu-a3",
"nightly-perf-4-npu-a3",
"nightly-perf-8-npu-a3",
"nightly-perf-16-npu-a3",
"full-4-npu-a2",
"full-1-npu-a3",
"full-2-npu-a3",
"full-4-npu-a3",
"full-8-npu-a3",
"full-16-npu-a3",
"full-acc-2-npu-a3",
"full-acc-4-npu-a3",
"full-acc-16-npu-a3",
"full-perf-2-npu-a3",
"full-perf-4-npu-a3",
"full-perf-8-npu-a3",
"full-perf-16-npu-a3",
],
HWBackend.XPU: [
"nightly-xpu-1-gpu",