diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 8e3b1510d..600d52ebb 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -463,6 +463,31 @@ jobs: runner: linux-aarch64-a3-2 test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type: 'perf' + # NPU accuracy tests + - name: glm4_7_flash_1p_gsm8k + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py + test_type: 'accuracy' + - name: qwen3_vl_30b_a3b_bf16_2p_gsm8k + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py + test_type: 'accuracy' + - name: glm5_top64_pruned_bf16_8p_gsm8k + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py + test_type: 'accuracy' + - name: moonshotai_moonlight_16b_a3b_bf16_1p_gsm8k + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py + test_type: 'accuracy' + - name: qwen3_5_9b_bf16_1p_gsm8k + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py + test_type: 'accuracy' + - name: qwen3_vl_8b_bf16_2p_gsm8k + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py + test_type: 'accuracy' uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml with: runner: ${{ matrix.test_config.runner }} diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index c0b733576..21cd57171 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -150,6 +150,13 @@ KIMI_K2_5_EAGLE3_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/lightseekorg/kimi-k2.5-eagle3" ) GLM_4_7_FLASH_MODEL_PATH = "/root/.cache/modelscope/hub/models/ZhipuAI/GLM-4.7-Flash" +QWEN3_5_9B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3.5-9B" +MOONLIGHT_16B_A3B_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/moonshotai/Moonlight-16B-A3B-Instruct" +) +GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/yzgjhdxf/GLM-5-top64-pruned-gsm8k" +) GLM_5_1_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.1-w4a8" MINIMAX_M2_5_W8A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/MiniMax-M2.5-w8a8-QuaRot" diff --git a/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py b/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py new file mode 100644 index 000000000..f2de0b3d2 --- /dev/null +++ b/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py @@ -0,0 +1,76 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "1000", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_SET_CPU_AFFINITY": "1", +} + +OTHER_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 2, + "--chunked-prefill-size", + 16384, + "--max-prefill-tokens", + 150000, + "--dtype", + "bfloat16", + "--max-running-requests", + 32, + "--trust-remote-code", + "--mem-fraction-static", + 0.75, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 16, + 32, + "--watchdog-timeout", + 9000, + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", +] + + +class TestNPUGlm4_7Flash_1P_GSM8K(TestNpuAccuracyTestCaseBase): + + model = GLM_4_7_FLASH_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.9560 + datasets = ["gsm8k"] + few_shot_num = 5 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_gsm8k(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py b/test/registered/ascend/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py new file mode 100644 index 000000000..d56b2d5ba --- /dev/null +++ b/test/registered/ascend/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py @@ -0,0 +1,71 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_NPU_USE_MULTI_STREAM": "1", + "HCCL_BUFFSIZE": "1000", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", +} +OTHER_ARGS = [ + "--attention-backend", + "ascend", + "--tp-size", + "16", + "--chunked-prefill-size", + "16384", + "--trust-remote-code", + "--disable-radix-cache", + "--mem-fraction-static", + "0.7", + "--served-model-name", + "glm-5", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--cuda-graph-bs", + 16, +] + + +class TestNPUGLM5_Top64_Pruned_GSM8K(TestNpuAccuracyTestCaseBase): + + model = GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.50 + datasets = ["gsm8k"] + generation_config = { + "max_tokens": 2048, + "temperature": 0.01, + } + eval_batch_size = 16 + limit = 100 + + def test_gsm8k(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py b/test/registered/ascend/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py new file mode 100644 index 000000000..ff490bb46 --- /dev/null +++ b/test/registered/ascend/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py @@ -0,0 +1,73 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + MOONLIGHT_16B_A3B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +MODEL_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_BUFFSIZE": "1536", + "HCCL_OP_EXPANSION_MODE": "AIV", +} + +MODEL_OTHER_ARGS = [ + "--tp-size", + 2, + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--dtype", + "bfloat16", + "--mem-fraction-static", + 0.8, + "--disable-radix-cache", + "--chunked-prefill-size", + 4096, + "--max-prefill-tokens", + 16384, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 16, + "--max-running-requests", + 128, + "--watchdog-timeout", + 9000, +] + + +class TestNPUMoonlight16B_A3B_GSM8K(TestNpuAccuracyTestCaseBase): + model = MOONLIGHT_16B_A3B_MODEL_PATH + envs = MODEL_ENVS + other_args = MODEL_OTHER_ARGS + accuracy = 0.8370 + datasets = ["gsm8k"] + few_shot_num = 5 + generation_config = {"max_tokens": 7168, "temperature": 1.0} + eval_batch_size = 64 + + def test_gsm8k(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/ascend/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py new file mode 100644 index 000000000..7b49dd52e --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -0,0 +1,76 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_5_9B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="stage-b-test-2-npu-a3", + nightly=True, + disabled="accuracy testcase", +) + +QWEN3_5_9B_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "ASCEND_LAUNCH_BLOCKING": "1", + "HCCL_BUFFSIZE": "1536", + "HCCL_OP_EXPANSION_MODE": "AIV", +} + +QWEN3_5_9B_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--enable-dp-attention", + "--chunked-prefill-size", + 4096, + "--max-prefill-tokens", + 280000, + "--disable-radix-cache", + "--trust-remote-code", + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 16, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", +] + + +class TestNPUQwen3_5_9B_GSM8K(TestNpuAccuracyTestCaseBase): + model = QWEN3_5_9B_MODEL_PATH + envs = QWEN3_5_9B_ENVS + other_args = QWEN3_5_9B_OTHER_ARGS + accuracy = 0.8350 + datasets = ["gsm8k"] + few_shot_num = 5 + generation_config = { + "max_tokens": 8192, + "temperature": 0.6, + } + eval_batch_size = 64 + limit = 100 + + def test_gsm8k(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py b/test/registered/ascend/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py new file mode 100644 index 000000000..ffaf7e9de --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_vl_30b_a3b/test_npu_qwen3_vl_30b_a3b_bf16_2p_gsm8k.py @@ -0,0 +1,75 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_VL_30B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="stage-b-test-4-npu-a3", + nightly=True, + disabled="accuracy testcase", +) + +QWEN3_VL_30B_A3B_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "ASCEND_LAUNCH_BLOCKING": "1", + "HCCL_BUFFSIZE": "1536", + "HCCL_OP_EXPANSION_MODE": "AIV", +} + +QWEN3_VL_30B_A3B_OTHER_ARGS = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--dtype", + "bfloat16", + "--device", + "npu", + "--mm-attention-backend", + "ascend_attn", + "--enable-multimodal", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 102400, + "--max-running-requests", + 512, + "--tp-size", + 4, + "--disable-radix-cache", + "--mem-fraction-static", + 0.78, + "--sampling-backend", + "ascend", +] + + +class TestNPUQwen3_VL_30B_A3B_GSM8K(TestNpuAccuracyTestCaseBase): + model = QWEN3_VL_30B_MODEL_PATH + envs = QWEN3_VL_30B_A3B_ENVS + other_args = QWEN3_VL_30B_A3B_OTHER_ARGS + accuracy = 0.9538 + datasets = ["gsm8k"] + few_shot_num = 5 + generation_config = { + "max_tokens": 40000, + "temperature": 0.0, + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + } + eval_batch_size = 64 + + def test_gsm8k(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py b/test/registered/ascend/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py new file mode 100644 index 000000000..6d44fc382 --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py @@ -0,0 +1,83 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_VL_8B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="stage-b-test-4-npu-a3", + nightly=True, + disabled="accuracy testcase", +) + +QWEN3_VL_8B_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "1536", + "HCCL_OP_EXPANSION_MODE": "AIV", +} + +QWEN3_VL_8B_OTHER_ARGS = [ + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--attention-backend", + "ascend", + "--device", + "npu", + "--trust-remote-code", + "--tp-size", + 4, + "--mem-fraction-static", + 0.8, + "--disable-radix-cache", + "--chunked-prefill-size", + -1, + "--sampling-backend", + "ascend", + "--tool-call-parser", + "qwen", + "--reasoning-parser", + "qwen3", + "--cuda-graph-bs", + 8, + 16, + 32, + 64, + 128, + 256, + "--dtype", + "bfloat16", +] + + +class TestNPUQwen3_VL_8B_GSM8K(TestNpuAccuracyTestCaseBase): + model = QWEN3_VL_8B_MODEL_PATH + envs = QWEN3_VL_8B_ENVS + other_args = QWEN3_VL_8B_OTHER_ARGS + accuracy = 0.9553 + datasets = ["gsm8k"] + few_shot_num = 5 + generation_config = { + "max_tokens": 32768, + "temperature": 1.0, + "top_p": 1.0, + "top_k": 40, + "repetition_penalty": 1.0, + "presence_penalty": 2.0, + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + } + eval_batch_size = 64 + + def test_gsm8k(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main()