diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 021a62280..fe82f4d96 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -83,12 +83,13 @@ /sgl-model-gateway/examples/wasm @slin1237 /test/registered/prefill_only @sundar24295s @chanh @fortunecookiee /benchmark/prefill_only/bench_score.py @sundar24295s @chanh @fortunecookiee -/test/srt/ascend @ping1jing2 @iforgetmyname +/test/registered/ascend @ping1jing2 @ssshinigami @e-martirosian /test/srt/test_modelopt* @Edwardf0t1 /python/sglang/srt/layers/gemma4_fused_ops.py @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 @kpham-sgl @pyc96 /python/sglang/srt/function_call/gemma4_detector.py @CatherineSue @JustinTong0323 @kpham-sgl @pyc96 /python/sglang/srt/models/gemma4_*.py @kpham-sgl @pyc96 /python/sglang/srt/multimodal/processors/gemma4.py @mickqian @JustinTong0323 @yhyang201 @yuan-luo @kpham-sgl @pyc96 +/python/sglang/test/ascend @ping1jing2 @ssshinigami @e-martirosian /docs_new/cookbook/autoregressive/Google/Gemma4.mdx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96 /docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @wisclmy0611 @zijiexia @Richardczl98 @kpham-sgl @pyc96 /python/sglang/srt/speculative/ngram_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl diff --git a/python/sglang/test/ascend/gsm8k_ascend_mixin.py b/python/sglang/test/ascend/gsm8k_ascend_mixin.py index f562172d9..44e663acf 100644 --- a/python/sglang/test/ascend/gsm8k_ascend_mixin.py +++ b/python/sglang/test/ascend/gsm8k_ascend_mixin.py @@ -29,6 +29,7 @@ class GSM8KAscendMixin(ABC): server_cmd = "" gsm8k_num_shots = 5 num_questions = 200 + gsm8k_parallel = 128 env = { **os.environ, @@ -82,13 +83,14 @@ class GSM8KAscendMixin(ABC): data_path=None, num_questions=self.num_questions, max_new_tokens=512, - parallel=128, + parallel=self.gsm8k_parallel, host="http://127.0.0.1", port=int(self.base_url.split(":")[-1]), ) metrics = run_eval(args) model_metrics["accuracy"] = metrics["accuracy"] model_metrics["output_throughput"] = metrics["output_throughput"] + model_metrics["latency"] = metrics["latency"] self.assertGreaterEqual( metrics["accuracy"], accuracy_threshold, diff --git a/python/sglang/test/ascend/test_ascend_utils.py b/python/sglang/test/ascend/test_ascend_utils.py index 4405f7d4d..02127c886 100644 --- a/python/sglang/test/ascend/test_ascend_utils.py +++ b/python/sglang/test/ascend/test_ascend_utils.py @@ -58,6 +58,9 @@ DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join( DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join( MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base" ) +ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS" +) ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT" ) @@ -154,6 +157,9 @@ QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE" ) QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8") +REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8" +) SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B") STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b" @@ -565,7 +571,6 @@ def run_bench_serving( HEADER = """ -### Models | Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status | | ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ | """ diff --git a/python/sglang/test/ascend/test_mmlu.py b/python/sglang/test/ascend/test_mmlu.py index 095137481..0b90f9c4f 100644 --- a/python/sglang/test/ascend/test_mmlu.py +++ b/python/sglang/test/ascend/test_mmlu.py @@ -7,6 +7,8 @@ from sglang.test.run_eval import run_eval class TestMMLU: + mmlu_num_examples = 128 + def test_mmlu(self): accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00) @@ -23,12 +25,14 @@ class TestMMLU: base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=128, + num_examples=self.mmlu_num_examples, num_threads=32, ) print("Starting mmlu test...") metrics = run_eval(args) model_metrics["accuracy"] = metrics["score"] + model_metrics["latency"] = metrics.get("latency", "-") + model_metrics["output_throughput"] = metrics.get("output_throughput", "-") self.assertGreater(metrics["score"], accuracy_mmlu_threshold) except Exception as e: model_metrics["error"] = e diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py index d8fa7fd92..166641984 100644 --- a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py +++ b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py @@ -1,25 +1,18 @@ import os import unittest -from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ascend.test_ascend_utils import ( QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH, ) +from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) -class TestDeepEpQwen(CustomTestCase): +class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): """ Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's auto mode enabled, and verify that there is no drop in accuracy compared to when DeepEP is not enabled. @@ -28,101 +21,64 @@ class TestDeepEpQwen(CustomTestCase): [Test Target] --moe-a2a-backend, --deepep-mode """ - @classmethod - def setUpClass(cls): - cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 96, - "--context-length", - 8192, - "--dtype", - "bfloat16", - "--chunked-prefill-size", - 28672, - "--max-prefill-tokens", - 458880, - "--disable-radix-cache", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - "--tp-size", - 16, - "--dp-size", - 4, - "--enable-dp-attention", - "--enable-dp-lm-head", - "--mem-fraction-static", - 0.7, - "--cuda-graph-bs", - 16, - 20, - 24, - ], - env={ - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "HCCL_BUFFSIZE": "2100", - "HCCL_OP_EXPANSION_MODE": "AIV", - "TRANSFORMERS_VERBOSITY": "error", - **os.environ, - }, - ) + model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 96, + "--context-length", + 8192, + "--dtype", + "bfloat16", + "--chunked-prefill-size", + 28672, + "--max-prefill-tokens", + 458880, + "--disable-radix-cache", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--tp-size", + 16, + "--dp-size", + 4, + "--enable-dp-attention", + "--enable-dp-lm-head", + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 16, + 20, + 24, + ] + env = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "HCCL_BUFFSIZE": "2100", + "HCCL_OP_EXPANSION_MODE": "AIV", + "TRANSFORMERS_VERBOSITY": "error", + **os.environ, + } - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + # MMLU Configs + mmlu_num_examples = 8 + accuracy_mmlu_threshold = 0.61 # MMLU accuracy ≥0.61 - def test_mmlu(self): - expect_score = 0.61 - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=8, - num_threads=32, - ) - metrics = run_eval(args) - self.assertGreater(metrics["score"], expect_score) - - def test_gsm8k(self): - expect_accuracy = 0.91 - - host = "http://127.0.0.1" - port = int(self.base_url.split(":")[-1]) - args = SimpleNamespace( - num_shots=8, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=host, - port=port, - ) - metrics = run_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - expect_accuracy, - f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', - ) + # GSM8K Configs + accuracy = 0.91 # GSM8K accuracy ≥0.91 + num_questions = 200 + gsm8k_num_shots = 8 if __name__ == "__main__": diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py index 65d74338d..8c3db8099 100644 --- a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py +++ b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py @@ -1,20 +1,13 @@ import os import unittest -from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ascend.test_ascend_utils import ( QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH, ) +from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=200, @@ -23,7 +16,7 @@ register_npu_ci( ) -class TestQwen3Next(CustomTestCase): +class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase): """ Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's auto mode enabled, and verify that there is no drop in accuracy compared to when DeepEP is not enabled. @@ -32,93 +25,60 @@ class TestQwen3Next(CustomTestCase): [Test Target] --moe-a2a-backend deepep, --deepep-mode auto """ - @classmethod - def setUpClass(cls): - cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--device", - "npu", - "--tp-size", - 8, - "--mem-fraction-static", - 0.8, - "--max-running-requests", - 80, - "--watchdog-timeout", - 9000, - "--disable-radix-cache", - "--cuda-graph-bs", - 2, - 4, - 6, - 8, - "--max-prefill-tokens", - 28672, - "--max-total-tokens", - 450560, - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - "--chunked-prefill-size", - -1, - ], - env={ - # In NPU scenarios, operators only support BF16 precision. - # This environment variable needs to be set for quantizing weights. - "SGLANG_DEEPEP_BF16_DISPATCH": "1", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "HCCL_OP_EXPANSION_MODE": "AIV", - "HCCL_ALGO": "level0:NA;level1:ring", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20", - "HCCL_BUFFSIZE": "2000", - "GDN_ATTN_BACKEND_TRITON": "1", - **os.environ, - }, - ) + model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 8, + "--mem-fraction-static", + 0.8, + "--max-running-requests", + 80, + "--watchdog-timeout", + 9000, + "--disable-radix-cache", + "--cuda-graph-bs", + 2, + 4, + 6, + 8, + "--max-prefill-tokens", + 28672, + "--max-total-tokens", + 450560, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--chunked-prefill-size", + -1, + ] + env = { + # In NPU scenarios, operators only support BF16 precision. + # This environment variable needs to be set for quantizing weights. + "SGLANG_DEEPEP_BF16_DISPATCH": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_ALGO": "level0:NA;level1:ring", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20", + "HCCL_BUFFSIZE": "2000", + "GDN_ATTN_BACKEND_TRITON": "1", + **os.environ, + } - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + # MMLU Configs + mmlu_num_examples = 8 + accuracy_mmlu_threshold = 0.56 # MMLU accuracy ≥0.56 - def test_mmlu(self): - expect_score = 0.56 - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=8, - num_threads=32, - ) - metrics = run_eval(args) - self.assertGreater(metrics["score"], expect_score) - - def test_gsm8k(self): - expect_accuracy = 0.9 - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - expect_accuracy, - f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', - ) + # GSM8K Configs + accuracy = 0.9 # GSM8K accuracy ≥0.9 + num_questions = 200 + gsm8k_num_shots = 5 if __name__ == "__main__": diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py index 1f3e19a4b..c7e2563e3 100644 --- a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py +++ b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py @@ -1,22 +1,16 @@ import os import unittest -from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH +from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) -class TestDeepEpDeepseekV32(CustomTestCase): +class TestDeepEpDeepseekV32(GSM8KAscendMixin, TestMMLU, CustomTestCase): """Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario, its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode low_latency is configured. @@ -25,85 +19,50 @@ class TestDeepEpDeepseekV32(CustomTestCase): [Test Suggestions] Mixing deployment + low_latency mode is not recommended. """ - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=6000, - other_args=[ - "--trust-remote-code", - "--tp-size", - "16", - "--quantization", - "modelslim", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "low_latency", - "--mem-fraction-static", - 0.82, - "--disable-cuda-graph", - "--disable-radix-cache", - "--context-length", - 40960, - "--max-prefill-tokens", - 128, - "--max-total-tokens", - 40960, - "--watchdog-timeout", - 900, - ], - env={ - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", - "HCCL_BUFFSIZE": "2048", - "HCCL_OP_EXPANSION_MODE": "AIV", - "TASK_QUEUE_ENABLE": "0", - "TRANSFORMERS_VERBOSITY": "error", - **os.environ, - }, - ) + model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH + timeout_for_server_launch = 6000 + other_args = [ + "--trust-remote-code", + "--tp-size", + "16", + "--quantization", + "modelslim", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", + "--mem-fraction-static", + 0.82, + "--disable-cuda-graph", + "--disable-radix-cache", + "--context-length", + 40960, + "--max-prefill-tokens", + 128, + "--max-total-tokens", + 40960, + "--watchdog-timeout", + 900, + ] + env = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "HCCL_BUFFSIZE": "2048", + "HCCL_OP_EXPANSION_MODE": "AIV", + "TASK_QUEUE_ENABLE": "0", + "TRANSFORMERS_VERBOSITY": "error", + **os.environ, + } - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + # MMLU Configs + mmlu_num_examples = 128 + accuracy_mmlu_threshold = 0.85 # MMLU accuracy ≥0.85 - def test_mmlu(self): - expect_score = 0.85 - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=128, - num_threads=32, - ) - print("Starting mmlu test...") - metrics = run_eval(args) - self.assertGreater(metrics["score"], expect_score) - - def test_gsm8k(self): - expect_accuracy = 0.95 - args = SimpleNamespace( - num_shots=8, - data_path=None, - timeout=60000, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - print("Starting gsm8k test...") - metrics = run_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - expect_accuracy, - f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', - ) + # GSM8K Configs + accuracy = 0.95 # GSM8K accuracy ≥0.95 + num_questions = 200 + gsm8k_num_shots = 8 if __name__ == "__main__": diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py index dafe3b269..54ababf38 100644 --- a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py +++ b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py @@ -1,25 +1,18 @@ import os import unittest -from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ascend.test_ascend_utils import ( QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH, ) +from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True) -class TestDeepEpQwen(CustomTestCase): +class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): """ Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's low_latency mode enabled, and verify that there is no drop in accuracy compared to when DeepEP is not enabled. @@ -29,97 +22,64 @@ class TestDeepEpQwen(CustomTestCase): [Test Suggestions] Mixing deployment + low_latency mode is not recommended. """ - @classmethod - def setUpClass(cls): - cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--max-running-requests", - 96, - "--context-length", - 8192, - "--dtype", - "bfloat16", - "--chunked-prefill-size", - 1024, - "--max-prefill-tokens", - 458880, - "--disable-radix-cache", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "low_latency", - "--tp-size", - 16, - "--dp-size", - 4, - "--enable-dp-attention", - "--enable-dp-lm-head", - "--mem-fraction-static", - 0.7, - "--cuda-graph-bs", - 16, - 20, - 24, - ], - env={ - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", - "HCCL_BUFFSIZE": "2100", - "HCCL_OP_EXPANSION_MODE": "AIV", - "TRANSFORMERS_VERBOSITY": "error", - **os.environ, - }, - ) + model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 96, + "--context-length", + 8192, + "--dtype", + "bfloat16", + "--chunked-prefill-size", + 1024, + "--max-prefill-tokens", + 458880, + "--disable-radix-cache", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", + "--tp-size", + 16, + "--dp-size", + 4, + "--enable-dp-attention", + "--enable-dp-lm-head", + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 16, + 20, + 24, + ] + env = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "HCCL_BUFFSIZE": "2100", + "HCCL_OP_EXPANSION_MODE": "AIV", + "TRANSFORMERS_VERBOSITY": "error", + **os.environ, + } - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + # MMLU Configs + mmlu_num_examples = 8 + accuracy_mmlu_threshold = 0.61 # MMLU accuracy ≥0.61 - def test_mmlu(self): - expect_score = 0.61 - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=8, - num_threads=32, - ) - metrics = run_eval(args) - self.assertGreater(metrics["score"], expect_score) - - def test_gsm8k(self): - expect_accuracy = 0.91 - args = SimpleNamespace( - num_shots=8, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - expect_accuracy, - f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', - ) + # GSM8K Configs + accuracy = 0.91 # GSM8K accuracy ≥0.91 + num_questions = 200 + gsm8k_num_shots = 8 if __name__ == "__main__": diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py index 2f405ae57..e1b804c1e 100644 --- a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py +++ b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py @@ -1,20 +1,13 @@ import os import unittest -from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ascend.test_ascend_utils import ( QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH, ) +from sglang.test.ascend.test_mmlu import TestMMLU from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase register_npu_ci( est_time=200, @@ -23,7 +16,7 @@ register_npu_ci( ) -class TestQwen3Next(CustomTestCase): +class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase): """ Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's low_latency mode enabled, and verify that there is no drop in accuracy compared to when DeepEP is not enabled. @@ -32,97 +25,64 @@ class TestQwen3Next(CustomTestCase): [Test Target] --moe-a2a-backend deepep, --deepep-mode low_latency """ - @classmethod - def setUpClass(cls): - cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--device", - "npu", - "--tp-size", - 8, - "--mem-fraction-static", - 0.8, - "--max-running-requests", - 80, - "--watchdog-timeout", - 9000, - "--disable-radix-cache", - "--cuda-graph-bs", - 2, - 4, - 6, - 8, - "--chunked-prefill-size", - 1024, - "--max-prefill-tokens", - 28672, - "--max-total-tokens", - 450560, - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "low_latency", - ], - env={ - # The product of the following two environment variables must be greater than --max-prefill-tokens - # divide by dp size - "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000", - "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10", - # In NPU scenarios, operators only support BF16 precision. - # This environment variable needs to be set for quantizing weights. - "SGLANG_DEEPEP_BF16_DISPATCH": "1", - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "HCCL_OP_EXPANSION_MODE": "AIV", - "HCCL_ALGO": "level0:NA;level1:ring", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160", - "HCCL_BUFFSIZE": "2048", - "GDN_ATTN_BACKEND_TRITON": "1", - **os.environ, - }, - ) + model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 8, + "--mem-fraction-static", + 0.8, + "--max-running-requests", + 80, + "--watchdog-timeout", + 9000, + "--disable-radix-cache", + "--cuda-graph-bs", + 2, + 4, + 6, + 8, + "--chunked-prefill-size", + 1024, + "--max-prefill-tokens", + 28672, + "--max-total-tokens", + 450560, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", + ] + env = { + # The product of the following two environment variables must be greater than --max-prefill-tokens + # divide by dp size + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10", + # In NPU scenarios, operators only support BF16 precision. + # This environment variable needs to be set for quantizing weights. + "SGLANG_DEEPEP_BF16_DISPATCH": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_ALGO": "level0:NA;level1:ring", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160", + "HCCL_BUFFSIZE": "2048", + "GDN_ATTN_BACKEND_TRITON": "1", + **os.environ, + } - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + # MMLU Configs + mmlu_num_examples = 8 + accuracy_mmlu_threshold = 0.56 # MMLU accuracy ≥0.56 - def test_mmlu(self): - expect_score = 0.56 - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=8, - num_threads=32, - ) - metrics = run_eval(args) - self.assertGreater(metrics["score"], expect_score) - - def test_gsm8k(self): - expect_accuracy = 0.9 - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - expect_accuracy, - f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', - ) + # GSM8K Configs + accuracy = 0.9 # GSM8K accuracy ≥0.9 + num_questions = 200 + gsm8k_num_shots = 5 if __name__ == "__main__": diff --git a/test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py b/test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py index e395ec4c8..133c7f308 100644 --- a/test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py +++ b/test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py @@ -6,79 +6,47 @@ python3 -m unittest test_ascend_w4a4_quantization.TestAscendW4A4.test_gsm8k import os import time import unittest -from types import SimpleNamespace -from urllib.parse import urlparse import requests -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin +from sglang.test.ascend.test_ascend_utils import ( + ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH, ) +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.test_utils import CustomTestCase, is_in_ci, write_github_step_summary register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False) register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) -if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ: - os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1,2,3" -DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( - 7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100 -) -DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}" +class TestAscendW4A4(GSM8KAscendMixin, CustomTestCase): -class TestAscendW4A4(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3-32B-w4a4-LAOS" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--device", - "npu", - "--attention-backend", - "ascend", - "--tp-size", - "4", - "--mem-fraction-static", - "0.8", - "--cuda-graph-bs", - "64", - "--disable-radix-cache", - ], - ) + model = ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--device", + "npu", + "--attention-backend", + "ascend", + "--tp-size", + "4", + "--mem-fraction-static", + "0.8", + "--cuda-graph-bs", + "64", + "--disable-radix-cache", + ] - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + env = { + **os.environ, + } - def test_gsm8k(self): - base_url = DEFAULT_URL_FOR_TEST - url = urlparse(base_url) - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=64, - host=f"http://{url.hostname}", - port=int(url.port), - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreaterEqual(metrics["accuracy"], 0.80) - self.assertGreaterEqual(metrics["output_throughput"], 1000) + # GSM8K Configs + accuracy = 0.80 # GSM8K accuracy ≥0.80 + num_questions = 1319 + gsm8k_num_shots = 5 + output_throughput = 1000 # GSM8K output throughput ≥1000 tokens/s def run_decode(self, max_new_tokens): response = requests.post( @@ -100,11 +68,12 @@ class TestAscendW4A4(CustomTestCase): tic = time.perf_counter() res = self.run_decode(max_tokens) tok = time.perf_counter() - print(res["text"]) throughput = max_tokens / (tok - tic) - print(f"Throughput: {throughput} tokens/s") + summary = res["text"] + f"\nThroughput: {throughput} tokens/s" + print(summary) if is_in_ci(): + write_github_step_summary(summary + "\nThroughput threshold: 35 tokens/s") self.assertGreaterEqual(throughput, 35) diff --git a/test/registered/ascend/basic_function/quant/test_npu_w8a8_quantization.py b/test/registered/ascend/basic_function/quant/test_npu_w8a8_quantization.py index 96bea7efb..9814dbeb5 100644 --- a/test/registered/ascend/basic_function/quant/test_npu_w8a8_quantization.py +++ b/test/registered/ascend/basic_function/quant/test_npu_w8a8_quantization.py @@ -6,74 +6,39 @@ python3 -m unittest test_ascend_w8a8_quantization.TestAscendW8A8.test_gsm8k import os import time import unittest -from types import SimpleNamespace -from urllib.parse import urlparse import requests -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin +from sglang.test.ascend.test_ascend_utils import ( + REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH, ) +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.test_utils import CustomTestCase, is_in_ci, write_github_step_summary register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) -if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ: - os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1" -DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( - 7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100 -) -DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}" +class TestAscendW8A8CompressedTensors(GSM8KAscendMixin, CustomTestCase): + model = REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--device", + "npu", + "--attention-backend", + "ascend", + ] + env = { + **os.environ, + } -class TestAscendW8A8CompressedTensors(CustomTestCase): - @classmethod - def setUpClass(cls): - # TODO: Move model to CI or Modelscope - cls.model = "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--disable-cuda-graph", - "--device", - "npu", - "--attention-backend", - "ascend", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - base_url = DEFAULT_URL_FOR_TEST - url = urlparse(base_url) - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{url.hostname}", - port=int(url.port), - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreaterEqual(metrics["accuracy"], 0.3) - self.assertGreaterEqual(metrics["output_throughput"], 700) + # GSM8K Configs + accuracy = 0.3 # GSM8K accuracy ≥0.3 + num_questions = 200 + gsm8k_num_shots = 5 + output_throughput = 700 # GSM8K output throughput >=700 tokens/s def run_decode(self, max_new_tokens): response = requests.post( @@ -95,11 +60,12 @@ class TestAscendW8A8CompressedTensors(CustomTestCase): tic = time.perf_counter() res = self.run_decode(max_tokens) tok = time.perf_counter() - print(res["text"]) throughput = max_tokens / (tok - tic) - print(f"Throughput: {throughput} tokens/s") + summary = f"\nThroughput: {throughput} tokens/s" + print(res["text"] + summary) if is_in_ci(): + write_github_step_summary(summary + "\nThroughput threshold: 25 tokens/s") self.assertGreaterEqual(throughput, 25) diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py b/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py index 0af226ece..20f0099bb 100644 --- a/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py +++ b/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py @@ -1,30 +1,15 @@ import os import unittest -from types import SimpleNamespace +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - kill_process_tree, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase register_npu_ci(est_time=500, suite="nightly-4-npu-a3", nightly=True) -QWEN3_30B_MODEL = QWEN3_30B_A3B_WEIGHTS_PATH -GSM8K_MIN_ACCURACY = 0.92 -GSM8K_NUM_QUESTIONS = 100 -_NPU_ENV_VARS = { - "ASCEND_USE_FIA": "1", -} - - -class TestQwen330BAttnCP(CustomTestCase): +class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase): """GSM8K accuracy test for Qwen3-30B-A3B mixed deployment on 4 NPUs. The test uses: @@ -37,58 +22,33 @@ class TestQwen330BAttnCP(CustomTestCase): environment variables from the PD GSM8K test. """ - @classmethod - def setUpClass(cls): - cls.model = QWEN3_30B_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.npu_env = {**os.environ, **_NPU_ENV_VARS} - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--mem-fraction-static", - "0.7", - "--max-running-requests", - "32", - "--attention-backend", - "ascend", - "--tp-size", - "4", - "--moe-dp-size", - "2", - "--attn-cp-size", - "2", - "--cuda-graph-max-bs", - "32", - "--enable-prefill-context-parallel", - ], - env=cls.npu_env, - ) + model = QWEN3_30B_A3B_WEIGHTS_PATH + other_args = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.7", + "--max-running-requests", + "32", + "--attention-backend", + "ascend", + "--tp-size", + "4", + "--moe-dp-size", + "2", + "--attn-cp-size", + "2", + "--cuda-graph-max-bs", + "32", + "--enable-prefill-context-parallel", + ] - @classmethod - def tearDownClass(cls): - if hasattr(cls, "process") and cls.process is not None: - kill_process_tree(cls.process.pid) + env = {**os.environ, "ASCEND_USE_FIA": "1"} - def test_gsm8k_accuracy(self): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=GSM8K_NUM_QUESTIONS, - max_new_tokens=512, - parallel=32, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print( - "GSM8K accuracy " - f"(mixed TP=4 MOE_DP=2 ATTN_CP=2, {GSM8K_NUM_QUESTIONS} samples): " - f"{metrics['accuracy']:.3f}" - ) - self.assertGreaterEqual(metrics["accuracy"], GSM8K_MIN_ACCURACY) + # GSM8K Configs + accuracy = 0.92 # GSM8K accuracy ≥0.92 + gsm8k_parallel = 32 + num_questions = 100 + gsm8k_num_shots = 5 if __name__ == "__main__":