NPU Nightly Pipeline Skip Test Case Adaptation and Recovery Testing (#26353)

test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mla.py

@@ -12,7 +12,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-16-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 
@@ -89,6 +88,7 @@ class TestNpuHierarchicalCacheMla(CustomTestCase):
                 gsp_question_len=1792,
                 gsp_output_len=1,
                 other_server_args=other_args,
+                timeout_for_server_launch=1000,
             )
             TTFT = res["mean_ttft_ms"]
             TTFTS.append(TTFT)

test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py

@@ -11,7 +11,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-2-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 

test/registered/ascend/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py

@@ -0,0 +1,74 @@
+import unittest
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.test_ascend_utils import LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+    run_bench_serving,
+)
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestNoChunkedPrefill(CustomTestCase):
+    """Testcase: Verify Llama-3.1-8B-Instruct accuracy ≥ 0.65 and serving normal with chunked prefill disabled.
+
+    [Test Category] Parameter
+    [Test Target] --chunked-prefill-size
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        cls.model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        other_args = [
+            "--attention-backend",
+            "ascend",
+            "--disable-cuda-graph",
+            "--chunked-prefill-size",
+            "-1",
+            "--disable-radix-cache",
+        ]
+
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=other_args,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_mmlu(self):
+        args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="mmlu",
+            num_examples=64,
+            num_threads=32,
+        )
+
+        metrics = run_eval(args)
+        self.assertGreaterEqual(metrics["score"], 0.65)
+
+    def test_no_chunked_prefill_without_radix_cache(self):
+        res = run_bench_serving(
+            model=LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH,
+            num_prompts=10,
+            request_rate=float("inf"),
+            other_server_args=["--disable-radix-cache", "--chunked-prefill-size", "-1"],
+        )
+
+        assert res["completed"] == 10
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py

@@ -20,7 +20,6 @@ register_npu_ci(
     est_time=200,
     suite="nightly-8-npu-a3",
     nightly=True,
-    disabled="https://github.com/Ascend/sglang/issues/58",
 )
 
 
@@ -56,7 +55,11 @@ class TestQwen3Next(CustomTestCase):
                 "--watchdog-timeout",
                 9000,
                 "--disable-radix-cache",
-                "--disable-cuda-graph",
+                "--cuda-graph-bs",
+                2,
+                4,
+                6,
+                8,
                 "--max-prefill-tokens",
                 28672,
                 "--max-total-tokens",
@@ -69,6 +72,9 @@ class TestQwen3Next(CustomTestCase):
                 -1,
             ],
             env={
+                # In NPU scenarios, operators only support BF16 precision.
+                # This environment variable needs to be set for quantizing weights.
+                "SGLANG_DEEPEP_BF16_DISPATCH": "1",
                 "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
                 "STREAMS_PER_DEVICE": "32",
                 "HCCL_OP_EXPANSION_MODE": "AIV",

test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py

@@ -20,7 +20,6 @@ register_npu_ci(
     est_time=200,
     suite="nightly-8-npu-a3",
     nightly=True,
-    disabled="https://github.com/Ascend/sglang/issues/58",
 )
 
 
@@ -56,7 +55,11 @@ class TestQwen3Next(CustomTestCase):
                 "--watchdog-timeout",
                 9000,
                 "--disable-radix-cache",
-                "--disable-cuda-graph",
+                "--cuda-graph-bs",
+                2,
+                4,
+                6,
+                8,
                 "--chunked-prefill-size",
                 1024,
                 "--max-prefill-tokens",
@@ -69,11 +72,18 @@ class TestQwen3Next(CustomTestCase):
                 "low_latency",
             ],
             env={
+                # The product of the following two environment variables must be greater than --max-prefill-tokens
+                # divide by dp size
+                "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3000",
+                "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
+                # In NPU scenarios, operators only support BF16 precision.
+                # This environment variable needs to be set for quantizing weights.
+                "SGLANG_DEEPEP_BF16_DISPATCH": "1",
                 "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
                 "STREAMS_PER_DEVICE": "32",
                 "HCCL_OP_EXPANSION_MODE": "AIV",
                 "HCCL_ALGO": "level0:NA;level1:ring",
-                "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
+                "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "160",
                 "HCCL_BUFFSIZE": "2048",
                 **os.environ,
             },

test/registered/ascend/basic_function/parameter/test_npu_fim_completion.py

@@ -20,7 +20,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-1-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 

test/registered/ascend/basic_function/parameter/test_npu_no_chunked_prefill.py

@@ -1,39 +0,0 @@
-import unittest
-
-from sglang.test.ascend.test_ascend_utils import LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH
-from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.test_utils import CustomTestCase, run_bench_serving, run_mmlu_test
-
-register_npu_ci(
-    est_time=400,
-    suite="nightly-1-npu-a3",
-    nightly=True,
-    disabled="run failed",
-)
-
-
-class TestNoChunkedPrefill(CustomTestCase):
-    """Testcase: Verify Llama-3.1-8B-Instruct accuracy ≥ 0.65 and serving normal with chunked prefill disabled.
-
-    [Test Category] Parameter
-    [Test Target] --chunked-prefill-size
-    """
-
-    def test_no_chunked_prefill(self):
-        run_mmlu_test(
-            disable_radix_cache=False, enable_mixed_chunk=False, chunked_prefill_size=-1
-        )
-
-    def test_no_chunked_prefill_without_radix_cache(self):
-        res = run_bench_serving(
-            model=LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH,
-            num_prompts=10,
-            request_rate=float("inf"),
-            other_server_args=["--disable-radix-cache", "--chunked-prefill-size", "-1"],
-        )
-
-        assert res["completed"] == 10
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/ascend/basic_function/parameter/test_npu_no_overlap_scheduler.py

@@ -1,13 +1,13 @@
 import unittest
 
+from sglang.test.ascend.test_ascend_utils import run_mmlu_test
 from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.test_utils import CustomTestCase, run_mmlu_test
+from sglang.test.test_utils import CustomTestCase
 
 register_npu_ci(
     est_time=400,
     suite="nightly-1-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 

test/registered/ascend/basic_function/parameter/test_npu_warmups.py

@@ -16,7 +16,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-4-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 

test/registered/ascend/interface/test_npu_enable_thinking.py

@@ -17,7 +17,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-2-npu-a3",
     nightly=True,
-    disabled="https://github.com/Ascend/sglang/issues/32",
 )
 
 
@@ -40,7 +39,9 @@ class TestEnableThinking(CustomTestCase):
             "--mem-fraction-static",
             0.95,
             "--tp",
-            16,
+            2,
+            "--reasoning-parser",
+            "qwen3",
         ]
         cls.process = popen_launch_server(
             cls.model,

test/registered/ascend/interface/test_npu_openai_function_calling.py

@@ -18,7 +18,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-1-npu-a3",
     nightly=True,
-    disabled="https://github.com/Ascend/sglang/issues/39",
 )
 
 
@@ -429,8 +428,10 @@ class TestOpenAIServerFunctionCalling(CustomTestCase):
 
     def test_function_call_required(self):
         """
-        Test: Whether tool_choice: "required" works as expected
+        Test: Whether tool_choice: "required" works as expected.
-        - When tool_choice == "required", the model should return one or more tool_calls.
+        - When tool_choice == "required", the model MUST return one or more tool_calls.
+        - The model may choose ANY of the provided tools; we only verify that
+          a tool call exists and the selected name is among the candidates.
         """
         client = openai.Client(api_key=self.api_key, base_url=self.base_url)
 
@@ -472,47 +473,42 @@ class TestOpenAIServerFunctionCalling(CustomTestCase):
                         },
                         "required": ["city"],
                     },
+                    "strict": True,
                 },
             },
         ]
 
-        messages = [{"role": "user", "content": "What is the capital of France?"}]
+        valid_tool_names = {t["function"]["name"] for t in tools}
+
+        messages = [{"role": "user", "content": "Tell me about Paris"}]
         response = client.chat.completions.create(
             model=self.model,
             max_tokens=2048,
             messages=messages,
-            temperature=0.8,
+            temperature=0,
-            top_p=0.8,
             stream=False,
             tools=tools,
             tool_choice="required",
         )
 
         tool_calls = response.choices[0].message.tool_calls
-        self.assertIsNotNone(tool_calls, "No tool_calls in the response")
+        self.assertIsNotNone(
+            tool_calls, "tool_choice='required' must produce tool_calls"
+        )
+        self.assertGreater(len(tool_calls), 0, "tool_calls list should be non-empty")
+
         function_name = tool_calls[0].function.name
+        self.assertIn(
+            function_name,
+            valid_tool_names,
+            f"Function name '{function_name}' is not among the provided tools: {valid_tool_names}",
+        )
+
+        # Verify the arguments are parseable JSON
         arguments = tool_calls[0].function.arguments
         args_obj = json.loads(arguments)
-
-        self.assertEqual(
-            function_name,
-            "get_weather",
-            f"Function name should be 'get_weather', got: {function_name}",
-        )
-        self.assertIn(
-            "city", args_obj, f"Function arguments should have 'city', got: {args_obj}"
-        )
-
-        # Make the test more robust by checking type and accepting valid responses
-        city_value = args_obj["city"]
         self.assertIsInstance(
-            city_value,
+            args_obj, dict, "Function arguments should be a JSON object"
-            str,
-            f"Parameter city should be a string, got: {type(city_value)}",
-        )
-        self.assertTrue(
-            "Paris" in city_value or "France" in city_value,
-            f"Parameter city should contain either 'Paris' or 'France', got: {city_value}",
         )
 
     def test_function_call_specific(self):
@@ -560,6 +556,7 @@ class TestOpenAIServerFunctionCalling(CustomTestCase):
                         },
                         "required": ["city"],
                     },
+                    "strict": True,
                 },
             },
         ]

test/registered/ascend/llm_models/test_npu_llama4_scount_17b_16e.py

@@ -7,12 +7,7 @@ from sglang.test.ascend.test_ascend_utils import (
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_npu_ci(
+register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
-    est_time=400,
-    suite="nightly-4-npu-a3",
-    nightly=True,
-    disabled="https://github.com/Ascend/sglang/issues/25",
-)
 
 
 class TestLlama4(GSM8KAscendMixin, CustomTestCase):
@@ -24,6 +19,7 @@ class TestLlama4(GSM8KAscendMixin, CustomTestCase):
 
     model = LLAMA_4_SCOUT_17B_16E_INSTRUCT_WEIGHTS_PATH
     accuracy = 0.9
+    timeout_for_server_launch = 1000
     other_args = [
         "--chat-template",
         "llama-4",

test/registered/ascend/llm_models/test_npu_minicpm3_4b.py

@@ -5,12 +5,7 @@ from sglang.test.ascend.test_ascend_utils import MINICPM3_4B_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_npu_ci(
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
-    est_time=400,
-    suite="nightly-1-npu-a3",
-    nightly=True,
-    disabled="https://github.com/Ascend/sglang/issues/23",
-)
 
 
 class TestMiniCPM3(GSM8KAscendMixin, CustomTestCase):

test/registered/ascend/vlm_models/test_npu_deepseek_vl2.py

@@ -8,7 +8,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-4-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 

test/registered/ascend/vlm_models/test_npu_kimi_vl_a3b_instruct.py

@@ -1,32 +1,28 @@
 import unittest
 
-from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
+from sglang.test.ascend.test_ascend_utils import KIMI_VL_A3B_INSTRUCT_WEIGHTS_PATH
+from sglang.test.ascend.vlm_utils import TestVLMModels
 from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.test_utils import CustomTestCase
 
 register_npu_ci(
     est_time=400,
     suite="nightly-4-npu-a3",
     nightly=True,
-    disabled="run failed",
 )
 
 
-class TestKimiVLA3BInstruct(GSM8KAscendMixin, CustomTestCase):
+class TestKimiVLA3BInstruct(TestVLMModels):
-    model = "/root/.cache/modelscope/hub/models/Kimi/Kimi-VL-A3B-Instruct"
+    """Testcase: Verify that the inference accuracy of the moonshotai/Kimi-VL-A3B-Instruct model on the MMMU dataset is no less than 0.2.
-    accuracy = 0.66
+
-    other_args = [
+    [Test Category] Model
-        "--trust-remote-code",
+    [Test Target] moonshotai/Kimi-VL-A3B-Instruct
-        "--max-running-requests",
+    """
-        2048,
+
-        "--mem-fraction-static",
+    model = KIMI_VL_A3B_INSTRUCT_WEIGHTS_PATH
-        0.7,
+    mmmu_accuracy = 0.2
-        "--attention-backend",
+
-        "ascend",
+    def test_vlm_mmmu_benchmark(self):
-        "--tp-size",
+        self._run_vlm_mmmu_test()
-        "4",
-        "--disable-cuda-graph",
-    ]
 
 
 if __name__ == "__main__":

test/registered/ascend/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py

@@ -1,20 +1,22 @@
 import unittest
 
+from sglang.test.ascend.test_ascend_utils import (
+    LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH,
+)
 from sglang.test.ascend.vlm_utils import TestVLMModels
 from sglang.test.ci.ci_register import register_npu_ci
 
-register_npu_ci(
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
-    est_time=400,
-    suite="nightly-1-npu-a3",
-    nightly=True,
-    disabled="run failed",
-)
 
 
 class TestLlama3211BVisionInstruct(TestVLMModels):
-    model = (
+    """Testcase: Verify that the inference accuracy of the LLM-Research/Llama-3.2-11B-Vision-Instruct model on the MMMU dataset is no less than 0.2.
-        "/root/.cache/modelscope/hub/models/LLM-Research/Llama-3.2-11B-Vision-Instruct"
+
-    )
+    [Test Category] Model
+    [Test Target] LLM-Research/Llama-3.2-11B-Vision-Instruct
+    """
+
+    model = LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH
     mmmu_accuracy = 0.2
     other_args = [
         "--trust-remote-code",

test/registered/ascend/vlm_models/test_npu_minicpm_o_2_6.py

@@ -8,7 +8,6 @@ register_npu_ci(
     est_time=400,
     suite="nightly-4-npu-a3",
     nightly=True,
-    disabled="run failed",
 )