[NPU] Fix failed test cases in pr‑test‑npu and improve execution efficiency (#38112)

Co-authored-by: Even Zhou <even.y.zhou@outlook.com>
Co-authored-by: sglang-npu-bot <sglangnpu@163.com>

.github/workflows/pr-test-npu.yml

@@ -227,7 +227,7 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
-  multimodal-gen-test-2-npu-a3:
+  multimodal-gen-test-4-npu-a3:
     needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
     if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }}
     runs-on: linux-aarch64-a3-800t-4
@@ -297,7 +297,7 @@ jobs:
     if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
-      runner: linux-aarch64-a3-800t-2
+      runner: linux-aarch64-a3-2-
       test_type: 'accuracy'
       test_suite: base-c-test-acc-2-npu-a3
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
@@ -311,7 +311,7 @@ jobs:
     if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }}
     uses: ./.github/workflows/_npu-single-node-test-stage.yml
     with:
-      runner: linux-aarch64-a3-800t-16
+      runner: linux-aarch64-a3-16-
       test_type: 'accuracy'
       test_suite: base-c-test-acc-16-npu-a3
       image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
@@ -359,7 +359,7 @@ jobs:
         base-b-test-16-npu-a3,
 
         multimodal-gen-test-1-npu-a3,
-        multimodal-gen-test-2-npu-a3,
+        multimodal-gen-test-4-npu-a3,
 
         base-c-test-acc-2-npu-a3,
         base-c-test-acc-16-npu-a3,

python/sglang/multimodal_gen/test/server/ascend/conftest.py

@@ -0,0 +1,111 @@
+"""Ascend NPU conftest: evict stale model page cache before each test case.
+
+Memory-capped CI runners (e.g. a 128 GiB cgroup on the 4-NPU A3 pool) count
+reclaimable page cache from previously loaded models in cgroup
+``memory.current``, so ``host_memory_available_bytes()`` reports ~0 GiB
+available and layerwise offload degrades to the slow checkpoint-mapping path
+(observed as multi-fold latency regressions on minimax/mova perf cases).
+Dropping the page cache of non-current models before each case keeps the host
+memory budget healthy for the upcoming model load.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import pytest
+
+from sglang.multimodal_gen.test.server.ascend.testcase_configs_npu import (
+    MODELSCOPE_MODEL_WEIGHTS_DIR,
+)
+
+_CGROUP_V2_CURRENT = "/sys/fs/cgroup/memory.current"
+_CGROUP_V1_USAGE = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
+
+
+def _read_cgroup_memory_current() -> str:
+    """Best-effort read of the container's cgroup memory usage in bytes."""
+    for path in (_CGROUP_V2_CURRENT, _CGROUP_V1_USAGE):
+        try:
+            with open(path, encoding="utf-8") as handle:
+                return handle.read().strip()
+        except OSError:
+            continue
+    return "unknown"
+
+
+def _collect_case_weight_paths(server_args) -> set[str]:
+    """Real paths of every weight dir/file the upcoming case will load."""
+    paths = set()
+    for attr in ("model_path", "transformer_weights_path", "lora_path"):
+        value = getattr(server_args, attr, None)
+        if value:
+            paths.add(os.path.realpath(value))
+    for attr in ("component_paths", "component_weights_paths"):
+        mapping = getattr(server_args, attr, None)
+        if mapping:
+            paths.update(os.path.realpath(value) for value in mapping.values() if value)
+    return paths
+
+
+def _evict_dir_page_cache(root: str, keep: set[str]) -> tuple[int, int]:
+    """Drop page cache of files under ``root`` except those under ``keep``.
+
+    ``posix_fadvise(POSIX_FADV_DONTNEED)`` only drops clean cache pages, so it
+    is safe on read-only model checkpoints.
+    """
+    evicted_files = 0
+    evicted_bytes = 0
+    for dirpath, _dirnames, filenames in os.walk(root):
+        for name in filenames:
+            path = os.path.realpath(os.path.join(dirpath, name))
+            if any(path == kept or path.startswith(kept + os.sep) for kept in keep):
+                continue
+            try:
+                fd = os.open(path, os.O_RDONLY)
+                try:
+                    os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+                    evicted_bytes += os.fstat(fd).st_size
+                    evicted_files += 1
+                finally:
+                    os.close(fd)
+            except OSError:
+                continue
+    return evicted_files, evicted_bytes
+
+
+@pytest.fixture(autouse=True)
+def _evict_stale_model_page_cache(request):
+    """Evict page cache of models not used by the upcoming test case.
+
+    For parametrized diffusion cases the current model's weights are kept so
+    the server load is not penalized; only stale cache from previous cases is
+    dropped. For standalone (non-parametrized) tests everything is dropped.
+    """
+    if sys.platform != "linux" or not os.path.isdir(MODELSCOPE_MODEL_WEIGHTS_DIR):
+        yield
+        return
+
+    keep: set[str] = set()
+    callspec = getattr(request.node, "callspec", None)
+    if callspec is not None:
+        for value in callspec.params.values():
+            server_args = getattr(value, "server_args", None)
+            if server_args is not None:
+                keep |= _collect_case_weight_paths(server_args)
+
+    before = _read_cgroup_memory_current()
+    print(
+        f"[CONFTEST] Evicting stale model page cache before {request.node.nodeid} "
+        f"(keep: {sorted(os.path.basename(p) for p in keep) or 'none'})"
+    )
+    print(f"[CONFTEST] cgroup memory.current before: {before}")
+
+    total_files, total_bytes = _evict_dir_page_cache(MODELSCOPE_MODEL_WEIGHTS_DIR, keep)
+    print(
+        f"[CONFTEST] Page cache eviction summary: {total_files} files, "
+        f"{total_bytes / 1024**3:.2f} GiB evicted"
+    )
+    print(f"[CONFTEST] cgroup memory.current after: {_read_cgroup_memory_current()}")
+    yield

python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py

@@ -176,6 +176,7 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [
                 "SGLANG_CACHE_DIT_TAYLORSEER": "true",
                 "SGLANG_CACHE_DIT_TS_ORDER": "2",
                 "HCCL_BUFFSIZE": "256",
+                "SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "64",
             },
         ),
         DiffusionSamplingParams(
@@ -227,6 +228,9 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [
             num_gpus=2,
             tp_size=2,
             dit_layerwise_offload=True,
+            env_vars={
+                "SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "96",
+            },
             extras=EXTRAS_DISABLE_WARMUP,
         ),
         run_consistency_check=False,

test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py

@@ -18,7 +18,6 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_npu_ci(est_time=700, suite="base-b-test-16-npu-a3")
 register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True)
 
 

test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py

@@ -23,7 +23,6 @@ from sglang.test.test_utils import (
     popen_launch_pd_server,
 )
 
-register_npu_ci(est_time=3600, suite="base-b-test-16-npu-a3")
 register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True)
 
 load_balance_method_options = [

test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py

@@ -1,105 +0,0 @@
-import os
-import unittest
-from types import SimpleNamespace
-from urllib.parse import urlparse
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke
-from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
-from sglang.test.test_utils import (
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
-register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
-
-TEST_MODEL_MATRIX = {
-    "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-R1-0528-W8A8": {
-        "accuracy": 0.95,
-        "latency": 1000,
-        "output_throughput": 6,
-    },
-}
-
-
-class TestAscendDeepEP(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.models = TEST_MODEL_MATRIX.keys()
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.url = urlparse(DEFAULT_URL_FOR_TEST)
-
-        cls.common_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "ascend",
-            "--mem-fraction-static",
-            0.8,
-            "--disable-radix-cache",
-            "--chunked-prefill-size",
-            32768,
-            "--tp-size",
-            16,
-            "--dp-size",
-            1,
-            "--ep-size",
-            16,
-            "--max-running-requests",
-            24,
-            "--moe-a2a-backend",
-            "deepep",
-            "--deepep-mode",
-            "auto",
-        ]
-
-        cls.extra_envs = {
-            "HCCL_BUFFSIZE": "1000",
-            "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
-            "SGLANG_NPU_USE_MLAPO": "1",
-            "TRANSFORMERS_VERBOSITY": "error",
-            "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
-        }
-        os.environ.update(cls.extra_envs)
-
-    def test_a_gsm8k(self):
-        for model in self.models:
-            with self.subTest(model=model):
-                process = popen_launch_server(
-                    model,
-                    self.base_url,
-                    timeout=2400,
-                    other_args=[
-                        *self.common_args,
-                    ],
-                )
-
-                try:
-                    if _is_pr_pipeline:
-                        run_npu_pr_smoke(self.base_url)
-                    else:
-                        print(f"##=== Testing accuracy: {model} ===##")
-
-                        args = SimpleNamespace(
-                            num_shots=5,
-                            data_path=None,
-                            num_questions=500,
-                            max_new_tokens=512,
-                            parallel=128,
-                            host=f"http://{self.url.hostname}",
-                            port=int(self.url.port),
-                        )
-
-                        metrics = run_eval_few_shot_gsm8k(args)
-                        self.assertGreaterEqual(
-                            metrics["accuracy"],
-                            TEST_MODEL_MATRIX[model]["accuracy"],
-                        )
-                finally:
-                    kill_process_tree(process.pid)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py

@@ -1,86 +0,0 @@
-import unittest
-from types import SimpleNamespace
-from urllib.parse import urlparse
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke
-from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
-from sglang.test.test_utils import (
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
-register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True)
-
-TEST_MODEL_MATRIX = {
-    "Qwen/Qwen3-30B-A3B-Instruct-2507": {
-        "accuracy": 0.90,
-        "latency": 180,
-        "output_throughput": 20,
-    },
-}
-
-
-class TestAscendTp4Bf16(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.models = TEST_MODEL_MATRIX.keys()
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.url = urlparse(DEFAULT_URL_FOR_TEST)
-        cls.common_args = [
-            "--trust-remote-code",
-            "--mem-fraction-static",
-            0.7,
-            "--max-running-requests",
-            32,
-            "--attention-backend",
-            "ascend",
-            "--disable-radix-cache",
-            "--cuda-graph-max-bs-decode",
-            32,
-            "--tp-size",
-            4,
-        ]
-
-    def test_a_gsm8k(self):
-        for model in self.models:
-            with self.subTest(model=model):
-                process = popen_launch_server(
-                    model,
-                    self.base_url,
-                    timeout=1800,
-                    other_args=[
-                        *self.common_args,
-                    ],
-                )
-
-                try:
-                    if _is_pr_pipeline:
-                        run_npu_pr_smoke(self.base_url)
-                    else:
-                        print(f"##=== Testing accuracy: {model} ===##")
-
-                        args = SimpleNamespace(
-                            num_shots=5,
-                            data_path=None,
-                            num_questions=1319,
-                            max_new_tokens=512,
-                            parallel=128,
-                            host=f"http://{self.url.hostname}",
-                            port=int(self.url.port),
-                        )
-
-                        metrics = run_eval_few_shot_gsm8k(args)
-                        self.assertGreaterEqual(
-                            metrics["accuracy"],
-                            TEST_MODEL_MATRIX[model]["accuracy"],
-                        )
-                finally:
-                    kill_process_tree(process.pid)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py

@@ -17,10 +17,9 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3")
 register_npu_ci(
     est_time=400,
-    suite="nightly-4-npu-a3",
+    suite="full-4-npu-a3",
     nightly=True,
 )
 
@@ -112,70 +111,6 @@ class TestNpuSpeculativeAttentionMode(CustomTestCase):
         finally:
             kill_process_tree(process.pid)
 
-    def test_speculative_attention_mode_prefill(self):
-        """Test --speculative-attention-mode prefill without PD disaggregation."""
-        args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "ascend",
-            "--device",
-            "npu",
-            "--quantization",
-            "modelslim",
-            "--disable-radix-cache",
-            "--speculative-draft-model-quantization",
-            "unquant",
-            "--speculative-algorithm",
-            "EAGLE3",
-            "--speculative-draft-model-path",
-            QWEN3_32B_EAGLE3_WEIGHTS_PATH,
-            "--speculative-num-steps",
-            "4",
-            "--speculative-eagle-topk",
-            "1",
-            "--speculative-num-draft-tokens",
-            "5",
-            "--speculative-attention-mode",
-            "prefill",
-            "--tp-size",
-            "4",
-            "--mem-fraction-static",
-            "0.7",
-            "--disable-cuda-graph",
-            "--dtype",
-            "bfloat16",
-        ]
-
-        env = os.environ.copy()
-        env.update(
-            {
-                "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
-                "SGLANG_ENABLE_SPEC_V2": "1",
-                "TRANSFORMERS_VERBOSITY": "error",
-            }
-        )
-
-        process = popen_launch_server(
-            QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH,
-            DEFAULT_URL_FOR_TEST,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
-            other_args=args,
-            env=env,
-        )
-
-        try:
-            if _is_pr_pipeline:
-                run_npu_pr_smoke(DEFAULT_URL_FOR_TEST)
-            else:
-                metrics = self._run_gsm8k_eval()
-                self.assertGreaterEqual(
-                    metrics["score"],
-                    0.83,
-                    f"GSM8K score {metrics['score']} below threshold 0.83",
-                )
-        finally:
-            kill_process_tree(process.pid)
-
 
 if __name__ == "__main__":
     unittest.main()

test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py

@@ -1,113 +0,0 @@
-import os
-import unittest
-from types import SimpleNamespace
-from urllib.parse import urlparse
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke
-from sglang.test.ascend.test_ascend_utils import (
-    DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH,
-)
-from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3")
-register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
-
-MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH
-
-
-class TestAscendSpeculativeDraftAttentionAndMoeRunner(CustomTestCase):
-    """Testcase: Test configuration '--speculative-draft-attention-backend' and '--speculative-moe-runner-backend' on the GSM8K dataset is no less than 0.9.
-
-    [Test Category] Parameter
-    [Test Target] --speculative-draft-attention-backend; --speculative-moe-runner-backend
-    """
-
-    os.environ["DEEP_NORMAL_MODE_USE_INT8_QUANT"] = "1"
-    os.environ["HCCL_BUFFSIZE"] = "2048"
-    os.environ["SGLANG_ENABLE_OVERLAP_PLAN_SITEAM"] = "1"
-    os.environ["SGLANG_ENABLE_SPEC_V2"] = "1"
-    env = os.environ.copy()
-
-    @classmethod
-    def setUpClass(cls):
-        cls.models = MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.url = urlparse(DEFAULT_URL_FOR_TEST)
-        cls.base_url = DEFAULT_URL_FOR_TEST
-
-        cls.common_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "ascend",
-            "--quantization",
-            "modelslim",
-            "--mem-fraction-static",
-            0.7,
-            "--disable-radix-cache",
-            "--chunked-prefill-size",
-            32768,
-            "--tp-size",
-            16,
-            "--speculative-algorithm",
-            "NEXTN",
-            "--speculative-num-steps",
-            1,
-            "--speculative-eagle-topk",
-            1,
-            "--speculative-num-draft-tokens",
-            2,
-            "--moe-a2a-backend",
-            "deepep",
-            "--deepep-mode",
-            "auto",
-            "--max-running-requests",
-            64,
-            "--speculative-draft-attention-backend",
-            "ascend",
-            "--speculative-moe-runner-backend",
-            "auto",
-        ]
-
-        cls.process = popen_launch_server(
-            MODEL_PATH,
-            cls.base_url,
-            timeout=1500,
-            other_args=cls.common_args,
-            env=cls.env,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(self):
-        if _is_pr_pipeline:
-            run_npu_pr_smoke(self.base_url)
-            return
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            eval_name="gsm8k",
-            api="completion",
-            num_examples=1319,
-            num_threads=128,
-            max_tokens=512,
-            num_shots=5,
-        )
-
-        metrics = run_eval(args)
-        score = metrics["score"]
-        print(f"GSM8K score for {MODEL_PATH}: {score:.4f}")
-        self.assertIsNotNone(score, "GSM8K evaluation returned no score")
-        self.assertIsInstance(score, float, "Score should be a float")
-        self.assertGreaterEqual(score, 0.9, f"GSM8K score {score} below threshold 0.9")
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py

@@ -29,7 +29,6 @@ class TestAscendDistTimeout(CustomTestCase):
         cls.url = urlparse(DEFAULT_URL_FOR_TEST)
         os.environ["HCCL_BUFFSIZE"] = "2048"
         os.environ["SGLANG_ENABLE_OVERLAP_PLAN_STREAM"] = "1"
-        os.environ["SGLANG_ENABLE_SPEC_V2"] = "1"
         os.environ["TRANSFORMERS_VERBOSITY"] = "error"
         cls.env = os.environ.copy()
         cls.common_args = [
@@ -62,6 +61,10 @@ class TestAscendDistTimeout(CustomTestCase):
             "auto",
             "--speculative-draft-model-quantization",
             "unquant",
+            "--speculative-draft-attention-backend",
+            "ascend",
+            "--speculative-moe-runner-backend",
+            "auto",
         ]
 
     def test_a_gsm8k(self):

test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py

@@ -20,8 +20,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_npu_ci(est_time=300, suite="base-b-test-4-npu-a3")
+register_npu_ci(est_time=300, suite="full-4-npu-a3", nightly=True)
-register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True)
 
 
 class TestNpuSpeculativeTokenMap(CustomTestCase):