diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 37224bd3e..6464945f6 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -227,7 +227,7 @@ jobs: if-no-files-found: ignore retention-days: 7 - multimodal-gen-test-2-npu-a3: + multimodal-gen-test-4-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }} runs-on: linux-aarch64-a3-800t-4 @@ -297,7 +297,7 @@ jobs: if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} uses: ./.github/workflows/_npu-single-node-test-stage.yml with: - runner: linux-aarch64-a3-800t-2 + runner: linux-aarch64-a3-2- test_type: 'accuracy' test_suite: base-c-test-acc-2-npu-a3 image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} @@ -311,7 +311,7 @@ jobs: if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} uses: ./.github/workflows/_npu-single-node-test-stage.yml with: - runner: linux-aarch64-a3-800t-16 + runner: linux-aarch64-a3-16- test_type: 'accuracy' test_suite: base-c-test-acc-16-npu-a3 image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} @@ -359,7 +359,7 @@ jobs: base-b-test-16-npu-a3, multimodal-gen-test-1-npu-a3, - multimodal-gen-test-2-npu-a3, + multimodal-gen-test-4-npu-a3, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, diff --git a/python/sglang/multimodal_gen/test/server/ascend/conftest.py b/python/sglang/multimodal_gen/test/server/ascend/conftest.py new file mode 100644 index 000000000..0e0afd054 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/ascend/conftest.py @@ -0,0 +1,111 @@ +"""Ascend NPU conftest: evict stale model page cache before each test case. + +Memory-capped CI runners (e.g. a 128 GiB cgroup on the 4-NPU A3 pool) count +reclaimable page cache from previously loaded models in cgroup +``memory.current``, so ``host_memory_available_bytes()`` reports ~0 GiB +available and layerwise offload degrades to the slow checkpoint-mapping path +(observed as multi-fold latency regressions on minimax/mova perf cases). +Dropping the page cache of non-current models before each case keeps the host +memory budget healthy for the upcoming model load. +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +from sglang.multimodal_gen.test.server.ascend.testcase_configs_npu import ( + MODELSCOPE_MODEL_WEIGHTS_DIR, +) + +_CGROUP_V2_CURRENT = "/sys/fs/cgroup/memory.current" +_CGROUP_V1_USAGE = "/sys/fs/cgroup/memory/memory.usage_in_bytes" + + +def _read_cgroup_memory_current() -> str: + """Best-effort read of the container's cgroup memory usage in bytes.""" + for path in (_CGROUP_V2_CURRENT, _CGROUP_V1_USAGE): + try: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + except OSError: + continue + return "unknown" + + +def _collect_case_weight_paths(server_args) -> set[str]: + """Real paths of every weight dir/file the upcoming case will load.""" + paths = set() + for attr in ("model_path", "transformer_weights_path", "lora_path"): + value = getattr(server_args, attr, None) + if value: + paths.add(os.path.realpath(value)) + for attr in ("component_paths", "component_weights_paths"): + mapping = getattr(server_args, attr, None) + if mapping: + paths.update(os.path.realpath(value) for value in mapping.values() if value) + return paths + + +def _evict_dir_page_cache(root: str, keep: set[str]) -> tuple[int, int]: + """Drop page cache of files under ``root`` except those under ``keep``. + + ``posix_fadvise(POSIX_FADV_DONTNEED)`` only drops clean cache pages, so it + is safe on read-only model checkpoints. + """ + evicted_files = 0 + evicted_bytes = 0 + for dirpath, _dirnames, filenames in os.walk(root): + for name in filenames: + path = os.path.realpath(os.path.join(dirpath, name)) + if any(path == kept or path.startswith(kept + os.sep) for kept in keep): + continue + try: + fd = os.open(path, os.O_RDONLY) + try: + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + evicted_bytes += os.fstat(fd).st_size + evicted_files += 1 + finally: + os.close(fd) + except OSError: + continue + return evicted_files, evicted_bytes + + +@pytest.fixture(autouse=True) +def _evict_stale_model_page_cache(request): + """Evict page cache of models not used by the upcoming test case. + + For parametrized diffusion cases the current model's weights are kept so + the server load is not penalized; only stale cache from previous cases is + dropped. For standalone (non-parametrized) tests everything is dropped. + """ + if sys.platform != "linux" or not os.path.isdir(MODELSCOPE_MODEL_WEIGHTS_DIR): + yield + return + + keep: set[str] = set() + callspec = getattr(request.node, "callspec", None) + if callspec is not None: + for value in callspec.params.values(): + server_args = getattr(value, "server_args", None) + if server_args is not None: + keep |= _collect_case_weight_paths(server_args) + + before = _read_cgroup_memory_current() + print( + f"[CONFTEST] Evicting stale model page cache before {request.node.nodeid} " + f"(keep: {sorted(os.path.basename(p) for p in keep) or 'none'})" + ) + print(f"[CONFTEST] cgroup memory.current before: {before}") + + total_files, total_bytes = _evict_dir_page_cache(MODELSCOPE_MODEL_WEIGHTS_DIR, keep) + print( + f"[CONFTEST] Page cache eviction summary: {total_files} files, " + f"{total_bytes / 1024**3:.2f} GiB evicted" + ) + print(f"[CONFTEST] cgroup memory.current after: {_read_cgroup_memory_current()}") + yield diff --git a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py index afccd1008..d3aac8fa6 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py +++ b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py @@ -176,6 +176,7 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [ "SGLANG_CACHE_DIT_TAYLORSEER": "true", "SGLANG_CACHE_DIT_TS_ORDER": "2", "HCCL_BUFFSIZE": "256", + "SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "64", }, ), DiffusionSamplingParams( @@ -227,6 +228,9 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [ num_gpus=2, tp_size=2, dit_layerwise_offload=True, + env_vars={ + "SGLANG_DIFFUSION_TEST_FORCE_HOST_AVAILABLE_GIB": "96", + }, extras=EXTRAS_DISABLE_WARMUP, ), run_consistency_check=False, diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py index cf347e0be..8bf6080c7 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py @@ -18,7 +18,6 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=700, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=700, suite="nightly-16-npu-a3", nightly=True) diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py index 730ae28ec..60fc0a61c 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py @@ -23,7 +23,6 @@ from sglang.test.test_utils import ( popen_launch_pd_server, ) -register_npu_ci(est_time=3600, suite="base-b-test-16-npu-a3") register_npu_ci(est_time=3600, suite="nightly-16-npu-a3", nightly=True) load_balance_method_options = [ diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py deleted file mode 100644 index cb38a6add..000000000 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py +++ /dev/null @@ -1,105 +0,0 @@ -import os -import unittest -from types import SimpleNamespace -from urllib.parse import urlparse - -from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3") -register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) - -TEST_MODEL_MATRIX = { - "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-R1-0528-W8A8": { - "accuracy": 0.95, - "latency": 1000, - "output_throughput": 6, - }, -} - - -class TestAscendDeepEP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.models = TEST_MODEL_MATRIX.keys() - cls.base_url = DEFAULT_URL_FOR_TEST - cls.url = urlparse(DEFAULT_URL_FOR_TEST) - - cls.common_args = [ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--mem-fraction-static", - 0.8, - "--disable-radix-cache", - "--chunked-prefill-size", - 32768, - "--tp-size", - 16, - "--dp-size", - 1, - "--ep-size", - 16, - "--max-running-requests", - 24, - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - ] - - cls.extra_envs = { - "HCCL_BUFFSIZE": "1000", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", - "SGLANG_NPU_USE_MLAPO": "1", - "TRANSFORMERS_VERBOSITY": "error", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - } - os.environ.update(cls.extra_envs) - - def test_a_gsm8k(self): - for model in self.models: - with self.subTest(model=model): - process = popen_launch_server( - model, - self.base_url, - timeout=2400, - other_args=[ - *self.common_args, - ], - ) - - try: - if _is_pr_pipeline: - run_npu_pr_smoke(self.base_url) - else: - print(f"##=== Testing accuracy: {model} ===##") - - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=500, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) - - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) - finally: - kill_process_tree(process.pid) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py deleted file mode 100644 index 638549082..000000000 --- a/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py +++ /dev/null @@ -1,86 +0,0 @@ -import unittest -from types import SimpleNamespace -from urllib.parse import urlparse - -from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") -register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) - -TEST_MODEL_MATRIX = { - "Qwen/Qwen3-30B-A3B-Instruct-2507": { - "accuracy": 0.90, - "latency": 180, - "output_throughput": 20, - }, -} - - -class TestAscendTp4Bf16(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.models = TEST_MODEL_MATRIX.keys() - cls.base_url = DEFAULT_URL_FOR_TEST - cls.url = urlparse(DEFAULT_URL_FOR_TEST) - cls.common_args = [ - "--trust-remote-code", - "--mem-fraction-static", - 0.7, - "--max-running-requests", - 32, - "--attention-backend", - "ascend", - "--disable-radix-cache", - "--cuda-graph-max-bs-decode", - 32, - "--tp-size", - 4, - ] - - def test_a_gsm8k(self): - for model in self.models: - with self.subTest(model=model): - process = popen_launch_server( - model, - self.base_url, - timeout=1800, - other_args=[ - *self.common_args, - ], - ) - - try: - if _is_pr_pipeline: - run_npu_pr_smoke(self.base_url) - else: - print(f"##=== Testing accuracy: {model} ===##") - - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=128, - host=f"http://{self.url.hostname}", - port=int(self.url.port), - ) - - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreaterEqual( - metrics["accuracy"], - TEST_MODEL_MATRIX[model]["accuracy"], - ) - finally: - kill_process_tree(process.pid) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py index cfc9dddec..a3cc30dae 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_attention_mode.py @@ -17,10 +17,9 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="base-b-test-4-npu-a3") register_npu_ci( est_time=400, - suite="nightly-4-npu-a3", + suite="full-4-npu-a3", nightly=True, ) @@ -112,70 +111,6 @@ class TestNpuSpeculativeAttentionMode(CustomTestCase): finally: kill_process_tree(process.pid) - def test_speculative_attention_mode_prefill(self): - """Test --speculative-attention-mode prefill without PD disaggregation.""" - args = [ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--device", - "npu", - "--quantization", - "modelslim", - "--disable-radix-cache", - "--speculative-draft-model-quantization", - "unquant", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - QWEN3_32B_EAGLE3_WEIGHTS_PATH, - "--speculative-num-steps", - "4", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "5", - "--speculative-attention-mode", - "prefill", - "--tp-size", - "4", - "--mem-fraction-static", - "0.7", - "--disable-cuda-graph", - "--dtype", - "bfloat16", - ] - - env = os.environ.copy() - env.update( - { - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_ENABLE_SPEC_V2": "1", - "TRANSFORMERS_VERBOSITY": "error", - } - ) - - process = popen_launch_server( - QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH, - DEFAULT_URL_FOR_TEST, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3, - other_args=args, - env=env, - ) - - try: - if _is_pr_pipeline: - run_npu_pr_smoke(DEFAULT_URL_FOR_TEST) - else: - metrics = self._run_gsm8k_eval() - self.assertGreaterEqual( - metrics["score"], - 0.83, - f"GSM8K score {metrics['score']} below threshold 0.83", - ) - finally: - kill_process_tree(process.pid) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py deleted file mode 100644 index 485e54b4f..000000000 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_draft_attention_backend.py +++ /dev/null @@ -1,113 +0,0 @@ -import os -import unittest -from types import SimpleNamespace -from urllib.parse import urlparse - -from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke -from sglang.test.ascend.test_ascend_utils import ( - DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH, -) -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_npu_ci(est_time=400, suite="base-b-test-16-npu-a3") -register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True) - -MODEL_PATH = DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH - - -class TestAscendSpeculativeDraftAttentionAndMoeRunner(CustomTestCase): - """Testcase: Test configuration '--speculative-draft-attention-backend' and '--speculative-moe-runner-backend' on the GSM8K dataset is no less than 0.9. - - [Test Category] Parameter - [Test Target] --speculative-draft-attention-backend; --speculative-moe-runner-backend - """ - - os.environ["DEEP_NORMAL_MODE_USE_INT8_QUANT"] = "1" - os.environ["HCCL_BUFFSIZE"] = "2048" - os.environ["SGLANG_ENABLE_OVERLAP_PLAN_SITEAM"] = "1" - os.environ["SGLANG_ENABLE_SPEC_V2"] = "1" - env = os.environ.copy() - - @classmethod - def setUpClass(cls): - cls.models = MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.url = urlparse(DEFAULT_URL_FOR_TEST) - cls.base_url = DEFAULT_URL_FOR_TEST - - cls.common_args = [ - "--trust-remote-code", - "--attention-backend", - "ascend", - "--quantization", - "modelslim", - "--mem-fraction-static", - 0.7, - "--disable-radix-cache", - "--chunked-prefill-size", - 32768, - "--tp-size", - 16, - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - 1, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 2, - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - "--max-running-requests", - 64, - "--speculative-draft-attention-backend", - "ascend", - "--speculative-moe-runner-backend", - "auto", - ] - - cls.process = popen_launch_server( - MODEL_PATH, - cls.base_url, - timeout=1500, - other_args=cls.common_args, - env=cls.env, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k(self): - if _is_pr_pipeline: - run_npu_pr_smoke(self.base_url) - return - args = SimpleNamespace( - base_url=self.base_url, - eval_name="gsm8k", - api="completion", - num_examples=1319, - num_threads=128, - max_tokens=512, - num_shots=5, - ) - - metrics = run_eval(args) - score = metrics["score"] - print(f"GSM8K score for {MODEL_PATH}: {score:.4f}") - self.assertIsNotNone(score, "GSM8K evaluation returned no score") - self.assertIsInstance(score, float, "Score should be a float") - self.assertGreaterEqual(score, 0.9, f"GSM8K score {score} below threshold 0.9") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py index f4a1e1b3f..9ebb2f6c4 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_moe_a2a_backend.py @@ -29,7 +29,6 @@ class TestAscendDistTimeout(CustomTestCase): cls.url = urlparse(DEFAULT_URL_FOR_TEST) os.environ["HCCL_BUFFSIZE"] = "2048" os.environ["SGLANG_ENABLE_OVERLAP_PLAN_STREAM"] = "1" - os.environ["SGLANG_ENABLE_SPEC_V2"] = "1" os.environ["TRANSFORMERS_VERBOSITY"] = "error" cls.env = os.environ.copy() cls.common_args = [ @@ -62,6 +61,10 @@ class TestAscendDistTimeout(CustomTestCase): "auto", "--speculative-draft-model-quantization", "unquant", + "--speculative-draft-attention-backend", + "ascend", + "--speculative-moe-runner-backend", + "auto", ] def test_a_gsm8k(self): diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py index 3cde909da..048371ed9 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_speculative_token_map.py @@ -20,8 +20,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=300, suite="base-b-test-4-npu-a3") -register_npu_ci(est_time=300, suite="nightly-4-npu-a3", nightly=True) +register_npu_ci(est_time=300, suite="full-4-npu-a3", nightly=True) class TestNpuSpeculativeTokenMap(CustomTestCase):