From 1b481deade53fd23ed787c4dbd5f389b1455a7b2 Mon Sep 17 00:00:00 2001 From: hhhh1252023 <139246842+hhhh1252023@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:41:15 +0800 Subject: [PATCH] feat: sync npu nightly test improvements from Ascend testcases (#29403) --- .../nightly-test-npu-e2e-single-node.yml | 29 +- .github/workflows/nightly-test-npu.yml | 6 +- .github/workflows/pr-test-npu.yml | 28 ++ .../ascend/e2e/test_npu_accuracy_utils.py | 252 +++++++++++++----- .../ascend/e2e/test_npu_multi_node_utils.py | 61 +++++ .../ascend/e2e/test_npu_performance_utils.py | 63 ++++- .../test_npu_deepseek_v3_2_8p_aime25.py | 4 + .../test_npu_glm4_6v_flash_1p_mmmu.py | 4 + .../test_npu_glm4_7_flash_1p_aime25.py | 4 + .../test_npu_qwen3_32b_bf16_8p_gpqa.py | 6 +- .../test_npu_qwen3_6_27b_1p_gpqa.py | 10 +- .../test_npu_qwen3_6_35b_a3b_1p_aime26.py | 10 +- ...t_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py | 57 +--- .../test_npu_qwen3_vl_8b_thinking_1p_mmmu.py | 39 +-- ...1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py | 10 +- ..._k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py | 6 +- ..._w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py | 8 +- ...max_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py | 6 +- ...qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 4 + ..._qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py | 6 +- ...n3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py | 4 + ...t_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py | 6 +- ..._qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py | 16 +- ...wen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py | 4 + ...n3_6_27b_1p_in1024x1024_30_out1024_50ms.py | 9 +- ...u_qwen3_6_27b_1p_in1080p_30_out256_50ms.py | 9 +- ...wen3_6_27b_2p_in64k_out1k_prefix90_50ms.py | 28 +- ...n3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 20 +- ...pu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py | 99 +++++++ ...u_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py | 6 +- ...pu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py | 28 +- ...pu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py | 29 +- ...6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py | 7 +- ...pu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py | 32 ++- ...npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py | 5 + ...a3b_1p_in64k_out1k_prefix90_50ms_aime26.py | 20 +- ...ext_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py | 8 +- 37 files changed, 708 insertions(+), 235 deletions(-) create mode 100644 test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py diff --git a/.github/workflows/nightly-test-npu-e2e-single-node.yml b/.github/workflows/nightly-test-npu-e2e-single-node.yml index 3d8ca0762..e288d3362 100644 --- a/.github/workflows/nightly-test-npu-e2e-single-node.yml +++ b/.github/workflows/nightly-test-npu-e2e-single-node.yml @@ -30,6 +30,16 @@ on: type: boolean default: false description: use sglang from source code or from docker image + install_sglang_deps: + required: false + type: boolean + default: false + description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation + device_type_for_deps: + required: false + type: string + default: 'a3' + description: device type for dependency installation (a3 or 910b) transformers_version: required: false type: string @@ -54,8 +64,25 @@ jobs: run: | npu-smi info + - name: Install sglang dependencies + if: ${{ inputs.install_sglang_deps == true }} + shell: bash + env: + TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" + PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" + RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082" + run: | + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host "${CACHING_URL}" + bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }} + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + - name: Run test - timeout-minutes: 120 + timeout-minutes: 300 env: SGLANG_USE_MODELSCOPE: true HF_ENDPOINT: https://hf-mirror.com diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index 0e6f49ce3..c743732af 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -1,4 +1,4 @@ -name: Nightly Test (NPU) +name: Nightly Test (NPU) on: schedule: - cron: '0 18 * * *' # Execute at 2:00 a.m. Beijing Time every day @@ -168,9 +168,9 @@ jobs: runner: linux-aarch64-a3-4 test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py test_type: 'perf' - - name: qwen3_6_27b_w8a8_2p_in64k_out1k_50ms + - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type: 'perf' - name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms runner: linux-aarch64-a3-4 diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index f2822495a..a1a2cf228 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -408,6 +408,32 @@ jobs: if-no-files-found: ignore retention-days: 7 + pr-single-node-tests: + name: single-node-poc + needs: [check-changes, pr-gate, set-image-config] + if: needs.check-changes.outputs.main_package == 'true' + strategy: + fail-fast: false + max-parallel: 6 + matrix: + test_config: + # qwen3_6_27b performance tests + - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py + test_type: 'perf' + uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml + with: + runner: ${{ matrix.test_config.runner }} + test_type: ${{ matrix.test_config.test_type }} + test_config_name: ${{ matrix.test_config.name }} + test_case: ${{ matrix.test_config.test_case }} + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + install_sglang_from_source: false + install_sglang_deps: true + device_type_for_deps: 'a3' + transformers_version: '' + pr-test-npu-finish: needs: [ @@ -420,6 +446,8 @@ jobs: multimodal-gen-test-1-npu-a3, multimodal-gen-test-2-npu-a3, + + pr-single-node-tests, ] if: always() runs-on: ubuntu-latest diff --git a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py index 58a63df18..6dc75173a 100644 --- a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py @@ -45,6 +45,21 @@ MAX_SERVER_KEEP_ALIVE_TIME = 3600 ACCURACY_TOLERANCE = 0.99 +# Dataset total question counts and allowed fluctuation (in questions) +DATASET_QUESTION_COUNTS = { + "aime25": 30, + "aime26": 30, + "gpqa_diamond": 198, +} + +DATASET_FLUCTUATION = { + "aime25": 2, + "aime26": 2, + "gpqa_diamond": 5, +} + +MAX_RETRY_COUNT = 3 + SERVER_INITIALIZATION_DELAY = 120 if os.environ.get("ASCEND_RT_VISIBLE_DEVICES"): @@ -58,6 +73,31 @@ else: DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_SERVER_PORT_FOR_TEST + 66}" +def get_accuracy_threshold(datasets, baseline_accuracy): + """Calculate accuracy threshold based on dataset fluctuation tolerance. + + For datasets with defined fluctuation (aime*, gpqa_diamond), use absolute + question count tolerance. For others (e.g. mmmu), use percentage tolerance. + """ + dataset = datasets[0] if datasets else None + if dataset in DATASET_FLUCTUATION and dataset in DATASET_QUESTION_COUNTS: + fluctuation = DATASET_FLUCTUATION[dataset] / DATASET_QUESTION_COUNTS[dataset] + return baseline_accuracy - fluctuation + return baseline_accuracy * ACCURACY_TOLERANCE + + +def get_max_retries(datasets): + """Return max retry count for accuracy tests. + + gpqa and aime datasets support up to MAX_RETRY_COUNT retries. + mmmu and others use 1 attempt (no retry). + """ + dataset = datasets[0] if datasets else None + if dataset in DATASET_FLUCTUATION: + return MAX_RETRY_COUNT + return 1 + + def run_evalscope( host, port, @@ -212,6 +252,7 @@ def assert_metrics(self, metrics): raise Exception("No metrics obtained from benchmark") if self.accuracy is not None: + threshold = get_accuracy_threshold(self.datasets, self.accuracy) dump_metric( "accuracy", float(metrics["accuracy"]), @@ -224,14 +265,11 @@ def assert_metrics(self, metrics): ) self.assertGreaterEqual( float(metrics["accuracy"]), - self.accuracy * ACCURACY_TOLERANCE, - f"Accuracy check failed. Expected >= {self.accuracy * ACCURACY_TOLERANCE}, Got: {metrics['accuracy']}", + threshold, + f"Accuracy check failed. Expected >= {threshold}, Got: {metrics['accuracy']}", ) -MMMU_LOCAL_PATH = "/root/.cache/modelscope/hub/datasets/AI-ModelScope___mmmu" - - class TestNpuAccuracyTestCaseBase(CustomTestCase): model = None benchmark_tool = BENCHMARK_TOOL_DEFAULT @@ -249,6 +287,7 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH envs = None max_attempts = 2 + n_runs = 3 accuracy = 0.1 @classmethod @@ -280,29 +319,67 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): except Exception as e: logger.error(f"Error during tearDown: {e}") - def _get_dataset_args(self): - if "mmmu" in self.datasets: - base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}} - if self.dataset_args: - if isinstance(self.dataset_args, dict): - base_args.update(self.dataset_args) - elif isinstance(self.dataset_args, str): - base_args.update(json.loads(self.dataset_args)) - return base_args - return self.dataset_args - def run_accuracy(self): parsed_url = urlparse(self.base_url) host = parsed_url.hostname port = parsed_url.port if self.benchmark_tool == EVALSCOPE: model_name = os.path.basename(self.model) + max_retries = get_max_retries(self.datasets) + best_metrics = None + for attempt in range(max_retries): + metrics = run_evalscope( + host=host, + port=port, + model=model_name, + datasets=self.datasets, + dataset_args=self.dataset_args, + eval_batch_size=self.eval_batch_size, + limit=self.limit, + generation_config=self.generation_config, + dataset_dir=self.dataset_dir, + stream=self.stream, + timeout=self.timeout, + eval_type=self.eval_type, + ) + if best_metrics is None or float(metrics.get("accuracy", 0)) > float( + best_metrics.get("accuracy", 0) + ): + best_metrics = metrics + threshold = get_accuracy_threshold(self.datasets, self.accuracy) + if float(best_metrics.get("accuracy", 0)) >= threshold: + break + if attempt < max_retries - 1: + logger.info( + f"Accuracy {best_metrics.get('accuracy')} below threshold " + f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..." + ) + assert_metrics(self, best_metrics) + + def run_accuracy_multiple(self, n_runs=None): + if n_runs is None: + n_runs = self.n_runs + + parsed_url = urlparse(self.base_url) + host = parsed_url.hostname + port = parsed_url.port + + if self.benchmark_tool != EVALSCOPE: + raise Exception( + "run_accuracy_multiple only supports evalscope benchmark tool" + ) + + model_name = os.path.basename(self.model) + all_metrics = [] + + for i in range(n_runs): + logger.info(f"=== Accuracy run {i + 1}/{n_runs} ===") metrics = run_evalscope( host=host, port=port, model=model_name, datasets=self.datasets, - dataset_args=self._get_dataset_args(), + dataset_args=self.dataset_args, eval_batch_size=self.eval_batch_size, limit=self.limit, generation_config=self.generation_config, @@ -311,7 +388,34 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase): timeout=self.timeout, eval_type=self.eval_type, ) - assert_metrics(self, metrics) + all_metrics.append(metrics) + if metrics and "accuracy" in metrics: + logger.info(f"Run {i + 1} accuracy: {metrics['accuracy']}") + else: + logger.warning(f"Run {i + 1} failed to get accuracy metric") + + valid_metrics = [m for m in all_metrics if m and "accuracy" in m] + if not valid_metrics: + raise Exception("No valid accuracy metrics obtained from any run") + + avg_accuracy = sum(float(m["accuracy"]) for m in valid_metrics) / len( + valid_metrics + ) + + logger.info("=" * 60) + logger.info("Multiple Run Accuracy Results:") + for i, m in enumerate(valid_metrics): + logger.info(f" Run {i + 1}: {m['accuracy']}") + logger.info(f" Average: {avg_accuracy}") + logger.info("=" * 60) + + avg_metrics = {"accuracy": avg_accuracy} + dump_metric( + "accuracy_avg", + avg_accuracy, + labels={"test_case": self.__class__.__name__, "type": "accuracy"}, + ) + assert_metrics(self, avg_metrics) class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase): @@ -378,17 +482,6 @@ class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase): ) time.sleep(MAX_SERVER_KEEP_ALIVE_TIME) - def _get_dataset_args(self): - if "mmmu" in self.datasets: - base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}} - if self.dataset_args: - if isinstance(self.dataset_args, dict): - base_args.update(self.dataset_args) - elif isinstance(self.dataset_args, str): - base_args.update(json.loads(self.dataset_args)) - return base_args - return self.dataset_args - @check_role(allowed_roles=["master", "worker"]) def run_accuracy(self): parsed_url = urlparse(self.base_url) @@ -396,21 +489,36 @@ class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase): port = parsed_url.port if self.benchmark_tool == EVALSCOPE: model_name = os.path.basename(self.model_config.get("model_path")) - metrics = run_evalscope( - host=self.host, - port=self.port, - model=model_name, - datasets=self.datasets, - dataset_args=self._get_dataset_args(), - eval_batch_size=self.eval_batch_size, - limit=self.limit, - generation_config=self.generation_config, - dataset_dir=self.dataset_dir, - stream=self.stream, - timeout=self.timeout, - eval_type=self.eval_type, - ) - assert_metrics(self, metrics) + max_retries = get_max_retries(self.datasets) + best_metrics = None + for attempt in range(max_retries): + metrics = run_evalscope( + host=self.host, + port=self.port, + model=model_name, + datasets=self.datasets, + dataset_args=self.dataset_args, + eval_batch_size=self.eval_batch_size, + limit=self.limit, + generation_config=self.generation_config, + dataset_dir=self.dataset_dir, + stream=self.stream, + timeout=self.timeout, + eval_type=self.eval_type, + ) + if best_metrics is None or float(metrics.get("accuracy", 0)) > float( + best_metrics.get("accuracy", 0) + ): + best_metrics = metrics + threshold = get_accuracy_threshold(self.datasets, self.accuracy) + if float(best_metrics.get("accuracy", 0)) >= threshold: + break + if attempt < max_retries - 1: + logger.info( + f"Accuracy {best_metrics.get('accuracy')} below threshold " + f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..." + ) + assert_metrics(self, best_metrics) class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase): @@ -490,17 +598,6 @@ class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase): f"Sglang process exited on node {cls.host} {cls.hostname} with exit code: {exit_code}" ) - def _get_dataset_args(self): - if "mmmu" in self.datasets: - base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}} - if self.dataset_args: - if isinstance(self.dataset_args, dict): - base_args.update(self.dataset_args) - elif isinstance(self.dataset_args, str): - base_args.update(json.loads(self.dataset_args)) - return base_args - return self.dataset_args - @check_role(allowed_roles=["router"]) def run_accuracy(self): parsed_url = urlparse(self.base_url) @@ -508,18 +605,33 @@ class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase): port = parsed_url.port if self.benchmark_tool == EVALSCOPE: model_name = os.path.basename(self.model_config.get("model_path")) - metrics = run_evalscope( - host=host, - port=port, - model=model_name, - datasets=self.datasets, - dataset_args=self._get_dataset_args(), - eval_batch_size=self.eval_batch_size, - limit=self.limit, - generation_config=self.generation_config, - dataset_dir=self.dataset_dir, - stream=self.stream, - timeout=self.timeout, - eval_type=self.eval_type, - ) - assert_metrics(self, metrics) + max_retries = get_max_retries(self.datasets) + best_metrics = None + for attempt in range(max_retries): + metrics = run_evalscope( + host=host, + port=port, + model=model_name, + datasets=self.datasets, + dataset_args=self.dataset_args, + eval_batch_size=self.eval_batch_size, + limit=self.limit, + generation_config=self.generation_config, + dataset_dir=self.dataset_dir, + stream=self.stream, + timeout=self.timeout, + eval_type=self.eval_type, + ) + if best_metrics is None or float(metrics.get("accuracy", 0)) > float( + best_metrics.get("accuracy", 0) + ): + best_metrics = metrics + threshold = get_accuracy_threshold(self.datasets, self.accuracy) + if float(best_metrics.get("accuracy", 0)) >= threshold: + break + if attempt < max_retries - 1: + logger.info( + f"Accuracy {best_metrics.get('accuracy')} below threshold " + f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..." + ) + assert_metrics(self, best_metrics) diff --git a/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py b/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py index 1fded0744..ed0f3191c 100644 --- a/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) NAMESPACE = os.environ.get("NAMESPACE") CONFIGMAP_NAME = os.environ.get("KUBE_CONFIG_MAP") +ACTIVE_TEST_CLASS = "active-test-class" LOCAL_TIMEOUT = 3600 ALL_ROLE_SET = {"prefill", "decode", "router", "master", "worker"} @@ -41,6 +42,7 @@ BOOTSTRAP_INIT_PORT = 8995 # Timeouts and delays ROUTER_CONFIGMAP_TIMEOUT = 300 SERVER_INITIALIZATION_DELAY = 30 +SERVICE_EXIT_WAIT_SECONDS = 120 def get_nic_name(): @@ -190,6 +192,65 @@ def query_configmap(name, namespace): return None +def upsert_configmap_field_strict( + name: str, + namespace: str, + key: str, + value: str, +): + """ + Add or update a field in ConfigMap using patch. + Strict mode: fail if ConfigMap does not exist. + """ + from kubernetes.client.rest import ApiException + + k8s_api = get_k8s_api() + patch = {"data": {key: value}} + + try: + k8s_api.patch_namespaced_config_map(name=name, namespace=namespace, body=patch) + logger.info(f"Upserted ConfigMap {name}: {key}={value}") + except ApiException as e: + if e.status == 404: + raise RuntimeError( + f"ConfigMap {name} does not exist in namespace {namespace}" + ) + logger.error(f"Failed to upsert ConfigMap {name}: {e}") + raise + + +def wait_for_prefill_decode_exit( + key: str, + value: str, + timeout: int = ROUTER_CONFIGMAP_TIMEOUT, + poll_interval: int = 15, +): + start_time = time.time() + + while time.time() - start_time < timeout: + configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE) + if not configmap or not configmap.data: + logger.info(f"ConfigMap data is not available yet, waiting for 15s...") + time.sleep(poll_interval) + continue + + existing_value = configmap.data.get(key) + + upsert_configmap_field_strict(CONFIGMAP_NAME, NAMESPACE, key, value) + + if existing_value is not None: + logger.info( + "%s already set (%s), waiting 120s for prefill/decode to exit ...", + key, + existing_value, + ) + time.sleep(SERVICE_EXIT_WAIT_SECONDS) + else: + logger.info("%s set for the first time (%s)", key, value) + + return + + # Get node count from Kubernetes def discover_worker_nodes(): """Discover worker nodes from Kubernetes. diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index 716945498..33a90b545 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -15,11 +15,16 @@ from sglang.test.ascend.e2e.gen_dataset_fixed_len import ( save_jsonl, ) from sglang.test.ascend.e2e.test_npu_multi_node_utils import ( + ACTIVE_TEST_CLASS, + CONFIGMAP_NAME, + NAMESPACE, SERVICE_PORT, check_role, launch_pd_mix_node, launch_pd_separation_node, launch_router, + query_configmap, + wait_for_prefill_decode_exit, wait_server_ready, ) from sglang.test.test_utils import ( @@ -338,6 +343,7 @@ def run_bench_serving( repeat_rate=None, temperature=None, top_p=None, + env=None, ): metrics_path = os.getenv("METRICS_DATA_FILE") result_file = ( @@ -449,7 +455,12 @@ def run_bench_serving( metrics = {"mean_ttft": None, "mean_tpot": None, "total_tps": None} process = subprocess.Popen( - cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1 + cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env, ) try: # Read output line by line @@ -883,6 +894,7 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): dp = None generation_kwargs = None + pop_sglang_is_in_ci_for_gsp = False @classmethod def setUpClass(cls): @@ -961,7 +973,15 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase): "top_p": self.top_p, } logger.info(f"Starting benchmark with parameters: {bench_params}") - metrics = run_bench_serving(**bench_params) + if ( + self.dataset_name == "generated-shared-prefix" + and self.pop_sglang_is_in_ci_for_gsp + ): + bench_env = os.environ.copy() + bench_env.pop("SGLANG_IS_IN_CI", None) + else: + bench_env = None + metrics = run_bench_serving(**bench_params, env=bench_env) assert_metrics(self, metrics) @@ -993,6 +1013,7 @@ class TestNpuPerfMultiNodePdMixTestCaseBase(CustomTestCase): dp = None generation_kwargs = None + pop_sglang_is_in_ci_for_gsp = False @classmethod def setUpClass(cls): @@ -1085,7 +1106,15 @@ class TestNpuPerfMultiNodePdMixTestCaseBase(CustomTestCase): "top_p": self.top_p, } logger.info(f"Starting benchmark with parameters: {bench_params}") - metrics = run_bench_serving(**bench_params) + if ( + self.dataset_name == "generated-shared-prefix" + and self.pop_sglang_is_in_ci_for_gsp + ): + bench_env = os.environ.copy() + bench_env.pop("SGLANG_IS_IN_CI", None) + else: + bench_env = None + metrics = run_bench_serving(**bench_params, env=bench_env) assert_metrics(self, metrics) @@ -1117,6 +1146,7 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase): dp = None generation_kwargs = None + pop_sglang_is_in_ci_for_gsp = False @classmethod def setUpClass(cls): @@ -1138,15 +1168,25 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase): @classmethod def tearDownClass(cls): + logger.info("Start exec tearDownClass") if cls.process: try: kill_process_tree(cls.process.pid) + for _ in range(60): + if cls.process.poll() is not None: + logger.info("Process fully exited") + break + time.sleep(1) + else: + logger.warning("Process did NOT exit in time") except Exception as e: logger.error(f"Error during tearDown: {e}") + logger.info("tearDownClass finished") @classmethod @check_role(allowed_roles=["router"]) def start_router_server(cls): + wait_for_prefill_decode_exit(key=ACTIVE_TEST_CLASS, value=cls.__name__) logger.info(f"Starting router in thread...") sglang_thread = threading.Thread(target=launch_router, args=(cls.model_config,)) sglang_thread.daemon = True @@ -1170,6 +1210,13 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase): # Loop to check if the process is still running while True: + configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE) + if configmap and configmap.data: + executing_class = configmap.data.get(ACTIVE_TEST_CLASS) + if executing_class and executing_class != cls.__name__: + logger.info(f"Retrieved ConfigMap data: {configmap.data}") + logger.info(f"[{cls.__name__}] exec completed, exiting waiter.") + return if cls.process.poll() is None: # Process is still running time.sleep(30) @@ -1226,5 +1273,13 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase): "top_p": self.top_p, } logger.info(f"Starting benchmark with parameters: {bench_params}") - metrics = run_bench_serving(**bench_params) + if ( + self.dataset_name == "generated-shared-prefix" + and self.pop_sglang_is_in_ci_for_gsp + ): + bench_env = os.environ.copy() + bench_env.pop("SGLANG_IS_IN_CI", None) + else: + bench_env = None + metrics = run_bench_serving(**bench_params, env=bench_env) assert_metrics(self, metrics) diff --git a/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py b/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py index f5d4756b6..bae1d231f 100644 --- a/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py +++ b/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py @@ -25,6 +25,10 @@ OTHER_ARGS = [ "--quantization", "modelslim", "--disable-radix-cache", + "--reasoning-parser", + "deepseek-v3", + "--tool-call-parser", + "deepseekv32", ] diff --git a/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py b/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py index c51f898fb..341470335 100644 --- a/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py +++ b/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py @@ -50,6 +50,10 @@ OTHER_ARGS = [ 32, "--watchdog-timeout", 9000, + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm45", ] diff --git a/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py b/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py index 9ba8bff31..678aa21e5 100644 --- a/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py +++ b/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py @@ -50,6 +50,10 @@ OTHER_ARGS = [ 32, "--watchdog-timeout", 9000, + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", ] diff --git a/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py b/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py index c76b0348e..2f71a8080 100644 --- a/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py +++ b/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py @@ -65,6 +65,10 @@ QWEN3_32B_OTHER_ARGS = [ 64, "--dtype", "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] @@ -72,7 +76,7 @@ class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): model = QWEN3_32B_MODEL_PATH envs = QWEN3_32B_ENVS other_args = QWEN3_32B_OTHER_ARGS - accuracy = 0.516 + accuracy = 0.4949 datasets = ["gpqa_diamond"] few_shot_num = 0 eval_batch_size = 64 diff --git a/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py b/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py index b24b05fdf..0c2514ebd 100644 --- a/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py +++ b/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py @@ -80,7 +80,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ ] -class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBase): +class TestNPUQwen3_6_27B_1P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBase): model = QWEN3_6_27B_MODEL_PATH envs = QWEN3_6_27B_64K_PREFIX_ENVS other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS @@ -88,7 +88,13 @@ class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBas datasets = ["gpqa_diamond"] few_shot_num = 0 eval_batch_size = 64 - generation_config = {"max_tokens": 81920, "temperature": 1.0} + generation_config = { + "max_tokens": 81920, + "temperature": 1.0, + "extra_body": { + "chat_template_kwargs": {"enable_thinking": True}, + }, + } def test_gpqa(self): self.run_accuracy() diff --git a/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py b/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py index 8b8032e05..cea8c3cdf 100644 --- a/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py +++ b/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py @@ -71,6 +71,10 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -84,7 +88,11 @@ class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_aime26(TestNpuAccuracyTestCaseBase): eval_batch_size = 4 generation_config = { "max_tokens": 131072, - "temperature": 0.2, + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + "min_p": 0.0, + "presence_penalty": 0.0, "repetition_penalty": 1.08, } diff --git a/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py b/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py index 04fb99519..f9720d552 100644 --- a/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py +++ b/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py @@ -16,71 +16,42 @@ register_npu_ci( ) ENVS = { - "ASCEND_LAUNCH_BLOCKING": "0", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", - "HCCL_BUFFSIZE": "400", + "HCCL_BUFFSIZE": "2000", } OTHER_ARGS = [ "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", "--attention-backend", "ascend", "--device", "npu", "--max-running-requests", - 162, + 128, "--disable-radix-cache", - # "--speculative-draft-model-quantization", - # "unquant", "--chunked-prefill-size", -1, "--max-prefill-tokens", - 35000, - # "--speculative-algorithm", - # "EAGLE3", - # "--speculative-draft-model-path", - # QWEN3_A3B_EAGLE_MODEL_PATH, - # "--speculative-num-steps", - # 3, - # "--speculative-eagle-topk", - # 1, - # "--speculative-num-draft-tokens", - # 4, + 12800, + "--prefill-max-requests", + 10, "--tp-size", 2, "--mem-fraction-static", - 0.87, - "--cuda-graph-bs", - 1, - 5, - 15, - 40, - 70, - 100, - 120, - 130, - 140, - 146, - 150, - 154, - 156, - 158, - 160, - 162, + 0.8, "--dtype", "bfloat16", + "--reasoning-parser", + "qwen3-thinking", + "--tool-call-parser", + "qwen", + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", ] @@ -91,7 +62,7 @@ class TestQwen3(TestNpuAccuracyTestCaseBase): accuracy = 0.76 datasets = ["mmmu"] few_shot_num = 0 - generation_config = {"max_tokens": 65536, "temperature": 1.0} + generation_config = {"max_tokens": 65536} eval_batch_size = 64 def test_mmmu(self): diff --git a/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py b/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py index fe49f2631..695e13041 100644 --- a/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py +++ b/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py @@ -21,22 +21,17 @@ ENVS = { "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_ENABLE_SPEC_V2": "1", + "HCCL_BUFFSIZE": "2000", } OTHER_ARGS = [ "--trust-remote-code", - "--nnodes", - "1", - "--node-rank", - "0", "--attention-backend", "ascend", "--device", "npu", "--max-running-requests", - 16, + 64, "--max-prefill-tokens", 16384, "--disable-radix-cache", @@ -45,26 +40,16 @@ OTHER_ARGS = [ "--tp-size", 2, "--mem-fraction-static", - 0.894, - "--cuda-graph-bs", - 1, - 5, - 15, - 16, + 0.8, "--dtype", "bfloat16", - # "--speculative-draft-model-quantization", - # "unquant", - # "--speculative-algorithm", - # "EAGLE3", - # "--speculative-draft-model-path", - # QWEN3_8B_EAGLE_MODEL_PATH, - # "--speculative-num-steps", - # 4, - # "--speculative-eagle-topk", - # 1, - # "--speculative-num-draft-tokens", - # 5, + "--reasoning-parser", + "qwen3-thinking", + "--tool-call-parser", + "qwen", + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", ] @@ -75,8 +60,8 @@ class TestQwen3(TestNpuAccuracyTestCaseBase): accuracy = 0.741 datasets = ["mmmu"] few_shot_num = 0 - generation_config = {"max_tokens": 65536, "temperature": 1.0} - eval_batch_size = 16 + generation_config = {"max_tokens": 65536} + eval_batch_size = 64 def test_mmmu(self): self.run_accuracy() diff --git a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index 1894dd8a9..9fcbcba73 100644 --- a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -97,6 +97,10 @@ GLM_5_1_PD_SEP_PREFILL_ARGS = [ 1, "--pp-size", 8, + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", ] GLM_5_1_PD_SEP_DECODE_ARGS = [ @@ -147,6 +151,10 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ "round_robin", "--speculative-draft-model-quantization", "unquant", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", ] GLM_5_1_PD_SEP_MODEL_CONFIG = { @@ -187,7 +195,7 @@ class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBa output_len = 1024 random_range_ratio = 1 tpot = 50 - output_token_throughput = 16 + output_token_throughput = 160 def test_npu_glm5_1_w4a8_pd_sep_in3k5_out1k5(self): """Run NPU performance test for GLM-5.1-w4a8 PD separation""" diff --git a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index f31529a02..7b20d2ff1 100644 --- a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -48,7 +48,7 @@ OTHER_ARGS = [ "--nnodes", 2, "--mem-fraction-static", - 0.55, + 0.662, "--max-running-requests", 32, "--chunked-prefill-size", @@ -82,6 +82,10 @@ OTHER_ARGS = [ 4, "--speculative-draft-model-quantization", "unquant", + "--reasoning-parser", + "kimi_k2", + "--tool-call-parser", + "kimi_k2", ] MODEL_CONFIG = { diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py index 0b7efbf89..2b2aa9dfa 100644 --- a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -86,6 +86,10 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ "--dtype", "bfloat16", "--trust-remote-code", + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", ] @@ -106,7 +110,7 @@ class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms( random_range_ratio = 1 repeat_rate = 0.9 tpot = 50 - output_token_throughput = 390.5839 + output_token_throughput = 390.5859 request_rate = float("inf") def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self): @@ -122,7 +126,7 @@ class TestNPUMiniMaxM2_5_W8A8_4P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): datasets = ["gpqa_diamond"] few_shot_num = 0 generation_config = {"max_tokens": 65536, "temperature": 1.0} - max_concurrency = 64 + eval_batch_size = 64 def test_accuracy(self): self.run_accuracy() diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index 219531538..27a15aaf6 100644 --- a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -87,6 +87,10 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ "unquant", "--dtype", "bfloat16", + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", ] @@ -122,7 +126,7 @@ class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): datasets = ["gpqa_diamond"] few_shot_num = 0 generation_config = {"max_tokens": 65536, "temperature": 1.0} - max_concurrency = 64 + eval_batch_size = 64 def test_accuracy(self): self.run_accuracy() diff --git a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index 4a16dc24c..3cec694f9 100644 --- a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -82,6 +82,10 @@ QWEN3_8B_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] diff --git a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py index 769a1cddb..9298a2f74 100644 --- a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py +++ b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py @@ -69,6 +69,10 @@ QWEN3_8B_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 5, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] @@ -84,7 +88,7 @@ class TestQwen8B(TestNpuPerformanceTestCaseBase): output_len = 1500 random_range_ratio = 1 tpot = 11.79 - output_token_throughput = 930 + output_token_throughput = 1040.96 def test_qwen3_8b(self): self.run_throughput() diff --git a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py index 1d5acf55e..3312bf040 100644 --- a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py +++ b/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -87,6 +87,10 @@ QWEN3_30B_A3B_OTHER_ARGS = [ 162, "--dtype", "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py index 675f7c832..54012c2d4 100644 --- a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py +++ b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py @@ -65,12 +65,16 @@ QWEN3_32B_OTHER_ARGS = [ 1, "--dtype", "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] class TestQwen32B(TestNpuPerformanceTestCaseBase): benchmark_tool = BENCHMARK_TOOL_DEFAULT - aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT model = QWEN3_32B_MODEL_PATH other_args = QWEN3_32B_OTHER_ARGS envs = QWEN3_32B_ENVS diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py index 7bf2ac958..fd18c6f70 100644 --- a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py +++ b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py @@ -84,6 +84,10 @@ QWEN3_32B_OTHER_ARGS = [ 101, "--dtype", "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] @@ -93,16 +97,12 @@ class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): model = QWEN3_32B_W8A8_MODEL_PATH other_args = QWEN3_32B_OTHER_ARGS envs = QWEN3_32B_ENVS - accuracy = 0.516 + accuracy = 0.4949 datasets = ["gpqa_diamond"] few_shot_num = 0 eval_batch_size = 64 generation_config = {"max_tokens": 40000, "temperature": 1.0} - @classmethod - def tearDownClass(cls): - pass - def test_qwen3_32b_qpqa(self): """Run NPU accuracy test for Qwen3-32B-W8A8 on qpqa""" self.run_accuracy() @@ -110,7 +110,7 @@ class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): class TestQwen32B(TestNpuPerformanceTestCaseBase): base_url = DEFAULT_URL_FOR_TEST - aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT model = QWEN3_32B_W8A8_MODEL_PATH other_args = QWEN3_32B_OTHER_ARGS envs = QWEN3_32B_ENVS @@ -123,10 +123,6 @@ class TestQwen32B(TestNpuPerformanceTestCaseBase): tpot = 50 output_token_throughput = 1600 - @classmethod - def setUpClass(cls): - pass - def test_qwen3_32b(self): self.run_throughput() diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py index 6a7cb216c..9724bb4d7 100644 --- a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py @@ -84,6 +84,10 @@ QWEN3_32B_OTHER_ARGS = [ 101, "--dtype", "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", ] diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py index b1b2ea62f..436ce4df4 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py @@ -80,6 +80,10 @@ QWEN3_6_27B_1024_OTHER_ARGS = [ "--speculative-num-draft-tokens", 4, "--mm-enable-dp-encoder", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -87,11 +91,12 @@ class TestNPUQwen3_6_27B_1P_In1024x1024_30_Out1024_50ms(TestNpuPerformanceTestCa """Test NPU performance for Qwen3.6-27B 1p in1024x1024 30 out1024 50ms""" benchmark_tool = BENCHMARK_TOOL_DEFAULT - aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN + dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN model = QWEN3_6_27B_MODEL_PATH other_args = QWEN3_6_27B_1024_OTHER_ARGS envs = QWEN3_6_27B_1024_ENVS - dataset_name = "random" + backend = "sglang-oai-chat" + dataset_name = "image" max_concurrency = 48 num_prompts = 48 input_len = 30 diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py index 72799ad68..3f15b99c0 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py @@ -77,6 +77,10 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [ "--speculative-num-draft-tokens", 4, "--mm-enable-dp-encoder", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -84,11 +88,12 @@ class TestNPUQwen3_6_27B_1P_In1080p_30_Out256_50ms(TestNpuPerformanceTestCaseBas """Test NPU performance for Qwen3.6-27B 1p in1080p 30 out256 50ms""" benchmark_tool = BENCHMARK_TOOL_DEFAULT - aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN + dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN model = QWEN3_6_27B_MODEL_PATH other_args = QWEN3_6_27B_1080P_OTHER_ARGS envs = QWEN3_6_27B_1080P_ENVS - dataset_name = "random" + backend = "sglang-oai-chat" + dataset_name = "image" max_concurrency = 30 num_prompts = 120 input_len = 30 diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py index 76c20a6d1..b775b4b00 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py @@ -46,24 +46,23 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ "--max-running-requests", 20, "--max-mamba-cache-size", - 120, + 160, "--mem-fraction-static", - 0.8, + 0.82, "--cuda-graph-bs", 1, 2, - 4, - 8, + 5, 10, - 12, - 16, - 18, + 15, + 17, + 19, 20, "--enable-prefill-delayer", "--prefill-delayer-queue-min-ratio", - 0.5, + 0.7, "--prefill-delayer-max-delay-ms", - 30000, + 20000, "--dtype", "bfloat16", "--mamba-ssm-dtype", @@ -76,11 +75,15 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] -class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCaseBase): - """Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k prefix90 50ms""" +class TestNPUQwen3_6_27B_1P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B 1p in64k out1k prefix90 50ms""" benchmark_tool = BENCHMARK_TOOL_DEFAULT model = QWEN3_6_27B_MODEL_PATH @@ -94,11 +97,12 @@ class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCase random_range_ratio = 1 repeat_rate = 0.9 request_rate = float("inf") + warmup_requests = 1 tpot = 50 output_token_throughput = 225 def test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms(self): - """Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k prefix90 50ms""" + """Run NPU performance test for Qwen3.6-27B in64k out1k prefix90 50ms""" self.run_throughput() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index 3eca45f15..6fc9a7e9e 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -58,8 +58,10 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [ 8, 16, 32, - 48, - 64, + 40, + 45, + 50, + 54, "--enable-multimodal", "--quantization", "modelslim", @@ -77,6 +79,10 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -106,11 +112,17 @@ class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase): model = QWEN3_6_27B_W8A8_MODEL_PATH envs = QWEN3_6_27B_3K5_1K5_ENVS other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS - accuracy = 0.855 + accuracy = 0.878 datasets = ["gpqa_diamond"] few_shot_num = 0 eval_batch_size = 8 - generation_config = {"max_tokens": 81920, "temperature": 1.0} + generation_config = { + "max_tokens": 81920, + "temperature": 1.0, + "extra_body": { + "chat_template_kwargs": {"enable_thinking": True}, + }, + } def test_accuracy(self): self.run_accuracy() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py new file mode 100644 index 000000000..4d9de7e18 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py @@ -0,0 +1,99 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_64K_1K_ENVS = { + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", +} + +QWEN3_6_27B_64K_1K_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 48000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 6, + "--max-mamba-cache-size", + 16, + "--mem-fraction-static", + 0.6, + "--cuda-graph-bs", + 1, + 2, + 4, + 5, + 6, + "--quantization", + "modelslim", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", +] + + +class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_27B_W8A8_MODEL_PATH + other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS + envs = QWEN3_6_27B_64K_1K_ENVS + dataset_name = "random" + max_concurrency = 6 + num_prompts = 12 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 57.85 + + def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self): + """Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py index 80b42a302..a17ff8781 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py @@ -65,6 +65,10 @@ QWEN3_6_27B_128K_OTHER_ARGS = [ "bfloat16", "--mamba-ssm-dtype", "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -72,7 +76,7 @@ class TestNPUQwen3_6_27B_2P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase): """Test NPU performance for Qwen3.6-27B-w8a8 2p in128k out1k 50ms""" benchmark_tool = BENCHMARK_TOOL_DEFAULT - aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT model = QWEN3_6_27B_W8A8_MODEL_PATH other_args = QWEN3_6_27B_128K_OTHER_ARGS envs = QWEN3_6_27B_128K_ENVS diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py index b869342af..2ff52e056 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py @@ -16,7 +16,6 @@ register_npu_ci( ) QWEN3_6_27B_16K_1k_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", @@ -25,7 +24,8 @@ QWEN3_6_27B_16K_1k_ENVS = { "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "GDN_ATTN_BACKEND_TRITON": "1", "ASCEND_USE_FIA": "1", } @@ -41,28 +41,28 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [ "--chunked-prefill-size", -1, "--max-prefill-tokens", - 50000, + 58000, "--disable-radix-cache", "--trust-remote-code", "--max-running-requests", - 28, + 29, "--max-mamba-cache-size", - 50, + 58, "--mem-fraction-static", - 0.7, + 0.68, "--cuda-graph-bs", + 1, 2, 8, 12, 16, 20, 24, + 26, 28, - "--enable-multimodal", + 29, "--quantization", "modelslim", - "--mm-attention-backend", - "ascend_attn", "--dtype", "bfloat16", "--mamba-ssm-dtype", @@ -75,6 +75,10 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -82,13 +86,13 @@ class TestNPUQwen3_6_27B_2P_In16k_Out1k_50ms(TestNpuPerformanceTestCaseBase): """Test NPU performance for Qwen3.6-27B-w8a8 2p in16k out1k 50ms""" benchmark_tool = BENCHMARK_TOOL_DEFAULT - aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT model = QWEN3_6_27B_W8A8_MODEL_PATH other_args = QWEN3_6_27B_16K_1k_OTHER_ARGS envs = QWEN3_6_27B_16K_1k_ENVS dataset_name = "random" - max_concurrency = 28 - num_prompts = 112 + max_concurrency = 29 + num_prompts = 116 input_len = 16000 output_len = 1000 random_range_ratio = 1 diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py index 3a5676f06..29d4f079d 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py @@ -18,6 +18,7 @@ register_npu_ci( QWEN3_6_35B_A3B_128K_1K_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "1600", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", @@ -26,6 +27,7 @@ QWEN3_6_35B_A3B_128K_1K_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "ASCEND_USE_FIA": "1", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20", + "GDN_ATTN_BACKEND_TRITON": "1", } QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [ @@ -40,21 +42,21 @@ QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [ "--chunked-prefill-size", -1, "--max-total-tokens", - 600000, + 420000, "--max-prefill-tokens", - 65536, + 128000, "--disable-radix-cache", "--trust-remote-code", - "--enable-prefill-delayer", "--max-running-requests", - 4, + 3, "--max-mamba-cache-size", - 12, + 3, "--mem-fraction-static", - 0.6, - "--max-mamba-cache-size", - 20, - "--disable-cuda-graph", + 0.9, + "--cuda-graph-bs", + 1, + 2, + 3, "--enable-multimodal", "--mm-attention-backend", "ascend_attn", @@ -70,6 +72,10 @@ QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -82,11 +88,12 @@ class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase) other_args = QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS envs = QWEN3_6_35B_A3B_128K_1K_ENVS dataset_name = "random" - max_concurrency = 4 - num_prompts = 16 + max_concurrency = 3 + num_prompts = 3 input_len = 128000 output_len = 1000 random_range_ratio = 1 + seed = 1 tpot = 50 output_token_throughput = 60.57 diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py index 5f3aacca9..978dc2279 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py @@ -76,6 +76,10 @@ QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -91,10 +95,11 @@ class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_Prefix90_50ms( dataset_name = "generated-shared-prefix" max_concurrency = 103 num_prompts = 412 - input_len = 64000 + input_len = 128000 output_len = 1000 random_range_ratio = 1 repeat_rate = 0.9 + seed = 1 tpot = 50 request_rate = float("inf") output_token_throughput = 308.2 diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py index 6813843ce..e82b14d44 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py @@ -18,7 +18,7 @@ register_npu_ci( QWEN3_6_35B_A3B_3K5_1K5_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", - "HCCL_BUFFSIZE": "800", + "HCCL_BUFFSIZE": "1", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", @@ -26,7 +26,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = { "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", "ASCEND_USE_FIA": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", } QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ @@ -40,25 +39,29 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ "npu", "--chunked-prefill-size", -1, + "--max-total-tokens", + 659840, "--max-prefill-tokens", - 35000, + 43400, "--disable-radix-cache", "--trust-remote-code", - "--enable-prefill-delayer", + "--prefill-max-requests", + "12", "--max-running-requests", - 110, + 122, "--max-mamba-cache-size", - 115, + 122, "--mem-fraction-static", - 0.78, + 0.9, "--cuda-graph-bs", 4, 16, 32, 64, - 84, - 105, - 110, + 96, + 116, + 120, + 122, "--enable-multimodal", "--mm-attention-backend", "ascend_attn", @@ -74,6 +77,10 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -86,11 +93,12 @@ class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase) other_args = QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS envs = QWEN3_6_35B_A3B_3K5_1K5_ENVS dataset_name = "random" - max_concurrency = 110 - num_prompts = 440 + max_concurrency = 122 + num_prompts = 122 input_len = 3500 output_len = 1500 random_range_ratio = 1 + seed = 1 tpot = 50 output_token_throughput = 2031.71 diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py index fa6b79f3c..208f23565 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py @@ -74,6 +74,10 @@ QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -91,6 +95,7 @@ class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase): input_len = 64000 output_len = 1000 random_range_ratio = 1 + seed = 1 tpot = 50 output_token_throughput = 141.72 diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py index 86bdd5ed1..7d4c0f942 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py @@ -43,18 +43,19 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ "npu", "--chunked-prefill-size", -1, + "--max-total-tokens", + 470784, "--max-prefill-tokens", 65536, "--trust-remote-code", - "--enable-prefill-delayer", "--mamba-scheduler-strategy", "extra_buffer", "--max-running-requests", - 42, + 40, "--max-mamba-cache-size", - 210, + 200, "--mem-fraction-static", - 0.71, + 0.9, "--cuda-graph-bs", 2, 8, @@ -63,7 +64,6 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ 32, 36, 40, - 42, "--enable-multimodal", "--mm-attention-backend", "ascend_attn", @@ -79,6 +79,10 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -118,15 +122,17 @@ class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_Prefix90_50ms( other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS dataset_name = "generated-shared-prefix" - max_concurrency = 42 - num_prompts = 42 + max_concurrency = 40 + num_prompts = 40 input_len = 65536 output_len = 1024 random_range_ratio = 1 repeat_rate = 0.9 + seed = 1 tpot = 50 request_rate = float("inf") output_token_throughput = 660 + pop_sglang_is_in_ci_for_gsp = True @classmethod def setUpClass(cls): diff --git a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 380507d99..2aef1850d 100644 --- a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -91,6 +91,10 @@ QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ "bfloat16", "--speculative-draft-model-path", QWEN3_NEXT_80B_A3B_MODEL_PATH, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", ] @@ -123,12 +127,12 @@ class TestQwen3Next80BA3B_aime25(TestNpuAccuracyTestCaseBase): "temperature": 0.7, "top_p": 0.8, "top_k": 20, - "extra_body": {"chat_template_kwargs": {"enable_thinking": True}}, + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, } max_concurrency = 16 def test_aime25(self): - self.run_accuracy() + self.run_accuracy_multiple(n_runs=3) if __name__ == "__main__":