minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

feat: sync npu nightly test improvements from Ascend testcases (#29403)

Browse Source
This commit is contained in:
hhhh1252023
2026-07-06 22:41:15 +08:00
committed by GitHub
parent 80decc78ec
commit 1b481deade
37 changed files with 708 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-npu-e2e-single-node.yml
+28 -1
View File
@@ -30,6 +30,16 @@ on:
type: boolean
default: false
description: use sglang from source code or from docker image
install_sglang_deps:
required: false
type: boolean
default: false
description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
device_type_for_deps:
required: false
type: string
default: 'a3'
description: device type for dependency installation (a3 or 910b)
transformers_version:
required: false
type: string
@@ -54,8 +64,25 @@ jobs:
run: |
npu-smi info
- name: Install sglang dependencies
if: ${{ inputs.install_sglang_deps == true }}
shell: bash
env:
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
run: |
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
pip config set global.index-url http://${CACHING_URL}/pypi/simple
pip config set global.trusted-host "${CACHING_URL}"
bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
- name: Run test
timeout-minutes: 120
timeout-minutes: 300
env:
SGLANG_USE_MODELSCOPE: true
HF_ENDPOINT: https://hf-mirror.com
.github/workflows/nightly-test-npu.yml
+3 -3
View File
@@ -1,4 +1,4 @@
name: Nightly Test (NPU)
name: Nightly Test (NPU)
on:
schedule:
- cron: '0 18 * * *' # Execute at 2:00 a.m. Beijing Time every day
@@ -168,9 +168,9 @@ jobs:
runner: linux-aarch64-a3-4
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_w8a8_2p_in64k_out1k_50ms
- name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
runner: linux-aarch64-a3-4
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
test_type: 'perf'
- name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms
runner: linux-aarch64-a3-4
.github/workflows/pr-test-npu.yml
+28
View File
@@ -408,6 +408,32 @@ jobs:
if-no-files-found: ignore
retention-days: 7
pr-single-node-tests:
name: single-node-poc
needs: [check-changes, pr-gate, set-image-config]
if: needs.check-changes.outputs.main_package == 'true'
strategy:
fail-fast: false
max-parallel: 6
matrix:
test_config:
# qwen3_6_27b performance tests
- name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
runner: linux-aarch64-a3-2
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
test_type: 'perf'
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
with:
runner: ${{ matrix.test_config.runner }}
test_type: ${{ matrix.test_config.test_type }}
test_config_name: ${{ matrix.test_config.name }}
test_case: ${{ matrix.test_config.test_case }}
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
install_sglang_from_source: false
install_sglang_deps: true
device_type_for_deps: 'a3'
transformers_version: ''
pr-test-npu-finish:
needs:
[
@@ -420,6 +446,8 @@ jobs:
multimodal-gen-test-1-npu-a3,
multimodal-gen-test-2-npu-a3,
pr-single-node-tests,
]
if: always()
runs-on: ubuntu-latest
python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py
+182 -70
View File
@@ -45,6 +45,21 @@ MAX_SERVER_KEEP_ALIVE_TIME = 3600
ACCURACY_TOLERANCE = 0.99
# Dataset total question counts and allowed fluctuation (in questions)
DATASET_QUESTION_COUNTS = {
"aime25": 30,
"aime26": 30,
"gpqa_diamond": 198,
}
DATASET_FLUCTUATION = {
"aime25": 2,
"aime26": 2,
"gpqa_diamond": 5,
}
MAX_RETRY_COUNT = 3
SERVER_INITIALIZATION_DELAY = 120
if os.environ.get("ASCEND_RT_VISIBLE_DEVICES"):
@@ -58,6 +73,31 @@ else:
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_SERVER_PORT_FOR_TEST + 66}"
def get_accuracy_threshold(datasets, baseline_accuracy):
"""Calculate accuracy threshold based on dataset fluctuation tolerance.
For datasets with defined fluctuation (aime*, gpqa_diamond), use absolute
question count tolerance. For others (e.g. mmmu), use percentage tolerance.
"""
dataset = datasets[0] if datasets else None
if dataset in DATASET_FLUCTUATION and dataset in DATASET_QUESTION_COUNTS:
fluctuation = DATASET_FLUCTUATION[dataset] / DATASET_QUESTION_COUNTS[dataset]
return baseline_accuracy - fluctuation
return baseline_accuracy * ACCURACY_TOLERANCE
def get_max_retries(datasets):
"""Return max retry count for accuracy tests.
gpqa and aime datasets support up to MAX_RETRY_COUNT retries.
mmmu and others use 1 attempt (no retry).
"""
dataset = datasets[0] if datasets else None
if dataset in DATASET_FLUCTUATION:
return MAX_RETRY_COUNT
return 1
def run_evalscope(
host,
port,
@@ -212,6 +252,7 @@ def assert_metrics(self, metrics):
raise Exception("No metrics obtained from benchmark")
if self.accuracy is not None:
threshold = get_accuracy_threshold(self.datasets, self.accuracy)
dump_metric(
"accuracy",
float(metrics["accuracy"]),
@@ -224,14 +265,11 @@ def assert_metrics(self, metrics):
)
self.assertGreaterEqual(
float(metrics["accuracy"]),
self.accuracy * ACCURACY_TOLERANCE,
f"Accuracy check failed. Expected >= {self.accuracy * ACCURACY_TOLERANCE}, Got: {metrics['accuracy']}",
threshold,
f"Accuracy check failed. Expected >= {threshold}, Got: {metrics['accuracy']}",
)
MMMU_LOCAL_PATH = "/root/.cache/modelscope/hub/datasets/AI-ModelScope___mmmu"
class TestNpuAccuracyTestCaseBase(CustomTestCase):
model = None
benchmark_tool = BENCHMARK_TOOL_DEFAULT
@@ -249,6 +287,7 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
envs = None
max_attempts = 2
n_runs = 3
accuracy = 0.1
@classmethod
@@ -280,29 +319,67 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
except Exception as e:
logger.error(f"Error during tearDown: {e}")
def _get_dataset_args(self):
if "mmmu" in self.datasets:
base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}}
if self.dataset_args:
if isinstance(self.dataset_args, dict):
base_args.update(self.dataset_args)
elif isinstance(self.dataset_args, str):
base_args.update(json.loads(self.dataset_args))
return base_args
return self.dataset_args
def run_accuracy(self):
parsed_url = urlparse(self.base_url)
host = parsed_url.hostname
port = parsed_url.port
if self.benchmark_tool == EVALSCOPE:
model_name = os.path.basename(self.model)
max_retries = get_max_retries(self.datasets)
best_metrics = None
for attempt in range(max_retries):
metrics = run_evalscope(
host=host,
port=port,
model=model_name,
datasets=self.datasets,
dataset_args=self.dataset_args,
eval_batch_size=self.eval_batch_size,
limit=self.limit,
generation_config=self.generation_config,
dataset_dir=self.dataset_dir,
stream=self.stream,
timeout=self.timeout,
eval_type=self.eval_type,
)
if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
best_metrics.get("accuracy", 0)
):
best_metrics = metrics
threshold = get_accuracy_threshold(self.datasets, self.accuracy)
if float(best_metrics.get("accuracy", 0)) >= threshold:
break
if attempt < max_retries - 1:
logger.info(
f"Accuracy {best_metrics.get('accuracy')} below threshold "
f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
)
assert_metrics(self, best_metrics)
def run_accuracy_multiple(self, n_runs=None):
if n_runs is None:
n_runs = self.n_runs
parsed_url = urlparse(self.base_url)
host = parsed_url.hostname
port = parsed_url.port
if self.benchmark_tool != EVALSCOPE:
raise Exception(
"run_accuracy_multiple only supports evalscope benchmark tool"
)
model_name = os.path.basename(self.model)
all_metrics = []
for i in range(n_runs):
logger.info(f"=== Accuracy run {i + 1}/{n_runs} ===")
metrics = run_evalscope(
host=host,
port=port,
model=model_name,
datasets=self.datasets,
dataset_args=self._get_dataset_args(),
dataset_args=self.dataset_args,
eval_batch_size=self.eval_batch_size,
limit=self.limit,
generation_config=self.generation_config,
@@ -311,7 +388,34 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
timeout=self.timeout,
eval_type=self.eval_type,
)
assert_metrics(self, metrics)
all_metrics.append(metrics)
if metrics and "accuracy" in metrics:
logger.info(f"Run {i + 1} accuracy: {metrics['accuracy']}")
else:
logger.warning(f"Run {i + 1} failed to get accuracy metric")
valid_metrics = [m for m in all_metrics if m and "accuracy" in m]
if not valid_metrics:
raise Exception("No valid accuracy metrics obtained from any run")
avg_accuracy = sum(float(m["accuracy"]) for m in valid_metrics) / len(
valid_metrics
)
logger.info("=" * 60)
logger.info("Multiple Run Accuracy Results:")
for i, m in enumerate(valid_metrics):
logger.info(f" Run {i + 1}: {m['accuracy']}")
logger.info(f" Average: {avg_accuracy}")
logger.info("=" * 60)
avg_metrics = {"accuracy": avg_accuracy}
dump_metric(
"accuracy_avg",
avg_accuracy,
labels={"test_case": self.__class__.__name__, "type": "accuracy"},
)
assert_metrics(self, avg_metrics)
class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase):
@@ -378,17 +482,6 @@ class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase):
)
time.sleep(MAX_SERVER_KEEP_ALIVE_TIME)
def _get_dataset_args(self):
if "mmmu" in self.datasets:
base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}}
if self.dataset_args:
if isinstance(self.dataset_args, dict):
base_args.update(self.dataset_args)
elif isinstance(self.dataset_args, str):
base_args.update(json.loads(self.dataset_args))
return base_args
return self.dataset_args
@check_role(allowed_roles=["master", "worker"])
def run_accuracy(self):
parsed_url = urlparse(self.base_url)
@@ -396,21 +489,36 @@ class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase):
port = parsed_url.port
if self.benchmark_tool == EVALSCOPE:
model_name = os.path.basename(self.model_config.get("model_path"))
metrics = run_evalscope(
host=self.host,
port=self.port,
model=model_name,
datasets=self.datasets,
dataset_args=self._get_dataset_args(),
eval_batch_size=self.eval_batch_size,
limit=self.limit,
generation_config=self.generation_config,
dataset_dir=self.dataset_dir,
stream=self.stream,
timeout=self.timeout,
eval_type=self.eval_type,
)
assert_metrics(self, metrics)
max_retries = get_max_retries(self.datasets)
best_metrics = None
for attempt in range(max_retries):
metrics = run_evalscope(
host=self.host,
port=self.port,
model=model_name,
datasets=self.datasets,
dataset_args=self.dataset_args,
eval_batch_size=self.eval_batch_size,
limit=self.limit,
generation_config=self.generation_config,
dataset_dir=self.dataset_dir,
stream=self.stream,
timeout=self.timeout,
eval_type=self.eval_type,
)
if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
best_metrics.get("accuracy", 0)
):
best_metrics = metrics
threshold = get_accuracy_threshold(self.datasets, self.accuracy)
if float(best_metrics.get("accuracy", 0)) >= threshold:
break
if attempt < max_retries - 1:
logger.info(
f"Accuracy {best_metrics.get('accuracy')} below threshold "
f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
)
assert_metrics(self, best_metrics)
class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase):
@@ -490,17 +598,6 @@ class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase):
f"Sglang process exited on node {cls.host} {cls.hostname} with exit code: {exit_code}"
)
def _get_dataset_args(self):
if "mmmu" in self.datasets:
base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}}
if self.dataset_args:
if isinstance(self.dataset_args, dict):
base_args.update(self.dataset_args)
elif isinstance(self.dataset_args, str):
base_args.update(json.loads(self.dataset_args))
return base_args
return self.dataset_args
@check_role(allowed_roles=["router"])
def run_accuracy(self):
parsed_url = urlparse(self.base_url)
@@ -508,18 +605,33 @@ class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase):
port = parsed_url.port
if self.benchmark_tool == EVALSCOPE:
model_name = os.path.basename(self.model_config.get("model_path"))
metrics = run_evalscope(
host=host,
port=port,
model=model_name,
datasets=self.datasets,
dataset_args=self._get_dataset_args(),
eval_batch_size=self.eval_batch_size,
limit=self.limit,
generation_config=self.generation_config,
dataset_dir=self.dataset_dir,
stream=self.stream,
timeout=self.timeout,
eval_type=self.eval_type,
)
assert_metrics(self, metrics)
max_retries = get_max_retries(self.datasets)
best_metrics = None
for attempt in range(max_retries):
metrics = run_evalscope(
host=host,
port=port,
model=model_name,
datasets=self.datasets,
dataset_args=self.dataset_args,
eval_batch_size=self.eval_batch_size,
limit=self.limit,
generation_config=self.generation_config,
dataset_dir=self.dataset_dir,
stream=self.stream,
timeout=self.timeout,
eval_type=self.eval_type,
)
if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
best_metrics.get("accuracy", 0)
):
best_metrics = metrics
threshold = get_accuracy_threshold(self.datasets, self.accuracy)
if float(best_metrics.get("accuracy", 0)) >= threshold:
break
if attempt < max_retries - 1:
logger.info(
f"Accuracy {best_metrics.get('accuracy')} below threshold "
f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
)
assert_metrics(self, best_metrics)
python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py
+61
View File
@@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
NAMESPACE = os.environ.get("NAMESPACE")
CONFIGMAP_NAME = os.environ.get("KUBE_CONFIG_MAP")
ACTIVE_TEST_CLASS = "active-test-class"
LOCAL_TIMEOUT = 3600
ALL_ROLE_SET = {"prefill", "decode", "router", "master", "worker"}
@@ -41,6 +42,7 @@ BOOTSTRAP_INIT_PORT = 8995
# Timeouts and delays
ROUTER_CONFIGMAP_TIMEOUT = 300
SERVER_INITIALIZATION_DELAY = 30
SERVICE_EXIT_WAIT_SECONDS = 120
def get_nic_name():
@@ -190,6 +192,65 @@ def query_configmap(name, namespace):
return None
def upsert_configmap_field_strict(
name: str,
namespace: str,
key: str,
value: str,
):
"""
Add or update a field in ConfigMap using patch.
Strict mode: fail if ConfigMap does not exist.
"""
from kubernetes.client.rest import ApiException
k8s_api = get_k8s_api()
patch = {"data": {key: value}}
try:
k8s_api.patch_namespaced_config_map(name=name, namespace=namespace, body=patch)
logger.info(f"Upserted ConfigMap {name}: {key}={value}")
except ApiException as e:
if e.status == 404:
raise RuntimeError(
f"ConfigMap {name} does not exist in namespace {namespace}"
)
logger.error(f"Failed to upsert ConfigMap {name}: {e}")
raise
def wait_for_prefill_decode_exit(
key: str,
value: str,
timeout: int = ROUTER_CONFIGMAP_TIMEOUT,
poll_interval: int = 15,
):
start_time = time.time()
while time.time() - start_time < timeout:
configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE)
if not configmap or not configmap.data:
logger.info(f"ConfigMap data is not available yet, waiting for 15s...")
time.sleep(poll_interval)
continue
existing_value = configmap.data.get(key)
upsert_configmap_field_strict(CONFIGMAP_NAME, NAMESPACE, key, value)
if existing_value is not None:
logger.info(
"%s already set (%s), waiting 120s for prefill/decode to exit ...",
key,
existing_value,
)
time.sleep(SERVICE_EXIT_WAIT_SECONDS)
else:
logger.info("%s set for the first time (%s)", key, value)
return
# Get node count from Kubernetes
def discover_worker_nodes():
"""Discover worker nodes from Kubernetes.
python/sglang/test/ascend/e2e/test_npu_performance_utils.py
+59 -4
View File
@@ -15,11 +15,16 @@ from sglang.test.ascend.e2e.gen_dataset_fixed_len import (
save_jsonl,
)
from sglang.test.ascend.e2e.test_npu_multi_node_utils import (
ACTIVE_TEST_CLASS,
CONFIGMAP_NAME,
NAMESPACE,
SERVICE_PORT,
check_role,
launch_pd_mix_node,
launch_pd_separation_node,
launch_router,
query_configmap,
wait_for_prefill_decode_exit,
wait_server_ready,
)
from sglang.test.test_utils import (
@@ -338,6 +343,7 @@ def run_bench_serving(
repeat_rate=None,
temperature=None,
top_p=None,
env=None,
):
metrics_path = os.getenv("METRICS_DATA_FILE")
result_file = (
@@ -449,7 +455,12 @@ def run_bench_serving(
metrics = {"mean_ttft": None, "mean_tpot": None, "total_tps": None}
process = subprocess.Popen(
cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
cmd_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
env=env,
)
try:
# Read output line by line
@@ -883,6 +894,7 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
dp = None
generation_kwargs = None
pop_sglang_is_in_ci_for_gsp = False
@classmethod
def setUpClass(cls):
@@ -961,7 +973,15 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
"top_p": self.top_p,
}
logger.info(f"Starting benchmark with parameters: {bench_params}")
metrics = run_bench_serving(**bench_params)
if (
self.dataset_name == "generated-shared-prefix"
and self.pop_sglang_is_in_ci_for_gsp
):
bench_env = os.environ.copy()
bench_env.pop("SGLANG_IS_IN_CI", None)
else:
bench_env = None
metrics = run_bench_serving(**bench_params, env=bench_env)
assert_metrics(self, metrics)
@@ -993,6 +1013,7 @@ class TestNpuPerfMultiNodePdMixTestCaseBase(CustomTestCase):
dp = None
generation_kwargs = None
pop_sglang_is_in_ci_for_gsp = False
@classmethod
def setUpClass(cls):
@@ -1085,7 +1106,15 @@ class TestNpuPerfMultiNodePdMixTestCaseBase(CustomTestCase):
"top_p": self.top_p,
}
logger.info(f"Starting benchmark with parameters: {bench_params}")
metrics = run_bench_serving(**bench_params)
if (
self.dataset_name == "generated-shared-prefix"
and self.pop_sglang_is_in_ci_for_gsp
):
bench_env = os.environ.copy()
bench_env.pop("SGLANG_IS_IN_CI", None)
else:
bench_env = None
metrics = run_bench_serving(**bench_params, env=bench_env)
assert_metrics(self, metrics)
@@ -1117,6 +1146,7 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase):
dp = None
generation_kwargs = None
pop_sglang_is_in_ci_for_gsp = False
@classmethod
def setUpClass(cls):
@@ -1138,15 +1168,25 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase):
@classmethod
def tearDownClass(cls):
logger.info("Start exec tearDownClass")
if cls.process:
try:
kill_process_tree(cls.process.pid)
for _ in range(60):
if cls.process.poll() is not None:
logger.info("Process fully exited")
break
time.sleep(1)
else:
logger.warning("Process did NOT exit in time")
except Exception as e:
logger.error(f"Error during tearDown: {e}")
logger.info("tearDownClass finished")
@classmethod
@check_role(allowed_roles=["router"])
def start_router_server(cls):
wait_for_prefill_decode_exit(key=ACTIVE_TEST_CLASS, value=cls.__name__)
logger.info(f"Starting router in thread...")
sglang_thread = threading.Thread(target=launch_router, args=(cls.model_config,))
sglang_thread.daemon = True
@@ -1170,6 +1210,13 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase):
# Loop to check if the process is still running
while True:
configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE)
if configmap and configmap.data:
executing_class = configmap.data.get(ACTIVE_TEST_CLASS)
if executing_class and executing_class != cls.__name__:
logger.info(f"Retrieved ConfigMap data: {configmap.data}")
logger.info(f"[{cls.__name__}] exec completed, exiting waiter.")
return
if cls.process.poll() is None:
# Process is still running
time.sleep(30)
@@ -1226,5 +1273,13 @@ class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase):
"top_p": self.top_p,
}
logger.info(f"Starting benchmark with parameters: {bench_params}")
metrics = run_bench_serving(**bench_params)
if (
self.dataset_name == "generated-shared-prefix"
and self.pop_sglang_is_in_ci_for_gsp
):
bench_env = os.environ.copy()
bench_env.pop("SGLANG_IS_IN_CI", None)
else:
bench_env = None
metrics = run_bench_serving(**bench_params, env=bench_env)
assert_metrics(self, metrics)
test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
+4
View File
@@ -25,6 +25,10 @@ OTHER_ARGS = [
"--quantization",
"modelslim",
"--disable-radix-cache",
"--reasoning-parser",
"deepseek-v3",
"--tool-call-parser",
"deepseekv32",
]
test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py
+4
View File
@@ -50,6 +50,10 @@ OTHER_ARGS = [
32,
"--watchdog-timeout",
9000,
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm45",
]
test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
+4
View File
@@ -50,6 +50,10 @@ OTHER_ARGS = [
32,
"--watchdog-timeout",
9000,
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
]
test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
+5 -1
View File
@@ -65,6 +65,10 @@ QWEN3_32B_OTHER_ARGS = [
64,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
@@ -72,7 +76,7 @@ class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase):
model = QWEN3_32B_MODEL_PATH
envs = QWEN3_32B_ENVS
other_args = QWEN3_32B_OTHER_ARGS
accuracy = 0.516
accuracy = 0.4949
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
+8 -2
View File
@@ -80,7 +80,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBase):
class TestNPUQwen3_6_27B_1P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBase):
model = QWEN3_6_27B_MODEL_PATH
envs = QWEN3_6_27B_64K_PREFIX_ENVS
other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS
@@ -88,7 +88,13 @@ class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBas
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 81920, "temperature": 1.0}
generation_config = {
"max_tokens": 81920,
"temperature": 1.0,
"extra_body": {
"chat_template_kwargs": {"enable_thinking": True},
},
}
def test_gpqa(self):
self.run_accuracy()
test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
+9 -1
View File
@@ -71,6 +71,10 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -84,7 +88,11 @@ class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_aime26(TestNpuAccuracyTestCaseBase):
eval_batch_size = 4
generation_config = {
"max_tokens": 131072,
"temperature": 0.2,
"temperature": 0.6,
"top_p": 0.95,
"top_k": 20,
"min_p": 0.0,
"presence_penalty": 0.0,
"repetition_penalty": 1.08,
}
test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
+14 -43
View File
@@ -16,71 +16,42 @@ register_npu_ci(
)
ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400",
"HCCL_BUFFSIZE": "2000",
}
OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
162,
128,
"--disable-radix-cache",
# "--speculative-draft-model-quantization",
# "unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
# "--speculative-algorithm",
# "EAGLE3",
# "--speculative-draft-model-path",
# QWEN3_A3B_EAGLE_MODEL_PATH,
# "--speculative-num-steps",
# 3,
# "--speculative-eagle-topk",
# 1,
# "--speculative-num-draft-tokens",
# 4,
12800,
"--prefill-max-requests",
10,
"--tp-size",
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
1,
5,
15,
40,
70,
100,
120,
130,
140,
146,
150,
154,
156,
158,
160,
162,
0.8,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3-thinking",
"--tool-call-parser",
"qwen",
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
]
@@ -91,7 +62,7 @@ class TestQwen3(TestNpuAccuracyTestCaseBase):
accuracy = 0.76
datasets = ["mmmu"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
generation_config = {"max_tokens": 65536}
eval_batch_size = 64
def test_mmmu(self):
test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
+12 -27
View File
@@ -21,22 +21,17 @@ ENVS = {
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"HCCL_BUFFSIZE": "2000",
}
OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
16,
64,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
@@ -45,26 +40,16 @@ OTHER_ARGS = [
"--tp-size",
2,
"--mem-fraction-static",
0.894,
"--cuda-graph-bs",
1,
5,
15,
16,
0.8,
"--dtype",
"bfloat16",
# "--speculative-draft-model-quantization",
# "unquant",
# "--speculative-algorithm",
# "EAGLE3",
# "--speculative-draft-model-path",
# QWEN3_8B_EAGLE_MODEL_PATH,
# "--speculative-num-steps",
# 4,
# "--speculative-eagle-topk",
# 1,
# "--speculative-num-draft-tokens",
# 5,
"--reasoning-parser",
"qwen3-thinking",
"--tool-call-parser",
"qwen",
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
]
@@ -75,8 +60,8 @@ class TestQwen3(TestNpuAccuracyTestCaseBase):
accuracy = 0.741
datasets = ["mmmu"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 16
generation_config = {"max_tokens": 65536}
eval_batch_size = 64
def test_mmmu(self):
self.run_accuracy()
test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
+9 -1
View File
@@ -97,6 +97,10 @@ GLM_5_1_PD_SEP_PREFILL_ARGS = [
1,
"--pp-size",
8,
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
]
GLM_5_1_PD_SEP_DECODE_ARGS = [
@@ -147,6 +151,10 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [
"round_robin",
"--speculative-draft-model-quantization",
"unquant",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
]
GLM_5_1_PD_SEP_MODEL_CONFIG = {
@@ -187,7 +195,7 @@ class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBa
output_len = 1024
random_range_ratio = 1
tpot = 50
output_token_throughput = 16
output_token_throughput = 160
def test_npu_glm5_1_w4a8_pd_sep_in3k5_out1k5(self):
"""Run NPU performance test for GLM-5.1-w4a8 PD separation"""
test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
+5 -1
View File
@@ -48,7 +48,7 @@ OTHER_ARGS = [
"--nnodes",
2,
"--mem-fraction-static",
0.55,
0.662,
"--max-running-requests",
32,
"--chunked-prefill-size",
@@ -82,6 +82,10 @@ OTHER_ARGS = [
4,
"--speculative-draft-model-quantization",
"unquant",
"--reasoning-parser",
"kimi_k2",
"--tool-call-parser",
"kimi_k2",
]
MODEL_CONFIG = {
test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
+6 -2
View File
@@ -86,6 +86,10 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
"--dtype",
"bfloat16",
"--trust-remote-code",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
]
@@ -106,7 +110,7 @@ class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms(
random_range_ratio = 1
repeat_rate = 0.9
tpot = 50
output_token_throughput = 390.5839
output_token_throughput = 390.5859
request_rate = float("inf")
def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self):
@@ -122,7 +126,7 @@ class TestNPUMiniMaxM2_5_W8A8_4P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase):
datasets = ["gpqa_diamond"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
max_concurrency = 64
eval_batch_size = 64
def test_accuracy(self):
self.run_accuracy()
test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
+5 -1
View File
@@ -87,6 +87,10 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [
"unquant",
"--dtype",
"bfloat16",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
]
@@ -122,7 +126,7 @@ class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase):
datasets = ["gpqa_diamond"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
max_concurrency = 64
eval_batch_size = 64
def test_accuracy(self):
self.run_accuracy()
test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+4
View File
@@ -82,6 +82,10 @@ QWEN3_8B_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
+5 -1
View File
@@ -69,6 +69,10 @@ QWEN3_8B_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
5,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
@@ -84,7 +88,7 @@ class TestQwen8B(TestNpuPerformanceTestCaseBase):
output_len = 1500
random_range_ratio = 1
tpot = 11.79
output_token_throughput = 930
output_token_throughput = 1040.96
def test_qwen3_8b(self):
self.run_throughput()
test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
+4
View File
@@ -87,6 +87,10 @@ QWEN3_30B_A3B_OTHER_ARGS = [
162,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
+5 -1
View File
@@ -65,12 +65,16 @@ QWEN3_32B_OTHER_ARGS = [
1,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
+6 -10
View File
@@ -84,6 +84,10 @@ QWEN3_32B_OTHER_ARGS = [
101,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
@@ -93,16 +97,12 @@ class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase):
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
accuracy = 0.516
accuracy = 0.4949
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 40000, "temperature": 1.0}
@classmethod
def tearDownClass(cls):
pass
def test_qwen3_32b_qpqa(self):
"""Run NPU accuracy test for Qwen3-32B-W8A8 on qpqa"""
self.run_accuracy()
@@ -110,7 +110,7 @@ class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase):
class TestQwen32B(TestNpuPerformanceTestCaseBase):
base_url = DEFAULT_URL_FOR_TEST
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
@@ -123,10 +123,6 @@ class TestQwen32B(TestNpuPerformanceTestCaseBase):
tpot = 50
output_token_throughput = 1600
@classmethod
def setUpClass(cls):
pass
def test_qwen3_32b(self):
self.run_throughput()
test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
+4
View File
@@ -84,6 +84,10 @@ QWEN3_32B_OTHER_ARGS = [
101,
"--dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen",
]
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
+7 -2
View File
@@ -80,6 +80,10 @@ QWEN3_6_27B_1024_OTHER_ARGS = [
"--speculative-num-draft-tokens",
4,
"--mm-enable-dp-encoder",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -87,11 +91,12 @@ class TestNPUQwen3_6_27B_1P_In1024x1024_30_Out1024_50ms(TestNpuPerformanceTestCa
"""Test NPU performance for Qwen3.6-27B 1p in1024x1024 30 out1024 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN
dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN
model = QWEN3_6_27B_MODEL_PATH
other_args = QWEN3_6_27B_1024_OTHER_ARGS
envs = QWEN3_6_27B_1024_ENVS
dataset_name = "random"
backend = "sglang-oai-chat"
dataset_name = "image"
max_concurrency = 48
num_prompts = 48
input_len = 30
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
+7 -2
View File
@@ -77,6 +77,10 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [
"--speculative-num-draft-tokens",
4,
"--mm-enable-dp-encoder",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -84,11 +88,12 @@ class TestNPUQwen3_6_27B_1P_In1080p_30_Out256_50ms(TestNpuPerformanceTestCaseBas
"""Test NPU performance for Qwen3.6-27B 1p in1080p 30 out256 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN
dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN
model = QWEN3_6_27B_MODEL_PATH
other_args = QWEN3_6_27B_1080P_OTHER_ARGS
envs = QWEN3_6_27B_1080P_ENVS
dataset_name = "random"
backend = "sglang-oai-chat"
dataset_name = "image"
max_concurrency = 30
num_prompts = 120
input_len = 30
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
+16 -12
View File
@@ -46,24 +46,23 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
"--max-running-requests",
20,
"--max-mamba-cache-size",
120,
160,
"--mem-fraction-static",
0.8,
0.82,
"--cuda-graph-bs",
1,
2,
4,
8,
5,
10,
12,
16,
18,
15,
17,
19,
20,
"--enable-prefill-delayer",
"--prefill-delayer-queue-min-ratio",
0.5,
0.7,
"--prefill-delayer-max-delay-ms",
30000,
20000,
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
@@ -76,11 +75,15 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k prefix90 50ms"""
class TestNPUQwen3_6_27B_1P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B 1p in64k out1k prefix90 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = QWEN3_6_27B_MODEL_PATH
@@ -94,11 +97,12 @@ class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCase
random_range_ratio = 1
repeat_rate = 0.9
request_rate = float("inf")
warmup_requests = 1
tpot = 50
output_token_throughput = 225
def test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k prefix90 50ms"""
"""Run NPU performance test for Qwen3.6-27B in64k out1k prefix90 50ms"""
self.run_throughput()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+16 -4
View File
@@ -58,8 +58,10 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [
8,
16,
32,
48,
64,
40,
45,
50,
54,
"--enable-multimodal",
"--quantization",
"modelslim",
@@ -77,6 +79,10 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -106,11 +112,17 @@ class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase):
model = QWEN3_6_27B_W8A8_MODEL_PATH
envs = QWEN3_6_27B_3K5_1K5_ENVS
other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS
accuracy = 0.855
accuracy = 0.878
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 8
generation_config = {"max_tokens": 81920, "temperature": 1.0}
generation_config = {
"max_tokens": 81920,
"temperature": 1.0,
"extra_body": {
"chat_template_kwargs": {"enable_thinking": True},
},
}
def test_accuracy(self):
self.run_accuracy()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
+99
View File
@@ -0,0 +1,99 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_64K_1K_ENVS = {
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
}
QWEN3_6_27B_64K_1K_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
48000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
6,
"--max-mamba-cache-size",
16,
"--mem-fraction-static",
0.6,
"--cuda-graph-bs",
1,
2,
4,
5,
6,
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS
envs = QWEN3_6_27B_64K_1K_ENVS
dataset_name = "random"
max_concurrency = 6
num_prompts = 12
input_len = 64000
output_len = 1000
random_range_ratio = 1
tpot = 50
output_token_throughput = 57.85
def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
+5 -1
View File
@@ -65,6 +65,10 @@ QWEN3_6_27B_128K_OTHER_ARGS = [
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -72,7 +76,7 @@ class TestNPUQwen3_6_27B_2P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in128k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_128K_OTHER_ARGS
envs = QWEN3_6_27B_128K_ENVS
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
+16 -12
View File
@@ -16,7 +16,6 @@ register_npu_ci(
)
QWEN3_6_27B_16K_1k_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
@@ -25,7 +24,8 @@ QWEN3_6_27B_16K_1k_ENVS = {
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"GDN_ATTN_BACKEND_TRITON": "1",
"ASCEND_USE_FIA": "1",
}
@@ -41,28 +41,28 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
50000,
58000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
28,
29,
"--max-mamba-cache-size",
50,
58,
"--mem-fraction-static",
0.7,
0.68,
"--cuda-graph-bs",
1,
2,
8,
12,
16,
20,
24,
26,
28,
"--enable-multimodal",
29,
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
@@ -75,6 +75,10 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -82,13 +86,13 @@ class TestNPUQwen3_6_27B_2P_In16k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in16k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_16K_1k_OTHER_ARGS
envs = QWEN3_6_27B_16K_1k_ENVS
dataset_name = "random"
max_concurrency = 28
num_prompts = 112
max_concurrency = 29
num_prompts = 116
input_len = 16000
output_len = 1000
random_range_ratio = 1
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
+18 -11
View File
@@ -18,6 +18,7 @@ register_npu_ci(
QWEN3_6_35B_A3B_128K_1K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "1600",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
@@ -26,6 +27,7 @@ QWEN3_6_35B_A3B_128K_1K_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
"GDN_ATTN_BACKEND_TRITON": "1",
}
QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [
@@ -40,21 +42,21 @@ QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [
"--chunked-prefill-size",
-1,
"--max-total-tokens",
600000,
420000,
"--max-prefill-tokens",
65536,
128000,
"--disable-radix-cache",
"--trust-remote-code",
"--enable-prefill-delayer",
"--max-running-requests",
4,
3,
"--max-mamba-cache-size",
12,
3,
"--mem-fraction-static",
0.6,
"--max-mamba-cache-size",
20,
"--disable-cuda-graph",
0.9,
"--cuda-graph-bs",
1,
2,
3,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
@@ -70,6 +72,10 @@ QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -82,11 +88,12 @@ class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase)
other_args = QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS
envs = QWEN3_6_35B_A3B_128K_1K_ENVS
dataset_name = "random"
max_concurrency = 4
num_prompts = 16
max_concurrency = 3
num_prompts = 3
input_len = 128000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 60.57
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
+6 -1
View File
@@ -76,6 +76,10 @@ QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -91,10 +95,11 @@ class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_Prefix90_50ms(
dataset_name = "generated-shared-prefix"
max_concurrency = 103
num_prompts = 412
input_len = 64000
input_len = 128000
output_len = 1000
random_range_ratio = 1
repeat_rate = 0.9
seed = 1
tpot = 50
request_rate = float("inf")
output_token_throughput = 308.2
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
+20 -12
View File
@@ -18,7 +18,7 @@ register_npu_ci(
QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "800",
"HCCL_BUFFSIZE": "1",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
@@ -26,7 +26,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
}
QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
@@ -40,25 +39,29 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
"npu",
"--chunked-prefill-size",
-1,
"--max-total-tokens",
659840,
"--max-prefill-tokens",
35000,
43400,
"--disable-radix-cache",
"--trust-remote-code",
"--enable-prefill-delayer",
"--prefill-max-requests",
"12",
"--max-running-requests",
110,
122,
"--max-mamba-cache-size",
115,
122,
"--mem-fraction-static",
0.78,
0.9,
"--cuda-graph-bs",
4,
16,
32,
64,
84,
105,
110,
96,
116,
120,
122,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
@@ -74,6 +77,10 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -86,11 +93,12 @@ class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase)
other_args = QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS
envs = QWEN3_6_35B_A3B_3K5_1K5_ENVS
dataset_name = "random"
max_concurrency = 110
num_prompts = 440
max_concurrency = 122
num_prompts = 122
input_len = 3500
output_len = 1500
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 2031.71
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
+5
View File
@@ -74,6 +74,10 @@ QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -91,6 +95,7 @@ class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
input_len = 64000
output_len = 1000
random_range_ratio = 1
seed = 1
tpot = 50
output_token_throughput = 141.72
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
+13 -7
View File
@@ -43,18 +43,19 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
"npu",
"--chunked-prefill-size",
-1,
"--max-total-tokens",
470784,
"--max-prefill-tokens",
65536,
"--trust-remote-code",
"--enable-prefill-delayer",
"--mamba-scheduler-strategy",
"extra_buffer",
"--max-running-requests",
42,
40,
"--max-mamba-cache-size",
210,
200,
"--mem-fraction-static",
0.71,
0.9,
"--cuda-graph-bs",
2,
8,
@@ -63,7 +64,6 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
32,
36,
40,
42,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
@@ -79,6 +79,10 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -118,15 +122,17 @@ class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_Prefix90_50ms(
other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS
envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 42
num_prompts = 42
max_concurrency = 40
num_prompts = 40
input_len = 65536
output_len = 1024
random_range_ratio = 1
repeat_rate = 0.9
seed = 1
tpot = 50
request_rate = float("inf")
output_token_throughput = 660
pop_sglang_is_in_ci_for_gsp = True
@classmethod
def setUpClass(cls):
test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
+6 -2
View File
@@ -91,6 +91,10 @@ QWEN3_NEXT_80B_A3B_OTHER_ARGS = [
"bfloat16",
"--speculative-draft-model-path",
QWEN3_NEXT_80B_A3B_MODEL_PATH,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
]
@@ -123,12 +127,12 @@ class TestQwen3Next80BA3B_aime25(TestNpuAccuracyTestCaseBase):
"temperature": 0.7,
"top_p": 0.8,
"top_k": 20,
"extra_body": {"chat_template_kwargs": {"enable_thinking": True}},
"extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
}
max_concurrency = 16
def test_aime25(self):
self.run_accuracy()
self.run_accuracy_multiple(n_runs=3)
if __name__ == "__main__":