Fix GLM4-7B-Flash accuracy test configuration, tune Qwen3.6-27B/35B performance test parameters, and harden Ascend NPU multi-node E2E test utilities against pod name format errors. (#32371)

This commit is contained in:
pllimax
2026-07-29 21:44:17 +08:00
committed by GitHub
parent 977f04aafe
commit d004a15a3e
6 changed files with 56 additions and 16 deletions
@@ -310,7 +310,9 @@ def create_or_update_configmap(cm_name: str, data: dict, namespace: str):
def prepare_cm_data(namespace, pod_string): def prepare_cm_data(namespace, pod_string):
"""Prepare a configmap data: {pod_name: pod_ip} by the running pod's information.""" """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information."""
pods = core_api.list_namespaced_pod(namespace=namespace) pods = core_api.list_namespaced_pod(
namespace=namespace, label_selector="app=sgl-ascend"
)
data = {} data = {}
for pod in pods.items: for pod in pods.items:
pod_name = pod.metadata.name pod_name = pod.metadata.name
@@ -426,7 +426,12 @@ def check_role(allowed_roles: Union[str, Iterable[str]]):
def launch_pd_mix_node(model_config): def launch_pd_mix_node(model_config):
logger.info(f"Launch pd mix node start ......") logger.info(f"Launch pd mix node start ......")
host_name = get_host_name() host_name = get_host_name()
pod_index = int(host_name.rsplit("-", 1)[-1]) last_part = host_name.rsplit("-", 1)[-1]
if not last_part.isdigit():
raise RuntimeError(
f"Unexpected hostname format, expected numeric suffix: {host_name}"
)
pod_index = int(last_part)
# Monitor ConfigMap to generate dist-init-addr and node-rank # Monitor ConfigMap to generate dist-init-addr and node-rank
is_ready = False is_ready = False
@@ -497,7 +502,12 @@ def launch_pd_mix_node(model_config):
def launch_pd_separation_node(model_config): def launch_pd_separation_node(model_config):
logger.info(f"Launch pd separation node start ......") logger.info(f"Launch pd separation node start ......")
host_name = get_host_name() host_name = get_host_name()
pod_index = int(host_name.rsplit("-", 1)[-1]) last_part = host_name.rsplit("-", 1)[-1]
if not last_part.isdigit():
raise RuntimeError(
f"Unexpected hostname format, expected numeric suffix: {host_name}"
)
pod_index = int(last_part)
role = "prefill" if "prefill" in host_name else "decode" role = "prefill" if "prefill" in host_name else "decode"
bootstrap_init_port = BOOTSTRAP_INIT_PORT bootstrap_init_port = BOOTSTRAP_INIT_PORT
@@ -717,7 +727,14 @@ def launch_router(model_config):
node_ip_list.clear() node_ip_list.clear()
for pod_name, pod_ip in configmap.data.items(): for pod_name, pod_ip in configmap.data.items():
pod_index = int(pod_name.rsplit("-", 1)[-1]) # Skip unexpected entries that don't end with a numeric index
last_part = pod_name.rsplit("-", 1)[-1]
if not last_part.isdigit():
logger.info(
"Skipping ConfigMap entry with non-numeric suffix: %s", pod_name
)
continue
pod_index = int(last_part)
if "prefill" in pod_name: if "prefill" in pod_name:
if is_multi_node_prefill_instance: if is_multi_node_prefill_instance:
@@ -16,7 +16,7 @@ register_npu_ci(
ENVS = { ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32", "STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "1000", "DEEPEP_HCCL_BUFFSIZE": "1000",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo", "HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
@@ -57,12 +57,12 @@ OTHER_ARGS = [
] ]
class TestNPUDeepSeek_V3_2_8P_AIME2025(TestNpuAccuracyTestCaseBase): class TestNPUGLM_4_7_FLASH_1P_AIME25(TestNpuAccuracyTestCaseBase):
model = GLM_4_7_FLASH_MODEL_PATH model = GLM_4_7_FLASH_MODEL_PATH
envs = ENVS envs = ENVS
other_args = OTHER_ARGS other_args = OTHER_ARGS
accuracy = 0.916 accuracy = 0.7667
datasets = ["aime25"] datasets = ["aime25"]
few_shot_num = 0 few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0} generation_config = {"max_tokens": 65536, "temperature": 1.0}
@@ -76,6 +76,10 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
1, 1,
"--speculative-num-draft-tokens", "--speculative-num-draft-tokens",
4, 4,
"--reasoning-parser",
"qwen3",
"--tool-call-parser",
"qwen3_coder",
] ]
@@ -21,9 +21,10 @@ QWEN3_6_27B_16K_1k_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"GDN_ATTN_BACKEND_TRITON": "1", "GDN_ATTN_BACKEND_TRITON": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
} }
@@ -44,22 +45,37 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [
"--disable-radix-cache", "--disable-radix-cache",
"--trust-remote-code", "--trust-remote-code",
"--max-running-requests", "--max-running-requests",
29, 37,
"--max-mamba-cache-size", "--max-mamba-cache-size",
58, 74,
"--mem-fraction-static", "--mem-fraction-static",
0.68, 0.70,
"--cuda-graph-bs", "--cuda-graph-bs",
1, 1,
2, 2,
3,
4,
6,
8, 8,
10,
12, 12,
14,
16, 16,
18,
20, 20,
21,
23,
24, 24,
25,
26, 26,
27,
28, 28,
29, 29,
30,
31,
33,
35,
37,
"--quantization", "--quantization",
"modelslim", "modelslim",
"--dtype", "--dtype",
@@ -69,11 +85,11 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [
"--speculative-algorithm", "--speculative-algorithm",
"NEXTN", "NEXTN",
"--speculative-num-steps", "--speculative-num-steps",
3, 4,
"--speculative-eagle-topk", "--speculative-eagle-topk",
1, 1,
"--speculative-num-draft-tokens", "--speculative-num-draft-tokens",
4, 5,
"--reasoning-parser", "--reasoning-parser",
"qwen3", "qwen3",
"--tool-call-parser", "--tool-call-parser",
@@ -90,8 +106,9 @@ class TestNPUQwen3_6_27B_2P_In16k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
other_args = QWEN3_6_27B_16K_1k_OTHER_ARGS other_args = QWEN3_6_27B_16K_1k_OTHER_ARGS
envs = QWEN3_6_27B_16K_1k_ENVS envs = QWEN3_6_27B_16K_1k_ENVS
dataset_name = "random" dataset_name = "random"
max_concurrency = 29 max_concurrency = 37
num_prompts = 116 warmup_requests = 4
num_prompts = 37
input_len = 16000 input_len = 16000
output_len = 1000 output_len = 1000
random_range_ratio = 1 random_range_ratio = 1
@@ -18,7 +18,7 @@ register_npu_ci(
QWEN3_6_35B_A3B_3K5_1K5_ENVS = { QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32", "STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "1", "HCCL_BUFFSIZE": "100",
"HCCL_SOCKET_IFNAME": "lo", "HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",