[NPU] Set DEEPEP_HYBRID_DEPLOYMENT for new DeepEP tests; switch glm5_2 to w8a8; tune nightly timeouts (#38775)

.github/workflows/nightly-test-npu.yml

@@ -131,7 +131,7 @@ jobs:
       self_name: nightly-1-npu-a3
       runner_config: linux-aarch64-a3-2-
       image: ${{ needs.set-image-config.outputs.image_a3 }}
-      run_timeout_minutes: '60'
+      run_timeout_minutes: '90'
       timeout_per_file: '3600'
       is_nightly_pipeline_job: true
       skip_pr_test_health_check: 'true'
@@ -146,7 +146,7 @@ jobs:
       self_name: nightly-2-npu-a3
       runner_config: linux-aarch64-a3-2-
       image: ${{ needs.set-image-config.outputs.image_a3 }}
-      run_timeout_minutes: '60'
+      run_timeout_minutes: '90'
       timeout_per_file: '3600'
       is_nightly_pipeline_job: true
       skip_pr_test_health_check: 'true'
@@ -292,20 +292,21 @@ jobs:
       matrix:
         test_config:
           # glm5_1 performance tests
-          - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms
+          # Disable test case temporarily due to feature refactor in PR #38293.
-            prefill_size: 2
+          # - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms
-            decode_size: 2
+          #   prefill_size: 2
-            router_size: 1
+          #   decode_size: 2
-            test_case: test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py
+          #   router_size: 1
-            test_type: 'perf'
+          #   test_case: test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py
-            prefill_decode_deployment: 'separation'
+          #   test_type: 'perf'
-          - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26
+          #   prefill_decode_deployment: 'separation'
-            prefill_size: 2
+          # - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26
-            decode_size: 2
+          #   prefill_size: 2
-            router_size: 1
+          #   decode_size: 2
-            test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
+          #   router_size: 1
-            test_type: 'accuracy'
+          #   test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
-            prefill_decode_deployment: 'separation'
+          #   test_type: 'accuracy'
+          #   prefill_decode_deployment: 'separation'
           # deepseek_v4_flash performance tests
           - name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
             prefill_size: 1
@@ -339,9 +340,9 @@ jobs:
       matrix:
         test_config:
           # glm_5_2 accuracy tests
-          - name: glm_5_2_w4a8_16p_gpqa
+          - name: glm_5_2_w8a8_16p_gpqa
             node_size: 2
-            test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
+            test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py
             test_type: 'accuracy'
           # kimi_k3 accuracy tests
           - name: kimi_k3_w4a8_32p_gpqa

python/sglang/test/ascend/e2e/test_npu_performance_utils.py

@@ -166,6 +166,7 @@ GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH = (
 )
 GLM_5_1_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.1-w4a8"
 GLM_5_2_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.2-w4a8"
+GLM_5_2_W8A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.2-w8a8"
 MINIMAX_M2_5_W8A8_MODEL_PATH = (
     "/root/.cache/modelscope/hub/models/Eco-Tech/MiniMax-M2.5-w8a8-QuaRot"
 )

test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py

@@ -6,7 +6,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
 )
 from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
 from sglang.test.ascend.e2e.test_npu_performance_utils import (
-    GLM_5_2_W4A8_MODEL_PATH,
+    GLM_5_2_W8A8_MODEL_PATH,
 )
 from sglang.test.ci.ci_register import register_npu_ci
 
@@ -17,7 +17,7 @@ register_npu_ci(
     disabled="accuracy testcase",
 )
 
-GLM_5_2_W4A8_16P_TWO_NODE_ENVS = {
+GLM_5_2_W8A8_16P_TWO_NODE_ENVS = {
     "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
     "STREAMS_PER_DEVICE": "32",
     "SGLANG_SET_CPU_AFFINITY": "1",
@@ -34,7 +34,7 @@ GLM_5_2_W4A8_16P_TWO_NODE_ENVS = {
     "GLOO_SOCKET_IFNAME": NIC_NAME,
 }
 
-GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
+GLM_5_2_W8A8_16P_TWO_NODE_OTHER_ARGS = [
     "--attention-backend",
     "ascend",
     "--device",
@@ -56,7 +56,7 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
     "--context-length",
     135000,
     "--served-model-name",
-    "glm-5.2-w4a8",
+    "glm-5.2-w8a8",
     "--cuda-graph-max-bs-decode",
     4,
     "--max-running-requests",
@@ -85,18 +85,18 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
     4,
 ]
 
-GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG = {
+GLM_5_2_W8A8_16P_TWO_NODE_MODEL_CONFIG = {
-    "model_path": GLM_5_2_W4A8_MODEL_PATH,
+    "model_path": GLM_5_2_W8A8_MODEL_PATH,
-    "other_args": GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS,
+    "other_args": GLM_5_2_W8A8_16P_TWO_NODE_OTHER_ARGS,
-    "node_envs": GLM_5_2_W4A8_16P_TWO_NODE_ENVS,
+    "node_envs": GLM_5_2_W8A8_16P_TWO_NODE_ENVS,
 }
 
 
-class TestNPUGLM_5_2_W4A8_16P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase):
+class TestNPUGLM_5_2_W8A8_16P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase):
-    """Test NPU accuracy for GLM-5.2-w4a8 16p two nodes on gpqa_diamond"""
+    """Test NPU accuracy for GLM-5.2-w8a8 16p two nodes on gpqa_diamond"""
 
     benchmark_tool = BENCHMARK_TOOL_DEFAULT
-    model_config = GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG
+    model_config = GLM_5_2_W8A8_16P_TWO_NODE_MODEL_CONFIG
     accuracy = 0.912
     datasets = ["gpqa_diamond"]
     # eval_batch_size = 16
@@ -109,8 +109,8 @@ class TestNPUGLM_5_2_W4A8_16P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase):
         "stream": True,
     }
 
-    def test_npu_glm_5_2_w4a8_16p_gpqa(self):
+    def test_npu_glm_5_2_w8a8_16p_gpqa(self):
-        """Run NPU accuracy test for GLM-5.2-w4a8 16p two nodes on gpqa_diamond"""
+        """Run NPU accuracy test for GLM-5.2-w8a8 16p two nodes on gpqa_diamond"""
         self.run_accuracy()
 
 

test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py

@@ -19,6 +19,7 @@ ENVS = {
     "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
     "SGLANG_NPU_USE_MULTI_STREAM": "1",
     "HCCL_BUFFSIZE": "1000",
+    "DEEPEP_HYBRID_DEPLOYMENT": "1",
     "HCCL_OP_EXPANSION_MODE": "AIV",
     "HCCL_SOCKET_IFNAME": "lo",
     "GLOO_SOCKET_IFNAME": "lo",

test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py

@@ -23,6 +23,7 @@ ENVS = {
     "STREAMS_PER_DEVICE": "32",
     "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
     "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
+    "DEEPEP_HYBRID_DEPLOYMENT": "1",
     "DEEPEP_HCCL_BUFFSIZE": "4400",
     "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
     "HCCL_SOCKET_IFNAME": NIC_NAME,

test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py

@@ -36,6 +36,7 @@ KIMI_K3_W4A8_32P_ENVS = {
     "STREAMS_PER_DEVICE": "32",
     "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
     "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
+    "DEEPEP_HYBRID_DEPLOYMENT": "1",
     "HCCL_BUFFSIZE": "2000",
     "DEEPEP_NORMAL_LONG_SEQ_ROUND": "64",
     "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "512",

test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_moe_runner_backend.py

@@ -58,6 +58,7 @@ class TestMoreRunnerBackendTriton(CustomTestCase):
             ],
             env={
                 "HCCL_BUFFSIZE": "1024",
+                "DEEPEP_HYBRID_DEPLOYMENT": "1",
             },
         )