[CI/NPU] Isolate multi-node tests by run_id to prevent concurrent-run… (#35500)

.github/workflows/nightly-test-npu-e2e-multi-node.yml

@@ -79,7 +79,7 @@ jobs:
         KUBECONFIG: /root/.cache/.cache/kb.yaml
         KUBECTL: /root/.cache/.cache/kubectl
         NAMESPACE: sgl-project
-        KUBE_JOB_NAME: ascend-sglang-${{ inputs.test_type }}-test
+        KUBE_JOB_NAME: ascend-sglang-test-${{ github.run_id }}
         SGLANG_IS_IN_CI: true
         ASCEND_E2E_TEST_CONFIG_PATH: python/sglang/test/ascend/e2e
         SGLANG_USE_MODELSCOPE: true
@@ -101,8 +101,9 @@ jobs:
       - name: Prepare code for testing
         run: |
           # copy source code to shared-disk
+          # Append run_id to isolate concurrent runs sharing the same PVC.
           current_path=$(pwd)
-          target_path=/root/.cache/tests/sglang
+          target_path=/root/.cache/tests/sglang-${{ github.run_id }}
           rm -rf ${target_path}
           mkdir -p ${target_path}
           cp -r ${current_path}/* ${target_path}/
@@ -129,7 +130,8 @@ jobs:
         timeout-minutes: 300
         run: |
           # sglang_source_relative_path is shared-disk path
-          sglang_source_relative_path=tests/sglang
+          # Append run_id so concurrent runs do not overwrite each other on the shared PVC.
+          sglang_source_relative_path=tests/sglang-${{ github.run_id }}
           sglang_source_path=/root/.cache/${sglang_source_relative_path}
           echo "Source code path: ${sglang_source_path}"
 
@@ -184,7 +186,8 @@ jobs:
             --test-case ${test_case} \
             --kube-name-space ${NAMESPACE} \
             --kube-job-type ${kube_job_type} \
-            --kube-job-name-prefix ${KUBE_JOB_NAME}"
+            --kube-job-name-prefix ${KUBE_JOB_NAME} \
+            --run-id ${{ github.run_id }}"
 
           if [ "$prefill_decode_deployment" = "separation" ];then
             CMD="${CMD} \
@@ -249,3 +252,5 @@ jobs:
               break
             fi
           done
+          # Cleanup the run-scoped source code directory on the shared PVC.
+          rm -rf /root/.cache/tests/sglang-${{ github.run_id }} || true

python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2

@@ -28,6 +28,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True

python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2

@@ -22,6 +22,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: pd-mix
     spec:
       hostNetwork: True

python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2

@@ -28,6 +28,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True
@@ -144,6 +145,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True
@@ -260,6 +262,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True

python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2

@@ -22,6 +22,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: prefill
     spec:
       restartPolicy: Always
@@ -125,6 +126,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: decode
     spec:
       restartPolicy: Always
@@ -228,6 +230,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: router
     spec:
       hostNetwork: True

python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2

@@ -22,6 +22,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         containers:

python/sglang/test/ascend/e2e/run_npu_e2e_test.py

@@ -308,10 +308,19 @@ def create_or_update_configmap(cm_name: str, data: dict, namespace: str):
             raise
 
 
-def prepare_cm_data(namespace, pod_string):
+def prepare_cm_data(namespace, pod_string, run_id=None):
-    """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information."""
+    """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information.
+
+    When run_id is provided, the label selector is scoped to that run_id so that
+    concurrent runs in the same namespace do not pollute each other's ConfigMap.
+    The pod_string filter (final_kube_job_name) still acts as a safety net.
+    """
+    if run_id:
+        label_selector = f"app=sgl-ascend,run-id={run_id}"
+    else:
+        label_selector = "app=sgl-ascend"
     pods = core_api.list_namespaced_pod(
-        namespace=namespace, label_selector="app=sgl-ascend"
+        namespace=namespace, label_selector=label_selector
     )
     data = {}
     for pod in pods.items:
@@ -563,6 +572,7 @@ def run_npu_e2e_test_case(
     env="debug",
     trouble_shotting=False,
     transformers_version="",
+    run_id: str = "",
 ):
     """The method for running a npu e2e test case.
     Args:
@@ -578,6 +588,8 @@ def run_npu_e2e_test_case(
         sglang_is_in_ci (bool): whether running in CI environment.
         install_sglang_from_source (bool): whether installing sglang from source or use docker image directly.
         env (str): the environment to run the test on.  Choose one in ["debug", "ci"]
+        run_id (str): the GitHub Actions run_id, used to label pods for run-scoped
+            isolation when multiple runs share the same k8s namespace.
     """
     random_str = get_unique_random_string(16, True)
 
@@ -629,6 +641,7 @@ def run_npu_e2e_test_case(
                 "trouble_shotting": trouble_shotting,
                 "transformers_version": transformers_version,
                 "run_label": run_label,
+                "run_id": run_id,
             }
             create_kube_yaml(
                 kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
@@ -652,6 +665,7 @@ def run_npu_e2e_test_case(
                 "trouble_shotting": trouble_shotting,
                 "transformers_version": transformers_version,
                 "run_label": run_label,
+                "run_id": run_id,
             }
             template_key = (
                 KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
@@ -680,6 +694,7 @@ def run_npu_e2e_test_case(
                 "trouble_shotting": trouble_shotting,
                 "transformers_version": transformers_version,
                 "run_label": run_label,
+                "run_id": run_id,
             }
             template_key = (
                 KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
@@ -699,7 +714,9 @@ def run_npu_e2e_test_case(
         ):
             if kube_job_type != "single":
                 matching_pod_string = final_kube_job_name
-                cm_data = prepare_cm_data(kube_name_space, matching_pod_string)
+                cm_data = prepare_cm_data(
+                    kube_name_space, matching_pod_string, run_id=run_id
+                )
                 if not cm_data:
                     logger.info(
                         f"No sglang pod found while matching {matching_pod_string}"
@@ -870,6 +887,14 @@ if __name__ == "__main__":
         help="The transformers version number for running sglang. Use default version in image if keep empty.",
     )
 
+    parser.add_argument(
+        "--run-id",
+        type=str,
+        required=False,
+        default="",
+        help="GitHub Actions run_id, used to label pods for run-scoped isolation.",
+    )
+
     args = parser.parse_args()
 
     docker_image_url = args.image
@@ -886,6 +911,7 @@ if __name__ == "__main__":
     env = args.env
     trouble_shotting = args.trouble_shotting
     transformers_version = args.transformers_version
+    run_id = args.run_id
 
     kube_name_space = args.kube_name_space
     kube_job_type = args.kube_job_type
@@ -915,4 +941,5 @@ if __name__ == "__main__":
         env=env,
         trouble_shotting=trouble_shotting,
         transformers_version=transformers_version,
+        run_id=run_id,
     )

python/sglang/test/ascend/e2e/run_npu_testcase.sh

@@ -143,9 +143,9 @@ fi
 source_plog_path="/root/ascend/log/debug/plog"
 if [ -d "$source_plog_path" ];then
     echo "Plog files found. Begin to backup them."
-    target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}"
+    target_plog_path="/root/sglang/debug/logs/plog/${run_label}/${tc_name}/${HOSTNAME}"
     if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
-        target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
+        target_plog_path="/root/.cache/tests/logs/plog/${run_label}/${tc_name}/${HOSTNAME}"
     fi
     rm -rf "${target_plog_path}"
     mkdir -p "${target_plog_path}"