[CI/NPU] Isolate multi-node tests by run_id to prevent concurrent-run… (#35500)

python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2

@@ -28,6 +28,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True

python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2

@@ -22,6 +22,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: pd-mix
     spec:
       hostNetwork: True

python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2

@@ -28,6 +28,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True
@@ -144,6 +145,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True
@@ -260,6 +262,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         hostNetwork: True

python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2

@@ -22,6 +22,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: prefill
     spec:
       restartPolicy: Always
@@ -125,6 +126,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: decode
     spec:
       restartPolicy: Always
@@ -228,6 +230,7 @@ spec:
     metadata:
       labels:
         app: sgl-ascend
+        run-id: "{{ run_id }}"
         task: router
     spec:
       hostNetwork: True

python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2

@@ -22,6 +22,7 @@ spec:
       metadata:
         labels:
           app: sgl-ascend
+          run-id: "{{ run_id }}"
           ring-controller.atlas: ascend-1980
       spec:
         containers:

python/sglang/test/ascend/e2e/run_npu_e2e_test.py

@@ -308,10 +308,19 @@ def create_or_update_configmap(cm_name: str, data: dict, namespace: str):
             raise
 
 
-def prepare_cm_data(namespace, pod_string):
-    """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information."""
+def prepare_cm_data(namespace, pod_string, run_id=None):
+    """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information.
+
+    When run_id is provided, the label selector is scoped to that run_id so that
+    concurrent runs in the same namespace do not pollute each other's ConfigMap.
+    The pod_string filter (final_kube_job_name) still acts as a safety net.
+    """
+    if run_id:
+        label_selector = f"app=sgl-ascend,run-id={run_id}"
+    else:
+        label_selector = "app=sgl-ascend"
     pods = core_api.list_namespaced_pod(
-        namespace=namespace, label_selector="app=sgl-ascend"
+        namespace=namespace, label_selector=label_selector
     )
     data = {}
     for pod in pods.items:
@@ -563,6 +572,7 @@ def run_npu_e2e_test_case(
     env="debug",
     trouble_shotting=False,
     transformers_version="",
+    run_id: str = "",
 ):
     """The method for running a npu e2e test case.
     Args:
@@ -578,6 +588,8 @@ def run_npu_e2e_test_case(
         sglang_is_in_ci (bool): whether running in CI environment.
         install_sglang_from_source (bool): whether installing sglang from source or use docker image directly.
         env (str): the environment to run the test on.  Choose one in ["debug", "ci"]
+        run_id (str): the GitHub Actions run_id, used to label pods for run-scoped
+            isolation when multiple runs share the same k8s namespace.
     """
     random_str = get_unique_random_string(16, True)
 
@@ -629,6 +641,7 @@ def run_npu_e2e_test_case(
                 "trouble_shotting": trouble_shotting,
                 "transformers_version": transformers_version,
                 "run_label": run_label,
+                "run_id": run_id,
             }
             create_kube_yaml(
                 kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
@@ -652,6 +665,7 @@ def run_npu_e2e_test_case(
                 "trouble_shotting": trouble_shotting,
                 "transformers_version": transformers_version,
                 "run_label": run_label,
+                "run_id": run_id,
             }
             template_key = (
                 KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
@@ -680,6 +694,7 @@ def run_npu_e2e_test_case(
                 "trouble_shotting": trouble_shotting,
                 "transformers_version": transformers_version,
                 "run_label": run_label,
+                "run_id": run_id,
             }
             template_key = (
                 KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
@@ -699,7 +714,9 @@ def run_npu_e2e_test_case(
         ):
             if kube_job_type != "single":
                 matching_pod_string = final_kube_job_name
-                cm_data = prepare_cm_data(kube_name_space, matching_pod_string)
+                cm_data = prepare_cm_data(
+                    kube_name_space, matching_pod_string, run_id=run_id
+                )
                 if not cm_data:
                     logger.info(
                         f"No sglang pod found while matching {matching_pod_string}"
@@ -870,6 +887,14 @@ if __name__ == "__main__":
         help="The transformers version number for running sglang. Use default version in image if keep empty.",
     )
 
+    parser.add_argument(
+        "--run-id",
+        type=str,
+        required=False,
+        default="",
+        help="GitHub Actions run_id, used to label pods for run-scoped isolation.",
+    )
+
     args = parser.parse_args()
 
     docker_image_url = args.image
@@ -886,6 +911,7 @@ if __name__ == "__main__":
     env = args.env
     trouble_shotting = args.trouble_shotting
     transformers_version = args.transformers_version
+    run_id = args.run_id
 
     kube_name_space = args.kube_name_space
     kube_job_type = args.kube_job_type
@@ -915,4 +941,5 @@ if __name__ == "__main__":
         env=env,
         trouble_shotting=trouble_shotting,
         transformers_version=transformers_version,
+        run_id=run_id,
     )

python/sglang/test/ascend/e2e/run_npu_testcase.sh

@@ -143,9 +143,9 @@ fi
 source_plog_path="/root/ascend/log/debug/plog"
 if [ -d "$source_plog_path" ];then
     echo "Plog files found. Begin to backup them."
-    target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}"
+    target_plog_path="/root/sglang/debug/logs/plog/${run_label}/${tc_name}/${HOSTNAME}"
     if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
-        target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
+        target_plog_path="/root/.cache/tests/logs/plog/${run_label}/${tc_name}/${HOSTNAME}"
     fi
     rm -rf "${target_plog_path}"
     mkdir -p "${target_plog_path}"