From 60f881b40c4138f2f4859d02cb3b4e7c0ee166f4 Mon Sep 17 00:00:00 2001 From: hhhh1252023 <139246842+hhhh1252023@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:08:27 +0800 Subject: [PATCH] =?UTF-8?q?[CI/NPU]=20Isolate=20multi-node=20tests=20by=20?= =?UTF-8?q?run=5Fid=20to=20prevent=20concurrent-run=E2=80=A6=20(#35500)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nightly-test-npu-e2e-multi-node.yml | 13 ++++--- .../ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 | 1 + .../e2e/k8s_multi_pd_mix_green.yaml.jinja2 | 1 + .../e2e/k8s_multi_pd_separation.yaml.jinja2 | 3 ++ .../k8s_multi_pd_separation_green.yaml.jinja2 | 3 ++ .../test/ascend/e2e/k8s_single.yaml.jinja2 | 1 + .../test/ascend/e2e/run_npu_e2e_test.py | 35 ++++++++++++++++--- .../test/ascend/e2e/run_npu_testcase.sh | 4 +-- 8 files changed, 51 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nightly-test-npu-e2e-multi-node.yml b/.github/workflows/nightly-test-npu-e2e-multi-node.yml index d591b1c27..079de0a13 100644 --- a/.github/workflows/nightly-test-npu-e2e-multi-node.yml +++ b/.github/workflows/nightly-test-npu-e2e-multi-node.yml @@ -79,7 +79,7 @@ jobs: KUBECONFIG: /root/.cache/.cache/kb.yaml KUBECTL: /root/.cache/.cache/kubectl NAMESPACE: sgl-project - KUBE_JOB_NAME: ascend-sglang-${{ inputs.test_type }}-test + KUBE_JOB_NAME: ascend-sglang-test-${{ github.run_id }} SGLANG_IS_IN_CI: true ASCEND_E2E_TEST_CONFIG_PATH: python/sglang/test/ascend/e2e SGLANG_USE_MODELSCOPE: true @@ -101,8 +101,9 @@ jobs: - name: Prepare code for testing run: | # copy source code to shared-disk + # Append run_id to isolate concurrent runs sharing the same PVC. current_path=$(pwd) - target_path=/root/.cache/tests/sglang + target_path=/root/.cache/tests/sglang-${{ github.run_id }} rm -rf ${target_path} mkdir -p ${target_path} cp -r ${current_path}/* ${target_path}/ @@ -129,7 +130,8 @@ jobs: timeout-minutes: 300 run: | # sglang_source_relative_path is shared-disk path - sglang_source_relative_path=tests/sglang + # Append run_id so concurrent runs do not overwrite each other on the shared PVC. + sglang_source_relative_path=tests/sglang-${{ github.run_id }} sglang_source_path=/root/.cache/${sglang_source_relative_path} echo "Source code path: ${sglang_source_path}" @@ -184,7 +186,8 @@ jobs: --test-case ${test_case} \ --kube-name-space ${NAMESPACE} \ --kube-job-type ${kube_job_type} \ - --kube-job-name-prefix ${KUBE_JOB_NAME}" + --kube-job-name-prefix ${KUBE_JOB_NAME} \ + --run-id ${{ github.run_id }}" if [ "$prefill_decode_deployment" = "separation" ];then CMD="${CMD} \ @@ -249,3 +252,5 @@ jobs: break fi done + # Cleanup the run-scoped source code directory on the shared PVC. + rm -rf /root/.cache/tests/sglang-${{ github.run_id }} || true diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 index 3ad20a9c6..3b21be13f 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 @@ -28,6 +28,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" ring-controller.atlas: ascend-1980 spec: hostNetwork: True diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 index eddcc878f..7218e2da6 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 @@ -22,6 +22,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" task: pd-mix spec: hostNetwork: True diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 index 9c25516c7..afdecf028 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 @@ -28,6 +28,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" ring-controller.atlas: ascend-1980 spec: hostNetwork: True @@ -144,6 +145,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" ring-controller.atlas: ascend-1980 spec: hostNetwork: True @@ -260,6 +262,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" ring-controller.atlas: ascend-1980 spec: hostNetwork: True diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 index d51dce478..c01290b3a 100644 --- a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 @@ -22,6 +22,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" task: prefill spec: restartPolicy: Always @@ -125,6 +126,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" task: decode spec: restartPolicy: Always @@ -228,6 +230,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" task: router spec: hostNetwork: True diff --git a/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 index af4246bab..500f786a1 100644 --- a/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 +++ b/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 @@ -22,6 +22,7 @@ spec: metadata: labels: app: sgl-ascend + run-id: "{{ run_id }}" ring-controller.atlas: ascend-1980 spec: containers: diff --git a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py index 81d53e8bd..9e24293df 100644 --- a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py +++ b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py @@ -308,10 +308,19 @@ def create_or_update_configmap(cm_name: str, data: dict, namespace: str): raise -def prepare_cm_data(namespace, pod_string): - """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information.""" +def prepare_cm_data(namespace, pod_string, run_id=None): + """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information. + + When run_id is provided, the label selector is scoped to that run_id so that + concurrent runs in the same namespace do not pollute each other's ConfigMap. + The pod_string filter (final_kube_job_name) still acts as a safety net. + """ + if run_id: + label_selector = f"app=sgl-ascend,run-id={run_id}" + else: + label_selector = "app=sgl-ascend" pods = core_api.list_namespaced_pod( - namespace=namespace, label_selector="app=sgl-ascend" + namespace=namespace, label_selector=label_selector ) data = {} for pod in pods.items: @@ -563,6 +572,7 @@ def run_npu_e2e_test_case( env="debug", trouble_shotting=False, transformers_version="", + run_id: str = "", ): """The method for running a npu e2e test case. Args: @@ -578,6 +588,8 @@ def run_npu_e2e_test_case( sglang_is_in_ci (bool): whether running in CI environment. install_sglang_from_source (bool): whether installing sglang from source or use docker image directly. env (str): the environment to run the test on. Choose one in ["debug", "ci"] + run_id (str): the GitHub Actions run_id, used to label pods for run-scoped + isolation when multiple runs share the same k8s namespace. """ random_str = get_unique_random_string(16, True) @@ -629,6 +641,7 @@ def run_npu_e2e_test_case( "trouble_shotting": trouble_shotting, "transformers_version": transformers_version, "run_label": run_label, + "run_id": run_id, } create_kube_yaml( kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type), @@ -652,6 +665,7 @@ def run_npu_e2e_test_case( "trouble_shotting": trouble_shotting, "transformers_version": transformers_version, "run_label": run_label, + "run_id": run_id, } template_key = ( KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type @@ -680,6 +694,7 @@ def run_npu_e2e_test_case( "trouble_shotting": trouble_shotting, "transformers_version": transformers_version, "run_label": run_label, + "run_id": run_id, } template_key = ( KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type @@ -699,7 +714,9 @@ def run_npu_e2e_test_case( ): if kube_job_type != "single": matching_pod_string = final_kube_job_name - cm_data = prepare_cm_data(kube_name_space, matching_pod_string) + cm_data = prepare_cm_data( + kube_name_space, matching_pod_string, run_id=run_id + ) if not cm_data: logger.info( f"No sglang pod found while matching {matching_pod_string}" @@ -870,6 +887,14 @@ if __name__ == "__main__": help="The transformers version number for running sglang. Use default version in image if keep empty.", ) + parser.add_argument( + "--run-id", + type=str, + required=False, + default="", + help="GitHub Actions run_id, used to label pods for run-scoped isolation.", + ) + args = parser.parse_args() docker_image_url = args.image @@ -886,6 +911,7 @@ if __name__ == "__main__": env = args.env trouble_shotting = args.trouble_shotting transformers_version = args.transformers_version + run_id = args.run_id kube_name_space = args.kube_name_space kube_job_type = args.kube_job_type @@ -915,4 +941,5 @@ if __name__ == "__main__": env=env, trouble_shotting=trouble_shotting, transformers_version=transformers_version, + run_id=run_id, ) diff --git a/python/sglang/test/ascend/e2e/run_npu_testcase.sh b/python/sglang/test/ascend/e2e/run_npu_testcase.sh index 4ce070093..ea1014193 100644 --- a/python/sglang/test/ascend/e2e/run_npu_testcase.sh +++ b/python/sglang/test/ascend/e2e/run_npu_testcase.sh @@ -143,9 +143,9 @@ fi source_plog_path="/root/ascend/log/debug/plog" if [ -d "$source_plog_path" ];then echo "Plog files found. Begin to backup them." - target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}" + target_plog_path="/root/sglang/debug/logs/plog/${run_label}/${tc_name}/${HOSTNAME}" if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then - target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}" + target_plog_path="/root/.cache/tests/logs/plog/${run_label}/${tc_name}/${HOSTNAME}" fi rm -rf "${target_plog_path}" mkdir -p "${target_plog_path}"