minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[CI/NPU] Isolate multi-node tests by run_id to prevent concurrent-run… (#35500)

Browse Source
This commit is contained in:
hhhh1252023
2026-09-01 11:08:27 +08:00
committed by GitHub
parent 5b04408784
commit 60f881b40c
8 changed files with 51 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-npu-e2e-multi-node.yml
+9 -4
View File
@@ -79,7 +79,7 @@ jobs:
KUBECONFIG: /root/.cache/.cache/kb.yaml
KUBECTL: /root/.cache/.cache/kubectl
NAMESPACE: sgl-project
KUBE_JOB_NAME: ascend-sglang-${{ inputs.test_type }}-test
KUBE_JOB_NAME: ascend-sglang-test-${{ github.run_id }}
SGLANG_IS_IN_CI: true
ASCEND_E2E_TEST_CONFIG_PATH: python/sglang/test/ascend/e2e
SGLANG_USE_MODELSCOPE: true
@@ -101,8 +101,9 @@ jobs:
- name: Prepare code for testing
run: |
# copy source code to shared-disk
# Append run_id to isolate concurrent runs sharing the same PVC.
current_path=$(pwd)
target_path=/root/.cache/tests/sglang
target_path=/root/.cache/tests/sglang-${{ github.run_id }}
rm -rf ${target_path}
mkdir -p ${target_path}
cp -r ${current_path}/* ${target_path}/
@@ -129,7 +130,8 @@ jobs:
timeout-minutes: 300
run: |
# sglang_source_relative_path is shared-disk path
sglang_source_relative_path=tests/sglang
# Append run_id so concurrent runs do not overwrite each other on the shared PVC.
sglang_source_relative_path=tests/sglang-${{ github.run_id }}
sglang_source_path=/root/.cache/${sglang_source_relative_path}
echo "Source code path: ${sglang_source_path}"
@@ -184,7 +186,8 @@ jobs:
--test-case ${test_case} \
--kube-name-space ${NAMESPACE} \
--kube-job-type ${kube_job_type} \
--kube-job-name-prefix ${KUBE_JOB_NAME}"
--kube-job-name-prefix ${KUBE_JOB_NAME} \
--run-id ${{ github.run_id }}"
if [ "$prefill_decode_deployment" = "separation" ];then
CMD="${CMD} \
@@ -249,3 +252,5 @@ jobs:
break
fi
done
# Cleanup the run-scoped source code directory on the shared PVC.
rm -rf /root/.cache/tests/sglang-${{ github.run_id }} || true
python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2
+1
View File
@@ -28,6 +28,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
ring-controller.atlas: ascend-1980
spec:
hostNetwork: True
python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2
+1
View File
@@ -22,6 +22,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
task: pd-mix
spec:
hostNetwork: True
python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2
+3
View File
@@ -28,6 +28,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
ring-controller.atlas: ascend-1980
spec:
hostNetwork: True
@@ -144,6 +145,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
ring-controller.atlas: ascend-1980
spec:
hostNetwork: True
@@ -260,6 +262,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
ring-controller.atlas: ascend-1980
spec:
hostNetwork: True
python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2
+3
View File
@@ -22,6 +22,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
task: prefill
spec:
restartPolicy: Always
@@ -125,6 +126,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
task: decode
spec:
restartPolicy: Always
@@ -228,6 +230,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
task: router
spec:
hostNetwork: True
python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2
+1
View File
@@ -22,6 +22,7 @@ spec:
metadata:
labels:
app: sgl-ascend
run-id: "{{ run_id }}"
ring-controller.atlas: ascend-1980
spec:
containers:
python/sglang/test/ascend/e2e/run_npu_e2e_test.py
+31 -4
View File
@@ -308,10 +308,19 @@ def create_or_update_configmap(cm_name: str, data: dict, namespace: str):
raise
def prepare_cm_data(namespace, pod_string):
"""Prepare a configmap data: {pod_name: pod_ip} by the running pod's information."""
def prepare_cm_data(namespace, pod_string, run_id=None):
"""Prepare a configmap data: {pod_name: pod_ip} by the running pod's information.
When run_id is provided, the label selector is scoped to that run_id so that
concurrent runs in the same namespace do not pollute each other's ConfigMap.
The pod_string filter (final_kube_job_name) still acts as a safety net.
"""
if run_id:
label_selector = f"app=sgl-ascend,run-id={run_id}"
else:
label_selector = "app=sgl-ascend"
pods = core_api.list_namespaced_pod(
namespace=namespace, label_selector="app=sgl-ascend"
namespace=namespace, label_selector=label_selector
)
data = {}
for pod in pods.items:
@@ -563,6 +572,7 @@ def run_npu_e2e_test_case(
env="debug",
trouble_shotting=False,
transformers_version="",
run_id: str = "",
):
"""The method for running a npu e2e test case.
Args:
@@ -578,6 +588,8 @@ def run_npu_e2e_test_case(
sglang_is_in_ci (bool): whether running in CI environment.
install_sglang_from_source (bool): whether installing sglang from source or use docker image directly.
env (str): the environment to run the test on. Choose one in ["debug", "ci"]
run_id (str): the GitHub Actions run_id, used to label pods for run-scoped
isolation when multiple runs share the same k8s namespace.
"""
random_str = get_unique_random_string(16, True)
@@ -629,6 +641,7 @@ def run_npu_e2e_test_case(
"trouble_shotting": trouble_shotting,
"transformers_version": transformers_version,
"run_label": run_label,
"run_id": run_id,
}
create_kube_yaml(
kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
@@ -652,6 +665,7 @@ def run_npu_e2e_test_case(
"trouble_shotting": trouble_shotting,
"transformers_version": transformers_version,
"run_label": run_label,
"run_id": run_id,
}
template_key = (
KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
@@ -680,6 +694,7 @@ def run_npu_e2e_test_case(
"trouble_shotting": trouble_shotting,
"transformers_version": transformers_version,
"run_label": run_label,
"run_id": run_id,
}
template_key = (
KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
@@ -699,7 +714,9 @@ def run_npu_e2e_test_case(
):
if kube_job_type != "single":
matching_pod_string = final_kube_job_name
cm_data = prepare_cm_data(kube_name_space, matching_pod_string)
cm_data = prepare_cm_data(
kube_name_space, matching_pod_string, run_id=run_id
)
if not cm_data:
logger.info(
f"No sglang pod found while matching {matching_pod_string}"
@@ -870,6 +887,14 @@ if __name__ == "__main__":
help="The transformers version number for running sglang. Use default version in image if keep empty.",
)
parser.add_argument(
"--run-id",
type=str,
required=False,
default="",
help="GitHub Actions run_id, used to label pods for run-scoped isolation.",
)
args = parser.parse_args()
docker_image_url = args.image
@@ -886,6 +911,7 @@ if __name__ == "__main__":
env = args.env
trouble_shotting = args.trouble_shotting
transformers_version = args.transformers_version
run_id = args.run_id
kube_name_space = args.kube_name_space
kube_job_type = args.kube_job_type
@@ -915,4 +941,5 @@ if __name__ == "__main__":
env=env,
trouble_shotting=trouble_shotting,
transformers_version=transformers_version,
run_id=run_id,
)
python/sglang/test/ascend/e2e/run_npu_testcase.sh
+2 -2
View File
@@ -143,9 +143,9 @@ fi
source_plog_path="/root/ascend/log/debug/plog"
if [ -d "$source_plog_path" ];then
echo "Plog files found. Begin to backup them."
target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}"
target_plog_path="/root/sglang/debug/logs/plog/${run_label}/${tc_name}/${HOSTNAME}"
if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
target_plog_path="/root/.cache/tests/logs/plog/${run_label}/${tc_name}/${HOSTNAME}"
fi
rm -rf "${target_plog_path}"
mkdir -p "${target_plog_path}"