[NPU] Nightly CI refactor and enhancement (#27433)
This commit is contained in:
hhhh1252023
@@ -0,0 +1,232 @@
|
||||
name: 'Nightly Test (NPU) Multi Node Template'
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runner:
|
||||
required: false
|
||||
type: string
|
||||
default: linux-amd64-cpu-8
|
||||
test_type:
|
||||
required: false
|
||||
type: string
|
||||
default: perf
|
||||
description: perf or accuracy
|
||||
test_config_name:
|
||||
required: true
|
||||
type: string
|
||||
description: test config name
|
||||
prefill_size:
|
||||
required: false
|
||||
type: number
|
||||
default: 0
|
||||
description: number of prefill nodes for pd-separation
|
||||
decode_size:
|
||||
required: false
|
||||
type: number
|
||||
default: 0
|
||||
description: number of decode nodes for pd-separation
|
||||
router_size:
|
||||
required: false
|
||||
type: number
|
||||
default: 0
|
||||
description: number of router nodes for pd-separation
|
||||
node_size:
|
||||
required: false
|
||||
type: number
|
||||
default: 0
|
||||
description: number of nodes for pd-mix
|
||||
test_case:
|
||||
required: true
|
||||
type: string
|
||||
description: path of test case file
|
||||
image:
|
||||
required: false
|
||||
type: string
|
||||
description: image for pods
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann8.5.0-a3"
|
||||
install_sglang_from_source:
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
description: use sglang from source code or from docker image
|
||||
prefill_decode_deployment:
|
||||
required: true
|
||||
type: string
|
||||
description: "The deployment of prefill and decode nodes. ['separation', 'mix']"
|
||||
transformers_version:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
description: "The transformers version number for running sglang. Use default version in image if keep empty."
|
||||
|
||||
concurrency:
|
||||
group: ascend-nightly-multi-node-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: ${{ inputs.test_config_name }}
|
||||
runs-on: ${{ inputs.runner }}
|
||||
container:
|
||||
image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/sglang:main-x86
|
||||
env:
|
||||
KUBECONFIG: /root/.cache/.cache/kb.yaml
|
||||
KUBECTL: /root/.cache/.cache/kubectl
|
||||
NAMESPACE: sgl-project
|
||||
KUBE_JOB_NAME: ascend-sglang-${{ inputs.test_type }}-test
|
||||
SGLANG_IS_IN_CI: true
|
||||
ASCEND_E2E_TEST_CONFIG_PATH: python/sglang/test/ascend/e2e
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TRANSFORMERS_VERBOSITY: "error"
|
||||
GDN_ATTN_BACKEND_TRITON: 1
|
||||
steps:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple jinja2
|
||||
pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple psutil
|
||||
# Install kubernetes
|
||||
pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple kubernetes
|
||||
cp $KUBECTL /usr/local/sbin/
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare code for testing
|
||||
run: |
|
||||
# copy source code to shared-disk
|
||||
current_path=$(pwd)
|
||||
target_path=/root/.cache/tests/sglang
|
||||
rm -rf ${target_path}
|
||||
mkdir -p ${target_path}
|
||||
cp -r ${current_path}/* ${target_path}/
|
||||
|
||||
- name: Clear resources
|
||||
run: |
|
||||
cd ${ASCEND_E2E_TEST_CONFIG_PATH}
|
||||
kubectl delete -f ./k8s_multi_pd_*.yaml --ignore-not-found=true || true
|
||||
|
||||
pod_name_prefix="${KUBE_JOB_NAME}-"
|
||||
echo "kube name space: $NAMESPACE, pod name prefix: ${pod_name_prefix}"
|
||||
while true; do
|
||||
if kubectl get po -A -n $NAMESPACE | grep -q "${pod_name_prefix}"; then
|
||||
echo "Found exist sglang job, sleeping for 30 seconds..."
|
||||
sleep 30
|
||||
kubectl get pods | grep "${pod_name_prefix}" | awk '{print $1}' | xargs kubectl delete pod -n $NAMESPACE || true
|
||||
else
|
||||
echo "No sglang job exist, start test case..."
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 300
|
||||
run: |
|
||||
# sglang_source_relative_path is shared-disk path
|
||||
sglang_source_relative_path=tests/sglang
|
||||
sglang_source_path=/root/.cache/${sglang_source_relative_path}
|
||||
echo "Source code path: ${sglang_source_path}"
|
||||
|
||||
test_case="${{ inputs.test_case }}"
|
||||
if [ ! -f "${sglang_source_path}/${test_case}" ]; then
|
||||
echo "The testcase does not exit: ${sglang_source_path}/${test_case}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# prepare for output data
|
||||
current_date=$(date +%Y%m%d)
|
||||
test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
|
||||
mkdir -p ${test_data_output_path}
|
||||
tc_name=${test_case##*/}
|
||||
tc_name=${tc_name%.*}
|
||||
metrics_data_path=${test_data_output_path}/${tc_name}
|
||||
mkdir -p ${metrics_data_path}
|
||||
echo "Metrics file path: ${metrics_data_path}"
|
||||
|
||||
prefill_decode_deployment="${{ inputs.prefill_decode_deployment }}"
|
||||
kube_job_type=""
|
||||
if [ "$prefill_decode_deployment" = "separation" ];then
|
||||
kube_job_type=multi-pd-separation
|
||||
elif [ "$prefill_decode_deployment" = "mix" ];then
|
||||
kube_job_type=multi-pd-mix
|
||||
else
|
||||
echo "Unsupported deployment type: ${prefill_decode_deployment}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
transformers_version="${{ inputs.transformers_version }}"
|
||||
cd ${ASCEND_E2E_TEST_CONFIG_PATH}
|
||||
CMD="python3 -u run_npu_e2e_test.py \
|
||||
--env ci \
|
||||
--image ${{ inputs.image }} \
|
||||
--sglang-source-relative-path ${sglang_source_relative_path} \
|
||||
--metrics-data-file ${metrics_data_path} \
|
||||
--test-case ${test_case} \
|
||||
--kube-name-space ${NAMESPACE} \
|
||||
--kube-job-type ${kube_job_type} \
|
||||
--kube-job-name-prefix ${KUBE_JOB_NAME}"
|
||||
|
||||
if [ "$prefill_decode_deployment" = "separation" ];then
|
||||
CMD="${CMD} \
|
||||
--prefill-size ${{ inputs.prefill_size }} \
|
||||
--decode-size ${{ inputs.decode_size }} \
|
||||
--router-size ${{ inputs.router_size }}"
|
||||
elif [ "$prefill_decode_deployment" = "mix" ];then
|
||||
CMD="${CMD} --node-size ${{ inputs.node_size }}"
|
||||
else
|
||||
echo "Unsupported deployment type: ${prefill_decode_deployment}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$SGLANG_IS_IN_CI" = "true" ] || [ "$SGLANG_IS_IN_CI" = "True" ];then
|
||||
echo "Run test in ci."
|
||||
CMD="${CMD} --sglang-is-in-ci"
|
||||
fi
|
||||
|
||||
install_sglang_from_source="${{ inputs.install_sglang_from_source }}"
|
||||
if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then
|
||||
echo "Install sglang from source."
|
||||
CMD="${CMD} --install-sglang-from-source"
|
||||
commit_id=${{ github.sha }}
|
||||
echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
|
||||
else
|
||||
echo "Use sglang from image: ${{ inputs.image }}"
|
||||
fi
|
||||
|
||||
if [ "$transformers_version" != "" ];then
|
||||
CMD="${CMD} --transformers-version ${transformers_version}"
|
||||
else
|
||||
echo "Use default transformers version in image."
|
||||
fi
|
||||
|
||||
echo "Run command: ${CMD}"
|
||||
eval "${CMD}"
|
||||
|
||||
- name: Upload metrics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: metrics-${{ inputs.test_config_name }}
|
||||
path: /tmp/metrics.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Post process
|
||||
if: always()
|
||||
run: |
|
||||
cd ${ASCEND_E2E_TEST_CONFIG_PATH}
|
||||
kubectl get pods -n $NAMESPACE | grep $KUBE_JOB_NAME
|
||||
kubectl delete -f ./k8s_multi_pd_*.yaml --ignore-not-found=true || true
|
||||
|
||||
pod_name_prefix="${KUBE_JOB_NAME}-"
|
||||
echo "kube name space: $NAMESPACE, pod name prefix: ${pod_name_prefix}"
|
||||
while true; do
|
||||
if kubectl get po -A -n $NAMESPACE | grep -q "${pod_name_prefix}"; then
|
||||
echo "Found exist sglang job, sleeping for 30 seconds..."
|
||||
sleep 30
|
||||
kubectl get pods | grep "${pod_name_prefix}" | awk '{print $1}' | xargs kubectl delete pod -n $NAMESPACE || true
|
||||
else
|
||||
echo "No sglang job exist, start test case..."
|
||||
break
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,228 @@
|
||||
name: 'Nightly Test (NPU) Single Node Template'
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
runner:
|
||||
required: true
|
||||
type: string
|
||||
default: linux-aarch64-a3-16
|
||||
test_type:
|
||||
required: false
|
||||
type: string
|
||||
default: perf
|
||||
description: perf or accuracy
|
||||
test_config_name:
|
||||
required: true
|
||||
type: string
|
||||
description: test config name
|
||||
test_case:
|
||||
required: true
|
||||
type: string
|
||||
description: path of test case file
|
||||
image:
|
||||
required: false
|
||||
type: string
|
||||
description: image for pods
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
|
||||
install_sglang_from_source:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
description: use sglang from source code or from docker image
|
||||
transformers_version:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
description: "The transformers version number for running sglang. Use default version in image if keep empty."
|
||||
|
||||
concurrency:
|
||||
group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: ${{ inputs.test_config_name }}
|
||||
runs-on: ${{ inputs.runner }}
|
||||
container:
|
||||
image: ${{ inputs.image }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Check npu info
|
||||
run: |
|
||||
npu-smi info
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
SGLANG_IS_IN_CI: true
|
||||
TRANSFORMERS_VERBOSITY: "error"
|
||||
GDN_ATTN_BACKEND_TRITON: 1
|
||||
SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
|
||||
shell: bash
|
||||
run: |
|
||||
sglang_source_path=$(pwd)
|
||||
echo "Source code path: ${sglang_source_path}"
|
||||
ln -sf ${sglang_source_path} /root/sglang
|
||||
|
||||
test_case=${{ inputs.test_case }}
|
||||
if [ ! -f "${sglang_source_path}/${test_case}" ]; then
|
||||
echo "The testcase does not exit: ${sglang_source_path}/${test_case}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# prepare for output data
|
||||
current_date=$(date +%Y%m%d)
|
||||
test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
|
||||
mkdir -p ${test_data_output_path}
|
||||
tc_name=${test_case##*/}
|
||||
tc_name=${tc_name%.*}
|
||||
export METRICS_DATA_FILE=${test_data_output_path}/${tc_name}
|
||||
mkdir -p ${METRICS_DATA_FILE}
|
||||
echo "Metrics file path: ${METRICS_DATA_FILE}"
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
|
||||
export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
|
||||
PYTHON_FOR_SGLANG="python"
|
||||
PIP_FOR_SGLANG="pip"
|
||||
if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
|
||||
echo "===== Install transformers for sglang - Begin ====="
|
||||
TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
|
||||
if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
|
||||
echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
|
||||
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
|
||||
pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
else
|
||||
echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
|
||||
TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
|
||||
mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
|
||||
cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
|
||||
pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
|
||||
fi
|
||||
echo "===== Install transformers for sglang in virtual env - End ====="
|
||||
fi
|
||||
echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
|
||||
|
||||
echo "scaling_governor performance num: \
|
||||
$(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
|
||||
echo "swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
|
||||
echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
|
||||
|
||||
export SGLANG_TEST_MAX_RETRY=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
|
||||
|
||||
install_sglang_from_source=${{ inputs.install_sglang_from_source }}
|
||||
if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then
|
||||
echo "Install sglang from source"
|
||||
commit_id=${{ github.sha }}
|
||||
echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
|
||||
export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH
|
||||
else
|
||||
echo "Use sglang from image: ${{ inputs.image }}"
|
||||
sglang_pkg_path=/sgl-workspace/sglang/python
|
||||
ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
|
||||
mkdir -p ${ascend_test_util_path}
|
||||
mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
|
||||
cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
|
||||
fi
|
||||
|
||||
source /usr/local/Ascend/cann/set_env.sh || true
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh || true
|
||||
|
||||
# Set environment of cann
|
||||
log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
|
||||
rm -rf ${log_path}
|
||||
mkdir -p ${log_path}
|
||||
echo "Log path: ${log_path}"
|
||||
|
||||
echo "Running test case ${test_case}"
|
||||
test_exit_code=0
|
||||
${PYTHON_FOR_SGLANG} -u ${test_case} 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
|
||||
echo "Finished test case ${test_case}"
|
||||
|
||||
if [ "${test_exit_code}" = "0" ]; then
|
||||
test_status="pass"
|
||||
status_icon="✅"
|
||||
else
|
||||
test_status="fail"
|
||||
status_icon="❌"
|
||||
fi
|
||||
|
||||
echo "test_status=${test_status}" >> $GITHUB_ENV
|
||||
echo "tc_name=${tc_name}" >> $GITHUB_ENV
|
||||
export test_status tc_name
|
||||
|
||||
echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
|
||||
if [ "${metric_count}" -gt 0 ]; then
|
||||
echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
|
||||
grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
|
||||
metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
|
||||
metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
|
||||
echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
|
||||
done
|
||||
else
|
||||
echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
echo "import json, re, os" > /tmp/dump_metrics.py
|
||||
echo "tc = os.environ.get('tc_name', 'unknown')" >> /tmp/dump_metrics.py
|
||||
echo "st = os.environ.get('test_status', 'unknown')" >> /tmp/dump_metrics.py
|
||||
echo "tp = '${{ inputs.test_type }}'" >> /tmp/dump_metrics.py
|
||||
echo "metrics = {}" >> /tmp/dump_metrics.py
|
||||
echo "with open('/tmp/test_output.log') as f:" >> /tmp/dump_metrics.py
|
||||
echo " for line in f:" >> /tmp/dump_metrics.py
|
||||
echo " m = re.match(r'\[METRIC\] (\S+)=(\S+)', line)" >> /tmp/dump_metrics.py
|
||||
echo " if m:" >> /tmp/dump_metrics.py
|
||||
echo " v = m.group(2)" >> /tmp/dump_metrics.py
|
||||
echo " try:" >> /tmp/dump_metrics.py
|
||||
echo " v = float(v)" >> /tmp/dump_metrics.py
|
||||
echo " except ValueError:" >> /tmp/dump_metrics.py
|
||||
echo " pass" >> /tmp/dump_metrics.py
|
||||
echo " metrics[m.group(1)] = v" >> /tmp/dump_metrics.py
|
||||
echo "baselines = {}" >> /tmp/dump_metrics.py
|
||||
echo "for k, v in list(metrics.items()):" >> /tmp/dump_metrics.py
|
||||
echo " if k.endswith('_baseline'):" >> /tmp/dump_metrics.py
|
||||
echo " baselines[k[:-9]] = v" >> /tmp/dump_metrics.py
|
||||
echo " del metrics[k]" >> /tmp/dump_metrics.py
|
||||
echo "with open('/tmp/metrics.json', 'w') as f:" >> /tmp/dump_metrics.py
|
||||
echo " json.dump({'test_case': tc, 'test_type': tp, 'status': st, 'metrics': metrics, 'baselines': baselines}, f)" >> /tmp/dump_metrics.py
|
||||
python3 /tmp/dump_metrics.py
|
||||
exit ${test_exit_code}
|
||||
|
||||
- name: Upload metrics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: metrics-${{ inputs.test_config_name }}
|
||||
path: /tmp/metrics.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Backup plog
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
plog_path="/root/ascend/log/debug/plog"
|
||||
if [ -d "$plog_path" ];then
|
||||
echo "Plog files found. Begin to backup them."
|
||||
tc_name=${{ inputs.test_case }}
|
||||
tc_name=${tc_name##*/}
|
||||
tc_name=${tc_name%.*}
|
||||
target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
|
||||
echo "Save path: ${target_plog_path}"
|
||||
rm -rf ${target_plog_path}
|
||||
mkdir -p ${target_plog_path}
|
||||
cp ${plog_path}/* ${target_plog_path}
|
||||
fi
|
||||
@@ -1,5 +1,4 @@
|
||||
name: Nightly Test (NPU)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 18 * * *' # Execute at 2:00 a.m. Beijing Time every day
|
||||
@@ -9,6 +8,32 @@ on:
|
||||
paths:
|
||||
- ".github/workflows/nightly-test-npu.yml"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
job_filter:
|
||||
description: 'Select which job to run (leave empty or "all" to run all jobs)'
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
|
||||
skip_install_flag:
|
||||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'true'
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
@@ -21,28 +46,31 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11'
|
||||
default: ''
|
||||
skip_install_flag:
|
||||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'false'
|
||||
|
||||
|
||||
default: ''
|
||||
concurrency:
|
||||
group: nightly-test-npu-${{ inputs.ref || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
||||
|
||||
jobs:
|
||||
set-image-config:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ref: ${{ steps.set-vars.outputs.ref }}
|
||||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||||
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
|
||||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||||
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
|
||||
steps:
|
||||
@@ -55,379 +83,324 @@ jobs:
|
||||
else
|
||||
echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.job_filter }}" ]; then
|
||||
echo "job_filter=all" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.image_a2 }}" ]; then
|
||||
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.skip_install_flag }}" ]; then
|
||||
echo "skip_install_flag=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
nightly-1-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-2
|
||||
nightly-poc-single-node-a2-tests:
|
||||
name: single-node-poc-a2
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [ set-image-config ]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 6
|
||||
matrix:
|
||||
part: [0, 1]
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
test_config:
|
||||
- name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2
|
||||
runner: linux-aarch64-a2-4
|
||||
test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
|
||||
test_type: 'perf'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
|
||||
with:
|
||||
runner: ${{ matrix.test_config.runner }}
|
||||
test_type: ${{ matrix.test_config.test_type }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a2 }}
|
||||
install_sglang_from_source: false
|
||||
transformers_version: ''
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
bash scripts/ci/npu/npu_log_print.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
pip install sglang_router
|
||||
hf download lmms-lab/MMMU --repo-type dataset
|
||||
pip install sentence_transformers torchaudio==2.10.0
|
||||
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
|
||||
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
|
||||
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
|
||||
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
|
||||
cd ./lmms-eval
|
||||
nohup pip install . > lmmslog.txt 2>&1 &
|
||||
sleep 120
|
||||
export PYTHONPATH=$PYTHONPATH:$(pwd)
|
||||
cd ../
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite nightly-1-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||
|
||||
nightly-2-npu-a3:
|
||||
nightly-poc-single-node-tests:
|
||||
name: single-node-poc
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-2
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 6
|
||||
matrix:
|
||||
part: [0]
|
||||
container:
|
||||
test_config:
|
||||
# qwen3_6_35b_a3b performance tests
|
||||
- name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_35b_a3b_1p_aime26
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
|
||||
- name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
|
||||
test_type: 'perf'
|
||||
# qwen3_6_27b performance tests
|
||||
- name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms
|
||||
runner: linux-aarch64-a3-4
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_w8a8_2p_in64k_out1k_50ms
|
||||
runner: linux-aarch64-a3-4
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms
|
||||
runner: linux-aarch64-a3-4
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms
|
||||
runner: linux-aarch64-a3-4
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_1p_in1080p_30_out256_50ms
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_6_27b_1p_gpqa
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
|
||||
# qwen3_32b performance tests
|
||||
- name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa
|
||||
runner: linux-aarch64-a3-4
|
||||
test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_32b_bf16_8p_in18k_out4k_6ms
|
||||
runner: linux-aarch64-a3-16
|
||||
test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_32b_bf16_8p_gpqa
|
||||
runner: linux-aarch64-a3-16
|
||||
test_case: test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
|
||||
# qwen3_30b_a3b performance tests
|
||||
- name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
|
||||
test_type: 'perf'
|
||||
# qwen3-8b performance tests
|
||||
- name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
|
||||
test_type: 'perf'
|
||||
- name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
|
||||
test_type: 'perf'
|
||||
# qwen3_next_80b_a3b_instruct performance tests
|
||||
- name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25
|
||||
runner: linux-aarch64-a3-4
|
||||
test_case: test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
|
||||
test_type: 'perf'
|
||||
# minimax_m2_5 performance tests
|
||||
- name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa
|
||||
runner: linux-aarch64-a3-16
|
||||
test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
|
||||
test_type: 'perf'
|
||||
- name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa
|
||||
runner: linux-aarch64-a3-16
|
||||
test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
|
||||
test_type: 'perf'
|
||||
# deepseek_v3_2 accuracy tests
|
||||
- name: deepseek_v3_2_8p_aime25
|
||||
runner: linux-aarch64-a3-16
|
||||
test_case: test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
|
||||
# glm4_7_flash accuracy tests
|
||||
- name: glm4_7_flash_1p_aime25
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
|
||||
# glm4_6v_flash accuracy tests
|
||||
- name: glm4_6v_flash_1p_mmmu
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py
|
||||
# qwen3_vl_8b_thinking accuracy tests
|
||||
- name: qwen3_vl_8b_thinking_1p_mmmu
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
|
||||
# qwen3_vl_30b_a3b_thinking accuracy tests
|
||||
- name: qwen3_vl_30b_a3b_thinking_1p_mmmu
|
||||
runner: linux-aarch64-a3-2
|
||||
test_case: test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
|
||||
with:
|
||||
runner: ${{ matrix.test_config.runner }}
|
||||
test_type: ${{ matrix.test_config.test_type }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
install_sglang_from_source: false
|
||||
transformers_version: ''
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
bash scripts/ci/npu/npu_log_print.sh
|
||||
- name: Run test
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
pip install sglang_router
|
||||
hf download lmms-lab/MMMU --repo-type dataset
|
||||
pip install sentence_transformers torchaudio==2.10.0
|
||||
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
|
||||
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
|
||||
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
|
||||
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
|
||||
cd ./lmms-eval
|
||||
nohup pip install . > lmmslog.txt 2>&1 &
|
||||
sleep 120
|
||||
export PYTHONPATH=$PYTHONPATH:$(pwd)
|
||||
cd ../
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite nightly-2-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1
|
||||
|
||||
nightly-4-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-4
|
||||
nightly-poc-multi-node-tests:
|
||||
name: multi-node-poc
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config, nightly-poc-single-node-tests]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
part: [0]
|
||||
container:
|
||||
test_config:
|
||||
# glm5_1 performance tests
|
||||
- name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26
|
||||
prefill_size: 2
|
||||
decode_size: 2
|
||||
router_size: 1
|
||||
test_case: test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
|
||||
test_type: 'perf'
|
||||
prefill_decode_deployment: 'separation'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||||
with:
|
||||
runner: linux-amd64-cpu-8
|
||||
test_type: ${{ matrix.test_config.test_type }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
prefill_size: ${{ matrix.test_config.prefill_size }}
|
||||
decode_size: ${{ matrix.test_config.decode_size }}
|
||||
router_size: ${{ matrix.test_config.router_size }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.set-image-config.outputs.ref|| github.ref }}
|
||||
install_sglang_from_source: false
|
||||
prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
|
||||
transformers_version: ''
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
bash scripts/ci/npu/npu_log_print.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
pip install sglang_router
|
||||
hf download lmms-lab/MMMU --repo-type dataset
|
||||
pip install sentence_transformers torchaudio==2.10.0
|
||||
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
|
||||
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
|
||||
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
|
||||
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
|
||||
cd ./lmms-eval
|
||||
nohup pip install . > lmmslog.txt 2>&1 &
|
||||
sleep 120
|
||||
export PYTHONPATH=$PYTHONPATH:$(pwd)
|
||||
cd ../
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite nightly-4-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1
|
||||
|
||||
nightly-8-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-8
|
||||
nightly-poc-multi-node-mix-tests:
|
||||
name: multi-node-mix-poc
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config, nightly-poc-single-node-tests, nightly-poc-multi-node-tests]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
part: [0]
|
||||
container:
|
||||
test_config:
|
||||
# kimi_k2_6 performance tests
|
||||
- name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
|
||||
node_size: 2
|
||||
test_case: test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
|
||||
test_type: 'perf'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||||
with:
|
||||
runner: linux-amd64-cpu-8
|
||||
test_type: ${{ matrix.test_config.test_type }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
node_size: ${{ matrix.test_config.node_size }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
bash scripts/ci/npu/npu_log_print.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
pip install sglang_router
|
||||
hf download lmms-lab/MMMU --repo-type dataset
|
||||
pip install sentence_transformers torchaudio==2.10.0
|
||||
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
|
||||
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
|
||||
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
|
||||
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
|
||||
cd ./lmms-eval
|
||||
nohup pip install . > lmmslog.txt 2>&1 &
|
||||
sleep 120
|
||||
export PYTHONPATH=$PYTHONPATH:$(pwd)
|
||||
cd ../
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite nightly-8-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1
|
||||
|
||||
nightly-16-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-16
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [0, 1]
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
bash scripts/ci/npu/npu_log_print.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
run: |
|
||||
pip install sglang_router
|
||||
hf download lmms-lab/MMMU --repo-type dataset
|
||||
pip install sentence_transformers torchaudio==2.10.0
|
||||
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
|
||||
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
|
||||
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
|
||||
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
|
||||
cd ./lmms-eval
|
||||
nohup pip install . > lmmslog.txt 2>&1 &
|
||||
sleep 120
|
||||
export PYTHONPATH=$PYTHONPATH:$(pwd)
|
||||
cd ../
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite nightly-16-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||
|
||||
install_sglang_from_source: false
|
||||
prefill_decode_deployment: 'mix'
|
||||
transformers_version: ''
|
||||
check-all-jobs:
|
||||
if: github.repository == 'sgl-project/sglang' && always()
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- nightly-1-npu-a3
|
||||
- nightly-2-npu-a3
|
||||
- nightly-4-npu-a3
|
||||
- nightly-8-npu-a3
|
||||
- nightly-16-npu-a3
|
||||
- nightly-poc-single-node-a2-tests
|
||||
- nightly-poc-single-node-tests
|
||||
- nightly-poc-multi-node-tests
|
||||
- nightly-poc-multi-node-mix-tests
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: docker.m.daocloud.io/ubuntu:22.04
|
||||
steps:
|
||||
- name: Check if any job failed
|
||||
- name: Download all metrics
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: metrics-*
|
||||
path: /tmp/metrics
|
||||
merge-multiple: false
|
||||
|
||||
- name: Generate results table
|
||||
run: |
|
||||
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
|
||||
echo "One or more nightly test jobs failed"
|
||||
exit 1
|
||||
status_emoji() {
|
||||
case "$1" in
|
||||
pass) echo "✅" ;;
|
||||
fail) echo "❌" ;;
|
||||
*) echo "❓" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
single_result_a2="${{ needs.nightly-poc-single-node-a2-tests.result }}"
|
||||
single_result="${{ needs.nightly-poc-single-node-tests.result }}"
|
||||
multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
|
||||
mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
|
||||
|
||||
group_icon() {
|
||||
case "$1" in
|
||||
success) echo "✅" ;;
|
||||
failure) echo "❌" ;;
|
||||
cancelled) echo "⏭️" ;;
|
||||
skipped) echo "⏭️" ;;
|
||||
*) echo "❓" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
echo "## Nightly Test Results" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| single-node-poc-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| single-node-poc | $(group_icon ${single_result}) ${single_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
echo "## Per-Test Metrics" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
has_metrics=false
|
||||
for dir in /tmp/metrics/metrics-*/; do
|
||||
file="${dir}metrics.json"
|
||||
if [ -f "$file" ]; then
|
||||
has_metrics=true
|
||||
echo "import json" > /tmp/parse_metrics.py
|
||||
echo "with open('$file') as f:" >> /tmp/parse_metrics.py
|
||||
echo " d = json.load(f)" >> /tmp/parse_metrics.py
|
||||
echo "tc = d.get('test_case', '-')" >> /tmp/parse_metrics.py
|
||||
echo "tp = d.get('test_type', '-')" >> /tmp/parse_metrics.py
|
||||
echo "st = d.get('status', 'unknown')" >> /tmp/parse_metrics.py
|
||||
echo "icon = '✅' if st == 'pass' else '❌'" >> /tmp/parse_metrics.py
|
||||
echo "metrics = d.get('metrics', {})" >> /tmp/parse_metrics.py
|
||||
echo "baselines = d.get('baselines', {})" >> /tmp/parse_metrics.py
|
||||
echo "mstr = ', '.join(f'{k}={v}' for k,v in metrics.items()) if metrics else '-'" >> /tmp/parse_metrics.py
|
||||
echo "bstr = ', '.join(f'{k}={v}' for k,v in baselines.items()) if baselines else '-'" >> /tmp/parse_metrics.py
|
||||
echo "print(f'| {tc} | {tp} | {icon} {st} | {mstr} | {bstr} |')" >> /tmp/parse_metrics.py
|
||||
python3 /tmp/parse_metrics.py
|
||||
fi
|
||||
done > /tmp/metrics_table.txt
|
||||
|
||||
if [ "${has_metrics}" = "true" ]; then
|
||||
echo "| Test Case | Type | Status | Metrics | Baseline |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|-----------|------|--------|---------|----------|" >> $GITHUB_STEP_SUMMARY
|
||||
cat /tmp/metrics_table.txt >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "No per-test metrics available (artifacts not found)." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
|
||||
echo "One or more nightly test jobs were cancelled"
|
||||
exit 1
|
||||
fi
|
||||
echo "All nightly test jobs passed"
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
FAIL=0
|
||||
if [ "${single_result}" != "success" ] && [ "${single_result}" != "skipped" ]; then FAIL=1; fi
|
||||
exit $FAIL
|
||||
|
||||
Reference in New Issue
Block a user