[NPU]Add a full test pipeline on NPU, resolve issues in the NPU test architecture (#20751)
This commit is contained in:
Cherry_ming
@@ -2,7 +2,7 @@ name: Nightly Test (NPU)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 17 * * *' # Execute at 1:00 a.m. Beijing Time every day
|
||||
- cron: '0 18 * * *' # Execute at 2:00 a.m. Beijing Time every day
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
@@ -21,13 +21,61 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11'
|
||||
skip_install_flag:
|
||||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'false'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: nightly-test-npu-${{ inputs.ref || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
|
||||
|
||||
jobs:
|
||||
set-image-config:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ref: ${{ steps.set-vars.outputs.ref }}
|
||||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||||
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
|
||||
steps:
|
||||
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
||||
- name: Set image config
|
||||
id: set-vars
|
||||
run: |
|
||||
if [ -z "${{ inputs.ref }}" ]; then
|
||||
echo "ref=" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.job_filter }}" ]; then
|
||||
echo "job_filter=all" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.skip_install_flag }}" ]; then
|
||||
echo "skip_install_flag=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
nightly-1-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-2
|
||||
strategy:
|
||||
@@ -35,26 +83,33 @@ jobs:
|
||||
matrix:
|
||||
part: [0, 1]
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn"
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy download through proxy
|
||||
curl -o /tmp/test.jsonl -L https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
@@ -86,6 +141,7 @@ jobs:
|
||||
python3 run_suite.py --hw npu --suite nightly-1-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||
|
||||
nightly-2-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-2
|
||||
strategy:
|
||||
@@ -93,26 +149,33 @@ jobs:
|
||||
matrix:
|
||||
part: [0]
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn"
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy download through proxy
|
||||
curl -o /tmp/test.jsonl -L https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
@@ -143,6 +206,7 @@ jobs:
|
||||
python3 run_suite.py --hw npu --suite nightly-2-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1
|
||||
|
||||
nightly-4-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-4
|
||||
strategy:
|
||||
@@ -150,25 +214,33 @@ jobs:
|
||||
matrix:
|
||||
part: [0]
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
ref: ${{ needs.set-image-config.outputs.ref|| github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn"
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy download through proxy
|
||||
curl -o /tmp/test.jsonl -L https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
@@ -200,6 +272,7 @@ jobs:
|
||||
python3 run_suite.py --hw npu --suite nightly-4-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1
|
||||
|
||||
nightly-8-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-8
|
||||
strategy:
|
||||
@@ -207,26 +280,33 @@ jobs:
|
||||
matrix:
|
||||
part: [0]
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn"
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy download through proxy
|
||||
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
@@ -258,6 +338,7 @@ jobs:
|
||||
python3 run_suite.py --hw npu --suite nightly-8-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1
|
||||
|
||||
nightly-16-npu-a3:
|
||||
needs: [set-image-config]
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
runs-on: linux-aarch64-a3-16
|
||||
strategy:
|
||||
@@ -265,26 +346,33 @@ jobs:
|
||||
matrix:
|
||||
part: [0, 1]
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11
|
||||
image: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
|
||||
PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
|
||||
GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
|
||||
run: |
|
||||
# speed up by using infra cache services
|
||||
CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
|
||||
sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
|
||||
pip config set global.index-url http://${CACHING_URL}/pypi/simple
|
||||
pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn"
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
pip config set global.trusted-host "${CACHING_URL}"
|
||||
|
||||
if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
|
||||
bash scripts/ci/npu/npu_ci_install_dependency.sh a3
|
||||
fi
|
||||
|
||||
# copy required file from our daily cache
|
||||
cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
|
||||
# copy download through proxy
|
||||
curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
|
||||
# copy gsm8k dataset
|
||||
cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
|
||||
|
||||
- name: Print Log Information
|
||||
run: |
|
||||
|
||||
Reference in New Issue
Block a user