[AMD] Deepseek v4 Flash / Pro nightly tests for MI35x ROCm 7.2 (#24203)
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
Bingxu Chen
co-authored by
YC Yen-Ching Tseng
YC Yen-Ching Tseng
parent
aea527afdc
commit
5eff3c489a
@@ -57,6 +57,8 @@ on:
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
|
||||
- nightly-8-gpu-mi35x-kimi-k25-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen35-rocm720
|
||||
@@ -1413,6 +1415,142 @@ jobs:
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU DeepSeek-V4-Flash FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2
|
||||
# NOTE on runtime sourcing: the DSv4 docker image (tag suffix `-DSv4`) bakes
|
||||
# in sglang built from a specific commit of the amd/deepseek_v4 branch (the
|
||||
# 7-char sha in the image tag is that commit). To keep the runtime as exactly
|
||||
# that image-frozen sglang/aiter, we pass `--skip-sglang-build` and
|
||||
# `--skip-aiter-build` so install_dependency.sh does NOT `pip install -e
|
||||
# /sglang-checkout/python` (which would override the image's sglang with
|
||||
# whatever this checkout happens to be) and does NOT rebuild aiter from this
|
||||
# checkout's docker/rocm.Dockerfile. The /sglang-checkout mount is still used
|
||||
# for shell scripts and for run_suite.py discovering test files; it does not
|
||||
# poison Python imports because the image's site-packages .pth points at
|
||||
# /sgl-workspace/sglang/python (a different path).
|
||||
nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Resolve DSv4 image tag
|
||||
id: dsv4_image
|
||||
run: |
|
||||
# Pick the latest Docker Hub tag matching rocm720-mi35x-<sha7>-<YYYYMMDD>-DSv4.
|
||||
# Docker Hub returns results sorted by last_updated DESC by default, so the
|
||||
# first regex match is the most recent daily build.
|
||||
AUTH_HEADER=()
|
||||
if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
|
||||
TOKEN=$(curl -s -H "Content-Type: application/json" \
|
||||
-X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \
|
||||
https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))")
|
||||
if [[ -n "$TOKEN" ]]; then
|
||||
AUTH_HEADER=(-H "Authorization: JWT $TOKEN")
|
||||
fi
|
||||
fi
|
||||
TAG=$(curl -s "${AUTH_HEADER[@]}" \
|
||||
"https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \
|
||||
| grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \
|
||||
| head -n 1 | cut -d'"' -f4)
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "::error::No DSv4 image found matching rocm720-mi35x-<sha7>-<YYYYMMDD>-DSv4 on Docker Hub"
|
||||
exit 1
|
||||
fi
|
||||
echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved DSv4 image: rocm/sgl-dev:$TAG"
|
||||
|
||||
- name: Setup docker (ROCm 7.2 DSv4)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies (preserve DSv4 sglang/aiter from image)
|
||||
run: |
|
||||
# --skip-sglang-build: keep the image's pre-installed DSv4 sglang
|
||||
# (default would `pip install -e /sglang-checkout/python` and clobber it with main's source).
|
||||
# --skip-aiter-build: keep the image's DSv4-tuned aiter
|
||||
# (default reads /sglang-checkout/docker/rocm.Dockerfile from main and rebuilds aiter to that commit).
|
||||
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh \
|
||||
--skip-sglang-build --skip-aiter-build --skip-test-time-deps
|
||||
# tabulate is the only thing run_suite.py imports that may not be in the DSv4 image.
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8 + FP4)
|
||||
timeout-minutes: 300
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# MI35x 8-GPU DeepSeek-V4-Pro FP8 + FP4 (Accuracy + Performance combined) ROCm 7.2
|
||||
# Pro is 1.6T (vs Flash 285B); load + warmup is much longer, so timeout-per-file
|
||||
# and the job timeout are both larger than the Flash job.
|
||||
# Same image / branch / install strategy as the Flash job above — see the comment
|
||||
# block on `nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720` for the rationale.
|
||||
nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720:
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Resolve DSv4 image tag
|
||||
id: dsv4_image
|
||||
run: |
|
||||
AUTH_HEADER=()
|
||||
if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
|
||||
TOKEN=$(curl -s -H "Content-Type: application/json" \
|
||||
-X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \
|
||||
https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))")
|
||||
if [[ -n "$TOKEN" ]]; then
|
||||
AUTH_HEADER=(-H "Authorization: JWT $TOKEN")
|
||||
fi
|
||||
fi
|
||||
TAG=$(curl -s "${AUTH_HEADER[@]}" \
|
||||
"https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \
|
||||
| grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \
|
||||
| head -n 1 | cut -d'"' -f4)
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "::error::No DSv4 image found matching rocm720-mi35x-<sha7>-<YYYYMMDD>-DSv4 on Docker Hub"
|
||||
exit 1
|
||||
fi
|
||||
echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved DSv4 image: rocm/sgl-dev:$TAG"
|
||||
|
||||
- name: Setup docker (ROCm 7.2 DSv4)
|
||||
run: |
|
||||
touch github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install dependencies (preserve DSv4 sglang/aiter from image)
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh \
|
||||
--skip-sglang-build --skip-aiter-build --skip-test-time-deps
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 + FP4)
|
||||
timeout-minutes: 480
|
||||
run: |
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-pro --nightly --timeout-per-file 14400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
check-all-jobs:
|
||||
if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
|
||||
needs:
|
||||
@@ -1453,6 +1591,8 @@ jobs:
|
||||
- nightly-accuracy-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-basic-rocm720
|
||||
- nightly-perf-8-gpu-mi35x-deepseek-v32-mtp-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-flash-rocm720
|
||||
- nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720
|
||||
- nightly-8-gpu-mi35x-kimi-k25-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
|
||||
- nightly-8-gpu-mi35x-qwen35-rocm720
|
||||
|
||||
Reference in New Issue
Block a user