[AMD] prepare for MI300x PR runner pool: registry mirror, runner routing, threshold tuning (#23156)
This commit is contained in:
Bingxu Chen
@@ -92,6 +92,8 @@ on:
|
||||
|
||||
env:
|
||||
AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
|
||||
DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
concurrency:
|
||||
# When called via workflow_call with ref set, use a unique group per caller run to avoid
|
||||
|
||||
@@ -92,6 +92,8 @@ on:
|
||||
|
||||
env:
|
||||
AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
|
||||
DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
concurrency:
|
||||
# When called via workflow_call with ref set, use a unique group per caller run to avoid
|
||||
|
||||
@@ -93,6 +93,8 @@ on:
|
||||
|
||||
env:
|
||||
AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
|
||||
DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
concurrency:
|
||||
# When called via workflow_call with run_all_tests=true, use a unique group per run to
|
||||
|
||||
@@ -59,6 +59,14 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
runner_arch:
|
||||
description: 'AMD runner pool to dispatch GPU jobs to'
|
||||
required: false
|
||||
type: choice
|
||||
default: mi300
|
||||
options:
|
||||
- mi300
|
||||
- mi325
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
@@ -84,6 +92,8 @@ on:
|
||||
|
||||
env:
|
||||
AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
|
||||
DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
concurrency:
|
||||
# Scheduled and run_all_tests runs get unique groups (never cancel each other).
|
||||
@@ -165,6 +175,7 @@ jobs:
|
||||
|
||||
# =============================================== sgl-kernel ====================================================
|
||||
sgl-kernel-unit-test-amd:
|
||||
name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -176,11 +187,7 @@ jobs:
|
||||
needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -226,6 +233,7 @@ jobs:
|
||||
exit $failures
|
||||
|
||||
sgl-kernel-unit-test-2-gpu-amd:
|
||||
name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -237,11 +245,7 @@ jobs:
|
||||
needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-2gpu-sglang]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -280,6 +284,7 @@ jobs:
|
||||
# =============================================== primary ====================================================
|
||||
|
||||
stage-a-test-1-gpu-small-amd:
|
||||
name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -291,11 +296,7 @@ jobs:
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -320,6 +321,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
jit-kernel-unit-test-amd:
|
||||
name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -331,11 +333,7 @@ jobs:
|
||||
needs.check-changes.outputs.jit_kernel == 'true'
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -386,6 +384,7 @@ jobs:
|
||||
max-wait-minutes: '240'
|
||||
|
||||
stage-b-test-1-gpu-small-amd:
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -400,9 +399,8 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -421,11 +419,12 @@ jobs:
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-1-gpu-small-amd-nondeterministic:
|
||||
name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -437,11 +436,7 @@ jobs:
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -460,7 +455,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
@@ -504,6 +499,7 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-1-gpu-large-amd:
|
||||
name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -518,9 +514,8 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
part: [0, 1, 2]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -539,11 +534,12 @@ jobs:
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-b-test-2-gpu-large-amd:
|
||||
name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, wait-for-stage-a-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -558,9 +554,8 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-2gpu-sglang]
|
||||
part: [0, 1]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -579,11 +574,12 @@ jobs:
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
multimodal-gen-test-1-gpu-amd:
|
||||
name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -599,9 +595,8 @@ jobs:
|
||||
fail-fast: false
|
||||
max-parallel: 1 # Run one at a time to avoid eviction from resource exhaustion during AITER kernel JIT
|
||||
matrix:
|
||||
runner: [linux-mi325-1gpu-sglang]
|
||||
part: [0, 1, 2, 3] # 2 partitions: 11 tests ÷ 2 = ~5-6 tests each
|
||||
runs-on: ${{matrix.runner}}
|
||||
part: [0, 1, 2, 3]
|
||||
runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -715,6 +710,7 @@ jobs:
|
||||
free -h
|
||||
|
||||
multimodal-gen-test-2-gpu-amd:
|
||||
name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() && !cancelled() &&
|
||||
@@ -730,9 +726,8 @@ jobs:
|
||||
fail-fast: false
|
||||
max-parallel: 1 # Run one at a time to avoid eviction from resource exhaustion during AITER kernel JIT
|
||||
matrix:
|
||||
runner: [linux-mi325-2gpu-sglang]
|
||||
part: [0, 1, 2] # 3 partitions: 2 parametrized + 1 standalone (test_disagg_server.py)
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -812,7 +807,7 @@ jobs:
|
||||
docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available"
|
||||
|
||||
- name: Run diffusion server tests (2-GPU)
|
||||
timeout-minutes: 80
|
||||
timeout-minutes: 90
|
||||
run: |
|
||||
# AMD CI: All 2-GPU tests including LoRA
|
||||
# Tests: T2V, T2I, I2V, LoRA
|
||||
@@ -871,6 +866,7 @@ jobs:
|
||||
max-wait-minutes: '480'
|
||||
|
||||
stage-c-test-4-gpu-amd:
|
||||
name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -885,9 +881,8 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-4gpu-sglang]
|
||||
part: [0]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -906,7 +901,7 @@ jobs:
|
||||
run: bash scripts/ci/amd/amd_ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 90
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh \
|
||||
-e NCCL_CUMEM_ENABLE=0 \
|
||||
@@ -919,7 +914,7 @@ jobs:
|
||||
--suite stage-c-test-4-gpu-amd \
|
||||
--auto-partition-id ${{ matrix.part }} \
|
||||
--auto-partition-size 1 \
|
||||
--timeout-per-file 1800 \
|
||||
--timeout-per-file 5400 \
|
||||
--enable-retry \
|
||||
--max-attempts 2 \
|
||||
--retry-wait-seconds 120 \
|
||||
@@ -927,6 +922,7 @@ jobs:
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
stage-c-test-large-8-gpu-amd:
|
||||
name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
|
||||
needs: [check-changes, call-gate, wait-for-stage-b-amd]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -939,13 +935,12 @@ jobs:
|
||||
)
|
||||
)
|
||||
env:
|
||||
RUNNER_LABELS: linux-mi325-8gpu-sglang
|
||||
RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [linux-mi325-8gpu-sglang]
|
||||
part: [0, 1, 2]
|
||||
runs-on: ${{matrix.runner}}
|
||||
runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -970,7 +965,7 @@ jobs:
|
||||
docker exec ci_sglang bash -c "cd /sglang-checkout && NCCL_DEBUG=INFO RCCL_DEBUG=INFO torchrun --nproc_per_node=8 scripts/ci/amd/test_rccl_multi_gpu.py"
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
|
||||
@@ -154,18 +154,15 @@ find_latest_image() {
|
||||
fi
|
||||
done
|
||||
|
||||
# Then try the local registry.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
|
||||
echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Finally, try the public registry.
|
||||
# If not found locally, fall back to pulling from public registry.
|
||||
# We intentionally do not probe ${LOCAL_DOCKER_REGISTRY} here with
|
||||
# `docker manifest inspect --insecure` because that command runs in the
|
||||
# runner pod's network namespace, which on every observed AMD scale set
|
||||
# cannot reach 172.29.8.23:5000 (every probe either fast-fails with TLS
|
||||
# reject or hits a 30s TCP timeout, multiplied across 7 daily candidates).
|
||||
# The actual local-registry pull still happens in the call site below via
|
||||
# `docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}"`, which goes through the
|
||||
# docker daemon on the host and inherits its insecure-registries config.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
||||
@@ -260,14 +257,17 @@ elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
|
||||
else
|
||||
# Find the latest pre-built image
|
||||
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
||||
echo "Pulling Docker image: ${IMAGE}"
|
||||
if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
|
||||
# Local registry is on-LAN; no need to retry.
|
||||
docker pull "${IMAGE}"
|
||||
docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
# Try the local docker registry first (avoids Docker Hub rate limits and is
|
||||
# faster on the LAN); if that fails for any reason, fall back to the
|
||||
# public registry with exponential-backoff retries. Capture stderr so the
|
||||
# real failure reason (TLS handshake, 404, connection refused, etc.) is
|
||||
# visible in the job log instead of being silently swallowed.
|
||||
if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
|
||||
echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
|
||||
docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
|
||||
else
|
||||
# Public registry pulls can hit rate limits; retry with backoff.
|
||||
echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
|
||||
printf '%s\n' "${local_pull_output}" | sed 's/^/ [local-pull] /' >&2
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -141,18 +141,11 @@ find_latest_image() {
|
||||
fi
|
||||
done
|
||||
|
||||
# Then try the local registry.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
|
||||
echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Finally, try the public registry.
|
||||
# If not found locally, fall back to pulling from public registry.
|
||||
# See amd_ci_start_container.sh for why we don't probe
|
||||
# ${LOCAL_DOCKER_REGISTRY} with `docker manifest inspect --insecure` from
|
||||
# the runner pod's network namespace; the actual local-registry pull
|
||||
# happens at the call site below via the docker daemon on the host.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
||||
@@ -210,14 +203,17 @@ find_latest_image() {
|
||||
|
||||
# Pull and run the latest image
|
||||
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
||||
echo "Pulling Docker image: ${IMAGE}"
|
||||
if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
|
||||
# Local registry is on-LAN; no need to retry.
|
||||
docker pull "${IMAGE}"
|
||||
docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
# Try the local docker registry first (avoids Docker Hub rate limits and is
|
||||
# faster on the LAN); if that fails for any reason, fall back to the
|
||||
# public registry with exponential-backoff retries. Capture stderr so the
|
||||
# real failure reason (TLS handshake, 404, connection refused, etc.) is
|
||||
# visible in the job log instead of being silently swallowed.
|
||||
if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
|
||||
echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
|
||||
docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
|
||||
else
|
||||
# Public registry pulls can hit rate limits; retry with backoff.
|
||||
echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
|
||||
printf '%s\n' "${local_pull_output}" | sed 's/^/ [local-pull] /' >&2
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
fi
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_amd_ci(est_time=1800, suite="stage-c-test-large-8-gpu-amd")
|
||||
register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd")
|
||||
|
||||
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
|
||||
|
||||
|
||||
@@ -84,7 +84,8 @@ class TestDeepseekV3MTP(CustomTestCase):
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
f"{avg_spec_accept_length=:.2f}\n"
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.935)
|
||||
# relax for mi300x
|
||||
self.assertGreaterEqual(metrics["accuracy"], 0.93)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(avg_spec_accept_length, 2.8)
|
||||
else:
|
||||
|
||||
@@ -60,7 +60,7 @@ common_args = [
|
||||
"1",
|
||||
"--enable-dp-lm-head",
|
||||
"--mem-fraction-static",
|
||||
"0.6",
|
||||
"0.72", # relax for mi300x
|
||||
"--chunked-prefill-size",
|
||||
"32768",
|
||||
"--max-running-requests",
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
@@ -73,6 +74,9 @@ class TestTorchCompileMoe(CustomTestCase):
|
||||
throughput = max_tokens / (tok - tic)
|
||||
if is_cuda():
|
||||
self.assertGreaterEqual(throughput, 285)
|
||||
elif is_in_amd_ci():
|
||||
# relax for mi300x
|
||||
self.assertGreaterEqual(throughput, 240)
|
||||
else:
|
||||
self.assertGreaterEqual(throughput, 270)
|
||||
|
||||
|
||||
@@ -142,9 +142,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
|
||||
self.assertLess(res["median_itl_ms"], 10)
|
||||
|
||||
def test_online_lora_latency(self):
|
||||
if is_in_amd_ci():
|
||||
pass
|
||||
|
||||
res = self._run_lora_latency_test(enable_background_task=False)
|
||||
|
||||
if is_in_ci():
|
||||
@@ -154,12 +151,13 @@ class TestBenchServing1GPUPart1(CustomTestCase):
|
||||
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 2400)
|
||||
self.assertLess(res["median_ttft_ms"], 58)
|
||||
# relax for mi300x (LoRA TTFT ~2x slower than mi325)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 100)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 58)
|
||||
|
||||
def test_online_lora_latency_with_concurrent_adapter_updates(self):
|
||||
if is_in_amd_ci():
|
||||
pass
|
||||
|
||||
res = self._run_lora_latency_test(enable_background_task=True)
|
||||
|
||||
if is_in_ci():
|
||||
@@ -169,7 +167,11 @@ class TestBenchServing1GPUPart1(CustomTestCase):
|
||||
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 4000)
|
||||
self.assertLess(res["median_ttft_ms"], 80)
|
||||
# relax for mi300x (LoRA TTFT ~2x slower than mi325)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 130)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 80)
|
||||
|
||||
def _run_lora_latency_test(self, enable_background_task: bool):
|
||||
"""
|
||||
|
||||
@@ -41,8 +41,9 @@ class TestBenchServing1GPUPart2(CustomTestCase):
|
||||
f"### test_vlm_offline_throughput\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 2000)
|
||||
self.assertGreater(res["output_throughput"], 900)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 2500)
|
||||
|
||||
@@ -116,12 +117,16 @@ class TestBenchServing1GPUPart2(CustomTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
bounds = {
|
||||
10: (45, 50),
|
||||
25: (50, 60),
|
||||
50: (60, 65),
|
||||
}
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (60, 65))
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
bounds = {10: (60, 65), 25: (70, 80), 50: (80, 90)}
|
||||
default_bounds = (90, 90)
|
||||
else:
|
||||
bounds = {10: (45, 50), 25: (50, 60), 50: (60, 65)}
|
||||
default_bounds = (60, 65)
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(
|
||||
batch_size, default_bounds
|
||||
)
|
||||
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
|
||||
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
|
||||
|
||||
@@ -146,9 +151,15 @@ class TestBenchServing1GPUPart2(CustomTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
self.assertLess(res["avg_latency_ms"], 20)
|
||||
self.assertLess(res["p95_latency_ms"], 25)
|
||||
self.assertGreater(res["throughput"], 60)
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["avg_latency_ms"], 35)
|
||||
self.assertLess(res["p95_latency_ms"], 40)
|
||||
self.assertGreater(res["throughput"], 30)
|
||||
else:
|
||||
self.assertLess(res["avg_latency_ms"], 20)
|
||||
self.assertLess(res["p95_latency_ms"], 25)
|
||||
self.assertGreater(res["throughput"], 60)
|
||||
|
||||
def test_embeddings_api_batch_scaling(self):
|
||||
"""Test embeddings API performance with different batch sizes"""
|
||||
@@ -173,12 +184,16 @@ class TestBenchServing1GPUPart2(CustomTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
bounds = {
|
||||
10: (60, 65),
|
||||
25: (115, 120),
|
||||
50: (190, 195),
|
||||
}
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (250, 250))
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
bounds = {10: (80, 90), 25: (140, 150), 50: (230, 240)}
|
||||
default_bounds = (300, 300)
|
||||
else:
|
||||
bounds = {10: (60, 65), 25: (115, 120), 50: (190, 195)}
|
||||
default_bounds = (250, 250)
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(
|
||||
batch_size, default_bounds
|
||||
)
|
||||
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
|
||||
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
@@ -47,6 +48,12 @@ class TestPyTorchSamplingBackend(CustomTestCase):
|
||||
metrics = run_eval(args)
|
||||
self.assertGreaterEqual(metrics["score"], 0.65)
|
||||
|
||||
@unittest.skipIf(
|
||||
is_in_amd_ci(),
|
||||
"Skip on MI300x: greedy decode is not bit-exact across runs on MI300x "
|
||||
"(kernel-level numerical jitter), so the assertEqual on identical "
|
||||
"regenerated text is flaky on this runner pool.",
|
||||
)
|
||||
def test_greedy(self):
|
||||
|
||||
first_text = None
|
||||
|
||||
@@ -10,6 +10,7 @@ from sglang.test.test_utils import (
|
||||
CustomTestCase,
|
||||
auto_config_device,
|
||||
get_benchmark_args,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
run_benchmark,
|
||||
@@ -70,7 +71,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 11000)
|
||||
self.assertLess(res["median_ttft_ms"], 86)
|
||||
# relax for mi300x
|
||||
self.assertLess(res["median_ttft_ms"], 130 if is_in_amd_ci() else 86)
|
||||
self.assertLess(res["median_itl_ms"], 10)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user