|
|
|
@@ -6,6 +6,7 @@ run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_
|
|
|
|
|
on:
|
|
|
|
|
schedule:
|
|
|
|
|
- cron: '30 17 * * *'
|
|
|
|
|
- cron: '0 */6 * * *' # Every 6 hours (UTC): target dsv4 accuracy jobs only
|
|
|
|
|
# push:
|
|
|
|
|
# branches: [ main ]
|
|
|
|
|
# paths:
|
|
|
|
@@ -15,15 +16,15 @@ on:
|
|
|
|
|
# - "sgl-kernel/**"
|
|
|
|
|
# - ".github/workflows/pr-test-amd-rocm720.yml"
|
|
|
|
|
# - "docker/rocm.Dockerfile"
|
|
|
|
|
# pull_request:
|
|
|
|
|
# branches: [ main ]
|
|
|
|
|
# paths:
|
|
|
|
|
# - "python/**"
|
|
|
|
|
# - "scripts/ci/**"
|
|
|
|
|
# - "test/**"
|
|
|
|
|
# - "sgl-kernel/**"
|
|
|
|
|
# - ".github/workflows/pr-test-amd-rocm720.yml"
|
|
|
|
|
# - "docker/rocm.Dockerfile"
|
|
|
|
|
pull_request:
|
|
|
|
|
branches: [ main ]
|
|
|
|
|
paths:
|
|
|
|
|
- "python/**"
|
|
|
|
|
- "scripts/ci/**"
|
|
|
|
|
- "test/**"
|
|
|
|
|
- "sgl-kernel/**"
|
|
|
|
|
- ".github/workflows/pr-test-amd-rocm720.yml"
|
|
|
|
|
- "docker/rocm.Dockerfile"
|
|
|
|
|
workflow_dispatch:
|
|
|
|
|
inputs:
|
|
|
|
|
target_stage_select:
|
|
|
|
@@ -48,6 +49,8 @@ on:
|
|
|
|
|
- stage-c-test-large-8-gpu-amd-mi35x-rocm720
|
|
|
|
|
- stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720
|
|
|
|
|
- stage-c-test-4-gpu-amd-rocm720
|
|
|
|
|
- dsv4-flash-fp4-fp8-amd-rocm720
|
|
|
|
|
- dsv4-pro-fp4-amd-rocm720
|
|
|
|
|
target_stage:
|
|
|
|
|
description: "Or type comma-separated stage names (overrides dropdown if non-empty)"
|
|
|
|
|
required: false
|
|
|
|
@@ -111,6 +114,13 @@ concurrency:
|
|
|
|
|
|
|
|
|
|
jobs:
|
|
|
|
|
call-gate:
|
|
|
|
|
# Runs on PRs (pr-gate.yml enforces the run-ci label / blocks drafts), the daily
|
|
|
|
|
# '30 17 * * *' cron, workflow_dispatch and workflow_call. It is skipped only on the
|
|
|
|
|
# 6h ('0 */6 * * *') cron, which cascades a skip to check-changes and every existing
|
|
|
|
|
# test job, so that cron runs the two dsv4 jobs only. On PRs the existing jobs are
|
|
|
|
|
# additionally kept off via the change-detection guard in check-changes, so only the
|
|
|
|
|
# dsv4 jobs run there -- and only when this gate passes (they `needs` it).
|
|
|
|
|
if: github.event.schedule != '0 */6 * * *'
|
|
|
|
|
uses: ./.github/workflows/pr-gate.yml
|
|
|
|
|
secrets: inherit
|
|
|
|
|
check-changes:
|
|
|
|
@@ -133,9 +143,12 @@ jobs:
|
|
|
|
|
run: |
|
|
|
|
|
# Run all tests for workflow_call (when ref input is provided)
|
|
|
|
|
# Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
|
|
|
|
|
if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
|
|
|
|
|
# Any scheduled run that reaches check-changes is the daily '30 17 * * *' cron
|
|
|
|
|
# (the 6h '0 */6 * * *' cron skips call-gate -> check-changes), so the daily run
|
|
|
|
|
# executes the full suite (existing jobs + dsv4), like pr-test-amd.yml.
|
|
|
|
|
if [[ "${{ inputs.run_all_tests }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then
|
|
|
|
|
echo "run_all_tests=true" >> $GITHUB_OUTPUT
|
|
|
|
|
echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})"
|
|
|
|
|
echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }}, event=${{ github.event_name }})"
|
|
|
|
|
else
|
|
|
|
|
echo "run_all_tests=false" >> $GITHUB_OUTPUT
|
|
|
|
|
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
|
|
|
|
@@ -145,11 +158,10 @@ jobs:
|
|
|
|
|
id: set-continue-on-error
|
|
|
|
|
run: |
|
|
|
|
|
# Mirror pr-test-amd.yml: continue-on-error when any of
|
|
|
|
|
# - run_all_tests was requested (workflow_dispatch checkbox or workflow_call input),
|
|
|
|
|
# - run_all_tests was requested (workflow_dispatch checkbox, workflow_call input,
|
|
|
|
|
# or a scheduled run, which run-mode above rolls into run_all_tests),
|
|
|
|
|
# - inputs.continue_on_error was explicitly set,
|
|
|
|
|
# - or this is a scheduled run.
|
|
|
|
|
# The schedule check is included explicitly here because (unlike pr-test-amd.yml's
|
|
|
|
|
# run-mode) the rocm720 run-mode does not roll schedule into run_all_tests.
|
|
|
|
|
# - or this is a scheduled run (kept as an explicit belt-and-suspenders guard).
|
|
|
|
|
if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then
|
|
|
|
|
echo "continue_on_error=true" >> $GITHUB_OUTPUT
|
|
|
|
|
echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, event=${{ github.event_name }})"
|
|
|
|
@@ -161,7 +173,9 @@ jobs:
|
|
|
|
|
- name: Detect file changes
|
|
|
|
|
id: filter
|
|
|
|
|
uses: dorny/paths-filter@v3
|
|
|
|
|
if: steps.run-mode.outputs.run_all_tests != 'true'
|
|
|
|
|
# On pull_request only the dsv4 jobs run; skip change detection on PRs so the
|
|
|
|
|
# per-stage outputs stay 'false' and the existing test jobs are gated off.
|
|
|
|
|
if: steps.run-mode.outputs.run_all_tests != 'true' && github.event_name != 'pull_request'
|
|
|
|
|
with:
|
|
|
|
|
filters: |
|
|
|
|
|
main_package:
|
|
|
|
@@ -1079,6 +1093,118 @@ jobs:
|
|
|
|
|
-e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \
|
|
|
|
|
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
|
|
|
|
|
|
|
|
|
# =============================================== DeepSeek-V4 (MI35x, 8-GPU) ====================================================
|
|
|
|
|
# On `pull_request` and the 6h ('0 */6 * * *') cron these are the ONLY jobs that run;
|
|
|
|
|
# they also run on the daily cron alongside the full suite. On PRs they run only when
|
|
|
|
|
# call-gate succeeds, i.e. the PR carries the run-ci label (and is not a draft), so
|
|
|
|
|
# they are gated just like every other PR test job. On PR they hard-fail (a failure
|
|
|
|
|
# blocks merge); on any scheduled run they pass --continue-on-error. They stay
|
|
|
|
|
# selectable via workflow_dispatch / run on workflow_call full runs.
|
|
|
|
|
dsv4-flash-fp4-fp8-amd-rocm720:
|
|
|
|
|
needs: [call-gate]
|
|
|
|
|
if: |
|
|
|
|
|
always() && !cancelled() &&
|
|
|
|
|
(
|
|
|
|
|
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-flash-fp4-fp8-amd-rocm720,')) ||
|
|
|
|
|
(
|
|
|
|
|
!(inputs.target_stage || inputs.target_stage_select) &&
|
|
|
|
|
(
|
|
|
|
|
(github.event_name == 'pull_request' && needs.call-gate.result == 'success') ||
|
|
|
|
|
(github.event_name == 'schedule') ||
|
|
|
|
|
inputs.run_all_tests
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
runs-on: linux-mi35x-gpu-8
|
|
|
|
|
steps:
|
|
|
|
|
- name: Checkout code
|
|
|
|
|
uses: actions/checkout@v4
|
|
|
|
|
with:
|
|
|
|
|
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
|
|
|
|
|
|
|
|
|
- name: Ensure VRAM is clear
|
|
|
|
|
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
|
|
|
|
|
|
|
|
|
- name: Setup docker (ROCm 7.2)
|
|
|
|
|
run: |
|
|
|
|
|
touch github_summary.md
|
|
|
|
|
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
|
|
|
|
|
env:
|
|
|
|
|
GITHUB_WORKSPACE: ${{ github.workspace }}
|
|
|
|
|
|
|
|
|
|
- name: Install dependencies
|
|
|
|
|
run: |
|
|
|
|
|
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
|
|
|
|
|
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
|
|
|
|
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
|
|
|
|
|
|
|
|
|
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP4 + FP8)
|
|
|
|
|
timeout-minutes: 300
|
|
|
|
|
run: |
|
|
|
|
|
> github_summary.md # Clear summary file
|
|
|
|
|
# SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test
|
|
|
|
|
# (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow.
|
|
|
|
|
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
|
|
|
|
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
|
|
|
|
-e SGLANG_DSV4_ACCURACY_ONLY=1 \
|
|
|
|
|
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
|
|
|
|
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
|
|
|
|
exit ${TEST_EXIT_CODE:-0}
|
|
|
|
|
|
|
|
|
|
dsv4-pro-fp4-amd-rocm720:
|
|
|
|
|
needs: [call-gate]
|
|
|
|
|
if: |
|
|
|
|
|
always() && !cancelled() &&
|
|
|
|
|
(
|
|
|
|
|
(contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-pro-fp4-amd-rocm720,')) ||
|
|
|
|
|
(
|
|
|
|
|
!(inputs.target_stage || inputs.target_stage_select) &&
|
|
|
|
|
(
|
|
|
|
|
(github.event_name == 'pull_request' && needs.call-gate.result == 'success') ||
|
|
|
|
|
(github.event_name == 'schedule') ||
|
|
|
|
|
inputs.run_all_tests
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
runs-on: linux-mi35x-gpu-8
|
|
|
|
|
steps:
|
|
|
|
|
- name: Checkout code
|
|
|
|
|
uses: actions/checkout@v4
|
|
|
|
|
with:
|
|
|
|
|
ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }}
|
|
|
|
|
|
|
|
|
|
- name: Ensure VRAM is clear
|
|
|
|
|
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
|
|
|
|
|
|
|
|
|
|
- name: Setup docker (ROCm 7.2)
|
|
|
|
|
run: |
|
|
|
|
|
touch github_summary.md
|
|
|
|
|
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
|
|
|
|
|
env:
|
|
|
|
|
GITHUB_WORKSPACE: ${{ github.workspace }}
|
|
|
|
|
|
|
|
|
|
- name: Install dependencies
|
|
|
|
|
run: |
|
|
|
|
|
# --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval.
|
|
|
|
|
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
|
|
|
|
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
|
|
|
|
|
|
|
|
|
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP4)
|
|
|
|
|
timeout-minutes: 480
|
|
|
|
|
run: |
|
|
|
|
|
> github_summary.md # Clear summary file
|
|
|
|
|
# SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test
|
|
|
|
|
# (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow.
|
|
|
|
|
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
|
|
|
|
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
|
|
|
|
-e SGLANG_DSV4_ACCURACY_ONLY=1 \
|
|
|
|
|
python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$?
|
|
|
|
|
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
|
|
|
|
if [[ "${{ github.event_name == 'schedule' || inputs.continue_on_error }}" == "true" ]]; then
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
|
|
|
|
exit ${TEST_EXIT_CODE:-0}
|
|
|
|
|
|
|
|
|
|
pr-test-amd-rocm720-finish:
|
|
|
|
|
needs:
|
|
|
|
|
[
|
|
|
|
@@ -1101,6 +1227,9 @@ jobs:
|
|
|
|
|
stage-c-test-4-gpu-amd-rocm720,
|
|
|
|
|
stage-c-test-large-8-gpu-amd-rocm720,
|
|
|
|
|
stage-c-test-large-8-gpu-amd-mi35x-rocm720,
|
|
|
|
|
|
|
|
|
|
dsv4-flash-fp4-fp8-amd-rocm720,
|
|
|
|
|
dsv4-pro-fp4-amd-rocm720,
|
|
|
|
|
]
|
|
|
|
|
if: always()
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
|
Bingxu Chen