diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index a3dd3999d..482d0b440 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -6,6 +6,7 @@ run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_ on: schedule: - cron: '30 17 * * *' + - cron: '0 */6 * * *' # Every 6 hours (UTC): target dsv4 accuracy jobs only # push: # branches: [ main ] # paths: @@ -15,15 +16,15 @@ on: # - "sgl-kernel/**" # - ".github/workflows/pr-test-amd-rocm720.yml" # - "docker/rocm.Dockerfile" - # pull_request: - # branches: [ main ] - # paths: - # - "python/**" - # - "scripts/ci/**" - # - "test/**" - # - "sgl-kernel/**" - # - ".github/workflows/pr-test-amd-rocm720.yml" - # - "docker/rocm.Dockerfile" + pull_request: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-amd-rocm720.yml" + - "docker/rocm.Dockerfile" workflow_dispatch: inputs: target_stage_select: @@ -48,6 +49,8 @@ on: - stage-c-test-large-8-gpu-amd-mi35x-rocm720 - stage-b-test-large-8-gpu-mi35x-disaggregation-amd-rocm720 - stage-c-test-4-gpu-amd-rocm720 + - dsv4-flash-fp4-fp8-amd-rocm720 + - dsv4-pro-fp4-amd-rocm720 target_stage: description: "Or type comma-separated stage names (overrides dropdown if non-empty)" required: false @@ -111,6 +114,13 @@ concurrency: jobs: call-gate: + # Runs on PRs (pr-gate.yml enforces the run-ci label / blocks drafts), the daily + # '30 17 * * *' cron, workflow_dispatch and workflow_call. It is skipped only on the + # 6h ('0 */6 * * *') cron, which cascades a skip to check-changes and every existing + # test job, so that cron runs the two dsv4 jobs only. On PRs the existing jobs are + # additionally kept off via the change-detection guard in check-changes, so only the + # dsv4 jobs run there -- and only when this gate passes (they `needs` it). + if: github.event.schedule != '0 */6 * * *' uses: ./.github/workflows/pr-gate.yml secrets: inherit check-changes: @@ -133,9 +143,12 @@ jobs: run: | # Run all tests for workflow_call (when ref input is provided) # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref - if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then + # Any scheduled run that reaches check-changes is the daily '30 17 * * *' cron + # (the 6h '0 */6 * * *' cron skips call-gate -> check-changes), so the daily run + # executes the full suite (existing jobs + dsv4), like pr-test-amd.yml. + if [[ "${{ inputs.run_all_tests }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then echo "run_all_tests=true" >> $GITHUB_OUTPUT - echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})" + echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }}, event=${{ github.event_name }})" else echo "run_all_tests=false" >> $GITHUB_OUTPUT echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" @@ -145,11 +158,10 @@ jobs: id: set-continue-on-error run: | # Mirror pr-test-amd.yml: continue-on-error when any of - # - run_all_tests was requested (workflow_dispatch checkbox or workflow_call input), + # - run_all_tests was requested (workflow_dispatch checkbox, workflow_call input, + # or a scheduled run, which run-mode above rolls into run_all_tests), # - inputs.continue_on_error was explicitly set, - # - or this is a scheduled run. - # The schedule check is included explicitly here because (unlike pr-test-amd.yml's - # run-mode) the rocm720 run-mode does not roll schedule into run_all_tests. + # - or this is a scheduled run (kept as an explicit belt-and-suspenders guard). if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then echo "continue_on_error=true" >> $GITHUB_OUTPUT echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, event=${{ github.event_name }})" @@ -161,7 +173,9 @@ jobs: - name: Detect file changes id: filter uses: dorny/paths-filter@v3 - if: steps.run-mode.outputs.run_all_tests != 'true' + # On pull_request only the dsv4 jobs run; skip change detection on PRs so the + # per-stage outputs stay 'false' and the existing test jobs are gated off. + if: steps.run-mode.outputs.run_all_tests != 'true' && github.event_name != 'pull_request' with: filters: | main_package: @@ -1079,6 +1093,118 @@ jobs: -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + # =============================================== DeepSeek-V4 (MI35x, 8-GPU) ==================================================== + # On `pull_request` and the 6h ('0 */6 * * *') cron these are the ONLY jobs that run; + # they also run on the daily cron alongside the full suite. On PRs they run only when + # call-gate succeeds, i.e. the PR carries the run-ci label (and is not a draft), so + # they are gated just like every other PR test job. On PR they hard-fail (a failure + # blocks merge); on any scheduled run they pass --continue-on-error. They stay + # selectable via workflow_dispatch / run on workflow_call full runs. + dsv4-flash-fp4-fp8-amd-rocm720: + needs: [call-gate] + if: | + always() && !cancelled() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-flash-fp4-fp8-amd-rocm720,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + ( + (github.event_name == 'pull_request' && needs.call-gate.result == 'success') || + (github.event_name == 'schedule') || + inputs.run_all_tests + ) + ) + ) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP4 + FP8) + timeout-minutes: 300 + run: | + > github_summary.md # Clear summary file + # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test + # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + -e SGLANG_DSV4_ACCURACY_ONLY=1 \ + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-deepseek-v4-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + dsv4-pro-fp4-amd-rocm720: + needs: [call-gate] + if: | + always() && !cancelled() && + ( + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',dsv4-pro-fp4-amd-rocm720,')) || + ( + !(inputs.target_stage || inputs.target_stage_select) && + ( + (github.event_name == 'pull_request' && needs.call-gate.result == 'success') || + (github.event_name == 'schedule') || + inputs.run_all_tests + ) + ) + ) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP4) + timeout-minutes: 480 + run: | + > github_summary.md # Clear summary file + # SGLANG_DSV4_ACCURACY_ONLY=1 makes the dsv4 test files skip their perf test + # (test_b_perf_8k_1k); only the GSM8K accuracy test runs in this workflow. + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + -e SGLANG_DSV4_ACCURACY_ONLY=1 \ + python3 registered/amd/test_deepseek_v4_pro_fp4.py || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + if [[ "${{ github.event_name == 'schedule' || inputs.continue_on_error }}" == "true" ]]; then + exit 0 + fi + exit ${TEST_EXIT_CODE:-0} + pr-test-amd-rocm720-finish: needs: [ @@ -1101,6 +1227,9 @@ jobs: stage-c-test-4-gpu-amd-rocm720, stage-c-test-large-8-gpu-amd-rocm720, stage-c-test-large-8-gpu-amd-mi35x-rocm720, + + dsv4-flash-fp4-fp8-amd-rocm720, + dsv4-pro-fp4-amd-rocm720, ] if: always() runs-on: ubuntu-latest diff --git a/test/registered/amd/test_deepseek_v4_flash_fp4.py b/test/registered/amd/test_deepseek_v4_flash_fp4.py index a407ba870..ad885bf9f 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp4.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp4.py @@ -44,7 +44,7 @@ COMMON_ENV_VARS = { "SGLANG_USE_ROCM700A": "1", "SGLANG_OPT_USE_FUSED_COMPRESS": "true", "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton", "SGLANG_OPT_FP8_WO_A_GEMM": "false", "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", "SGLANG_OPT_USE_TOPK_V2": "false", @@ -131,6 +131,10 @@ class TestDeepseekV4Fp4(CustomTestCase): ) self.assertGreater(metrics["accuracy"], 0.91) + @unittest.skipIf( + os.environ.get("SGLANG_DSV4_ACCURACY_ONLY") == "1", + "SGLANG_DSV4_ACCURACY_ONLY=1: accuracy-only run (skipping perf)", + ) def test_b_perf_8k_1k(self): json_output = "/tmp/deepseek_v4_flash_fp4_perf.json" if os.path.exists(json_output): diff --git a/test/registered/amd/test_deepseek_v4_flash_fp8.py b/test/registered/amd/test_deepseek_v4_flash_fp8.py index 53a51bc7d..5f23bf497 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp8.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp8.py @@ -44,7 +44,7 @@ COMMON_ENV_VARS = { "SGLANG_USE_ROCM700A": "1", "SGLANG_OPT_USE_FUSED_COMPRESS": "true", "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton", "SGLANG_OPT_FP8_WO_A_GEMM": "false", "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", "SGLANG_OPT_USE_TOPK_V2": "false", @@ -131,6 +131,10 @@ class TestDeepseekV4Fp8(CustomTestCase): ) self.assertGreater(metrics["accuracy"], 0.91) + @unittest.skipIf( + os.environ.get("SGLANG_DSV4_ACCURACY_ONLY") == "1", + "SGLANG_DSV4_ACCURACY_ONLY=1: accuracy-only run (skipping perf)", + ) def test_b_perf_8k_1k(self): json_output = "/tmp/deepseek_v4_flash_fp8_perf.json" if os.path.exists(json_output): diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4.py b/test/registered/amd/test_deepseek_v4_pro_fp4.py index b1d91e66e..ad17c1c3d 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4.py @@ -46,7 +46,7 @@ COMMON_ENV_VARS = { "SGLANG_USE_ROCM700A": "1", "SGLANG_OPT_USE_FUSED_COMPRESS": "true", "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton", "SGLANG_OPT_FP8_WO_A_GEMM": "false", "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", "SGLANG_OPT_USE_TOPK_V2": "false", @@ -133,6 +133,10 @@ class TestDeepseekV4ProFp4(CustomTestCase): ) self.assertGreater(metrics["accuracy"], 0.92) + @unittest.skipIf( + os.environ.get("SGLANG_DSV4_ACCURACY_ONLY") == "1", + "SGLANG_DSV4_ACCURACY_ONLY=1: accuracy-only run (skipping perf)", + ) def test_b_perf_8k_1k(self): json_output = "/tmp/deepseek_v4_pro_fp4_perf.json" if os.path.exists(json_output): diff --git a/test/registered/amd/test_deepseek_v4_pro_fp8.py b/test/registered/amd/test_deepseek_v4_pro_fp8.py index 6ef73267d..afab21a8d 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp8.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp8.py @@ -46,7 +46,7 @@ COMMON_ENV_VARS = { "SGLANG_USE_ROCM700A": "1", "SGLANG_OPT_USE_FUSED_COMPRESS": "true", "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton", "SGLANG_OPT_FP8_WO_A_GEMM": "false", "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", "SGLANG_OPT_USE_TOPK_V2": "false", @@ -133,6 +133,10 @@ class TestDeepseekV4ProFp8(CustomTestCase): ) self.assertGreater(metrics["accuracy"], 0.91) + @unittest.skipIf( + os.environ.get("SGLANG_DSV4_ACCURACY_ONLY") == "1", + "SGLANG_DSV4_ACCURACY_ONLY=1: accuracy-only run (skipping perf)", + ) def test_b_perf_8k_1k(self): json_output = "/tmp/deepseek_v4_pro_fp8_perf.json" if os.path.exists(json_output):