diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 6f0de0439..d64ea2fea 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -4,15 +4,8 @@ name: PR Test (AMD) run-name: ${{ (inputs.target_stage || inputs.target_stage_select) && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage || inputs.target_stage_select, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage || inputs.target_stage_select)) || '' }} on: - push: - branches: [ main ] - paths: - - "python/**" - - "scripts/ci/**" - - "test/**" - - "sgl-kernel/**" - - ".github/workflows/pr-test-amd.yml" - - "docker/rocm.Dockerfile" + schedule: + - cron: '0 */6 * * *' # Run every 6 hours (UTC) pull_request: branches: [ main ] paths: @@ -93,24 +86,26 @@ env: AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} concurrency: - # When called via workflow_call with run_all_tests=true, use a unique group per run to - # avoid collisions with direct push/PR triggers. We use run_all_tests (not github.event_name) - # to detect this, because github.event_name inherits from the caller in workflow_call. - group: pr-test-amd-${{ inputs.run_all_tests && format('full-{0}', github.run_id) || inputs.pr_head_sha || inputs.ref || github.ref }} - cancel-in-progress: ${{ !inputs.run_all_tests && github.event_name != 'workflow_call' }} + # Scheduled and run_all_tests runs get unique groups (never cancel each other). + # PR runs share a group per branch so new pushes cancel stale runs. + group: pr-test-amd-${{ (inputs.run_all_tests || github.event_name == 'schedule') && format('full-{0}', github.run_id) || inputs.pr_head_sha || inputs.ref || github.ref }} + cancel-in-progress: ${{ !inputs.run_all_tests && github.event_name != 'workflow_call' && github.event_name != 'schedule' }} jobs: call-gate: + if: github.event_name != 'schedule' uses: ./.github/workflows/pr-gate.yml secrets: inherit check-changes: needs: [call-gate] + if: always() runs-on: ubuntu-latest outputs: main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }} jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} + continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -120,16 +115,25 @@ jobs: - name: Determine run mode id: run-mode run: | - # Run all tests for workflow_call (when ref input is provided) - # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref - if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then + if [[ "${{ inputs.run_all_tests }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then echo "run_all_tests=true" >> $GITHUB_OUTPUT - echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})" + echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }}, event=${{ github.event_name }})" else echo "run_all_tests=false" >> $GITHUB_OUTPUT echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" fi + - name: Set continue-on-error for schedule/full runs + id: set-continue-on-error + run: | + if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.continue_on_error }}" == "true" ]]; then + echo "continue_on_error=true" >> $GITHUB_OUTPUT + echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }})" + else + echo "continue_on_error=false" >> $GITHUB_OUTPUT + echo "Continue-on-error: DISABLED" + fi + - name: Detect file changes id: filter uses: dorny/paths-filter@v3 @@ -287,7 +291,7 @@ jobs: - name: Run test timeout-minutes: 10 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} jit-kernel-unit-test-amd: needs: [check-changes] @@ -328,15 +332,41 @@ jobs: run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout" python3 -m pytest -q python/sglang/jit_kernel/tests/test_store_cache.py + # =============================================== Wait Jobs for Sequential PR Execution ==================================================== + # These jobs poll GitHub API to wait for previous stages to complete. + # For PR runs: wait jobs run and enforce sequential execution via polling. + # For scheduled runs: wait jobs are skipped, enabling parallel execution of all stages. + + wait-for-stage-a-amd: + needs: [check-changes, call-gate] + if: | + always() && + !cancelled() && + github.event_name == 'pull_request' && + !(inputs.target_stage || inputs.target_stage_select) && + (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') && + (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') + runs-on: ubuntu-latest + outputs: + stage_a_result: ${{ steps.wait.outputs.result }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/wait-for-jobs + id: wait + with: + stage-name: stage-a-amd + jobs: '[{"prefix": "stage-a-test-1-gpu-small-amd", "expected_count": 1}]' + max-wait-minutes: '240' + stage-b-test-1-gpu-small-amd: - needs: [check-changes, stage-a-test-1-gpu-small-amd] + needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -366,17 +396,17 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-nondeterministic: - needs: [check-changes, stage-a-test-1-gpu-small-amd] + needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-nondeterministic,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -405,17 +435,17 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-small-amd-mi35x: - needs: [check-changes, stage-a-test-1-gpu-small-amd] + needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-small-amd-mi35x,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -444,17 +474,17 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-1-gpu-large-amd: - needs: [check-changes, stage-a-test-1-gpu-small-amd] + needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-1-gpu-large-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -484,17 +514,17 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-b-test-2-gpu-large-amd: - needs: [check-changes, stage-a-test-1-gpu-small-amd] + needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-2-gpu-large-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -524,7 +554,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} multimodal-gen-test-1-gpu-amd: needs: [check-changes] @@ -783,15 +813,41 @@ jobs: free -h + wait-for-stage-b-amd: + needs: [check-changes, call-gate, wait-for-stage-a-amd] + if: | + always() && + !cancelled() && + github.event_name == 'pull_request' && + !(inputs.target_stage || inputs.target_stage_select) && + (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') && + (needs.wait-for-stage-a-amd.result == 'success' || needs.wait-for-stage-a-amd.result == 'skipped') && + (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') + runs-on: ubuntu-latest + outputs: + stage_b_result: ${{ steps.wait.outputs.result }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/wait-for-jobs + id: wait + with: + stage-name: stage-b-amd + jobs: | + [ + {"prefix": "stage-b-test-1-gpu-small-amd", "expected_count": 14}, + {"prefix": "stage-b-test-2-gpu-large-amd", "expected_count": 2} + ] + max-wait-minutes: '480' + stage-c-test-4-gpu-amd: - needs: [check-changes, call-gate, stage-b-test-1-gpu-small-amd, stage-b-test-2-gpu-large-amd] + needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-4-gpu-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -837,17 +893,17 @@ jobs: --max-attempts 2 \ --retry-wait-seconds 120 \ --retry-timeout-increase 0 \ - ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd: - needs: [check-changes, call-gate, stage-b-test-1-gpu-small-amd, stage-b-test-2-gpu-large-amd] + needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-large-8-gpu-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -885,17 +941,17 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x: - needs: [check-changes, call-gate, stage-b-test-1-gpu-small-amd, stage-b-test-2-gpu-large-amd] + needs: [check-changes, call-gate, wait-for-stage-b-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-c-test-large-8-gpu-amd-mi35x,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -925,18 +981,18 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-35x-disaggregation-amd: - needs: [check-changes, stage-a-test-1-gpu-small-amd] + needs: [check-changes, wait-for-stage-a-amd] if: | always() && ( (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-b-test-large-8-gpu-disaggregation-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && - (!failure() && !cancelled()) && + ((github.event_name == 'schedule') || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) @@ -1037,7 +1093,7 @@ jobs: run: | bash scripts/ci/amd/amd_ci_exec.sh \ -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ - -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} pr-test-amd-finish: needs: @@ -1050,8 +1106,10 @@ jobs: multimodal-gen-test-1-gpu-amd, multimodal-gen-test-2-gpu-amd, + wait-for-stage-a-amd, stage-a-test-1-gpu-small-amd, jit-kernel-unit-test-amd, + wait-for-stage-b-amd, stage-b-test-1-gpu-small-amd, stage-b-test-1-gpu-small-amd-nondeterministic, stage-b-test-1-gpu-small-amd-mi35x,