diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index fb8888c15..1033b5198 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -159,107 +159,32 @@ jobs: rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} # No `secrets: inherit`: this hosted CPU job has no secret consumer. - # =============================================== extra-a (1-/2-gpu) =============================================== - extra-a-test-1-gpu-small: + # =============================================== extra-a / extra-b =============================================== + # `name:` keeps the job names wait-for-jobs and check-pr-test-health match on. + extra-test: + name: ${{ matrix.stage }}-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} + strategy: + fail-fast: false + matrix: + include: + - { stage: extra-a, runner_config: 1-gpu-small, timeout_per_file: '' } + - { stage: extra-a, runner_config: 1-gpu-large, timeout_per_file: '1800' } + - { stage: extra-a, runner_config: 2-gpu-large, timeout_per_file: '' } + - { stage: extra-b, runner_config: 4-gpu-h100, timeout_per_file: '3600' } + - { stage: extra-b, runner_config: 4-gpu-b200, timeout_per_file: '1800' } + - { stage: extra-b, runner_config: 8-gpu-h200, timeout_per_file: '' } + - { stage: extra-b, runner_config: 8-gpu-b300, timeout_per_file: '3600' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: extra-a-test-1-gpu-small - runner_config: 1-gpu-small + self_name: ${{ matrix.stage }}-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-a-test-1-gpu-large: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-a-test-1-gpu-large - runner_config: 1-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-a-test-2-gpu-large: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-a-test-2-gpu-large - runner_config: 2-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - # =============================================== extra-b (4-/8-gpu) =============================================== - extra-b-test-4-gpu-h100: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-4-gpu-h100 - runner_config: 4-gpu-h100 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '3600' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-b-test-4-gpu-b200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-4-gpu-b200 - runner_config: 4-gpu-b200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-b-test-8-gpu-h200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-8-gpu-h200 - runner_config: 8-gpu-h200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-b-test-8-gpu-b300: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-8-gpu-b300 - runner_config: 8-gpu-b300 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '3600' + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit @@ -275,13 +200,7 @@ jobs: sgl-kernel-build-wheels, rust-ext-build, simulator-test-cpu, - extra-a-test-1-gpu-small, - extra-a-test-1-gpu-large, - extra-a-test-2-gpu-large, - extra-b-test-4-gpu-h100, - extra-b-test-4-gpu-b200, - extra-b-test-8-gpu-h200, - extra-b-test-8-gpu-b300, + extra-test, ] if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/pr-test-jit-kernel.yml b/.github/workflows/pr-test-jit-kernel.yml index 9a5b17182..8f123dea1 100644 --- a/.github/workflows/pr-test-jit-kernel.yml +++ b/.github/workflows/pr-test-jit-kernel.yml @@ -13,9 +13,6 @@ on: description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, falls back to the cache; a miss there compiles during install.' type: string default: '' - runner_config: - required: true - type: string runs_on_map: required: true type: string @@ -43,15 +40,36 @@ env: SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }} jobs: - jit-kernel-unit-test: + # `name:` reproduces the job names this workflow published before the table. + jit-kernel-test: + name: ${{ matrix.job_name }} # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. strategy: fail-fast: false matrix: - partition: [0, 1] - runs-on: 1-gpu-h100 + include: + - job_name: jit-kernel-unit-test (0) + runner_config: 1-gpu-large + suite: base-b-kernel-unit-test-1-gpu-large + suite_args: --auto-partition-id 0 --auto-partition-size 2 --fork-worker-batch-size 20 + - job_name: jit-kernel-unit-test (1) + runner_config: 1-gpu-large + suite: base-b-kernel-unit-test-1-gpu-large + suite_args: --auto-partition-id 1 --auto-partition-size 2 --fork-worker-batch-size 20 + - job_name: jit-kernel-multigpu-unit-test + runner_config: 8-gpu-h200 + suite: base-b-kernel-unit-test-8-gpu-h200 + # Alone among these, it has never run the health check. + skip_health_check: true + - job_name: jit-kernel-benchmark-test + runner_config: 1-gpu-large + suite: base-b-kernel-benchmark-test-1-gpu-large + - job_name: jit-kernel-b200-test + runner_config: 4-gpu-b200 + suite: base-b-kernel-unit-test-4-gpu-b200 + runs-on: ${{ fromJson(inputs.runs_on_map)[matrix.runner_config] }} timeout-minutes: 240 steps: - uses: actions/checkout@v4 @@ -59,6 +77,7 @@ jobs: ref: ${{ inputs.git_ref || github.sha }} - uses: ./.github/actions/check-pr-test-health + if: ${{ !matrix.skip_health_check }} - uses: ./.github/actions/check-maintenance @@ -89,138 +108,4 @@ jobs: timeout-minutes: 60 run: | cd test/ - python3 run_suite.py --hw cuda \ - --suite base-b-kernel-unit-test-1-gpu-large \ - --auto-partition-id ${{ matrix.partition }} \ - --auto-partition-size 2 \ - --fork-worker-batch-size 20 - - jit-kernel-multigpu-unit-test: - # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the - # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so - # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. - runs-on: 8-gpu-h200 - timeout-minutes: 240 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-maintenance - - - name: Cleanup - if: inputs.sgl_kernel == 'true' - run: | - ls -alh python/sglang/kernels/aot/dist || true - rm -rf python/sglang/kernels/aot/dist/* || true - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda13.0 - - - uses: ./.github/actions/download-rust-ext - with: - artifact_name: ${{ inputs.rust_ext_artifact }} - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run multi-GPU test - timeout-minutes: 45 - run: | - cd test/ - python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-8-gpu-h200 - - jit-kernel-benchmark-test: - # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the - # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so - # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. - runs-on: 1-gpu-h100 - timeout-minutes: 240 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-pr-test-health - - - uses: ./.github/actions/check-maintenance - - - name: Cleanup - if: inputs.sgl_kernel == 'true' - run: | - ls -alh python/sglang/kernels/aot/dist || true - rm -rf python/sglang/kernels/aot/dist/* || true - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda13.0 - - - uses: ./.github/actions/download-rust-ext - with: - artifact_name: ${{ inputs.rust_ext_artifact }} - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run benchmark tests - timeout-minutes: 45 - run: | - cd test/ - python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large - - jit-kernel-b200-test: - # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the - # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so - # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. - runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }} - timeout-minutes: 240 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-pr-test-health - - - uses: ./.github/actions/check-maintenance - - - name: Cleanup - if: inputs.sgl_kernel == 'true' - run: | - ls -alh python/sglang/kernels/aot/dist || true - rm -rf python/sglang/kernels/aot/dist/* || true - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda13.0 - - - uses: ./.github/actions/download-rust-ext - with: - artifact_name: ${{ inputs.rust_ext_artifact }} - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run B200 diffusion test - timeout-minutes: 30 - run: | - cd test/ - python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-4-gpu-b200 + python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} ${{ matrix.suite_args }} diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 1b8ba5f38..fc81cc3aa 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -290,7 +290,6 @@ jobs: needs.check-changes.outputs.jit_kernel == 'true' uses: ./.github/workflows/pr-test-jit-kernel.yml with: - runner_config: 4-gpu-b200 runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }} jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }} # On scheduled/parallel-dispatch runs sgl-kernel-build-wheels is skipped, so the wheel @@ -305,18 +304,27 @@ jobs: # =============================================== primary ==================================================== - # Runs on 5090 (32GB, SM120) - base-a-test-1-gpu-small: + # `name:` is load-bearing: wait-for-jobs gates a stage by job-name prefix, + # and check-pr-test-health carves h20 out by the same name. + base-a-test: + name: base-a-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + # 5090 (32GB, SM120) + - { runner_config: 1-gpu-small, run_timeout: '10', timeout_per_file: '' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-a-test-1-gpu-small - runner_config: 1-gpu-small + self_name: base-a-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '10' + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit @@ -337,62 +345,28 @@ jobs: # No `secrets: inherit`: this stage has no secret consumer, unlike the GPU # stages' coredump upload. GITHUB_TOKEN and permissions inherit regardless. - # Runs on 5090 (32GB, SM120) - base-b-test-1-gpu-small: + base-b-test: + name: base-b-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + # 5090 (32GB, SM120) + - { runner_config: 1-gpu-small, run_timeout: '30', timeout_per_file: '' } + - { runner_config: 1-gpu-large, run_timeout: '60', timeout_per_file: '1800' } + - { runner_config: 2-gpu-large, run_timeout: '30', timeout_per_file: '' } + - { runner_config: 4-gpu-b200, run_timeout: '40', timeout_per_file: '' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-b-test-1-gpu-small - runner_config: 1-gpu-small + self_name: base-b-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.) - base-b-test-1-gpu-large: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-b-test-1-gpu-large - runner_config: 1-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-b-test-2-gpu-large: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-b-test-2-gpu-large - runner_config: 2-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-b-test-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-b-test-4-gpu-b200 - runner_config: 4-gpu-b200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '40' + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit @@ -416,104 +390,70 @@ jobs: skip_pr_test_health_check: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }} secrets: inherit - base-c-test-4-gpu-h100: + base-c-test: + name: base-c-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + - { + runner_config: 4-gpu-h100, + run_timeout: '30', + timeout_per_file: '', + warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4', + warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4', + } + - { + runner_config: 8-gpu-h200, + run_timeout: '30', + timeout_per_file: '1800', + warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8', + warmup_timeout: '60', + } + - { runner_config: 8-gpu-h20, run_timeout: '30', timeout_per_file: '' } + - { runner_config: 4-gpu-b200, run_timeout: '30', timeout_per_file: '1800' } + - { runner_config: 8-gpu-b300, run_timeout: '60', timeout_per_file: '3600' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-c-test-4-gpu-h100 - runner_config: 4-gpu-h100 + self_name: base-c-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4' - warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4' + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} + warmup_deep_gemm_models: ${{ matrix.warmup_deep_gemm_models }} + warmup_server_models: ${{ matrix.warmup_server_models }} + # Rows that do not set it keep _pr-test-stage.yml's own default. + warmup_timeout_minutes: ${{ matrix.warmup_timeout || '25' }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit - base-c-test-8-gpu-h200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-8-gpu-h200 - runner_config: 8-gpu-h200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - # Per-model TP must match the test's launch in test/registered/ -- see - # FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep - # flags. Only models that actually invoke DeepGEMM kernels at runtime - # are listed. Cold-cache ~13 min; warm-cache <=30 s via marker file. - # Server CUDA Graph warmup is combined into this step (warmup_server_models unset). - warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8' - warmup_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-c-test-8-gpu-h20: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-8-gpu-h20 - runner_config: 8-gpu-h20 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-c-test-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-4-gpu-b200 - runner_config: 4-gpu-b200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-c-test-4-gpu-gb300: + # Separate only because `needs` cannot vary per matrix row: this stage takes + # the aarch64 build, and neither table should wait on the other's. + base-c-test-aarch64: + name: base-c-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + - { runner_config: 4-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-c-test-4-gpu-gb300 - runner_config: 4-gpu-gb300 + self_name: base-c-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - # The one aarch64 stage, so it takes the aarch64 build, not rust-ext-build's. + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build-aarch64.outputs.artifact_name }} secrets: inherit - base-c-test-8-gpu-b300: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-8-gpu-b300 - runner_config: 8-gpu-b300 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '3600' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - # List every build and test job: `skipped` passes here, so an omission turns that # job's failure into a green run with no tests. pr-test-finish: @@ -536,18 +476,11 @@ jobs: call-multimodal-gen-tests, - base-a-test-1-gpu-small, + base-a-test, base-a-test-cpu, - base-b-test-1-gpu-small, - base-b-test-1-gpu-large, - base-b-test-2-gpu-large, - base-b-test-4-gpu-b200, - base-c-test-4-gpu-h100, - base-c-test-8-gpu-h20, - base-c-test-8-gpu-h200, - base-c-test-4-gpu-b200, - base-c-test-4-gpu-gb300, - base-c-test-8-gpu-b300, + base-b-test, + base-c-test, + base-c-test-aarch64, ] if: always() runs-on: ubuntu-latest