diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 738aba085..bf0b0fb6d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -92,6 +92,7 @@ jobs: max_parallel_small: ${{ steps.set-parallel.outputs.max_parallel_small }} max_parallel_2gpu: ${{ steps.set-parallel.outputs.max_parallel_2gpu }} b200_runner: ${{ steps.set-runner.outputs.b200_runner }} + b200_low_disk_runner: ${{ steps.set-runner.outputs.b200_low_disk_runner }} enable_retry: ${{ steps.set-retry.outputs.enable_retry }} continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} steps: @@ -273,8 +274,10 @@ jobs: target_stage="${{ inputs.target_stage }}" if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT + echo "b200_low_disk_runner=4-gpu-b200-kernel-low-disk" >> $GITHUB_OUTPUT else echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT + echo "b200_low_disk_runner=4-gpu-b200-low-disk" >> $GITHUB_OUTPUT fi - name: Enable retry for CI @@ -330,6 +333,7 @@ jobs: echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |" echo "| max_parallel | ${{ steps.set-parallel.outputs.parallel_level }} (h100=${{ steps.set-parallel.outputs.max_parallel }}, 5090=${{ steps.set-parallel.outputs.max_parallel_small }}, 2gpu=${{ steps.set-parallel.outputs.max_parallel_2gpu }}) |" echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" + echo "| b200_low_disk_runner | ${{ steps.set-runner.outputs.b200_low_disk_runner }} |" echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |" } >> $GITHUB_STEP_SUMMARY @@ -1293,7 +1297,7 @@ jobs: strategy: fail-fast: false matrix: - part: [0, 1, 2, 3, 4, 5] + part: [0, 1, 2] steps: - name: Checkout code @@ -1324,7 +1328,69 @@ jobs: CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG + + - uses: ./.github/actions/upload-cuda-coredumps + if: failure() + with: + artifact-suffix: ${{ matrix.part }} + + - name: Cleanup venv + if: always() + run: bash scripts/ci/cuda/ci_cleanup_venv.sh + + stage-c-test-4-gpu-b200-small: + needs: [check-changes, call-gate, wait-for-stage-b] + if: | + always() && + ( + (inputs.target_stage == 'stage-c-test-4-gpu-b200-small') || + ( + !inputs.target_stage && + ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + # The `*-low-disk` label (resolved by `set-runner` to `4-gpu-b200-low-disk` or + # `4-gpu-b200-kernel-low-disk`) is advertised by both the existing large-disk B200 + # runners and the new low-disk runner, so this job can land on either pool. + runs-on: ${{ needs.check-changes.outputs.b200_low_disk_runner }} + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + part: [0, 1, 2] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + + - uses: ./.github/actions/check-stage-health + + - uses: ./.github/actions/check-maintenance + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v6 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda13.0 + + - name: Install dependencies + timeout-minutes: 20 + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + run: | + cd test + python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200-small --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps if: failure() @@ -1416,6 +1482,7 @@ jobs: stage-c-test-deepep-4-gpu-h100, stage-c-test-deepep-8-gpu-h200, stage-c-test-4-gpu-b200, + stage-c-test-4-gpu-b200-small, # stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner ] if: always() diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 66b813bb2..35e7ebae0 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -294,6 +294,7 @@ def handle_rerun_stage( "stage-c-test-8-gpu-h200", "stage-c-test-8-gpu-h20", "stage-c-test-4-gpu-b200", + "stage-c-test-4-gpu-b200-small", "stage-c-test-4-gpu-gb200", "stage-c-test-deepep-4-gpu-h100", "stage-c-test-deepep-8-gpu-h200", @@ -460,6 +461,7 @@ CUDA_SUITE_TO_RUNNER = { "stage-c-test-8-gpu-h200": "8-gpu-h200", "stage-c-test-8-gpu-h20": "8-gpu-h20", "stage-c-test-4-gpu-b200": "4-gpu-b200", + "stage-c-test-4-gpu-b200-small": "4-gpu-b200-low-disk", "stage-c-test-deepep-4-gpu-h100": "4-gpu-h100", "stage-c-test-deepep-8-gpu-h200": "8-gpu-h200", # Nightly test suites (NVIDIA) diff --git a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py b/test/registered/4-gpu-models/test_gpt_oss_4gpu.py index 86c132547..c31deb720 100644 --- a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py +++ b/test/registered/4-gpu-models/test_gpt_oss_4gpu.py @@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.gpt_oss_common import BaseTestGptOss register_cuda_ci(est_time=328, suite="stage-c-test-4-gpu-h100") -register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200-small") class TestGptOss4Gpu(BaseTestGptOss): diff --git a/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py index 38c1395bb..613369c7b 100644 --- a/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=710, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=710, suite="nightly-4-gpu-b200", nightly=True) NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py b/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py index 669ed41f4..53c8e7917 100644 --- a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py +++ b/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200-small") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_triton.py b/test/registered/4-gpu-models/test_qwen35_fp4_triton.py index 8ed90d175..29b7e5ce0 100644 --- a/test/registered/4-gpu-models/test_qwen35_fp4_triton.py +++ b/test/registered/4-gpu-models/test_qwen35_fp4_triton.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( ModelLaunchSettings, ) -register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200-small") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} diff --git a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py index f6d3fab97..851e5463d 100644 --- a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py +++ b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=300, - suite="stage-c-test-4-gpu-b200", + suite="stage-c-test-4-gpu-b200-small", ) BASE_MODEL = "lmsys/gpt-oss-20b-bf16" diff --git a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py index c9647f524..9d188ff9b 100644 --- a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=160, - suite="stage-c-test-4-gpu-b200", + suite="stage-c-test-4-gpu-b200-small", ) BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507" diff --git a/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py b/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py index ca52832c7..b20ed4d9d 100644 --- a/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=160, - suite="stage-c-test-4-gpu-b200", + suite="stage-c-test-4-gpu-b200-small", ) BASE_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct" diff --git a/test/registered/moe/test_cutedsl_moe.py b/test/registered/moe/test_cutedsl_moe.py index 25d91bc45..f64db08f0 100644 --- a/test/registered/moe/test_cutedsl_moe.py +++ b/test/registered/moe/test_cutedsl_moe.py @@ -16,7 +16,7 @@ except ImportError: CuteDslMoEWrapper = None convert_sf_to_mma_layout = None -register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200-small") SKIP_TEST = torch.cuda.get_device_capability() < (10, 0) SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above." diff --git a/test/registered/quant/test_fp8_blockwise_gemm.py b/test/registered/quant/test_fp8_blockwise_gemm.py index ae1446866..6c586af06 100644 --- a/test/registered/quant/test_fp8_blockwise_gemm.py +++ b/test/registered/quant/test_fp8_blockwise_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=630, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=630, suite="nightly-4-gpu-b200", nightly=True) MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8" MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8" diff --git a/test/registered/quant/test_nvfp4_gemm.py b/test/registered/quant/test_nvfp4_gemm.py index d91154c9c..bff9cb042 100644 --- a/test/registered/quant/test_nvfp4_gemm.py +++ b/test/registered/quant/test_nvfp4_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=550, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=550, suite="nightly-4-gpu-b200", nightly=True) MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4" diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index 58d79c375..935d8ba31 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -1,6 +1,6 @@ from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=149, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=149, suite="stage-c-test-4-gpu-b200-small") import unittest diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index 183a3c1b1..ba21aabc3 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( ) # EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs) -register_cuda_ci(est_time=136, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=136, suite="stage-c-test-4-gpu-b200-small") def test_gsm8k(base_url: str, model: str): diff --git a/test/run_suite.py b/test/run_suite.py index 7fe58ad91..d5447088a 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -48,6 +48,7 @@ PER_COMMIT_SUITES = { "stage-b-kernel-benchmark-1-gpu-large", "stage-c-test-4-gpu-h100", "stage-c-test-4-gpu-b200", + "stage-c-test-4-gpu-b200-small", "stage-c-test-4-gpu-gb200", "stage-c-test-8-gpu-h20", "stage-c-test-8-gpu-h200",