From 694ef516cb36d97ce48c0155b2be59844d6a68a5 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:57:19 -0700 Subject: [PATCH] Revert "[ci] split stage-c-test-4-gpu-b200 to enable a low-disk runner pool" (#24163) Co-authored-by: Alison Shao --- .github/workflows/pr-test.yml | 97 +++---------------- scripts/ci/utils/slash_command_handler.py | 2 - .../4-gpu-models/test_gpt_oss_4gpu.py | 2 +- .../test_nvidia_nemotron_3_super_nvfp4.py | 2 +- .../4-gpu-models/test_qwen35_fp4_mtp_v2.py | 2 +- .../4-gpu-models/test_qwen35_fp4_triton.py | 2 +- .../test_lora_gpt_oss_20b_logprob_diff.py | 2 +- ...nemotron_3_super_120b_a12b_logprob_diff.py | 2 +- ...wen3_30b_a3b_instruct_2507_logprob_diff.py | 2 +- .../test_lora_qwen3_5_35b_a3b_logprob_diff.py | 2 +- ..._qwen3_vl_30b_a3b_instruct_logprob_diff.py | 2 +- test/registered/moe/test_cutedsl_moe.py | 2 +- .../quant/test_fp8_blockwise_gemm.py | 2 +- test/registered/quant/test_nvfp4_gemm.py | 2 +- ...test_update_weights_from_disk_blackwell.py | 2 +- .../test_eagle_infer_beta_dp_attention.py | 2 +- test/run_suite.py | 1 - 17 files changed, 29 insertions(+), 99 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 2674ef83f..04333eb45 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -101,7 +101,6 @@ jobs: max_parallel_small: ${{ steps.set-parallel.outputs.max_parallel_small }} max_parallel_2gpu: ${{ steps.set-parallel.outputs.max_parallel_2gpu }} b200_runner: ${{ steps.set-runner.outputs.b200_runner }} - b200_low_disk_runner: ${{ steps.set-runner.outputs.b200_low_disk_runner }} enable_retry: ${{ steps.set-retry.outputs.enable_retry }} continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} steps: @@ -283,10 +282,8 @@ jobs: target_stage="${{ inputs.target_stage }}" if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT - echo "b200_low_disk_runner=4-gpu-b200-kernel-low-disk" >> $GITHUB_OUTPUT else echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT - echo "b200_low_disk_runner=4-gpu-b200-low-disk" >> $GITHUB_OUTPUT fi - name: Enable retry for CI @@ -337,20 +334,19 @@ jobs: { echo "## Change Detection" echo "" - echo "| Component | Changed |" - echo "|----------------------|---------|" - echo "| main_package | ${{ steps.filter-api.outputs.main_package || steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |" - echo "| sgl_kernel (raw) | ${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }} |" - echo "| sgl_kernel (used) | ${{ (!inputs.target_stage || inputs.include_wheel_build) && (steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel) }} |" - echo "| jit_kernel | ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |" - echo "| multimodal_gen | ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |" - echo "| target_stage | ${{ inputs.target_stage || '(none)' }} |" - echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |" - echo "| max_parallel | ${{ steps.set-parallel.outputs.parallel_level }} (h100=${{ steps.set-parallel.outputs.max_parallel }}, 5090=${{ steps.set-parallel.outputs.max_parallel_small }}, 2gpu=${{ steps.set-parallel.outputs.max_parallel_2gpu }}) |" - echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" - echo "| b200_low_disk_runner | ${{ steps.set-runner.outputs.b200_low_disk_runner }} |" - echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" - echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |" + echo "| Component | Changed |" + echo "|-------------------|---------|" + echo "| main_package | ${{ steps.filter-api.outputs.main_package || steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |" + echo "| sgl_kernel (raw) | ${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }} |" + echo "| sgl_kernel (used) | ${{ (!inputs.target_stage || inputs.include_wheel_build) && (steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel) }} |" + echo "| jit_kernel | ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |" + echo "| multimodal_gen | ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |" + echo "| target_stage | ${{ inputs.target_stage || '(none)' }} |" + echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |" + echo "| max_parallel | ${{ steps.set-parallel.outputs.parallel_level }} (h100=${{ steps.set-parallel.outputs.max_parallel }}, 5090=${{ steps.set-parallel.outputs.max_parallel_small }}, 2gpu=${{ steps.set-parallel.outputs.max_parallel_2gpu }}) |" + echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" + echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" + echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |" } >> $GITHUB_STEP_SUMMARY # =============================================== Wait Jobs for Sequential PR Execution ==================================================== @@ -1382,7 +1378,7 @@ jobs: strategy: fail-fast: false matrix: - part: [0, 1, 2] + part: [0, 1, 2, 3, 4, 5] steps: - name: Checkout code @@ -1413,69 +1409,7 @@ jobs: CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.part }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh - - stage-c-test-4-gpu-b200-small: - needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-4-gpu-b200-small') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - # The `*-low-disk` label (resolved by `set-runner` to `4-gpu-b200-low-disk` or - # `4-gpu-b200-kernel-low-disk`) is advertised by both the existing large-disk B200 - # runners and the new low-disk runner, so this job can land on either pool. - runs-on: ${{ needs.check-changes.outputs.b200_low_disk_runner }} - timeout-minutes: 240 - strategy: - fail-fast: false - matrix: - part: [0, 1, 2] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v6 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200-small --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps if: failure() @@ -1567,7 +1501,6 @@ jobs: stage-c-test-deepep-4-gpu-h100, stage-c-test-deepep-8-gpu-h200, stage-c-test-4-gpu-b200, - stage-c-test-4-gpu-b200-small, # stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner ] if: always() diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 2f53608ec..3c4a70cad 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -303,7 +303,6 @@ def handle_rerun_stage( "stage-c-test-8-gpu-h200", "stage-c-test-8-gpu-h20", "stage-c-test-4-gpu-b200", - "stage-c-test-4-gpu-b200-small", "stage-c-test-4-gpu-gb200", "stage-c-test-deepep-4-gpu-h100", "stage-c-test-deepep-8-gpu-h200", @@ -486,7 +485,6 @@ CUDA_SUITE_TO_RUNNER = { "stage-c-test-8-gpu-h200": "8-gpu-h200", "stage-c-test-8-gpu-h20": "8-gpu-h20", "stage-c-test-4-gpu-b200": "4-gpu-b200", - "stage-c-test-4-gpu-b200-small": "4-gpu-b200-low-disk", "stage-c-test-deepep-4-gpu-h100": "4-gpu-h100", "stage-c-test-deepep-8-gpu-h200": "8-gpu-h200", # Nightly test suites (NVIDIA) diff --git a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py b/test/registered/4-gpu-models/test_gpt_oss_4gpu.py index 58c71e72b..e2e9ac475 100644 --- a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py +++ b/test/registered/4-gpu-models/test_gpt_oss_4gpu.py @@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.gpt_oss_common import BaseTestGptOss register_cuda_ci(est_time=392, suite="stage-c-test-4-gpu-h100") -register_cuda_ci(est_time=584, suite="stage-c-test-4-gpu-b200-small") +register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200") class TestGptOss4Gpu(BaseTestGptOss): diff --git a/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py index 613369c7b..38c1395bb 100644 --- a/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=710, suite="nightly-4-gpu-b200", nightly=True) +register_cuda_ci(est_time=710, suite="stage-c-test-4-gpu-b200") NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py b/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py index 1f7034286..669ed41f4 100644 --- a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py +++ b/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=422, suite="stage-c-test-4-gpu-b200-small") +register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_triton.py b/test/registered/4-gpu-models/test_qwen35_fp4_triton.py index 96363bf69..8ed90d175 100644 --- a/test/registered/4-gpu-models/test_qwen35_fp4_triton.py +++ b/test/registered/4-gpu-models/test_qwen35_fp4_triton.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( ModelLaunchSettings, ) -register_cuda_ci(est_time=563, suite="stage-c-test-4-gpu-b200-small") +register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} diff --git a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py index 851e5463d..f6d3fab97 100644 --- a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py +++ b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=300, - suite="stage-c-test-4-gpu-b200-small", + suite="stage-c-test-4-gpu-b200", ) BASE_MODEL = "lmsys/gpt-oss-20b-bf16" diff --git a/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py b/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py index 1da641680..60f247b08 100644 --- a/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py +++ b/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=300, - suite="stage-c-test-4-gpu-b200-small", + suite="stage-c-test-4-gpu-b200", ) BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" diff --git a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py index 9d188ff9b..c9647f524 100644 --- a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=160, - suite="stage-c-test-4-gpu-b200-small", + suite="stage-c-test-4-gpu-b200", ) BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507" diff --git a/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py b/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py index f13107f2c..cd4001a43 100644 --- a/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=160, - suite="stage-c-test-4-gpu-b200-small", + suite="stage-c-test-4-gpu-b200", ) BASE_MODEL = "Qwen/Qwen3.5-35B-A3B" diff --git a/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py b/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py index b20ed4d9d..ca52832c7 100644 --- a/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py @@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase register_cuda_ci( est_time=160, - suite="stage-c-test-4-gpu-b200-small", + suite="stage-c-test-4-gpu-b200", ) BASE_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct" diff --git a/test/registered/moe/test_cutedsl_moe.py b/test/registered/moe/test_cutedsl_moe.py index d0ba9682d..fcc0ff0e2 100644 --- a/test/registered/moe/test_cutedsl_moe.py +++ b/test/registered/moe/test_cutedsl_moe.py @@ -16,7 +16,7 @@ except ImportError: CuteDslMoEWrapper = None convert_sf_to_mma_layout = None -register_cuda_ci(est_time=427, suite="stage-c-test-4-gpu-b200-small") +register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200") SKIP_TEST = torch.cuda.get_device_capability() < (10, 0) SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above." diff --git a/test/registered/quant/test_fp8_blockwise_gemm.py b/test/registered/quant/test_fp8_blockwise_gemm.py index 6c586af06..ae1446866 100644 --- a/test/registered/quant/test_fp8_blockwise_gemm.py +++ b/test/registered/quant/test_fp8_blockwise_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=630, suite="nightly-4-gpu-b200", nightly=True) +register_cuda_ci(est_time=630, suite="stage-c-test-4-gpu-b200") MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8" MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8" diff --git a/test/registered/quant/test_nvfp4_gemm.py b/test/registered/quant/test_nvfp4_gemm.py index bff9cb042..d91154c9c 100644 --- a/test/registered/quant/test_nvfp4_gemm.py +++ b/test/registered/quant/test_nvfp4_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=550, suite="nightly-4-gpu-b200", nightly=True) +register_cuda_ci(est_time=550, suite="stage-c-test-4-gpu-b200") MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4" diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index ba13c5bbd..58b896695 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -1,6 +1,6 @@ from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=400, suite="stage-c-test-4-gpu-b200-small") +register_cuda_ci(est_time=400, suite="stage-c-test-4-gpu-b200") import unittest diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index 0dcb7c5a2..b0dacf452 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( ) # EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs) -register_cuda_ci(est_time=123, suite="stage-c-test-4-gpu-b200-small") +register_cuda_ci(est_time=123, suite="stage-c-test-4-gpu-b200") def test_gsm8k(base_url: str, model: str): diff --git a/test/run_suite.py b/test/run_suite.py index d5447088a..7fe58ad91 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -48,7 +48,6 @@ PER_COMMIT_SUITES = { "stage-b-kernel-benchmark-1-gpu-large", "stage-c-test-4-gpu-h100", "stage-c-test-4-gpu-b200", - "stage-c-test-4-gpu-b200-small", "stage-c-test-4-gpu-gb200", "stage-c-test-8-gpu-h20", "stage-c-test-8-gpu-h200",