[ci] split stage-c-test-4-gpu-b200 to enable a low-disk runner pool (#23417)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
6cf0b004ca
commit
77fd86f89e
@@ -92,6 +92,7 @@ jobs:
|
||||
max_parallel_small: ${{ steps.set-parallel.outputs.max_parallel_small }}
|
||||
max_parallel_2gpu: ${{ steps.set-parallel.outputs.max_parallel_2gpu }}
|
||||
b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
|
||||
b200_low_disk_runner: ${{ steps.set-runner.outputs.b200_low_disk_runner }}
|
||||
enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
|
||||
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
|
||||
steps:
|
||||
@@ -273,8 +274,10 @@ jobs:
|
||||
target_stage="${{ inputs.target_stage }}"
|
||||
if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then
|
||||
echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT
|
||||
echo "b200_low_disk_runner=4-gpu-b200-kernel-low-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT
|
||||
echo "b200_low_disk_runner=4-gpu-b200-low-disk" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Enable retry for CI
|
||||
@@ -330,6 +333,7 @@ jobs:
|
||||
echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |"
|
||||
echo "| max_parallel | ${{ steps.set-parallel.outputs.parallel_level }} (h100=${{ steps.set-parallel.outputs.max_parallel }}, 5090=${{ steps.set-parallel.outputs.max_parallel_small }}, 2gpu=${{ steps.set-parallel.outputs.max_parallel_2gpu }}) |"
|
||||
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
|
||||
echo "| b200_low_disk_runner | ${{ steps.set-runner.outputs.b200_low_disk_runner }} |"
|
||||
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
|
||||
echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
|
||||
} >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1293,7 +1297,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [0, 1, 2, 3, 4, 5]
|
||||
part: [0, 1, 2]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -1324,7 +1328,69 @@ jobs:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
- name: Cleanup venv
|
||||
if: always()
|
||||
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
||||
|
||||
stage-c-test-4-gpu-b200-small:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
(inputs.target_stage == 'stage-c-test-4-gpu-b200-small') ||
|
||||
(
|
||||
!inputs.target_stage &&
|
||||
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
# The `*-low-disk` label (resolved by `set-runner` to `4-gpu-b200-low-disk` or
|
||||
# `4-gpu-b200-kernel-low-disk`) is advertised by both the existing large-disk B200
|
||||
# runners and the new low-disk runner, so this job can land on either pool.
|
||||
runs-on: ${{ needs.check-changes.outputs.b200_low_disk_runner }}
|
||||
timeout-minutes: 240
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [0, 1, 2]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Download artifacts
|
||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
path: sgl-kernel/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda13.0
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200-small --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
@@ -1416,6 +1482,7 @@ jobs:
|
||||
stage-c-test-deepep-4-gpu-h100,
|
||||
stage-c-test-deepep-8-gpu-h200,
|
||||
stage-c-test-4-gpu-b200,
|
||||
stage-c-test-4-gpu-b200-small,
|
||||
# stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
||||
]
|
||||
if: always()
|
||||
|
||||
@@ -294,6 +294,7 @@ def handle_rerun_stage(
|
||||
"stage-c-test-8-gpu-h200",
|
||||
"stage-c-test-8-gpu-h20",
|
||||
"stage-c-test-4-gpu-b200",
|
||||
"stage-c-test-4-gpu-b200-small",
|
||||
"stage-c-test-4-gpu-gb200",
|
||||
"stage-c-test-deepep-4-gpu-h100",
|
||||
"stage-c-test-deepep-8-gpu-h200",
|
||||
@@ -460,6 +461,7 @@ CUDA_SUITE_TO_RUNNER = {
|
||||
"stage-c-test-8-gpu-h200": "8-gpu-h200",
|
||||
"stage-c-test-8-gpu-h20": "8-gpu-h20",
|
||||
"stage-c-test-4-gpu-b200": "4-gpu-b200",
|
||||
"stage-c-test-4-gpu-b200-small": "4-gpu-b200-low-disk",
|
||||
"stage-c-test-deepep-4-gpu-h100": "4-gpu-h100",
|
||||
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
|
||||
# Nightly test suites (NVIDIA)
|
||||
|
||||
@@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.gpt_oss_common import BaseTestGptOss
|
||||
|
||||
register_cuda_ci(est_time=328, suite="stage-c-test-4-gpu-h100")
|
||||
register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200-small")
|
||||
|
||||
|
||||
class TestGptOss4Gpu(BaseTestGptOss):
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=710, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=710, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200-small")
|
||||
|
||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
|
||||
ModelLaunchSettings,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200-small")
|
||||
|
||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||
|
||||
@@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
suite="stage-c-test-4-gpu-b200",
|
||||
suite="stage-c-test-4-gpu-b200-small",
|
||||
)
|
||||
|
||||
BASE_MODEL = "lmsys/gpt-oss-20b-bf16"
|
||||
|
||||
@@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=160,
|
||||
suite="stage-c-test-4-gpu-b200",
|
||||
suite="stage-c-test-4-gpu-b200-small",
|
||||
)
|
||||
|
||||
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
|
||||
|
||||
@@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=160,
|
||||
suite="stage-c-test-4-gpu-b200",
|
||||
suite="stage-c-test-4-gpu-b200-small",
|
||||
)
|
||||
|
||||
BASE_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||
|
||||
@@ -16,7 +16,7 @@ except ImportError:
|
||||
CuteDslMoEWrapper = None
|
||||
convert_sf_to_mma_layout = None
|
||||
|
||||
register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200-small")
|
||||
|
||||
SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
|
||||
SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above."
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=630, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=630, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
|
||||
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=550, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=550, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=149, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=149, suite="stage-c-test-4-gpu-b200-small")
|
||||
|
||||
import unittest
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
# EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs)
|
||||
register_cuda_ci(est_time=136, suite="stage-c-test-4-gpu-b200")
|
||||
register_cuda_ci(est_time=136, suite="stage-c-test-4-gpu-b200-small")
|
||||
|
||||
|
||||
def test_gsm8k(base_url: str, model: str):
|
||||
|
||||
@@ -48,6 +48,7 @@ PER_COMMIT_SUITES = {
|
||||
"stage-b-kernel-benchmark-1-gpu-large",
|
||||
"stage-c-test-4-gpu-h100",
|
||||
"stage-c-test-4-gpu-b200",
|
||||
"stage-c-test-4-gpu-b200-small",
|
||||
"stage-c-test-4-gpu-gb200",
|
||||
"stage-c-test-8-gpu-h20",
|
||||
"stage-c-test-8-gpu-h200",
|
||||
|
||||
Reference in New Issue
Block a user