[ci] split stage-c-test-4-gpu-b200 to enable a low-disk runner pool (#23417)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
6cf0b004ca
commit
77fd86f89e
@@ -92,6 +92,7 @@ jobs:
|
|||||||
max_parallel_small: ${{ steps.set-parallel.outputs.max_parallel_small }}
|
max_parallel_small: ${{ steps.set-parallel.outputs.max_parallel_small }}
|
||||||
max_parallel_2gpu: ${{ steps.set-parallel.outputs.max_parallel_2gpu }}
|
max_parallel_2gpu: ${{ steps.set-parallel.outputs.max_parallel_2gpu }}
|
||||||
b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
|
b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
|
||||||
|
b200_low_disk_runner: ${{ steps.set-runner.outputs.b200_low_disk_runner }}
|
||||||
enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
|
enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
|
||||||
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
|
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
|
||||||
steps:
|
steps:
|
||||||
@@ -273,8 +274,10 @@ jobs:
|
|||||||
target_stage="${{ inputs.target_stage }}"
|
target_stage="${{ inputs.target_stage }}"
|
||||||
if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then
|
if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then
|
||||||
echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT
|
echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT
|
||||||
|
echo "b200_low_disk_runner=4-gpu-b200-kernel-low-disk" >> $GITHUB_OUTPUT
|
||||||
else
|
else
|
||||||
echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT
|
echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT
|
||||||
|
echo "b200_low_disk_runner=4-gpu-b200-low-disk" >> $GITHUB_OUTPUT
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Enable retry for CI
|
- name: Enable retry for CI
|
||||||
@@ -330,6 +333,7 @@ jobs:
|
|||||||
echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |"
|
echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |"
|
||||||
echo "| max_parallel | ${{ steps.set-parallel.outputs.parallel_level }} (h100=${{ steps.set-parallel.outputs.max_parallel }}, 5090=${{ steps.set-parallel.outputs.max_parallel_small }}, 2gpu=${{ steps.set-parallel.outputs.max_parallel_2gpu }}) |"
|
echo "| max_parallel | ${{ steps.set-parallel.outputs.parallel_level }} (h100=${{ steps.set-parallel.outputs.max_parallel }}, 5090=${{ steps.set-parallel.outputs.max_parallel_small }}, 2gpu=${{ steps.set-parallel.outputs.max_parallel_2gpu }}) |"
|
||||||
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
|
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
|
||||||
|
echo "| b200_low_disk_runner | ${{ steps.set-runner.outputs.b200_low_disk_runner }} |"
|
||||||
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
|
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
|
||||||
echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
|
echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
|
||||||
} >> $GITHUB_STEP_SUMMARY
|
} >> $GITHUB_STEP_SUMMARY
|
||||||
@@ -1293,7 +1297,7 @@ jobs:
|
|||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
part: [0, 1, 2, 3, 4, 5]
|
part: [0, 1, 2]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
@@ -1324,7 +1328,69 @@ jobs:
|
|||||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||||
run: |
|
run: |
|
||||||
cd test
|
cd test
|
||||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 6 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
|
- uses: ./.github/actions/upload-cuda-coredumps
|
||||||
|
if: failure()
|
||||||
|
with:
|
||||||
|
artifact-suffix: ${{ matrix.part }}
|
||||||
|
|
||||||
|
- name: Cleanup venv
|
||||||
|
if: always()
|
||||||
|
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
||||||
|
|
||||||
|
stage-c-test-4-gpu-b200-small:
|
||||||
|
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||||
|
if: |
|
||||||
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'stage-c-test-4-gpu-b200-small') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# The `*-low-disk` label (resolved by `set-runner` to `4-gpu-b200-low-disk` or
|
||||||
|
# `4-gpu-b200-kernel-low-disk`) is advertised by both the existing large-disk B200
|
||||||
|
# runners and the new low-disk runner, so this job can land on either pool.
|
||||||
|
runs-on: ${{ needs.check-changes.outputs.b200_low_disk_runner }}
|
||||||
|
timeout-minutes: 240
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
part: [0, 1, 2]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
|
||||||
|
|
||||||
|
- uses: ./.github/actions/check-stage-health
|
||||||
|
|
||||||
|
- uses: ./.github/actions/check-maintenance
|
||||||
|
|
||||||
|
- name: Download artifacts
|
||||||
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
|
uses: actions/download-artifact@v6
|
||||||
|
with:
|
||||||
|
path: sgl-kernel/dist/
|
||||||
|
merge-multiple: true
|
||||||
|
pattern: wheel-python3.10-cuda13.0
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
timeout-minutes: 20
|
||||||
|
run: |
|
||||||
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
|
||||||
|
|
||||||
|
- name: Run test
|
||||||
|
timeout-minutes: 30
|
||||||
|
env:
|
||||||
|
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||||
|
run: |
|
||||||
|
cd test
|
||||||
|
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200-small --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
- uses: ./.github/actions/upload-cuda-coredumps
|
- uses: ./.github/actions/upload-cuda-coredumps
|
||||||
if: failure()
|
if: failure()
|
||||||
@@ -1416,6 +1482,7 @@ jobs:
|
|||||||
stage-c-test-deepep-4-gpu-h100,
|
stage-c-test-deepep-4-gpu-h100,
|
||||||
stage-c-test-deepep-8-gpu-h200,
|
stage-c-test-deepep-8-gpu-h200,
|
||||||
stage-c-test-4-gpu-b200,
|
stage-c-test-4-gpu-b200,
|
||||||
|
stage-c-test-4-gpu-b200-small,
|
||||||
# stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
# stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
|
|||||||
@@ -294,6 +294,7 @@ def handle_rerun_stage(
|
|||||||
"stage-c-test-8-gpu-h200",
|
"stage-c-test-8-gpu-h200",
|
||||||
"stage-c-test-8-gpu-h20",
|
"stage-c-test-8-gpu-h20",
|
||||||
"stage-c-test-4-gpu-b200",
|
"stage-c-test-4-gpu-b200",
|
||||||
|
"stage-c-test-4-gpu-b200-small",
|
||||||
"stage-c-test-4-gpu-gb200",
|
"stage-c-test-4-gpu-gb200",
|
||||||
"stage-c-test-deepep-4-gpu-h100",
|
"stage-c-test-deepep-4-gpu-h100",
|
||||||
"stage-c-test-deepep-8-gpu-h200",
|
"stage-c-test-deepep-8-gpu-h200",
|
||||||
@@ -460,6 +461,7 @@ CUDA_SUITE_TO_RUNNER = {
|
|||||||
"stage-c-test-8-gpu-h200": "8-gpu-h200",
|
"stage-c-test-8-gpu-h200": "8-gpu-h200",
|
||||||
"stage-c-test-8-gpu-h20": "8-gpu-h20",
|
"stage-c-test-8-gpu-h20": "8-gpu-h20",
|
||||||
"stage-c-test-4-gpu-b200": "4-gpu-b200",
|
"stage-c-test-4-gpu-b200": "4-gpu-b200",
|
||||||
|
"stage-c-test-4-gpu-b200-small": "4-gpu-b200-low-disk",
|
||||||
"stage-c-test-deepep-4-gpu-h100": "4-gpu-h100",
|
"stage-c-test-deepep-4-gpu-h100": "4-gpu-h100",
|
||||||
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
|
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
|
||||||
# Nightly test suites (NVIDIA)
|
# Nightly test suites (NVIDIA)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
|||||||
from sglang.test.gpt_oss_common import BaseTestGptOss
|
from sglang.test.gpt_oss_common import BaseTestGptOss
|
||||||
|
|
||||||
register_cuda_ci(est_time=328, suite="stage-c-test-4-gpu-h100")
|
register_cuda_ci(est_time=328, suite="stage-c-test-4-gpu-h100")
|
||||||
register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=740, suite="stage-c-test-4-gpu-b200-small")
|
||||||
|
|
||||||
|
|
||||||
class TestGptOss4Gpu(BaseTestGptOss):
|
class TestGptOss4Gpu(BaseTestGptOss):
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(est_time=710, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=710, suite="nightly-4-gpu-b200", nightly=True)
|
||||||
|
|
||||||
NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
|
|||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-b200-small")
|
||||||
|
|
||||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
|
|||||||
ModelLaunchSettings,
|
ModelLaunchSettings,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200-small")
|
||||||
|
|
||||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase
|
|||||||
|
|
||||||
register_cuda_ci(
|
register_cuda_ci(
|
||||||
est_time=300,
|
est_time=300,
|
||||||
suite="stage-c-test-4-gpu-b200",
|
suite="stage-c-test-4-gpu-b200-small",
|
||||||
)
|
)
|
||||||
|
|
||||||
BASE_MODEL = "lmsys/gpt-oss-20b-bf16"
|
BASE_MODEL = "lmsys/gpt-oss-20b-bf16"
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase
|
|||||||
|
|
||||||
register_cuda_ci(
|
register_cuda_ci(
|
||||||
est_time=160,
|
est_time=160,
|
||||||
suite="stage-c-test-4-gpu-b200",
|
suite="stage-c-test-4-gpu-b200-small",
|
||||||
)
|
)
|
||||||
|
|
||||||
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
|
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from sglang.test.test_utils import CustomTestCase
|
|||||||
|
|
||||||
register_cuda_ci(
|
register_cuda_ci(
|
||||||
est_time=160,
|
est_time=160,
|
||||||
suite="stage-c-test-4-gpu-b200",
|
suite="stage-c-test-4-gpu-b200-small",
|
||||||
)
|
)
|
||||||
|
|
||||||
BASE_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct"
|
BASE_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ except ImportError:
|
|||||||
CuteDslMoEWrapper = None
|
CuteDslMoEWrapper = None
|
||||||
convert_sf_to_mma_layout = None
|
convert_sf_to_mma_layout = None
|
||||||
|
|
||||||
register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=590, suite="stage-c-test-4-gpu-b200-small")
|
||||||
|
|
||||||
SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
|
SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
|
||||||
SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above."
|
SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above."
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
|||||||
try_cached_model,
|
try_cached_model,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(est_time=630, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=630, suite="nightly-4-gpu-b200", nightly=True)
|
||||||
|
|
||||||
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
|
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
|
||||||
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
|
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
|||||||
try_cached_model,
|
try_cached_model,
|
||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(est_time=550, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=550, suite="nightly-4-gpu-b200", nightly=True)
|
||||||
|
|
||||||
MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from sglang.test.ci.ci_register import register_cuda_ci
|
from sglang.test.ci.ci_register import register_cuda_ci
|
||||||
|
|
||||||
register_cuda_ci(est_time=149, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=149, suite="stage-c-test-4-gpu-b200-small")
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
# EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs)
|
# EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs)
|
||||||
register_cuda_ci(est_time=136, suite="stage-c-test-4-gpu-b200")
|
register_cuda_ci(est_time=136, suite="stage-c-test-4-gpu-b200-small")
|
||||||
|
|
||||||
|
|
||||||
def test_gsm8k(base_url: str, model: str):
|
def test_gsm8k(base_url: str, model: str):
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ PER_COMMIT_SUITES = {
|
|||||||
"stage-b-kernel-benchmark-1-gpu-large",
|
"stage-b-kernel-benchmark-1-gpu-large",
|
||||||
"stage-c-test-4-gpu-h100",
|
"stage-c-test-4-gpu-h100",
|
||||||
"stage-c-test-4-gpu-b200",
|
"stage-c-test-4-gpu-b200",
|
||||||
|
"stage-c-test-4-gpu-b200-small",
|
||||||
"stage-c-test-4-gpu-gb200",
|
"stage-c-test-4-gpu-gb200",
|
||||||
"stage-c-test-8-gpu-h20",
|
"stage-c-test-8-gpu-h20",
|
||||||
"stage-c-test-8-gpu-h200",
|
"stage-c-test-8-gpu-h200",
|
||||||
|
|||||||
Reference in New Issue
Block a user