ci: B200 conditional split + LPT_SLOP removal (stage-c partition 8→3) (#25203)

This commit is contained in:
Alison Shao
2026-05-13 18:40:45 -07:00
committed by GitHub
parent 5fb6bde6c0
commit 1e308aec66
19 changed files with 33 additions and 34 deletions
+9 -10
View File
@@ -39,15 +39,13 @@ _STAGE_A_OVERRIDES = {
"stage-a-test-1-gpu-small": 1,
}
# Per-partition wall-clock target + ceiling. Single knob for the whole
# pipeline. ~17 min avg under perfect LPT (TARGET / LPT_SLOP), ~22 min under
# worst-case LPT 4/3 imbalance, fail-fast above 30 min.
# Per-partition wall-clock target. ~20 min avg naive; worst-case LPT 4/3
# imbalance is ~27 min, still below the 30-min job-level timeout that acts
# as the real safety net. No LPT slop applied — we lean on the runtime
# timeout + the explicit MAX_PARTITION_SECONDS sanity check rather than
# padding partition count.
TARGET_SECONDS = 20 * 60
# LPT (Longest Processing Time first) worst case is 4/3 * OPT; pad ~15% so a
# slightly-unlucky LPT result still fits inside MAX_PARTITION_SECONDS.
LPT_SLOP = 1.15
# Hard ceiling. Exceeded → raise, forcing the maintainer to split a slow file
# or bump TARGET_SECONDS deliberately.
MAX_PARTITION_SECONDS = 30 * 60
@@ -97,10 +95,11 @@ def compute_partitions(tests, full_parallel=False):
size = _STAGE_A_OVERRIDES[suite]
max_parallel = size
else:
size = max(1, math.ceil(total * LPT_SLOP / TARGET_SECONDS))
size = max(1, math.ceil(total / TARGET_SECONDS))
max_parallel = size if full_parallel else compute_max_parallel(size)
# Check naive average (total/size). LPT can be ~4/3 of that, but the
# ceil + LPT_SLOP padding above absorbs that slack.
# Check naive average (total/size). LPT can be ~4/3 of that in
# worst case; the 30-min job timeout enforces the real ceiling at
# runtime. This build-time check fails fast on egregious misconfigs.
if total / size > MAX_PARTITION_SECONDS:
raise RuntimeError(
f"Suite {suite!r}: total est_time {total:.0f}s / size {size} "
@@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=392, stage="stage-c", runner_config="4-gpu-h100")
register_cuda_ci(est_time=740, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=350, stage="stage-c", runner_config="4-gpu-b200")
class TestGptOss4Gpu(BaseTestGptOss):
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=710, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=540, suite="nightly-4-gpu-b200", nightly=True)
NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=540, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=340, stage="stage-c", runner_config="4-gpu-b200")
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=260, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=370, stage="stage-c", runner_config="4-gpu-b200")
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
)
# FlashAttention4 integration test (requires SM 100+ / Blackwell B200)
register_cuda_ci(est_time=265, stage="stage-b", runner_config="4-gpu-b200")
register_cuda_ci(est_time=260, stage="stage-b", runner_config="4-gpu-b200")
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
@@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=300,
stage="stage-c",
runner_config="4-gpu-b200",
est_time=90,
suite="nightly-4-gpu-b200",
nightly=True,
)
BASE_MODEL = "lmsys/gpt-oss-20b-bf16"
@@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=300,
stage="stage-c",
runner_config="4-gpu-b200",
est_time=100,
suite="nightly-4-gpu-b200",
nightly=True,
)
BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
@@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=160,
stage="stage-c",
runner_config="4-gpu-b200",
est_time=100,
suite="nightly-4-gpu-b200",
nightly=True,
)
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
@@ -35,7 +35,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=160,
est_time=110,
stage="stage-c",
runner_config="4-gpu-b200",
)
@@ -35,7 +35,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(
est_time=160,
est_time=110,
stage="stage-c",
runner_config="4-gpu-b200",
)
+1 -1
View File
@@ -16,7 +16,7 @@ except ImportError:
CuteDslMoEWrapper = None
convert_sf_to_mma_layout = None
register_cuda_ci(est_time=590, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=24, suite="nightly-4-gpu-b200", nightly=True)
SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above."
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
)
register_cuda_ci(
est_time=1060,
est_time=690,
stage="stage-c",
runner_config="4-gpu-b200",
)
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=1190, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=960, stage="stage-c", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=630, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=430, suite="nightly-4-gpu-b200", nightly=True)
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=420, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=350, stage="stage-c", runner_config="4-gpu-b200")
MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
@@ -1,6 +1,6 @@
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=400, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=320, suite="nightly-4-gpu-b200", nightly=True)
import unittest
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=420, stage="stage-b", runner_config="4-gpu-b200")
register_cuda_ci(est_time=340, stage="stage-b", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1200
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
)
# EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs)
register_cuda_ci(est_time=123, stage="stage-c", runner_config="4-gpu-b200")
register_cuda_ci(est_time=90, stage="stage-c", runner_config="4-gpu-b200")
def test_gsm8k(base_url: str, model: str):