ci: B200 conditional split + LPT_SLOP removal (stage-c partition 8→3) (#25203)
This commit is contained in:
@@ -39,15 +39,13 @@ _STAGE_A_OVERRIDES = {
|
||||
"stage-a-test-1-gpu-small": 1,
|
||||
}
|
||||
|
||||
# Per-partition wall-clock target + ceiling. Single knob for the whole
|
||||
# pipeline. ~17 min avg under perfect LPT (TARGET / LPT_SLOP), ~22 min under
|
||||
# worst-case LPT 4/3 imbalance, fail-fast above 30 min.
|
||||
# Per-partition wall-clock target. ~20 min avg naive; worst-case LPT 4/3
|
||||
# imbalance is ~27 min, still below the 30-min job-level timeout that acts
|
||||
# as the real safety net. No LPT slop applied — we lean on the runtime
|
||||
# timeout + the explicit MAX_PARTITION_SECONDS sanity check rather than
|
||||
# padding partition count.
|
||||
TARGET_SECONDS = 20 * 60
|
||||
|
||||
# LPT (Longest Processing Time first) worst case is 4/3 * OPT; pad ~15% so a
|
||||
# slightly-unlucky LPT result still fits inside MAX_PARTITION_SECONDS.
|
||||
LPT_SLOP = 1.15
|
||||
|
||||
# Hard ceiling. Exceeded → raise, forcing the maintainer to split a slow file
|
||||
# or bump TARGET_SECONDS deliberately.
|
||||
MAX_PARTITION_SECONDS = 30 * 60
|
||||
@@ -97,10 +95,11 @@ def compute_partitions(tests, full_parallel=False):
|
||||
size = _STAGE_A_OVERRIDES[suite]
|
||||
max_parallel = size
|
||||
else:
|
||||
size = max(1, math.ceil(total * LPT_SLOP / TARGET_SECONDS))
|
||||
size = max(1, math.ceil(total / TARGET_SECONDS))
|
||||
max_parallel = size if full_parallel else compute_max_parallel(size)
|
||||
# Check naive average (total/size). LPT can be ~4/3 of that, but the
|
||||
# ceil + LPT_SLOP padding above absorbs that slack.
|
||||
# Check naive average (total/size). LPT can be ~4/3 of that in
|
||||
# worst case; the 30-min job timeout enforces the real ceiling at
|
||||
# runtime. This build-time check fails fast on egregious misconfigs.
|
||||
if total / size > MAX_PARTITION_SECONDS:
|
||||
raise RuntimeError(
|
||||
f"Suite {suite!r}: total est_time {total:.0f}s / size {size} "
|
||||
|
||||
@@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.gpt_oss_common import BaseTestGptOss
|
||||
|
||||
register_cuda_ci(est_time=392, stage="stage-c", runner_config="4-gpu-h100")
|
||||
register_cuda_ci(est_time=740, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=350, stage="stage-c", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
class TestGptOss4Gpu(BaseTestGptOss):
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=710, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=540, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=540, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=340, stage="stage-c", runner_config="4-gpu-b200")
|
||||
|
||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=260, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=370, stage="stage-c", runner_config="4-gpu-b200")
|
||||
|
||||
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
# FlashAttention4 integration test (requires SM 100+ / Blackwell B200)
|
||||
register_cuda_ci(est_time=265, stage="stage-b", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=260, stage="stage-b", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
|
||||
|
||||
@@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
stage="stage-c",
|
||||
runner_config="4-gpu-b200",
|
||||
est_time=90,
|
||||
suite="nightly-4-gpu-b200",
|
||||
nightly=True,
|
||||
)
|
||||
|
||||
BASE_MODEL = "lmsys/gpt-oss-20b-bf16"
|
||||
|
||||
@@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
stage="stage-c",
|
||||
runner_config="4-gpu-b200",
|
||||
est_time=100,
|
||||
suite="nightly-4-gpu-b200",
|
||||
nightly=True,
|
||||
)
|
||||
|
||||
BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
|
||||
|
||||
@@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=160,
|
||||
stage="stage-c",
|
||||
runner_config="4-gpu-b200",
|
||||
est_time=100,
|
||||
suite="nightly-4-gpu-b200",
|
||||
nightly=True,
|
||||
)
|
||||
|
||||
BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507"
|
||||
|
||||
@@ -35,7 +35,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=160,
|
||||
est_time=110,
|
||||
stage="stage-c",
|
||||
runner_config="4-gpu-b200",
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=160,
|
||||
est_time=110,
|
||||
stage="stage-c",
|
||||
runner_config="4-gpu-b200",
|
||||
)
|
||||
|
||||
@@ -16,7 +16,7 @@ except ImportError:
|
||||
CuteDslMoEWrapper = None
|
||||
convert_sf_to_mma_layout = None
|
||||
|
||||
register_cuda_ci(est_time=590, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=24, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
|
||||
SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above."
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=1060,
|
||||
est_time=690,
|
||||
stage="stage-c",
|
||||
runner_config="4-gpu-b200",
|
||||
)
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=1190, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=960, stage="stage-c", runner_config="4-gpu-b200")
|
||||
|
||||
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
|
||||
SERVER_LAUNCH_TIMEOUT = 1200
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=630, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=430, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8"
|
||||
MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8"
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=420, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=350, stage="stage-c", runner_config="4-gpu-b200")
|
||||
|
||||
MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=400, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=320, suite="nightly-4-gpu-b200", nightly=True)
|
||||
|
||||
import unittest
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=420, stage="stage-b", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=340, stage="stage-b", runner_config="4-gpu-b200")
|
||||
|
||||
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
|
||||
SERVER_LAUNCH_TIMEOUT = 1200
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
|
||||
# EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs)
|
||||
register_cuda_ci(est_time=123, stage="stage-c", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=90, stage="stage-c", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
def test_gsm8k(base_url: str, model: str):
|
||||
|
||||
Reference in New Issue
Block a user