From 1e308aec6630528b300e5bbf5e29c94597488e04 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Wed, 13 May 2026 18:40:45 -0700 Subject: [PATCH] =?UTF-8?q?ci:=20B200=20conditional=20split=20+=20LPT=5FSL?= =?UTF-8?q?OP=20removal=20(stage-c=20partition=208=E2=86=923)=20(#25203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci/utils/compute_partitions.py | 19 +++++++++---------- .../4-gpu-models/test_gpt_oss_4gpu.py | 2 +- .../test_nvidia_nemotron_3_super_nvfp4.py | 2 +- .../4-gpu-models/test_qwen35_fp4_mtp_v2.py | 2 +- .../4-gpu-models/test_qwen35_models.py | 2 +- .../attention/test_flash_attention_4.py | 2 +- .../test_lora_gpt_oss_20b_logprob_diff.py | 6 +++--- ...nemotron_3_super_120b_a12b_logprob_diff.py | 6 +++--- ...wen3_30b_a3b_instruct_2507_logprob_diff.py | 6 +++--- .../test_lora_qwen3_5_35b_a3b_logprob_diff.py | 2 +- ..._qwen3_vl_30b_a3b_instruct_logprob_diff.py | 2 +- test/registered/moe/test_cutedsl_moe.py | 2 +- .../quant/test_deepseek_v32_fp4_mtp_4gpu.py | 2 +- .../quant/test_deepseek_v3_fp4_4gpu.py | 2 +- .../quant/test_fp8_blockwise_gemm.py | 2 +- test/registered/quant/test_nvfp4_gemm.py | 2 +- ...test_update_weights_from_disk_blackwell.py | 2 +- .../eagle/test_deepseek_v3_fp4_mtp_small.py | 2 +- .../test_eagle_infer_beta_dp_attention.py | 2 +- 19 files changed, 33 insertions(+), 34 deletions(-) diff --git a/scripts/ci/utils/compute_partitions.py b/scripts/ci/utils/compute_partitions.py index 88754c464..5cb48faa7 100644 --- a/scripts/ci/utils/compute_partitions.py +++ b/scripts/ci/utils/compute_partitions.py @@ -39,15 +39,13 @@ _STAGE_A_OVERRIDES = { "stage-a-test-1-gpu-small": 1, } -# Per-partition wall-clock target + ceiling. Single knob for the whole -# pipeline. ~17 min avg under perfect LPT (TARGET / LPT_SLOP), ~22 min under -# worst-case LPT 4/3 imbalance, fail-fast above 30 min. +# Per-partition wall-clock target. ~20 min avg naive; worst-case LPT 4/3 +# imbalance is ~27 min, still below the 30-min job-level timeout that acts +# as the real safety net. No LPT slop applied — we lean on the runtime +# timeout + the explicit MAX_PARTITION_SECONDS sanity check rather than +# padding partition count. TARGET_SECONDS = 20 * 60 -# LPT (Longest Processing Time first) worst case is 4/3 * OPT; pad ~15% so a -# slightly-unlucky LPT result still fits inside MAX_PARTITION_SECONDS. -LPT_SLOP = 1.15 - # Hard ceiling. Exceeded → raise, forcing the maintainer to split a slow file # or bump TARGET_SECONDS deliberately. MAX_PARTITION_SECONDS = 30 * 60 @@ -97,10 +95,11 @@ def compute_partitions(tests, full_parallel=False): size = _STAGE_A_OVERRIDES[suite] max_parallel = size else: - size = max(1, math.ceil(total * LPT_SLOP / TARGET_SECONDS)) + size = max(1, math.ceil(total / TARGET_SECONDS)) max_parallel = size if full_parallel else compute_max_parallel(size) - # Check naive average (total/size). LPT can be ~4/3 of that, but the - # ceil + LPT_SLOP padding above absorbs that slack. + # Check naive average (total/size). LPT can be ~4/3 of that in + # worst case; the 30-min job timeout enforces the real ceiling at + # runtime. This build-time check fails fast on egregious misconfigs. if total / size > MAX_PARTITION_SECONDS: raise RuntimeError( f"Suite {suite!r}: total est_time {total:.0f}s / size {size} " diff --git a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py b/test/registered/4-gpu-models/test_gpt_oss_4gpu.py index 2d13560e6..06a53d646 100644 --- a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py +++ b/test/registered/4-gpu-models/test_gpt_oss_4gpu.py @@ -4,7 +4,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.gpt_oss_common import BaseTestGptOss register_cuda_ci(est_time=392, stage="stage-c", runner_config="4-gpu-h100") -register_cuda_ci(est_time=740, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=350, stage="stage-c", runner_config="4-gpu-b200") class TestGptOss4Gpu(BaseTestGptOss): diff --git a/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py index 96361446b..854c25771 100644 --- a/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/4-gpu-models/test_nvidia_nemotron_3_super_nvfp4.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=710, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=540, suite="nightly-4-gpu-b200", nightly=True) NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py b/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py index d9ee7586a..6e4c39c42 100644 --- a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py +++ b/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=540, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=340, stage="stage-c", runner_config="4-gpu-b200") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} diff --git a/test/registered/4-gpu-models/test_qwen35_models.py b/test/registered/4-gpu-models/test_qwen35_models.py index 695d25c30..beee76658 100644 --- a/test/registered/4-gpu-models/test_qwen35_models.py +++ b/test/registered/4-gpu-models/test_qwen35_models.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=260, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=370, stage="stage-c", runner_config="4-gpu-b200") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} diff --git a/test/registered/attention/test_flash_attention_4.py b/test/registered/attention/test_flash_attention_4.py index 0d2e4aeb3..b0210afbc 100644 --- a/test/registered/attention/test_flash_attention_4.py +++ b/test/registered/attention/test_flash_attention_4.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( ) # FlashAttention4 integration test (requires SM 100+ / Blackwell B200) -register_cuda_ci(est_time=265, stage="stage-b", runner_config="4-gpu-b200") +register_cuda_ci(est_time=260, stage="stage-b", runner_config="4-gpu-b200") @unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") diff --git a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py index c7aa96095..a57b8ffcd 100644 --- a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py +++ b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py @@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci( - est_time=300, - stage="stage-c", - runner_config="4-gpu-b200", + est_time=90, + suite="nightly-4-gpu-b200", + nightly=True, ) BASE_MODEL = "lmsys/gpt-oss-20b-bf16" diff --git a/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py b/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py index 4fd458f48..41a4c47fb 100644 --- a/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py +++ b/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py @@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci( - est_time=300, - stage="stage-c", - runner_config="4-gpu-b200", + est_time=100, + suite="nightly-4-gpu-b200", + nightly=True, ) BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" diff --git a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py index a4ea150f5..3f5a7e55c 100644 --- a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py @@ -35,9 +35,9 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci( - est_time=160, - stage="stage-c", - runner_config="4-gpu-b200", + est_time=100, + suite="nightly-4-gpu-b200", + nightly=True, ) BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507" diff --git a/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py b/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py index b11b38ec0..ea959653b 100644 --- a/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_5_35b_a3b_logprob_diff.py @@ -35,7 +35,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci( - est_time=160, + est_time=110, stage="stage-c", runner_config="4-gpu-b200", ) diff --git a/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py b/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py index 63ca79ef4..90cafd6a5 100644 --- a/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_vl_30b_a3b_instruct_logprob_diff.py @@ -35,7 +35,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci( - est_time=160, + est_time=110, stage="stage-c", runner_config="4-gpu-b200", ) diff --git a/test/registered/moe/test_cutedsl_moe.py b/test/registered/moe/test_cutedsl_moe.py index 7e566b2a4..52139ed3c 100644 --- a/test/registered/moe/test_cutedsl_moe.py +++ b/test/registered/moe/test_cutedsl_moe.py @@ -16,7 +16,7 @@ except ImportError: CuteDslMoEWrapper = None convert_sf_to_mma_layout = None -register_cuda_ci(est_time=590, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=24, suite="nightly-4-gpu-b200", nightly=True) SKIP_TEST = torch.cuda.get_device_capability() < (10, 0) SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above." diff --git a/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py b/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py index c087c8bfd..6ded6338b 100644 --- a/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py +++ b/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( ) register_cuda_ci( - est_time=1060, + est_time=690, stage="stage-c", runner_config="4-gpu-b200", ) diff --git a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py index 1cfd18f09..fd716f862 100644 --- a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py +++ b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=1190, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=960, stage="stage-c", runner_config="4-gpu-b200") FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" SERVER_LAUNCH_TIMEOUT = 1200 diff --git a/test/registered/quant/test_fp8_blockwise_gemm.py b/test/registered/quant/test_fp8_blockwise_gemm.py index ebc158c67..a7257fd21 100644 --- a/test/registered/quant/test_fp8_blockwise_gemm.py +++ b/test/registered/quant/test_fp8_blockwise_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=630, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=430, suite="nightly-4-gpu-b200", nightly=True) MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8" MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8" diff --git a/test/registered/quant/test_nvfp4_gemm.py b/test/registered/quant/test_nvfp4_gemm.py index 433ccc6c9..c8b39167c 100644 --- a/test/registered/quant/test_nvfp4_gemm.py +++ b/test/registered/quant/test_nvfp4_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=420, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=350, stage="stage-c", runner_config="4-gpu-b200") MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4" diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index ca5af2e2e..00ee93076 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -1,6 +1,6 @@ from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=400, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=320, suite="nightly-4-gpu-b200", nightly=True) import unittest diff --git a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py index e300218d0..faa4ed476 100644 --- a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py +++ b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=420, stage="stage-b", runner_config="4-gpu-b200") +register_cuda_ci(est_time=340, stage="stage-b", runner_config="4-gpu-b200") FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" SERVER_LAUNCH_TIMEOUT = 1200 diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index 85d22235f..3f9906b9f 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( ) # EAGLE with DP attention on B200 (tp=2, dp=2, requires 4 B200 GPUs) -register_cuda_ci(est_time=123, stage="stage-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=90, stage="stage-c", runner_config="4-gpu-b200") def test_gsm8k(base_url: str, model: str):