From 9bd146175761280c3e4383f0ae2d467c1e0ce938 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 5 Aug 2026 20:00:30 -0700 Subject: [PATCH] [CI] Bound the CUDA graph capture range in test launches and lift the spec fixture's admission cap (#33776) --- python/sglang/test/server_fixtures/spec_eagle_fixture.py | 8 ++++++-- python/sglang/test/test_utils.py | 9 +++++++++ test/registered/cpu/test_spec_eagle_cpu.py | 2 ++ test/registered/cpu/test_spec_eagle_parity_cpu.py | 2 ++ test/registered/cpu/test_spec_eagle_topk_cpu.py | 2 ++ test/registered/spec/eagle/test_spec_eagle_stress.py | 2 -- test/registered/spec/eagle/test_spec_eagle_topk.py | 3 +++ test/registered/spec/eagle/test_spec_eagle_topk_page.py | 2 ++ test/registered/spec/eagle/test_spec_eagle_triton.py | 2 -- 9 files changed, 26 insertions(+), 6 deletions(-) diff --git a/python/sglang/test/server_fixtures/spec_eagle_fixture.py b/python/sglang/test/server_fixtures/spec_eagle_fixture.py index 4ca00b001..bc73d019b 100644 --- a/python/sglang/test/server_fixtures/spec_eagle_fixture.py +++ b/python/sglang/test/server_fixtures/spec_eagle_fixture.py @@ -58,8 +58,12 @@ class SpecEagleServerBase(CustomTestCase): attention_backend = "flashinfer" # Primary axis: False -> overlap scheduler; True -> synchronous (non-overlap). disable_overlap = False - mem_fraction_static = 0.85 - max_running_requests = 8 + # Leaves ~3.3GB on a 32GB card for the verify logits and activations at a + # cap of 64; higher OOMs, lower starves the KV pool into capping the batch. + mem_fraction_static = 0.80 + # The eval kits drive 128 client threads, so a small cap just serializes them. + # Capture follows: capture_bs is clipped to req_to_token_pool.size (cap + 1). + max_running_requests = 64 chunked_prefill_size = 128 # bf16 rather than fp16: fp16 activations can overflow (-> Inf -> NaN) on # degenerate draft branches in verify and trip the CI NaN asserts. diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 2a9c1a68d..ae7ac405d 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -952,6 +952,15 @@ def popen_launch_server( other_args = list(other_args) other_args += ["--device", str(device)] + # Prefill dominates capture time: the bucket list runs to chunked_prefill_size + # (8192 on H100-class GPUs) and its largest buckets cost seconds each, while + # 97% of CI prefill batches are under 1024 tokens -- a server that serves one + # test file captures the rest and never replays it. Decode is left alone: its + # capture cost is per-phase, not per-bucket. Pass the flag to opt out. + prefill_flag = "--cuda-graph-max-bs-prefill" + if not any(str(arg).startswith(prefill_flag) for arg in other_args): + other_args = list(other_args) + [prefill_flag, "1024"] + # CI-specific: Validate cache and enable offline mode if complete if env is None: env = os.environ.copy() diff --git a/test/registered/cpu/test_spec_eagle_cpu.py b/test/registered/cpu/test_spec_eagle_cpu.py index eebf1b242..06817361d 100644 --- a/test/registered/cpu/test_spec_eagle_cpu.py +++ b/test/registered/cpu/test_spec_eagle_cpu.py @@ -40,6 +40,8 @@ class _Core(EagleLlama2Base): attention_backend = "intel_amx" disable_overlap = True mem_fraction_static = 0.3 + # CPU decode is compute-bound; a wider batch buys nothing here. + max_running_requests = 8 gsm8k_num_examples = 64 env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/cpu/test_spec_eagle_parity_cpu.py b/test/registered/cpu/test_spec_eagle_parity_cpu.py index 977ad9c66..724e702d9 100644 --- a/test/registered/cpu/test_spec_eagle_parity_cpu.py +++ b/test/registered/cpu/test_spec_eagle_parity_cpu.py @@ -20,6 +20,8 @@ class TestEagle3ParityCPU(SpecParityKit, Eagle3Base): attention_backend = "intel_amx" disable_overlap = True mem_fraction_static = 0.3 + # CPU decode is compute-bound; a wider batch buys nothing here. + max_running_requests = 8 env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/cpu/test_spec_eagle_topk_cpu.py b/test/registered/cpu/test_spec_eagle_topk_cpu.py index 12558f714..bc1949c5b 100644 --- a/test/registered/cpu/test_spec_eagle_topk_cpu.py +++ b/test/registered/cpu/test_spec_eagle_topk_cpu.py @@ -32,6 +32,8 @@ class _Core(EagleLlama2Base): attention_backend = "intel_amx" disable_overlap = True mem_fraction_static = 0.3 + # CPU decode is compute-bound; a wider batch buys nothing here. + max_running_requests = 8 gsm8k_num_examples = 64 env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/spec/eagle/test_spec_eagle_stress.py b/test/registered/spec/eagle/test_spec_eagle_stress.py index e17b4ff0b..5e1a4d109 100644 --- a/test/registered/spec/eagle/test_spec_eagle_stress.py +++ b/test/registered/spec/eagle/test_spec_eagle_stress.py @@ -30,7 +30,6 @@ class TestEagle3Perf(Eagle3Base, SpecPerfKit): class TestEagleLlama2Retract(EagleLlama2Base, SpecAccuracyKit, SpecFeatureKit): """Retract under a small KV budget; must not leak.""" - max_running_requests = 64 extra_args = ("--max-total-tokens", 4500) # small KV to trigger retract env_overrides = ( (envs.SGLANG_TEST_RETRACT, True), @@ -46,7 +45,6 @@ class TestEagle3Topk16V2Retract(Eagle3Base, SpecAccuracyKit, SpecFeatureKit): spec_tokens = 64 disable_overlap = False cuda_graph_max_bs_decode = 5 - max_running_requests = 64 gsm8k_accept_len_thres = 2.4 extra_args = ("--max-total-tokens", 4500) # small KV to trigger retract env_overrides = ( diff --git a/test/registered/spec/eagle/test_spec_eagle_topk.py b/test/registered/spec/eagle/test_spec_eagle_topk.py index 2ac9e9a23..7bf7113ea 100644 --- a/test/registered/spec/eagle/test_spec_eagle_topk.py +++ b/test/registered/spec/eagle/test_spec_eagle_topk.py @@ -40,6 +40,9 @@ class TestEagle3Topk16( spec_tokens = 64 disable_overlap = False enable_return_hidden_states = True + # Verify materializes bs * spec_tokens fp32 logit rows: 262MB here, but + # 2.1GB at the fixture's 64 -- OOM on a 32GB card. + max_running_requests = 8 cuda_graph_max_bs_decode = 5 acc_length_thres = 3.1 batch_accept_len_thres = 1.75 diff --git a/test/registered/spec/eagle/test_spec_eagle_topk_page.py b/test/registered/spec/eagle/test_spec_eagle_topk_page.py index 3fd20f376..7d38c42eb 100644 --- a/test/registered/spec/eagle/test_spec_eagle_topk_page.py +++ b/test/registered/spec/eagle/test_spec_eagle_topk_page.py @@ -31,6 +31,8 @@ class TestEagle3Page4Topk8(Eagle3Base, SpecAccuracyKit, SpecLogprobKit, SpecFeat # Preset accept-length values are topk=1 numbers -- loose for a topk=8 # tree; tighten once CI reports the actuals. gsm8k_accept_len_thres = 2.0 + # Same fp32 logits budget as the topk=16 classes (bs * spec_tokens * vocab). + max_running_requests = 16 cuda_graph_max_bs_decode = 5 env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/spec/eagle/test_spec_eagle_triton.py b/test/registered/spec/eagle/test_spec_eagle_triton.py index e57f74090..e8f94e987 100644 --- a/test/registered/spec/eagle/test_spec_eagle_triton.py +++ b/test/registered/spec/eagle/test_spec_eagle_triton.py @@ -32,8 +32,6 @@ class TestEagle3Triton( """Overlap scheduler on triton (kits listed in bases).""" attention_backend = "triton" - max_running_requests = 64 - cuda_graph_max_bs_decode = 64 gsm8k_num_examples = 200 gsm8k_check_accept_len = False env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),)