minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[misc] Use --cuda-graph-max-bs-decode in tests, examples, and docs (#29591)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-06-28 18:38:28 -07:00
committed by GitHub
parent 3217410cf6
commit 909123ddb8
140 changed files with 304 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/manual/attention/test_fa3.py
+5 -5
View File
@@ -45,7 +45,7 @@ if OFFLINE_MODE:
# Default server arguments shared across all tests
DEFAULT_SERVER_ARGS = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--attention-backend",
"fa3",
@@ -144,7 +144,7 @@ class TestFlashAttention3SpeculativeDecode(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE3",
@@ -178,7 +178,7 @@ class TestFlashAttention3SpeculativeDecodeTopk(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE3",
@@ -210,7 +210,7 @@ class TestFlashAttention3MLASpeculativeDecode(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE",
@@ -242,7 +242,7 @@ class TestFlashAttention3MLASpeculativeDecodeTopk(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE",
test/manual/attention/test_local_attn.py
+1 -1
View File
@@ -25,7 +25,7 @@ class TestFlashAttention3LocalAttn(CustomTestCase):
@classmethod
def get_server_args(cls):
return [
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
"--attention-backend",
"fa3",
test/manual/dsv4/test_b200_pro.py
+3 -3
View File
@@ -60,7 +60,7 @@ class TestB200ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"128",
@@ -83,7 +83,7 @@ class TestB200ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"256",
@@ -108,7 +108,7 @@ class TestB200ProCP(DSV4ProAime25TestBase):
"16384",
"--mem-fraction-static",
"0.78",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/manual/dsv4/test_b300_pro.py
+3 -3
View File
@@ -62,7 +62,7 @@ class TestB300ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"128",
@@ -85,7 +85,7 @@ class TestB300ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"256",
@@ -110,7 +110,7 @@ class TestB300ProCP(DSV4ProAime25TestBase):
"16384",
"--mem-fraction-static",
"0.78",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_flash_mtp_dp4.py
+2 -2
View File
@@ -37,7 +37,7 @@ DSV4_FLASH_ENV = {
"SGLANG_DSV4_FP4_EXPERTS": "0",
# MTP runs ~num_draft_tokens forward passes per step, so the deepep
# dispatch input size scales by that factor. Default 256 (used by the
# plain server) overflows once cuda-graph-max-bs * num_draft_tokens
# plain server) overflows once cuda-graph-max-bs-decode * num_draft_tokens
# > 256. 1024 covers bs=128 * 4 draft tokens with headroom.
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "1024",
}
@@ -66,7 +66,7 @@ class DSV4FlashMTPServerBase(CustomTestCase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_flash_sanity_dp4.py
+3 -3
View File
@@ -65,7 +65,7 @@ class TestDSV4FlashTP4DP4(
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -101,7 +101,7 @@ class TestDSV4FlashTP4EP(
# No --enable-dp-attention by design: covers TP-attn path.
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"64",
@@ -139,7 +139,7 @@ class TestDSV4FlashTP4DP4ChunkedPrefillLarge(
"deepep",
"--chunked-prefill-size",
"16384",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_pd_disagg_nixl.py
+2 -2
View File
@@ -61,7 +61,7 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase):
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -103,7 +103,7 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase):
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_swa_radix_retract.py
+1 -1
View File
@@ -66,7 +66,7 @@ class TestDSV4FlashSWARadixRetract(CustomTestCase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_gb300_pro.py
+3 -3
View File
@@ -62,7 +62,7 @@ class TestGB300ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -85,7 +85,7 @@ class TestGB300ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -110,7 +110,7 @@ class TestGB300ProCP(DSV4ProAime25TestBase):
"16384",
"--mem-fraction-static",
"0.88",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/manual/dsv4/test_h200_fp8_flash.py
+2 -2
View File
@@ -53,7 +53,7 @@ class TestH200Fp8FlashBalanced(DSV4FlashAime25TestBase):
"1",
"--speculative-num-draft-tokens",
"2",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"128",
@@ -77,7 +77,7 @@ class TestH200Fp8FlashMaxThroughput(DSV4FlashAime25TestBase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_h200_fp8_pro.py
+3 -3
View File
@@ -44,7 +44,7 @@ class TestH200Fp8ProLowLatency(DSV4ProAime25TestBase):
*multinode_args(2),
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--max-running-requests",
"32",
@@ -88,7 +88,7 @@ class TestH200Fp8ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.88",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--max-running-requests",
"32",
@@ -114,7 +114,7 @@ class TestH200Fp8ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.88",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/ep/test_eplb_mori.py
+1 -1
View File
@@ -69,7 +69,7 @@ common_args = [
"12288",
"--attention-backend",
"aiter",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
]
test/manual/ep/test_moe_deepep_eval_accuracy_large.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase):
"8",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
test/manual/ep/test_mooncake_expert_backup.py
+1 -1
View File
@@ -58,7 +58,7 @@ class TestBackup(CustomTestCase):
"50",
"--chunked-prefill-size",
"512",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
test/manual/mla/test_flashmla_archived.py
+1 -1
View File
@@ -35,7 +35,7 @@ class TestFlashMLAAttnBackend(unittest.TestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
"--attention-backend",
"flashmla",
test/manual/mla/test_mla_deepseek_v3.py
+6 -4
View File
@@ -22,7 +22,9 @@ class TestMLADeepseekV3(CustomTestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"]
if is_cuda():
other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"])
other_args.extend(
["--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"]
)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
@@ -59,7 +61,7 @@ class TestMLADeepseekV3DisableFusedFunc(CustomTestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"]
if is_cuda():
other_args.extend(["--cuda-graph-max-bs", "2"])
other_args.extend(["--cuda-graph-max-bs-decode", "2"])
cls.process = popen_launch_server(
cls.model,
cls.base_url,
@@ -107,7 +109,7 @@ class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase):
"fa3",
"--mem-fraction-static",
"0.8",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
]
)
@@ -145,7 +147,7 @@ class TestDeepseekV3MTP(CustomTestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
"--disable-radix",
"--enable-torch-compile",
test/manual/mla/test_mla_flashinfer_archived.py
+1 -1
View File
@@ -31,7 +31,7 @@ class TestFlashinferMLA(CustomTestCase):
other_args.extend(
[
"--enable-torch-compile",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--attention-backend",
"flashinfer",
test/manual/mla/test_mla_int8_deepseek_v3_archived.py
+2 -2
View File
@@ -31,7 +31,7 @@ class TestMLADeepseekV3ChannelInt8(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
@@ -75,7 +75,7 @@ class TestMLADeepseekV3BlockInt8(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
test/manual/perf/test_bench_one_batch_1gpu.py
+2 -2
View File
@@ -20,7 +20,7 @@ class TestBenchOneBatch1GPU(CustomTestCase):
def test_bs1_small(self):
_, output_throughput, _ = run_bench_one_batch(
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"]
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs-decode", "2"]
)
self.assertGreater(output_throughput, 50)
@@ -42,7 +42,7 @@ class TestBenchOneBatch1GPU(CustomTestCase):
"1024",
"--model-path",
DEFAULT_MODEL_NAME_FOR_TEST,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
]
test/manual/test_kv_events.py
+3 -3
View File
@@ -46,7 +46,7 @@ class TestKvEvents(CustomTestCase):
'{"publisher": "zmq", "topic": "kv-events"}',
"--max-total-tokens",
32,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
2,
"--enable-dp-attention",
"--dp-size",
@@ -190,7 +190,7 @@ class TestKvEvents(CustomTestCase):
'{"publisher": "zmq", "topic": "kv-events"}',
"--max-total-tokens",
64,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
4,
"--enable-dp-attention",
"--dp-size",
@@ -327,7 +327,7 @@ class TestKvEvents(CustomTestCase):
"--max-running-requests",
4,
"--disable-cuda-graph",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
4,
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
test/manual/test_mla_tp.py
+7 -1
View File
@@ -21,7 +21,13 @@ class TestDeepseekTP2(CustomTestCase):
other_args = ["--trust-remote-code"]
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"]
[
"--tp",
"2",
"--enable-torch-compile",
"--cuda-graph-max-bs-decode",
"2",
]
)
cls.process = popen_launch_server(
cls.model,
test/manual/test_qwen3_235b.py
+1 -1
View File
@@ -84,7 +84,7 @@ class TestQwen3235BFP8(unittest.TestCase):
]
MTP_ARGS = [
"--cuda-graph-max-bs=32",
"--cuda-graph-max-bs-decode=32",
"--max-running-requests=32",
]
variants = [
test/manual/test_vertex_endpoint.py
+1 -1
View File
@@ -26,7 +26,7 @@ class TestVertexEndpoint(CustomTestCase):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--cuda-graph-max-bs", 2],
other_args=["--cuda-graph-max-bs-decode", 2],
)
@classmethod
test/manual/vlm/test_anthropic_vision.py
+1 -1
View File
@@ -44,7 +44,7 @@ class TestAnthropicVision(CustomTestCase):
other_args=[
"--trust-remote-code",
"--enable-multimodal",
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
],
)
cls.messages_url = cls.base_url + "/v1/messages"
test/registered/8-gpu-models/test_gpt_oss_120b.py
+2 -2
View File
@@ -26,14 +26,14 @@ class TestGptOss120B(unittest.TestCase):
base_args = [
"--tp=8",
"--trust-remote-code",
"--cuda-graph-max-bs=200",
"--cuda-graph-max-bs-decode=200",
"--mem-fraction-static=0.93",
]
# Lower batch size for EAGLE3 variants to avoid OOM
base_args_eagle3 = [
"--tp=8",
"--trust-remote-code",
"--cuda-graph-max-bs=100",
"--cuda-graph-max-bs-decode=100",
"--mem-fraction-static=0.85",
]
parser_args = [
test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py
+1 -1
View File
@@ -138,7 +138,7 @@ DEEPSEEK_R1_MODELS = [
"131072",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--enable-torch-compile",
"--disable-cuda-graph",
test/registered/amd/accuracy/mi30x/test_deepseek_v32_tc_eval_amd.py
+1 -1
View File
@@ -56,7 +56,7 @@ class TestDeepseekV32TC(CustomTestCase):
"131072",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--enable-torch-compile",
"--disable-cuda-graph",
test/registered/amd/test_moriep_small.py
+1 -1
View File
@@ -72,7 +72,7 @@ common_args = [
"32768",
"--attention-backend",
"aiter",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
]
test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py
+1 -1
View File
@@ -39,7 +39,7 @@ class TestAscendTp4Bf16(CustomTestCase):
"--attention-backend",
"ascend",
"--disable-radix-cache",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
4,
test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py
+1 -1
View File
@@ -25,7 +25,7 @@ class TestLlama(GSM8KAscendMixin, CustomTestCase):
32,
"--attention-backend",
"ascend",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
2,
test/registered/ascend/llm_models/test_npu_qwen3_30b.py
+1 -1
View File
@@ -28,7 +28,7 @@ class TestQwen330B(GSM8KAscendMixin, CustomTestCase):
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
2,
test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py
+1 -1
View File
@@ -37,7 +37,7 @@ class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase):
"2",
"--attn-cp-size",
"2",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--enable-prefill-context-parallel",
]
test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py
+1 -1
View File
@@ -26,7 +26,7 @@ class TestQwen317BGPTQInt8(GSM8KAscendMixin, CustomTestCase):
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
2,
test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py
+1 -1
View File
@@ -25,7 +25,7 @@ class TestQwen38BCommQuantization(GSM8KAscendMixin, CustomTestCase):
32,
"--attention-backend",
"ascend",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
32,
"--tp-size",
2,
test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py
+1 -1
View File
@@ -11,7 +11,7 @@ class TestGLM4Models(TestVLMModels):
mmmu_accuracy = 0.2
other_args = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--enable-multimodal",
"--mem-fraction-static",
test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py
+1 -1
View File
@@ -18,7 +18,7 @@ class TestQwen25VL72B(TestVLMModels):
mmmu_accuracy = 0.2
other_args = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--enable-multimodal",
"--mem-fraction-static",
test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py
+1 -1
View File
@@ -20,7 +20,7 @@ class TestQwen3VL235BA22B(TestVLMModels):
mmmu_accuracy = 0.2
other_args = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--enable-multimodal",
"--mem-fraction-static",
test/registered/attention/test_triton_attention_backend.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TestTritonAttnBackend(CustomTestCase):
"--attention-backend",
"triton",
"--enable-torch-compile",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
4,
],
)
test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py
+1 -1
View File
@@ -31,7 +31,7 @@ class TestDeepseekR1Fp8Flashinfer(CustomTestCase):
"8192",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-prefill-tokens",
"8192",
test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py
+1 -1
View File
@@ -28,7 +28,7 @@ class TestFlashinferTrtllmGenAttnBackend(CustomTestCase):
other_args=[
"--attention-backend",
"trtllm_mha",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"512",
"--tp-size",
"4",
test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py
+2 -2
View File
@@ -78,7 +78,7 @@ class FlashinferTrtllmGenMoeBackendBF16Base:
"triton",
"--moe-runner-backend",
cls.backend,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"512",
"--tp-size",
"4",
@@ -261,7 +261,7 @@ class FlashinferTrtllmGenMoeBackendNvFp4OnlineBase:
"triton",
"--moe-runner-backend",
cls.backend,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--tp-size",
"4",
test/registered/backends/test_torch_compile.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TestTorchCompile(CustomTestCase, MMLUMixin):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-torch-compile", "--cuda-graph-max-bs", "4"],
other_args=["--enable-torch-compile", "--cuda-graph-max-bs-decode", "4"],
)
@classmethod
test/registered/core/test_basic_sanity.py
+1 -1
View File
@@ -48,7 +48,7 @@ class TestBasicSanity(
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--mem-fraction-static",
"0.7",
test/registered/core/test_basic_sanity_eagle3.py
+1 -1
View File
@@ -70,7 +70,7 @@ class TestBasicSanityEagle3(
"1",
"--speculative-num-draft-tokens",
"2",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--mem-fraction-static",
"0.7",
test/registered/core/test_srt_endpoint.py
+1 -1
View File
@@ -48,7 +48,7 @@ class TestSRTEndpoint(CustomTestCase):
"--enable-custom-logit-processor",
"--mem-fraction-static",
"0.7",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
),
)
test/registered/cp/test_deepseek_v32_cp_single_node.py
+2 -2
View File
@@ -44,7 +44,7 @@ class TestDeepseekV32CPInSeqSplit(CustomTestCase):
"4",
"--mem-frac",
"0.85",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -110,7 +110,7 @@ class TestDeepseekV32CPRoundRobinSplit(CustomTestCase):
"4",
"--mem-frac",
"0.85",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
test/registered/cp/test_deepseek_v3_cp_single_node.py
+1 -1
View File
@@ -42,7 +42,7 @@ class TestDeepseekV3CPInSeqSplit(CustomTestCase):
"fa3",
"--mem-frac",
"0.7",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
test/registered/cp/test_gqa_prefill_cp_legacy.py
+2 -2
View File
@@ -37,7 +37,7 @@ class TestGQACP2TP2EP2(CustomTestCase):
"--attn-cp-size",
"2",
"--enable-prefill-context-parallel",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -94,7 +94,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase):
"--attn-cp-size",
"2",
"--enable-prefill-context-parallel",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
test/registered/cp/test_gqa_preill_cp.py
+3 -3
View File
@@ -39,7 +39,7 @@ class TestGQACP2TP2EP2(CustomTestCase):
"--enable-prefill-cp",
"--cp-strategy",
"zigzag",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -98,7 +98,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase):
"--enable-prefill-cp",
"--cp-strategy",
"zigzag",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -159,7 +159,7 @@ class TestGQACPCP4EP4(CustomTestCase):
"deepep",
"--attention-backend",
"fa3",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py
+1 -1
View File
@@ -69,7 +69,7 @@ class TestPCGWithNGRAM(PCGSpecBase, unittest.TestCase):
"NGRAM",
"--speculative-num-draft-tokens",
"16",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--mem-fraction-static",
"0.8",
test/registered/dcp/test_dsv31_dcp8_gsm8k.py
+1 -1
View File
@@ -73,7 +73,7 @@ _COMMON_SERVER_ARGS = [
"16384",
"--max-running-requests",
"256",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--attention-backend",
"flashinfer",
test/registered/disaggregation/test_disaggregation_basic.py
+1 -1
View File
@@ -240,7 +240,7 @@ class TestDisaggregationMooncakeSpec(
"4",
"--speculative-num-draft-tokens",
"16",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--dtype=float16",
]
test/registered/disaggregation/test_disaggregation_dsv4.py
+2 -2
View File
@@ -71,7 +71,7 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"128",
@@ -107,7 +107,7 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"128",
test/registered/ep/test_deepep_large.py
+1 -1
View File
@@ -176,7 +176,7 @@ class TestDeepseekV32TBO(CustomTestCase):
"--enable-two-batch-overlap",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
test/registered/ep/test_deepep_small.py
+10 -10
View File
@@ -37,7 +37,7 @@ class TestPureDP(CustomTestCase):
"4",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
@@ -84,7 +84,7 @@ class TestHybridDPTP(CustomTestCase):
"2",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -126,7 +126,7 @@ class TestTP(CustomTestCase):
"4",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"128",
@@ -175,7 +175,7 @@ class TestNoGatherdBuffer(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"512",
@@ -223,7 +223,7 @@ class TestTBO(CustomTestCase):
"--moe-a2a-backend",
"deepep",
"--enable-two-batch-overlap",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
@@ -270,7 +270,7 @@ class TestTBOWithTPAttn(CustomTestCase):
"--moe-a2a-backend",
"deepep",
"--enable-two-batch-overlap",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
@@ -323,7 +323,7 @@ class TestTBOWithTPAttnAndDenseDP(CustomTestCase):
"--moe-a2a-backend",
"deepep",
"--enable-two-batch-overlap",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
@@ -386,7 +386,7 @@ class TestMTP(CustomTestCase):
"3",
"--speculative-num-draft-tokens",
"3",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"64",
@@ -456,7 +456,7 @@ class TestMTPWithTBO(CustomTestCase):
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
"--chunked-prefill-size",
"256",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"128",
@@ -530,7 +530,7 @@ class TestMTPWithTPAttnAndTBO(CustomTestCase):
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
"--chunked-prefill-size",
"256",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"128",
test/registered/ep/test_mooncake_ep_small.py
+1 -1
View File
@@ -53,7 +53,7 @@ class TestTP(CustomTestCase):
"72",
"--chunked-prefill-size",
"512",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
test/registered/gb300/test_deepseek_v4_pro_fp4.py
+2 -2
View File
@@ -53,7 +53,7 @@ BALANCED_ARGS = [
"2",
"--mem-fraction-static",
"0.85",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -70,7 +70,7 @@ HIGH_THROUGHPUT_ARGS = [
"megamoe",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/registered/hicache/test_hicache_storage_mooncake_backend.py
+1 -1
View File
@@ -281,7 +281,7 @@ class TestMooncakeBackendQwen330BCP2(
"--attn-cp-size": 2,
"--enable-prefill-context-parallel": True,
"--trust-remote-code": True,
"--cuda-graph-max-bs": 32,
"--cuda-graph-max-bs-decode": 32,
"--max-running-requests": 32,
"--max-total-tokens": 8192,
"--model-loader-extra-config": (
test/registered/input_embedding/test_input_embeddings.py
+1 -1
View File
@@ -31,7 +31,7 @@ class TestInputEmbeds(CustomTestCase):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--disable-radix", "--cuda-graph-max-bs", 4],
other_args=["--disable-radix", "--cuda-graph-max-bs-decode", 4],
)
cls.texts = [
"The capital of France is",
test/registered/input_embedding/test_input_embeds_chunked.py
+1 -1
View File
@@ -94,7 +94,7 @@ class TestInputEmbedsChunkedAndRetract(CustomTestCase):
"--disable-radix-cache",
"--chunked-prefill-size",
str(CHUNKED_PREFILL_SIZE),
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
],
)
test/registered/jit/test_deepseek_v4_compress_state_runtime_shapes.py
+1 -1
View File
@@ -53,7 +53,7 @@
# --chunked-prefill-size 4096 \
# --max-prefill-tokens 16384 \
# --max-running-requests 32 \
# --cuda-graph-max-bs 16 \
# --cuda-graph-max-bs-decode 16 \
# --enable-metrics \
# --disable-radix-cache
#
test/registered/kv_canary/test_self_e2e_pr_25015.py
+1 -1
View File
@@ -30,7 +30,7 @@ _SPEC_EAGLE_SERVER_ARGS = (
"token_oracle",
"--speculative-algorithm",
"EAGLE",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
str(_CUDA_GRAPH_MAX_BS),
"--max-running-requests",
"32",
test/registered/kv_canary/test_self_e2e_pr_26329.py
+1 -1
View File
@@ -21,7 +21,7 @@ _EAGLE_CHUNKED_SERVER_ARGS = (
"EAGLE",
"--chunked-prefill-size",
str(_CHUNKED_PREFILL_SIZE),
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"1",
"--max-running-requests",
"4",
test/registered/mla/test_flashmla.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TestFlashMLAMTP(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--disable-radix",
"--enable-torch-compile",
test/registered/mla/test_mla_flashinfer.py
+1 -1
View File
@@ -27,7 +27,7 @@ class TestFlashinferMLAMTP(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--enable-torch-compile",
"--torch-compile-max-bs",
test/registered/mla/test_mla_int8_deepseek_v3.py
+2 -2
View File
@@ -28,7 +28,7 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
@@ -91,7 +91,7 @@ class TestDeepseekV3MTPBlockInt8(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
test/registered/model_loading/test_load_weights_from_remote_instance.py
+1 -1
View File
@@ -211,7 +211,7 @@ def init_process_dst(
str(base_gpu_id),
"--tp-size",
str(tp_size),
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
2,
"--tokenizer-path",
model_name,
test/registered/model_loading/test_load_weights_from_remote_instance_npu.py
+1 -1
View File
@@ -222,7 +222,7 @@ def init_process_dst(
str(base_gpu_id),
"--tp-size",
str(tp_size),
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
2,
"--tokenizer-path",
model_name,
test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py
+1 -1
View File
@@ -198,7 +198,7 @@ class TestDSV4FlashFP4BreakableCudaGraphB200(
"1024",
"--mem-fraction-static",
"0.80",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--max-running-requests",
"128",
test/registered/models_e2e/test_deepseek_v4_flash_fp8_h200.py
+1 -1
View File
@@ -66,7 +66,7 @@ class TestDSV4FlashFP8H200(
"1",
"--speculative-num-draft-tokens",
"2",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"128",
test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py
+1 -1
View File
@@ -15,7 +15,7 @@ class TestGptOss4GpuBf16(BaseTestGptOss):
expected_score_of_reasoning_effort={
"low": 0.58,
},
other_args=["--tp", "4", "--cuda-graph-max-bs", "200"],
other_args=["--tp", "4", "--cuda-graph-max-bs-decode", "200"],
)
test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py
+1 -1
View File
@@ -18,7 +18,7 @@ class TestGptOss4GpuMxfp4(BaseTestGptOss):
other_args=[
"--tp",
"4",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"200",
],
)
test/registered/models_e2e/test_mimo_v2_flash.py
+1 -1
View File
@@ -26,7 +26,7 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
"fa3",
"--max-running-requests",
"128",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--page-size",
"64",
test/registered/moe/test_hybrid_dp_ep_tp_mtp.py
+20 -20
View File
@@ -414,7 +414,7 @@ class Test10(CustomTestCase):
"8",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -455,7 +455,7 @@ class Test11(CustomTestCase):
"4",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -496,7 +496,7 @@ class Test12(CustomTestCase):
"8",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -536,7 +536,7 @@ class Test13(CustomTestCase):
"1",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -579,7 +579,7 @@ class Test14(CustomTestCase):
"1",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -622,7 +622,7 @@ class Test15(CustomTestCase):
"1",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -664,7 +664,7 @@ class Test16(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -706,7 +706,7 @@ class Test17(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -750,7 +750,7 @@ class Test18(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -794,7 +794,7 @@ class Test19(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
@@ -1708,7 +1708,7 @@ class Test40(CustomTestCase):
"8",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -1761,7 +1761,7 @@ class Test41(CustomTestCase):
"4",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -1814,7 +1814,7 @@ class Test42(CustomTestCase):
"8",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -1866,7 +1866,7 @@ class Test43(CustomTestCase):
"1",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -1921,7 +1921,7 @@ class Test44(CustomTestCase):
"1",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -1976,7 +1976,7 @@ class Test45(CustomTestCase):
"1",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -2030,7 +2030,7 @@ class Test46(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -2084,7 +2084,7 @@ class Test47(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -2140,7 +2140,7 @@ class Test48(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
@@ -2196,7 +2196,7 @@ class Test49(CustomTestCase):
"--enable-dp-lm-head",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
test/registered/musa/test_llm_server_smoke_musa.py
+1 -1
View File
@@ -28,7 +28,7 @@ class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase):
served_model_name,
"--attention-backend",
"fa3",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--tp-size",
"1",
test/registered/observability/test_metrics.py
+6 -1
View File
@@ -103,7 +103,12 @@ class TestEnableMetrics(CustomTestCase):
envs.SGLANG_ENABLE_METRICS_DEVICE_TIMER.override(True),
envs.SGLANG_TEST_RETRACT.override(True),
):
launch_args = ["--enable-metrics", "--cuda-graph-max-bs", 2, *other_args]
launch_args = [
"--enable-metrics",
"--cuda-graph-max-bs-decode",
2,
*other_args,
]
if enable_mfu_metrics:
launch_args.insert(1, "--enable-mfu-metrics")
process = popen_launch_server(
test/registered/perf/test_bench_one_batch_2gpu.py
+3 -2
View File
@@ -19,7 +19,8 @@ class TestBenchOneBatch2GPU(CustomTestCase):
def test_moe_tp2_bs1(self):
output_throughput = run_bench_offline_throughput(
DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"]
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
["--tp", "2", "--cuda-graph-max-bs-decode", "2"],
)
if is_in_ci():
@@ -35,7 +36,7 @@ class TestBenchOneBatch2GPU(CustomTestCase):
def test_torch_compile_tp2_bs1(self):
output_throughput = run_bench_offline_throughput(
DEFAULT_MODEL_NAME_FOR_TEST,
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"],
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"],
)
if is_in_ci():
test/registered/perf/test_gpt_oss_4gpu_perf.py
+1 -1
View File
@@ -18,7 +18,7 @@ class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
[
"--tp",
"4",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"200",
"--mem-fraction-static",
"0.93",
test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py
+1 -1
View File
@@ -75,7 +75,7 @@ class TestUnifiedQwen3HiCachePP(UnifiedRadixTreeTestMixin, CustomTestCase):
"2",
"--mem-fraction-static",
"0.8",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
str(cls.max_running_requests),
test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py
+1 -1
View File
@@ -43,7 +43,7 @@ class TestUnifiedQwen3HiCacheCP(UnifiedRadixTreeTestMixin, CustomTestCase):
"--enable-prefill-context-parallel",
"--mem-fraction-static",
"0.8",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
str(cls.max_running_requests),
test/registered/rl/test_return_routed_experts.py
+1 -1
View File
@@ -66,7 +66,7 @@ class TestReturnRoutedExperts(CustomTestCase):
"deepep",
# Force normal-mode dispatch: deepep auto routes decode through
# low_latency mode whose buffer (num_max_dispatch_tokens_per_rank)
# is undersized for cuda graph capture at default --cuda-graph-max-bs.
# is undersized for cuda graph capture at default --cuda-graph-max-bs-decode.
"--deepep-mode",
"normal",
]
test/registered/rl/test_update_weights_from_distributed.py
+1 -1
View File
@@ -334,7 +334,7 @@ def init_process_sgl(
str(base_gpu_id),
"--tp-size",
str(tp_size),
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
2,
),
)
test/registered/spec/eagle/test_eagle_dp_attention.py
+1 -1
View File
@@ -57,7 +57,7 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
"triton" if is_in_amd_ci() else "fa3",
"--mem-fraction-static",
"0.75",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
]
with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1):
test/registered/spec/eagle/test_spec_eagle_fa3.py
+1 -1
View File
@@ -44,7 +44,7 @@ class TestEagleLlama2Fa3Page256(
attention_backend = "fa3"
page_size = 256
chunked_prefill_size = 4096 # must be divisible by page_size (256)
cuda_graph_max_bs = 5
cuda_graph_max_bs_decode = 5
env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),)
test/registered/spec/eagle/test_spec_eagle_stress.py
+1 -1
View File
@@ -48,7 +48,7 @@ class TestEagle3Topk16V2Retract(Eagle3Base, SpecAccuracyKit, SpecFeatureKit):
spec_topk = 16
spec_tokens = 64
disable_overlap = False
cuda_graph_max_bs = 5
cuda_graph_max_bs_decode = 5
max_running_requests = 64
gsm8k_accept_len_thres = 2.4
extra_args = ("--max-total-tokens", 4500) # small KV to trigger retract
test/registered/spec/eagle/test_spec_eagle_topk.py
+2 -2
View File
@@ -28,7 +28,7 @@ class TestEagle3Topk16(Eagle3Base, SpecCorrectnessKit, SpecAccuracyKit, SpecLogp
spec_topk = 16
spec_tokens = 64
disable_overlap = True # synchronous baseline; SpecV2 subclass flips overlap on
cuda_graph_max_bs = 5
cuda_graph_max_bs_decode = 5
acc_length_thres = 3.1
batch_accept_len_thres = 1.75
gsm8k_accept_len_thres = 2.4 # EAGLE3 topk16 gsm8k accept ~2.48
@@ -69,7 +69,7 @@ class TestEagleLlama3TokenMap(EagleLlama2Base, SpecAccuracyKit):
draft_model = "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B"
spec_topk = 4
spec_tokens = 8
cuda_graph_max_bs = 5
cuda_graph_max_bs_decode = 5
gsm8k_accept_len_thres = 2.5 # FR-Spec token map lowers accept (~2.57)
extra_args = (
"--speculative-token-map",
test/registered/spec/eagle/test_spec_eagle_topk_page.py
+1 -1
View File
@@ -26,7 +26,7 @@ class TestEagle3Page64Topk8(Eagle3Base, SpecAccuracyKit, SpecFeatureKit):
spec_topk = 8
spec_tokens = 32
disable_overlap = False
cuda_graph_max_bs = 5
cuda_graph_max_bs_decode = 5
env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),)
test/registered/spec/eagle/test_spec_eagle_triton.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TestEagle3Triton(
attention_backend = "triton"
max_running_requests = 64
cuda_graph_max_bs = 64
cuda_graph_max_bs_decode = 64
gsm8k_num_examples = 1000
gsm8k_check_accept_len = False
env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),)
test/registered/vlm/test_encoder_dp.py
+2 -2
View File
@@ -19,11 +19,11 @@ MODELS = [
class TestVLMEncoderDP(MMMUMultiModelTestBase):
# --cuda-graph-max-bs 32 last-wins over the kit's default 64.
# --cuda-graph-max-bs-decode 32 last-wins over the kit's default 64.
other_args = [
"--mm-enable-dp-encoder",
"--tp=4",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
]
test/registered/vlm/test_vision_openai_server_a.py
+8 -8
View File
@@ -33,13 +33,13 @@ class TestLfm2VlServer(ImageOpenAITestMixin):
class TestQwen25VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
model = "Qwen/Qwen2.5-VL-7B-Instruct"
extra_args = [
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
]
class TestQwen3VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
model = "Qwen/Qwen3-VL-30B-A3B-Instruct"
extra_args = ["--cuda-graph-max-bs=4"]
extra_args = ["--cuda-graph-max-bs-decode=4"]
class TestQwen3OmniServer(OmniOpenAITestMixin):
@@ -66,7 +66,7 @@ class TestQwen2VLContextLengthServer(CustomTestCase):
other_args=[
"--context-length",
"300",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
],
)
@@ -116,7 +116,7 @@ class TestQwen2VLContextLengthServer(CustomTestCase):
class TestInternVL25Server(ImageOpenAITestMixin):
model = "OpenGVLab/InternVL2_5-2B"
extra_args = [
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
]
@@ -124,7 +124,7 @@ class TestInternVL25Server(ImageOpenAITestMixin):
class TestMiniCPMV4Server(ImageOpenAITestMixin):
model = "openbmb/MiniCPM-V-4"
extra_args = [
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
]
@@ -132,14 +132,14 @@ class TestMiniCPMV4Server(ImageOpenAITestMixin):
class TestMiniCPMo26Server(ImageOpenAITestMixin, AudioOpenAITestMixin):
model = "openbmb/MiniCPM-o-2_6"
extra_args = [
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
]
class TestGemma3itServer(ImageOpenAITestMixin):
model = "google/gemma-3-4b-it"
extra_args = [
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
]
@@ -175,7 +175,7 @@ class TestDeepseekOCRServer(TestOpenAIMLLMServerBase):
trust_remote_code = False
extra_args = [
"--mem-fraction-static=0.70",
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
]
def verify_single_image_response_for_ocr(self, response):
test/registered/vlm/test_vlm_tp4.py
+1 -1
View File
@@ -34,7 +34,7 @@ class TestVLMTP4(CustomTestCase):
other_args=[
"--tp-size",
"4",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--mem-fraction-static",
"0.8",