minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[misc] Use --cuda-graph-max-bs-decode in tests, examples, and docs (#29591)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-06-28 18:38:28 -07:00
committed by GitHub
parent 3217410cf6
commit 909123ddb8
140 changed files with 304 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/manual/attention/test_fa3.py
+5 -5
View File
@@ -45,7 +45,7 @@ if OFFLINE_MODE:
# Default server arguments shared across all tests
DEFAULT_SERVER_ARGS = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--attention-backend",
"fa3",
@@ -144,7 +144,7 @@ class TestFlashAttention3SpeculativeDecode(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE3",
@@ -178,7 +178,7 @@ class TestFlashAttention3SpeculativeDecodeTopk(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE3",
@@ -210,7 +210,7 @@ class TestFlashAttention3MLASpeculativeDecode(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE",
@@ -242,7 +242,7 @@ class TestFlashAttention3MLASpeculativeDecodeTopk(BaseFlashAttentionTest):
args = DEFAULT_SERVER_ARGS
args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--speculative-algorithm",
"EAGLE",
test/manual/attention/test_local_attn.py
+1 -1
View File
@@ -25,7 +25,7 @@ class TestFlashAttention3LocalAttn(CustomTestCase):
@classmethod
def get_server_args(cls):
return [
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
"--attention-backend",
"fa3",
test/manual/dsv4/test_b200_pro.py
+3 -3
View File
@@ -60,7 +60,7 @@ class TestB200ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"128",
@@ -83,7 +83,7 @@ class TestB200ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"256",
@@ -108,7 +108,7 @@ class TestB200ProCP(DSV4ProAime25TestBase):
"16384",
"--mem-fraction-static",
"0.78",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/manual/dsv4/test_b300_pro.py
+3 -3
View File
@@ -62,7 +62,7 @@ class TestB300ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"128",
@@ -85,7 +85,7 @@ class TestB300ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.82",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"64",
"--max-running-requests",
"256",
@@ -110,7 +110,7 @@ class TestB300ProCP(DSV4ProAime25TestBase):
"16384",
"--mem-fraction-static",
"0.78",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_flash_mtp_dp4.py
+2 -2
View File
@@ -37,7 +37,7 @@ DSV4_FLASH_ENV = {
"SGLANG_DSV4_FP4_EXPERTS": "0",
# MTP runs ~num_draft_tokens forward passes per step, so the deepep
# dispatch input size scales by that factor. Default 256 (used by the
# plain server) overflows once cuda-graph-max-bs * num_draft_tokens
# plain server) overflows once cuda-graph-max-bs-decode * num_draft_tokens
# > 256. 1024 covers bs=128 * 4 draft tokens with headroom.
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "1024",
}
@@ -66,7 +66,7 @@ class DSV4FlashMTPServerBase(CustomTestCase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_flash_sanity_dp4.py
+3 -3
View File
@@ -65,7 +65,7 @@ class TestDSV4FlashTP4DP4(
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -101,7 +101,7 @@ class TestDSV4FlashTP4EP(
# No --enable-dp-attention by design: covers TP-attn path.
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"64",
@@ -139,7 +139,7 @@ class TestDSV4FlashTP4DP4ChunkedPrefillLarge(
"deepep",
"--chunked-prefill-size",
"16384",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_pd_disagg_nixl.py
+2 -2
View File
@@ -61,7 +61,7 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase):
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -103,7 +103,7 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase):
"deepep",
"--deepep-config",
DEEPEP_CONFIG,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_dsv4_swa_radix_retract.py
+1 -1
View File
@@ -66,7 +66,7 @@ class TestDSV4FlashSWARadixRetract(CustomTestCase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_gb300_pro.py
+3 -3
View File
@@ -62,7 +62,7 @@ class TestGB300ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -85,7 +85,7 @@ class TestGB300ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.9",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
@@ -110,7 +110,7 @@ class TestGB300ProCP(DSV4ProAime25TestBase):
"16384",
"--mem-fraction-static",
"0.88",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/manual/dsv4/test_h200_fp8_flash.py
+2 -2
View File
@@ -53,7 +53,7 @@ class TestH200Fp8FlashBalanced(DSV4FlashAime25TestBase):
"1",
"--speculative-num-draft-tokens",
"2",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"128",
@@ -77,7 +77,7 @@ class TestH200Fp8FlashMaxThroughput(DSV4FlashAime25TestBase):
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/dsv4/test_h200_fp8_pro.py
+3 -3
View File
@@ -44,7 +44,7 @@ class TestH200Fp8ProLowLatency(DSV4ProAime25TestBase):
*multinode_args(2),
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--max-running-requests",
"32",
@@ -88,7 +88,7 @@ class TestH200Fp8ProBalanced(DSV4ProAime25TestBase):
"2",
"--mem-fraction-static",
"0.88",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"8",
"--max-running-requests",
"32",
@@ -114,7 +114,7 @@ class TestH200Fp8ProMaxThroughput(DSV4ProAime25TestBase):
"deepep",
"--mem-fraction-static",
"0.88",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"256",
test/manual/ep/test_eplb_mori.py
+1 -1
View File
@@ -69,7 +69,7 @@ common_args = [
"12288",
"--attention-backend",
"aiter",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
]
test/manual/ep/test_moe_deepep_eval_accuracy_large.py
+1 -1
View File
@@ -32,7 +32,7 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase):
"8",
"--moe-a2a-backend",
"deepep",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
],
)
test/manual/ep/test_mooncake_expert_backup.py
+1 -1
View File
@@ -58,7 +58,7 @@ class TestBackup(CustomTestCase):
"50",
"--chunked-prefill-size",
"512",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"128",
"--max-running-requests",
"512",
test/manual/mla/test_flashmla_archived.py
+1 -1
View File
@@ -35,7 +35,7 @@ class TestFlashMLAAttnBackend(unittest.TestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
"--attention-backend",
"flashmla",
test/manual/mla/test_mla_deepseek_v3.py
+6 -4
View File
@@ -22,7 +22,9 @@ class TestMLADeepseekV3(CustomTestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"]
if is_cuda():
other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"])
other_args.extend(
["--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"]
)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
@@ -59,7 +61,7 @@ class TestMLADeepseekV3DisableFusedFunc(CustomTestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"]
if is_cuda():
other_args.extend(["--cuda-graph-max-bs", "2"])
other_args.extend(["--cuda-graph-max-bs-decode", "2"])
cls.process = popen_launch_server(
cls.model,
cls.base_url,
@@ -107,7 +109,7 @@ class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase):
"fa3",
"--mem-fraction-static",
"0.8",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
]
)
@@ -145,7 +147,7 @@ class TestDeepseekV3MTP(CustomTestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
"--disable-radix",
"--enable-torch-compile",
test/manual/mla/test_mla_flashinfer_archived.py
+1 -1
View File
@@ -31,7 +31,7 @@ class TestFlashinferMLA(CustomTestCase):
other_args.extend(
[
"--enable-torch-compile",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--attention-backend",
"flashinfer",
test/manual/mla/test_mla_int8_deepseek_v3_archived.py
+2 -2
View File
@@ -31,7 +31,7 @@ class TestMLADeepseekV3ChannelInt8(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
@@ -75,7 +75,7 @@ class TestMLADeepseekV3BlockInt8(CustomTestCase):
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
[
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"16",
"--enable-torch-compile",
"--torch-compile-max-bs",
test/manual/perf/test_bench_one_batch_1gpu.py
+2 -2
View File
@@ -20,7 +20,7 @@ class TestBenchOneBatch1GPU(CustomTestCase):
def test_bs1_small(self):
_, output_throughput, _ = run_bench_one_batch(
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"]
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs-decode", "2"]
)
self.assertGreater(output_throughput, 50)
@@ -42,7 +42,7 @@ class TestBenchOneBatch1GPU(CustomTestCase):
"1024",
"--model-path",
DEFAULT_MODEL_NAME_FOR_TEST,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"2",
]
test/manual/test_kv_events.py
+3 -3
View File
@@ -46,7 +46,7 @@ class TestKvEvents(CustomTestCase):
'{"publisher": "zmq", "topic": "kv-events"}',
"--max-total-tokens",
32,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
2,
"--enable-dp-attention",
"--dp-size",
@@ -190,7 +190,7 @@ class TestKvEvents(CustomTestCase):
'{"publisher": "zmq", "topic": "kv-events"}',
"--max-total-tokens",
64,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
4,
"--enable-dp-attention",
"--dp-size",
@@ -327,7 +327,7 @@ class TestKvEvents(CustomTestCase):
"--max-running-requests",
4,
"--disable-cuda-graph",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
4,
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
test/manual/test_mla_tp.py
+7 -1
View File
@@ -21,7 +21,13 @@ class TestDeepseekTP2(CustomTestCase):
other_args = ["--trust-remote-code"]
if torch.cuda.is_available() and torch.version.cuda:
other_args.extend(
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"]
[
"--tp",
"2",
"--enable-torch-compile",
"--cuda-graph-max-bs-decode",
"2",
]
)
cls.process = popen_launch_server(
cls.model,
test/manual/test_qwen3_235b.py
+1 -1
View File
@@ -84,7 +84,7 @@ class TestQwen3235BFP8(unittest.TestCase):
]
MTP_ARGS = [
"--cuda-graph-max-bs=32",
"--cuda-graph-max-bs-decode=32",
"--max-running-requests=32",
]
variants = [
test/manual/test_vertex_endpoint.py
+1 -1
View File
@@ -26,7 +26,7 @@ class TestVertexEndpoint(CustomTestCase):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--cuda-graph-max-bs", 2],
other_args=["--cuda-graph-max-bs-decode", 2],
)
@classmethod
test/manual/vlm/test_anthropic_vision.py
+1 -1
View File
@@ -44,7 +44,7 @@ class TestAnthropicVision(CustomTestCase):
other_args=[
"--trust-remote-code",
"--enable-multimodal",
"--cuda-graph-max-bs=4",
"--cuda-graph-max-bs-decode=4",
],
)
cls.messages_url = cls.base_url + "/v1/messages"