minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[cookbook] drop redundant serve flags (GLM-5.2) + fix M3 page-size note (#28731)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-06-29 13:34:13 +08:00
committed by GitHub
parent bb74ed4a8d
commit 38d4ffcd86
5 changed files with 63 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx
+4
View File
@@ -52,6 +52,10 @@ import { benchmarks } from "/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx
<Deployment config={config} benchmarks={benchmarks} />
<Note>
Speed numbers are measured with `--random-range-ratio 1.0`, `--flush-cache`, on `main @ 09ca4fc`. Spec cells pin the EAGLE acceptance length via the serve env `SGLANG_SIMULATE_ACC_LEN` (low-latency 5-1-6 = 3.5, balanced 1-1-2 = 2); high-throughput has no spec.
</Note>
<Note>
**NVFP4 (B300 / GB300)** deploys on the dev image **`lmsysorg/sglang:dev-glm52-nvfp4`** — the command panel's **Docker** toggle selects it automatically. The FP8 / BF16 recipes use the release **`lmsysorg/sglang:latest`** (the `docker pull` in Install above).
</Note>
docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx
+2 -2
View File
@@ -117,10 +117,10 @@ The first import JIT-compiles the kernels, which can take 30 s to a few minutes
**Warm the JIT cache before a multi-GPU launch.** On a *cold* cache, several tensor-parallel ranks racing to JIT-compile MSA's plan kernel can leave one rank loading a half-linked module (`AttributeError: Module has no function 'plan'` at CUDA-graph capture). Run the gate-check `python -c "..."` (or any single-process `fmha_sm100_plan` call) once before launching the server — that compiles the kernel single-process, and every rank then hits the warm cache.
</Warning>
The gate requires `--attention-backend fa4 --page-size 128` (already part of the Blackwell recipe above; on current `main` these are also the auto-selected M3 defaults on SM100 GPUs). Force the Triton path at any time with the env var `SGLANG_DISABLE_MSA=1`. MSA is a Blackwell (SM100) kernel and does not apply to the AMD ROCm paths.
The gate requires `--attention-backend fa4` (MSA's sparse blocks are 128 tokens, so the page size must be 128). SGLang auto-forces `page_size` to 128 for the `fa4` backend — including the combined `--attention-backend fa4` the M3 recipe uses (#28976) — so `--page-size 128` is omitted from the Blackwell cells below. Force the Triton path at any time with the env var `SGLANG_DISABLE_MSA=1`. MSA is a Blackwell (SM100) kernel and does not apply to the AMD ROCm paths.
<Note>
For multimodal (image) serving, keep the same text recipe above — `--attention-backend fa4 --page-size 128` (MSA) is unchanged — and add `--mm-attention-backend flashinfer_cudnn` for the vision tower. The text and vision-tower attention backends are independent knobs; MSA only touches the language-model sparse attention, not image handling.
For multimodal (image) serving, keep the same text recipe above — `--attention-backend fa4` (MSA) is unchanged — and add `--mm-attention-backend flashinfer_cudnn` for the vision tower. The text and vision-tower attention backends are independent knobs; MSA only touches the language-model sparse attention, not image handling.
</Note>
### 2.2 Memory and workload tuning
docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx
-4
View File
@@ -211,7 +211,6 @@ sgl-eval run mmmu_pro \\
"--tool-call-parser auto",
"--tp 8",
"--attention-backend fa4",
"--page-size 128",
"--moe-runner-backend deep_gemm",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.65",
@@ -230,7 +229,6 @@ sgl-eval run mmmu_pro \\
"--tool-call-parser auto",
"--tp 4",
"--attention-backend fa4",
"--page-size 128",
"--moe-runner-backend deep_gemm",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.75",
@@ -250,7 +248,6 @@ sgl-eval run mmmu_pro \\
"--tool-call-parser auto",
"--tp 4",
"--attention-backend fa4",
"--page-size 128",
"--moe-runner-backend deep_gemm",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.75",
@@ -269,7 +266,6 @@ sgl-eval run mmmu_pro \\
"--tool-call-parser auto",
"--tp 4",
"--attention-backend fa4",
"--page-size 128",
"--moe-runner-backend deep_gemm",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.75",
docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx
+49 -156
View File
@@ -3,164 +3,95 @@
// Numbers pending: each entry is a bare `match` stub (renders "pending") until measured
// end-to-end on the corresponding hardware, then filled with sglang_version + speed/accuracy.
export const benchmarks = [
// ---- H200 + FP8 ---- (measured on the v0.5.13.post1 release image, flush-cache on every run)
// ---- H200 + FP8 ---- (serve recipe in glm-5.2.jsx; benchmark pending re-measurement)
{ match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } },
{ match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
{ match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } },
// ---- B200 + FP8 ---- (8-GPU single node, TP8; real weights, --random-range-ratio 1.0, flush-cache every run)
{
// EAGLE MTP 5-1-6 (was 3-1-4): accept ~5.96/6 → +31%/+15% throughput, -25%/-11% TPOT vs 3-1-4.
// KV stays bf16 (Hopper auto-default). fp8 KV measured worse on H200 (slower flashmla_kv prefill
// + lower decode throughput): conc=1 31 gpu / TTFT 838, conc=16 96 gpu / TTFT 6650.
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 662, tpot_ms: 3.03, tokens_per_sec_per_gpu: 34 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 5080, tpot_ms: 12.44, tokens_per_sec_per_gpu: 113 },
],
},
{
// Tuned prefill (chunked-prefill 32768 + max-running 80): +44%/+78% throughput and
// -59%/-49% TTFT vs the untuned default-chunked (2048) baseline (152/133 gpu) on post1.
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 8013, tpot_ms: 25.57, tokens_per_sec_per_gpu: 219 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 77790, tpot_ms: 29.08, tokens_per_sec_per_gpu: 236 },
],
},
{
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 450276, tpot_ms: 86.71, tokens_per_sec_per_gpu: 184 },
],
},
// ---- B200 + FP8 ---- (measured on the v0.5.13.post1 release image, flush-cache on every run)
{
// EAGLE MTP 5-1-6 (was 3-1-4): accept length ~5.98/6 → +33%/+22% throughput, -26%/-15% TPOT vs 3-1-4.
// EAGLE MTP 5-1-6, mfs 0.8, no cuda-graph-max-bs. env SGLANG_SIMULATE_ACC_LEN=3.5
// (match-expected: 50% accept 3 / 50% accept 4) fixes the acceptance length.
match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
sglang_version: "0.5.13.post1",
sglang_version: "main @ 09ca4fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 334, tpot_ms: 2.30, tokens_per_sec_per_gpu: 48 },
ttft_ms: 757, tpot_ms: 3.22, tokens_per_sec_per_gpu: 32 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 2777, tpot_ms: 6.84, tokens_per_sec_per_gpu: 209 },
ttft_ms: 3188, tpot_ms: 9.12, tokens_per_sec_per_gpu: 164 },
],
},
{
// Re-measured on v0.5.13.post1 with tuned prefill (chunked-prefill 32768 + max-running 80):
// +34%/+44% throughput and -55%/-39% TTFT vs the untuned default-chunked (2048) baseline.
// Balanced: DP8 + deepep + mfs 0.85 + chunked-prefill 32768 + max-running 256, 1-1-2 EAGLE.
// env SGLANG_SIMULATE_ACC_LEN=2 (match-expected: accept 2 of 2 draft tokens).
match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
sglang_version: "0.5.13.post1",
sglang_version: "main @ 09ca4fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 5680, tpot_ms: 18.76, tokens_per_sec_per_gpu: 285 },
ttft_ms: 5742, tpot_ms: 17.65, tokens_per_sec_per_gpu: 342 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 60665, tpot_ms: 23.91, tokens_per_sec_per_gpu: 297 },
ttft_ms: 18744, tpot_ms: 32.61, tokens_per_sec_per_gpu: 558 },
],
},
{
// HT: DP8 + deepep + mfs 0.85 + max-running 256. B200 (178GB) keeps --max-running-requests 256
// (clamps the decode capture list to <=32 < the default 128 DeepEP buffer); no env buffer bump.
// No spec, so no SIMULATE_ACC_LEN.
match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
sglang_version: "0.5.13.post1",
sglang_version: "main @ 09ca4fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 178249, tpot_ms: 48.28, tokens_per_sec_per_gpu: 449 },
ttft_ms: 177620, tpot_ms: 47.99, tokens_per_sec_per_gpu: 451 },
],
},
// ---- GB300 + FP8 ---- (4-GPU single node, TP4; measured on the v0.5.13.post1 release image, flush-cache on every run)
// ---- GB300 + FP8 ---- (4-GPU single node, TP4; real weights, --random-range-ratio 1.0, flush-cache every run)
{
// EAGLE MTP 5-1-6 (was 3-1-4): accept length ~5.98/6 → +34%/+24% throughput, -28%/-18% TPOT vs 3-1-4.
// EAGLE MTP 5-1-6, mfs 0.85, no cuda-graph-max-bs; mrr auto-capped 48. env
// SGLANG_SIMULATE_ACC_LEN=3.5 (match-expected: 50% accept 3 / 50% accept 4) fixes the
// acceptance length so the spec numbers are comparable across runs.
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
sglang_version: "0.5.13.post1",
sglang_version: "main @ 09ca4fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 393, tpot_ms: 2.78, tokens_per_sec_per_gpu: 79 },
ttft_ms: 374, tpot_ms: 4.55, tokens_per_sec_per_gpu: 51 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 3201, tpot_ms: 8.53, tokens_per_sec_per_gpu: 341 },
ttft_ms: 3719, tpot_ms: 11.5, tokens_per_sec_per_gpu: 264 },
],
},
{
// Balanced uses the tuned prefill (chunked-prefill 32768 + max-running 80), same lever as H200/B200.
// Balanced: DP4 + deepep + mfs 0.85 + chunked-prefill 32768 (÷dp4 = 8192) + max-running 256,
// 1-1-2 EAGLE. env SGLANG_SIMULATE_ACC_LEN=2 (match-expected: accept 2 of 2 draft tokens).
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
sglang_version: "0.5.13.post1",
sglang_version: "main @ 09ca4fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 7678, tpot_ms: 27.77, tokens_per_sec_per_gpu: 411 },
ttft_ms: 7429, tpot_ms: 25.21, tokens_per_sec_per_gpu: 493 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 76359, tpot_ms: 31.08, tokens_per_sec_per_gpu: 483 },
ttft_ms: 27488, tpot_ms: 48.43, tokens_per_sec_per_gpu: 756 },
],
},
// GB300 HT: drop-flags (mfs/cgbs/mrr dropped) + env SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=512.
// DeepEP low_latency asserts x.size(0) <= num_max_dispatch_tokens_per_rank (deep_ep.cpp:1262,
// default 128). Decode cuda-graph capture builds a dummy batch of `bs` tokens per rank (not
// DP-split), so the 256/512 capture buckets trip the assert at default 128; GB300 (DP4) also
// hits bs/4 = 256 > 128 at c1024 runtime. Raising the buffer to 512 fixes both and lets HT
// drop --max-running-requests. Verified on main: with env=512 capture + serve pass; without
// env the assert trips at the bs=512 capture bucket. No spec, so no SIMULATE_ACC_LEN.
{
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
sglang_version: "0.5.13.post1",
sglang_version: "main @ 09ca4fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 250727, tpot_ms: 68.55, tokens_per_sec_per_gpu: 641 },
],
},
// ---- B300 + FP8 ---- (8-GPU single node, TP8; measured on v0.5.13.post1, flush-cache every run.
// B300 (sm103) trails B200 (sm100) per-GPU here — the deep_gemm/DSA kernels are tuned for sm100 and
// fall to a slower path on sm103; the gap should close as sm103 gets first-class kernels.)
{
match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 503, tpot_ms: 3.24, tokens_per_sec_per_gpu: 34 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 4731, tpot_ms: 9.56, tokens_per_sec_per_gpu: 140 },
],
},
{
match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 6465, tpot_ms: 23.36, tokens_per_sec_per_gpu: 245 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 67814, tpot_ms: 26.19, tokens_per_sec_per_gpu: 265 },
],
},
{
match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 206246, tpot_ms: 56.11, tokens_per_sec_per_gpu: 388 },
],
},
// ---- B300 + BF16 ---- (unquantized zai-org/GLM-5.2, TP8; measured on v0.5.13.post1, flush-cache every run.
// balanced/HT run plain TP8 (no DP-Attention/DeepEP), so they trail the FP8 dp-attention recipe at high concurrency.)
{
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 470, tpot_ms: 2.93, tokens_per_sec_per_gpu: 37 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 3474, tpot_ms: 10.33, tokens_per_sec_per_gpu: 146 },
],
},
{
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 14123, tpot_ms: 35.47, tokens_per_sec_per_gpu: 157 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 116633, tpot_ms: 40.65, tokens_per_sec_per_gpu: 167 },
],
},
{
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
sglang_version: "0.5.13.post1",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 525108, tpot_ms: 82.52, tokens_per_sec_per_gpu: 168 },
ttft_ms: 231101, tpot_ms: 86.01, tokens_per_sec_per_gpu: 671 },
],
},
// ---- B300 + FP8 ---- (8-GPU single node, TP8; serve recipe in glm-5.2.jsx; benchmark pending re-measurement)
{ match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } },
{ match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
{ match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } },
// ---- B300 + BF16 ---- (unquantized zai-org/GLM-5.2, TP8; serve recipe in glm-5.2.jsx; benchmark pending re-measurement)
{ match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } },
{ match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } },
{ match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } },
// ---- BF16 multi-node (inferred) ---- benchmarks pending
{ match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } },
{ match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } },
@@ -171,42 +102,4 @@ export const benchmarks = [
{ match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } },
{ match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } },
{ match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" } },
// ---- NVFP4 (Blackwell Ultra) ----
{ match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } },
{ match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } },
// ---- GB300 + NVFP4 ---- (4-GPU single node, TP4; nvidia/GLM-5.2-NVFP4 via --quantization modelopt_fp4,
// measured on the lmsysorg/sglang:dev-glm52-nvfp4 preview image, flush-cache every run.
// tokens_per_sec_per_gpu = total server output tok/s / 4 GPUs (337→84, 1248→312, 1162→291, 1695→424, 1730→433).
// aime25 overrides the variant default (87.7 → 89.58, measured on this NVFP4 build); gsm8k inherits the default.)
{
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "dev-glm52-nvfp4",
accuracy: { aime25_pct: 89.58 },
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 238, tpot_ms: 2.23, tokens_per_sec_per_gpu: 84 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 315, tpot_ms: 11.9, tokens_per_sec_per_gpu: 312 },
],
},
{
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
sglang_version: "dev-glm52-nvfp4",
accuracy: { aime25_pct: 89.58 },
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 1169, tpot_ms: 58, tokens_per_sec_per_gpu: 291 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 6389, tpot_ms: 167, tokens_per_sec_per_gpu: 424 },
],
},
{
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
sglang_version: "dev-glm52-nvfp4",
accuracy: { aime25_pct: 89.58 },
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 156000, tpot_ms: 321, tokens_per_sec_per_gpu: 433 },
],
},
];
docs_new/src/snippets/configs/zai-org/glm-5.2.jsx
+8 -30
View File
@@ -55,6 +55,7 @@ export const config = {
--model {{MODEL_NAME}} \\
--dataset-name {{DATASET}} \\
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
--random-range-ratio 1.0 \\
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
--warmup-requests 64 --flush-cache`,
accuracy: {
@@ -185,7 +186,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.8",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -205,11 +205,10 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
// Large chunked-prefill is the dominant balanced lever (prefill-bound at this
// concurrency); max-running tracks KV capacity (~60-80 for 8K+1K reqs on 8xH200).
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -225,7 +224,6 @@ sgl-eval run aime25 \\
"--enable-dp-attention",
"--moe-a2a-backend deepep",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
@@ -247,7 +245,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.8",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -267,11 +264,10 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
// Large chunked-prefill is the dominant balanced lever (prefill-bound at this
// concurrency); max-running tracks KV capacity (~89 for 8K+1K reqs on 8xB200).
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -287,7 +283,6 @@ sgl-eval run aime25 \\
"--enable-dp-attention",
"--moe-a2a-backend deepep",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
@@ -312,7 +307,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -332,10 +326,9 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
// Same prefill lever as H200/B200 balanced; max-running tracks the TP4 KV capacity.
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -343,7 +336,9 @@ sgl-eval run aime25 \\
{
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
verified: true,
env: [],
env: [
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=512",
],
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 4",
@@ -351,8 +346,6 @@ sgl-eval run aime25 \\
"--enable-dp-attention",
"--moe-a2a-backend deepep",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -375,7 +368,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.8",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -395,9 +387,8 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -413,7 +404,6 @@ sgl-eval run aime25 \\
"--enable-dp-attention",
"--moe-a2a-backend deepep",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
@@ -438,7 +428,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.9",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -455,7 +444,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.9",
"--cuda-graph-max-bs-decode 128",
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--host {{HOST_IP}}",
@@ -470,7 +458,6 @@ sgl-eval run aime25 \\
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--mem-fraction-static 0.9",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
@@ -495,7 +482,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -512,7 +498,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--host {{HOST_IP}}",
@@ -527,7 +512,6 @@ sgl-eval run aime25 \\
"--model-path {{MODEL_NAME}}",
"--tp 16",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
@@ -545,7 +529,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -562,7 +545,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--host {{HOST_IP}}",
@@ -577,7 +559,6 @@ sgl-eval run aime25 \\
"--model-path {{MODEL_NAME}}",
"--tp 16",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",
@@ -595,7 +576,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 32",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -612,7 +592,6 @@ sgl-eval run aime25 \\
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 2",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 128",
"--chunked-prefill-size 32768",
"--max-running-requests 80",
"--host {{HOST_IP}}",
@@ -627,7 +606,6 @@ sgl-eval run aime25 \\
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--mem-fraction-static 0.85",
"--cuda-graph-max-bs-decode 256",
"--max-running-requests 256",
"--host {{HOST_IP}}",
"--port {{PORT}}",