diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx index 79b8bcdcd..0e9dc263f 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -52,6 +52,10 @@ import { benchmarks } from "/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx + + Speed numbers are measured with `--random-range-ratio 1.0`, `--flush-cache`, on `main @ 09ca4fc`. Spec cells pin the EAGLE acceptance length via the serve env `SGLANG_SIMULATE_ACC_LEN` (low-latency 5-1-6 = 3.5, balanced 1-1-2 = 2); high-throughput has no spec. + + **NVFP4 (B300 / GB300)** deploys on the dev image **`lmsysorg/sglang:dev-glm52-nvfp4`** — the command panel's **Docker** toggle selects it automatically. The FP8 / BF16 recipes use the release **`lmsysorg/sglang:latest`** (the `docker pull` in Install above). diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx index a0123e0b3..bc081ee38 100644 --- a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx @@ -117,10 +117,10 @@ The first import JIT-compiles the kernels, which can take 30 s to a few minutes **Warm the JIT cache before a multi-GPU launch.** On a *cold* cache, several tensor-parallel ranks racing to JIT-compile MSA's plan kernel can leave one rank loading a half-linked module (`AttributeError: Module has no function 'plan'` at CUDA-graph capture). Run the gate-check `python -c "..."` (or any single-process `fmha_sm100_plan` call) once before launching the server — that compiles the kernel single-process, and every rank then hits the warm cache. -The gate requires `--attention-backend fa4 --page-size 128` (already part of the Blackwell recipe above; on current `main` these are also the auto-selected M3 defaults on SM100 GPUs). Force the Triton path at any time with the env var `SGLANG_DISABLE_MSA=1`. MSA is a Blackwell (SM100) kernel and does not apply to the AMD ROCm paths. +The gate requires `--attention-backend fa4` (MSA's sparse blocks are 128 tokens, so the page size must be 128). SGLang auto-forces `page_size` to 128 for the `fa4` backend — including the combined `--attention-backend fa4` the M3 recipe uses (#28976) — so `--page-size 128` is omitted from the Blackwell cells below. Force the Triton path at any time with the env var `SGLANG_DISABLE_MSA=1`. MSA is a Blackwell (SM100) kernel and does not apply to the AMD ROCm paths. -For multimodal (image) serving, keep the same text recipe above — `--attention-backend fa4 --page-size 128` (MSA) is unchanged — and add `--mm-attention-backend flashinfer_cudnn` for the vision tower. The text and vision-tower attention backends are independent knobs; MSA only touches the language-model sparse attention, not image handling. +For multimodal (image) serving, keep the same text recipe above — `--attention-backend fa4` (MSA) is unchanged — and add `--mm-attention-backend flashinfer_cudnn` for the vision tower. The text and vision-tower attention backends are independent knobs; MSA only touches the language-model sparse attention, not image handling. ### 2.2 Memory and workload tuning diff --git a/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx b/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx index f6396c707..a710982fd 100644 --- a/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx +++ b/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx @@ -211,7 +211,6 @@ sgl-eval run mmmu_pro \\ "--tool-call-parser auto", "--tp 8", "--attention-backend fa4", - "--page-size 128", "--moe-runner-backend deep_gemm", "--chunked-prefill-size 8192", "--mem-fraction-static 0.65", @@ -230,7 +229,6 @@ sgl-eval run mmmu_pro \\ "--tool-call-parser auto", "--tp 4", "--attention-backend fa4", - "--page-size 128", "--moe-runner-backend deep_gemm", "--chunked-prefill-size 8192", "--mem-fraction-static 0.75", @@ -250,7 +248,6 @@ sgl-eval run mmmu_pro \\ "--tool-call-parser auto", "--tp 4", "--attention-backend fa4", - "--page-size 128", "--moe-runner-backend deep_gemm", "--chunked-prefill-size 8192", "--mem-fraction-static 0.75", @@ -269,7 +266,6 @@ sgl-eval run mmmu_pro \\ "--tool-call-parser auto", "--tp 4", "--attention-backend fa4", - "--page-size 128", "--moe-runner-backend deep_gemm", "--chunked-prefill-size 8192", "--mem-fraction-static 0.75", diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx index f404d5aa0..562df1039 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx @@ -3,164 +3,95 @@ // Numbers pending: each entry is a bare `match` stub (renders "pending") until measured // end-to-end on the corresponding hardware, then filled with sglang_version + speed/accuracy. export const benchmarks = [ - // ---- H200 + FP8 ---- (measured on the v0.5.13.post1 release image, flush-cache on every run) + // ---- H200 + FP8 ---- (serve recipe in glm-5.2.jsx; benchmark pending re-measurement) + { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + // ---- B200 + FP8 ---- (8-GPU single node, TP8; real weights, --random-range-ratio 1.0, flush-cache every run) { - // EAGLE MTP 5-1-6 (was 3-1-4): accept ~5.96/6 → +31%/+15% throughput, -25%/-11% TPOT vs 3-1-4. - // KV stays bf16 (Hopper auto-default). fp8 KV measured worse on H200 (slower flashmla_kv prefill - // + lower decode throughput): conc=1 31 gpu / TTFT 838, conc=16 96 gpu / TTFT 6650. - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 662, tpot_ms: 3.03, tokens_per_sec_per_gpu: 34 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 5080, tpot_ms: 12.44, tokens_per_sec_per_gpu: 113 }, - ], - }, - { - // Tuned prefill (chunked-prefill 32768 + max-running 80): +44%/+78% throughput and - // -59%/-49% TTFT vs the untuned default-chunked (2048) baseline (152/133 gpu) on post1. - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 8013, tpot_ms: 25.57, tokens_per_sec_per_gpu: 219 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 77790, tpot_ms: 29.08, tokens_per_sec_per_gpu: 236 }, - ], - }, - { - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 450276, tpot_ms: 86.71, tokens_per_sec_per_gpu: 184 }, - ], - }, - // ---- B200 + FP8 ---- (measured on the v0.5.13.post1 release image, flush-cache on every run) - { - // EAGLE MTP 5-1-6 (was 3-1-4): accept length ~5.98/6 → +33%/+22% throughput, -26%/-15% TPOT vs 3-1-4. + // EAGLE MTP 5-1-6, mfs 0.8, no cuda-graph-max-bs. env SGLANG_SIMULATE_ACC_LEN=3.5 + // (match-expected: 50% accept 3 / 50% accept 4) fixes the acceptance length. match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "main @ 09ca4fc", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 334, tpot_ms: 2.30, tokens_per_sec_per_gpu: 48 }, + ttft_ms: 757, tpot_ms: 3.22, tokens_per_sec_per_gpu: 32 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 2777, tpot_ms: 6.84, tokens_per_sec_per_gpu: 209 }, + ttft_ms: 3188, tpot_ms: 9.12, tokens_per_sec_per_gpu: 164 }, ], }, { - // Re-measured on v0.5.13.post1 with tuned prefill (chunked-prefill 32768 + max-running 80): - // +34%/+44% throughput and -55%/-39% TTFT vs the untuned default-chunked (2048) baseline. + // Balanced: DP8 + deepep + mfs 0.85 + chunked-prefill 32768 + max-running 256, 1-1-2 EAGLE. + // env SGLANG_SIMULATE_ACC_LEN=2 (match-expected: accept 2 of 2 draft tokens). match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "main @ 09ca4fc", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 5680, tpot_ms: 18.76, tokens_per_sec_per_gpu: 285 }, + ttft_ms: 5742, tpot_ms: 17.65, tokens_per_sec_per_gpu: 342 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 60665, tpot_ms: 23.91, tokens_per_sec_per_gpu: 297 }, + ttft_ms: 18744, tpot_ms: 32.61, tokens_per_sec_per_gpu: 558 }, ], }, { + // HT: DP8 + deepep + mfs 0.85 + max-running 256. B200 (178GB) keeps --max-running-requests 256 + // (clamps the decode capture list to <=32 < the default 128 DeepEP buffer); no env buffer bump. + // No spec, so no SIMULATE_ACC_LEN. match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "main @ 09ca4fc", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 178249, tpot_ms: 48.28, tokens_per_sec_per_gpu: 449 }, + ttft_ms: 177620, tpot_ms: 47.99, tokens_per_sec_per_gpu: 451 }, ], }, - // ---- GB300 + FP8 ---- (4-GPU single node, TP4; measured on the v0.5.13.post1 release image, flush-cache on every run) + // ---- GB300 + FP8 ---- (4-GPU single node, TP4; real weights, --random-range-ratio 1.0, flush-cache every run) { - // EAGLE MTP 5-1-6 (was 3-1-4): accept length ~5.98/6 → +34%/+24% throughput, -28%/-18% TPOT vs 3-1-4. + // EAGLE MTP 5-1-6, mfs 0.85, no cuda-graph-max-bs; mrr auto-capped 48. env + // SGLANG_SIMULATE_ACC_LEN=3.5 (match-expected: 50% accept 3 / 50% accept 4) fixes the + // acceptance length so the spec numbers are comparable across runs. match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "main @ 09ca4fc", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 393, tpot_ms: 2.78, tokens_per_sec_per_gpu: 79 }, + ttft_ms: 374, tpot_ms: 4.55, tokens_per_sec_per_gpu: 51 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 3201, tpot_ms: 8.53, tokens_per_sec_per_gpu: 341 }, + ttft_ms: 3719, tpot_ms: 11.5, tokens_per_sec_per_gpu: 264 }, ], }, { - // Balanced uses the tuned prefill (chunked-prefill 32768 + max-running 80), same lever as H200/B200. + // Balanced: DP4 + deepep + mfs 0.85 + chunked-prefill 32768 (÷dp4 = 8192) + max-running 256, + // 1-1-2 EAGLE. env SGLANG_SIMULATE_ACC_LEN=2 (match-expected: accept 2 of 2 draft tokens). match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "main @ 09ca4fc", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 7678, tpot_ms: 27.77, tokens_per_sec_per_gpu: 411 }, + ttft_ms: 7429, tpot_ms: 25.21, tokens_per_sec_per_gpu: 493 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 76359, tpot_ms: 31.08, tokens_per_sec_per_gpu: 483 }, + ttft_ms: 27488, tpot_ms: 48.43, tokens_per_sec_per_gpu: 756 }, ], }, + // GB300 HT: drop-flags (mfs/cgbs/mrr dropped) + env SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=512. + // DeepEP low_latency asserts x.size(0) <= num_max_dispatch_tokens_per_rank (deep_ep.cpp:1262, + // default 128). Decode cuda-graph capture builds a dummy batch of `bs` tokens per rank (not + // DP-split), so the 256/512 capture buckets trip the assert at default 128; GB300 (DP4) also + // hits bs/4 = 256 > 128 at c1024 runtime. Raising the buffer to 512 fixes both and lets HT + // drop --max-running-requests. Verified on main: with env=512 capture + serve pass; without + // env the assert trips at the bs=512 capture bucket. No spec, so no SIMULATE_ACC_LEN. { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "main @ 09ca4fc", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 250727, tpot_ms: 68.55, tokens_per_sec_per_gpu: 641 }, - ], - }, - // ---- B300 + FP8 ---- (8-GPU single node, TP8; measured on v0.5.13.post1, flush-cache every run. - // B300 (sm103) trails B200 (sm100) per-GPU here — the deep_gemm/DSA kernels are tuned for sm100 and - // fall to a slower path on sm103; the gap should close as sm103 gets first-class kernels.) - { - match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 503, tpot_ms: 3.24, tokens_per_sec_per_gpu: 34 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 4731, tpot_ms: 9.56, tokens_per_sec_per_gpu: 140 }, - ], - }, - { - match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 6465, tpot_ms: 23.36, tokens_per_sec_per_gpu: 245 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 67814, tpot_ms: 26.19, tokens_per_sec_per_gpu: 265 }, - ], - }, - { - match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 206246, tpot_ms: 56.11, tokens_per_sec_per_gpu: 388 }, - ], - }, - // ---- B300 + BF16 ---- (unquantized zai-org/GLM-5.2, TP8; measured on v0.5.13.post1, flush-cache every run. - // balanced/HT run plain TP8 (no DP-Attention/DeepEP), so they trail the FP8 dp-attention recipe at high concurrency.) - { - match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 470, tpot_ms: 2.93, tokens_per_sec_per_gpu: 37 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 3474, tpot_ms: 10.33, tokens_per_sec_per_gpu: 146 }, - ], - }, - { - match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 14123, tpot_ms: 35.47, tokens_per_sec_per_gpu: 157 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 116633, tpot_ms: 40.65, tokens_per_sec_per_gpu: 167 }, - ], - }, - { - match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.13.post1", - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 525108, tpot_ms: 82.52, tokens_per_sec_per_gpu: 168 }, + ttft_ms: 231101, tpot_ms: 86.01, tokens_per_sec_per_gpu: 671 }, ], }, + // ---- B300 + FP8 ---- (8-GPU single node, TP8; serve recipe in glm-5.2.jsx; benchmark pending re-measurement) + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + // ---- B300 + BF16 ---- (unquantized zai-org/GLM-5.2, TP8; serve recipe in glm-5.2.jsx; benchmark pending re-measurement) + { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, // ---- BF16 multi-node (inferred) ---- benchmarks pending { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } }, { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } }, @@ -171,42 +102,4 @@ export const benchmarks = [ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" } }, - // ---- NVFP4 (Blackwell Ultra) ---- - { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, - // ---- GB300 + NVFP4 ---- (4-GPU single node, TP4; nvidia/GLM-5.2-NVFP4 via --quantization modelopt_fp4, - // measured on the lmsysorg/sglang:dev-glm52-nvfp4 preview image, flush-cache every run. - // tokens_per_sec_per_gpu = total server output tok/s / 4 GPUs (337→84, 1248→312, 1162→291, 1695→424, 1730→433). - // aime25 overrides the variant default (87.7 → 89.58, measured on this NVFP4 build); gsm8k inherits the default.) - { - match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "dev-glm52-nvfp4", - accuracy: { aime25_pct: 89.58 }, - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 238, tpot_ms: 2.23, tokens_per_sec_per_gpu: 84 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 315, tpot_ms: 11.9, tokens_per_sec_per_gpu: 312 }, - ], - }, - { - match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, - sglang_version: "dev-glm52-nvfp4", - accuracy: { aime25_pct: 89.58 }, - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 1169, tpot_ms: 58, tokens_per_sec_per_gpu: 291 }, - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 6389, tpot_ms: 167, tokens_per_sec_per_gpu: 424 }, - ], - }, - { - match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, - sglang_version: "dev-glm52-nvfp4", - accuracy: { aime25_pct: 89.58 }, - speed: [ - { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 156000, tpot_ms: 321, tokens_per_sec_per_gpu: 433 }, - ], - }, ]; diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index d3846488e..2915a10ee 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -55,6 +55,7 @@ export const config = { --model {{MODEL_NAME}} \\ --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --random-range-ratio 1.0 \\ --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ --warmup-requests 64 --flush-cache`, accuracy: { @@ -185,7 +186,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.8", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -205,11 +205,10 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", // Large chunked-prefill is the dominant balanced lever (prefill-bound at this // concurrency); max-running tracks KV capacity (~60-80 for 8K+1K reqs on 8xH200). "--chunked-prefill-size 32768", - "--max-running-requests 80", + "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -225,7 +224,6 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -247,7 +245,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.8", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -267,11 +264,10 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", // Large chunked-prefill is the dominant balanced lever (prefill-bound at this // concurrency); max-running tracks KV capacity (~89 for 8K+1K reqs on 8xB200). "--chunked-prefill-size 32768", - "--max-running-requests 80", + "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -287,7 +283,6 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -312,7 +307,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -332,10 +326,9 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", // Same prefill lever as H200/B200 balanced; max-running tracks the TP4 KV capacity. "--chunked-prefill-size 32768", - "--max-running-requests 80", + "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -343,7 +336,9 @@ sgl-eval run aime25 \\ { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, - env: [], + env: [ + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=512", + ], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", @@ -351,8 +346,6 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", - "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -375,7 +368,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.8", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -395,9 +387,8 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", - "--max-running-requests 80", + "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -413,7 +404,6 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -438,7 +428,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -455,7 +444,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -470,7 +458,6 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -495,7 +482,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -512,7 +498,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -527,7 +512,6 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -545,7 +529,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -562,7 +545,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -577,7 +559,6 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -595,7 +576,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -612,7 +592,6 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -627,7 +606,6 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}",