minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[cookbook] GLM-5.2 NVFP4 B300: TP8 recipe + 3 strategies (#29557)

Browse Source 
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
zijiexia
2026-06-29 14:42:19 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 98d0e702c3
commit 5106b42cbd
2 changed files with 74 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx
+35
View File
@@ -134,4 +134,39 @@ export const benchmarks = [
ttft_ms: 130174, tpot_ms: 67.12, tokens_per_sec_per_gpu: 589.4 },
],
},
// ---- B300 + NVFP4 ---- (8-GPU single node, TP8; nvidia/GLM-5.2-NVFP4 via --quantization modelopt_fp4,
// measured on the lmsysorg/sglang:dev-glm52-nvfp4 preview image, flush-cache every run.
// tokens_per_sec_per_gpu = total server output tok/s / 8 GPUs (410→51, 1793→224, 1220→153, 1641→205, 3439→430).
// aime25 overrides the variant default (87.7 → 89.58, measured on this NVFP4 build); gsm8k inherits the default.)
{
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "dev-glm52-nvfp4",
accuracy: { aime25_pct: 89.58 },
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 196, tpot_ms: 1.86, tokens_per_sec_per_gpu: 51 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 274, tpot_ms: 6.95, tokens_per_sec_per_gpu: 224 },
],
},
{
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
sglang_version: "dev-glm52-nvfp4",
accuracy: { aime25_pct: 89.58 },
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 680, tpot_ms: 48.9, tokens_per_sec_per_gpu: 153 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
ttft_ms: 3010, tpot_ms: 149, tokens_per_sec_per_gpu: 205 },
],
},
{
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
sglang_version: "dev-glm52-nvfp4",
accuracy: { aime25_pct: 89.58 },
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
ttft_ms: 6370, tpot_ms: 280, tokens_per_sec_per_gpu: 430 },
],
},
];
docs_new/src/snippets/configs/zai-org/glm-5.2.jsx
+39 -8
View File
@@ -614,11 +614,12 @@ sgl-eval run aime25 \\
},
// ====================================================================
// NVFP4 — nvidia/GLM-5.2-NVFP4 (Model Optimizer).
// B200: 8-GPU single node, TP8 (low-latency / balanced / high-throughput); balanced &
// NVFP4 — nvidia/GLM-5.2-NVFP4 (Model Optimizer). TP8 on B200/B300, TP4 on GB300.
// B200/B300: 8-GPU single node, TP8 (low-latency / balanced / high-throughput); balanced &
// high-throughput add DP-Attention (dp8). low-latency uses MTP 5-1-6, balanced MTP 2-1-3.
// B300/GB300: 4-GPU single node, TP4 (the node fits the ~381 GB build); GB300 adds dp4 on
// balanced & high-throughput. Blackwell NVFP4 measured on the dev-glm52-nvfp4 preview image.
// GB300: 4-GPU single node, TP4 (the node fits the ~381 GB build); GB300 adds dp4 on
// balanced & high-throughput; low-latency uses MTP 5-1-6.
// Blackwell NVFP4 measured on the dev-glm52-nvfp4 preview image.
// ====================================================================
{
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
@@ -685,14 +686,14 @@ sgl-eval run aime25 \\
env: [],
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--tp 8",
"--quantization modelopt_fp4",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 5",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 6",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.8",
"--mem-fraction-static 0.85",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
@@ -703,10 +704,40 @@ sgl-eval run aime25 \\
env: [],
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--tp 8",
"--dp 8",
"--enable-dp-attention",
"--quantization modelopt_fp4",
// Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the
// verify overhead of a long draft outweighs the accept-length gain.
"--speculative-algorithm EAGLE",
"--speculative-num-steps 2",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 3",
// Two required flags for DP-Attention + MTP here: `decode`-mode spec attention
// avoids a CUDA-graph capture deadlock, and max-running 256 lifts the default
// ~48-request throttle so DP-Attention can fill all 8 ranks.
"--speculative-attention-mode decode",
"--max-running-requests 256",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.8",
"--mem-fraction-static 0.85",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
verified: true,
env: [],
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--dp 8",
"--enable-dp-attention",
"--quantization modelopt_fp4",
"--max-running-requests 1024",
"--chunked-prefill-size 8192",
"--mem-fraction-static 0.85",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],