From 5106b42cbd98a14813f76f3c5c1f23f0012bd2fd Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:42:19 -0700 Subject: [PATCH] [cookbook] GLM-5.2 NVFP4 B300: TP8 recipe + 3 strategies (#29557) Co-authored-by: Claude Opus 4.8 --- .../configs/zai-org/glm-5.2-benchmarks.jsx | 35 ++++++++++++++ .../src/snippets/configs/zai-org/glm-5.2.jsx | 47 +++++++++++++++---- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx index 410b84021..e107bc61f 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx @@ -134,4 +134,39 @@ export const benchmarks = [ ttft_ms: 130174, tpot_ms: 67.12, tokens_per_sec_per_gpu: 589.4 }, ], }, + // ---- B300 + NVFP4 ---- (8-GPU single node, TP8; nvidia/GLM-5.2-NVFP4 via --quantization modelopt_fp4, + // measured on the lmsysorg/sglang:dev-glm52-nvfp4 preview image, flush-cache every run. + // tokens_per_sec_per_gpu = total server output tok/s / 8 GPUs (410→51, 1793→224, 1220→153, 1641→205, 3439→430). + // aime25 overrides the variant default (87.7 → 89.58, measured on this NVFP4 build); gsm8k inherits the default.) + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + accuracy: { aime25_pct: 89.58 }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 196, tpot_ms: 1.86, tokens_per_sec_per_gpu: 51 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 274, tpot_ms: 6.95, tokens_per_sec_per_gpu: 224 }, + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + accuracy: { aime25_pct: 89.58 }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 680, tpot_ms: 48.9, tokens_per_sec_per_gpu: 153 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 3010, tpot_ms: 149, tokens_per_sec_per_gpu: 205 }, + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + accuracy: { aime25_pct: 89.58 }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 6370, tpot_ms: 280, tokens_per_sec_per_gpu: 430 }, + ], + }, ]; diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index 4aa913203..8d8a8ac25 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -614,11 +614,12 @@ sgl-eval run aime25 \\ }, // ==================================================================== - // NVFP4 — nvidia/GLM-5.2-NVFP4 (Model Optimizer). - // B200: 8-GPU single node, TP8 (low-latency / balanced / high-throughput); balanced & + // NVFP4 — nvidia/GLM-5.2-NVFP4 (Model Optimizer). TP8 on B200/B300, TP4 on GB300. + // B200/B300: 8-GPU single node, TP8 (low-latency / balanced / high-throughput); balanced & // high-throughput add DP-Attention (dp8). low-latency uses MTP 5-1-6, balanced MTP 2-1-3. - // B300/GB300: 4-GPU single node, TP4 (the node fits the ~381 GB build); GB300 adds dp4 on - // balanced & high-throughput. Blackwell NVFP4 measured on the dev-glm52-nvfp4 preview image. + // GB300: 4-GPU single node, TP4 (the node fits the ~381 GB build); GB300 adds dp4 on + // balanced & high-throughput; low-latency uses MTP 5-1-6. + // Blackwell NVFP4 measured on the dev-glm52-nvfp4 preview image. // ==================================================================== { match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, @@ -685,14 +686,14 @@ sgl-eval run aime25 \\ env: [], flags: [ "--model-path {{MODEL_NAME}}", - "--tp 4", + "--tp 8", "--quantization modelopt_fp4", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--chunked-prefill-size 8192", - "--mem-fraction-static 0.8", + "--mem-fraction-static 0.85", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -703,10 +704,40 @@ sgl-eval run aime25 \\ env: [], flags: [ "--model-path {{MODEL_NAME}}", - "--tp 4", + "--tp 8", + "--dp 8", + "--enable-dp-attention", "--quantization modelopt_fp4", + // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the + // verify overhead of a long draft outweighs the accept-length gain. + "--speculative-algorithm EAGLE", + "--speculative-num-steps 2", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 3", + // Two required flags for DP-Attention + MTP here: `decode`-mode spec attention + // avoids a CUDA-graph capture deadlock, and max-running 256 lifts the default + // ~48-request throttle so DP-Attention can fill all 8 ranks. + "--speculative-attention-mode decode", + "--max-running-requests 256", "--chunked-prefill-size 8192", - "--mem-fraction-static 0.8", + "--mem-fraction-static 0.85", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--quantization modelopt_fp4", + "--max-running-requests 1024", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.85", "--host {{HOST_IP}}", "--port {{PORT}}", ],