From e0c0c0a45cb1bda90392bfa2bba4184f5b0638a0 Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Sat, 27 Jun 2026 01:01:28 -0700 Subject: [PATCH] [Cookbook] GLM-5.2: tune GB300 NVFP4 recipes + fill benchmarks (#29486) Co-authored-by: Claude Opus 4.8 --- .../configs/zai-org/glm-5.2-benchmarks.jsx | 39 ++++++++++++++++-- .../src/snippets/configs/zai-org/glm-5.2.jsx | 41 +++++++++++++++---- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx index 0324576a6..f404d5aa0 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx @@ -171,9 +171,42 @@ export const benchmarks = [ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" } }, - // ---- NVFP4 (Blackwell Ultra) ---- benchmarks pending + // ---- NVFP4 (Blackwell Ultra) ---- { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, + // ---- GB300 + NVFP4 ---- (4-GPU single node, TP4; nvidia/GLM-5.2-NVFP4 via --quantization modelopt_fp4, + // measured on the lmsysorg/sglang:dev-glm52-nvfp4 preview image, flush-cache every run. + // tokens_per_sec_per_gpu = total server output tok/s / 4 GPUs (337→84, 1248→312, 1162→291, 1695→424, 1730→433). + // aime25 overrides the variant default (87.7 → 89.58, measured on this NVFP4 build); gsm8k inherits the default.) + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + accuracy: { aime25_pct: 89.58 }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 238, tpot_ms: 2.23, tokens_per_sec_per_gpu: 84 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 315, tpot_ms: 11.9, tokens_per_sec_per_gpu: 312 }, + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + accuracy: { aime25_pct: 89.58 }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 1169, tpot_ms: 58, tokens_per_sec_per_gpu: 291 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 6389, tpot_ms: 167, tokens_per_sec_per_gpu: 424 }, + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + accuracy: { aime25_pct: 89.58 }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 156000, tpot_ms: 321, tokens_per_sec_per_gpu: 433 }, + ], + }, ]; diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index 6b5b545ff..bc0a66660 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -635,16 +635,16 @@ sgl-eval run aime25 \\ }, // ==================================================================== - // NVFP4 (Blackwell Ultra) — nvidia/GLM-5.2-NVFP4 (Model Optimizer). - // TP4 on B300 / GB300, low-latency + balanced. GB300 mirrors the B300 - // recipe (same TP4 / flags; the 4-GPU GB300 node fits the ~381 GB build). + // NVFP4 (Blackwell Ultra) — nvidia/GLM-5.2-NVFP4 (Model Optimizer). TP4. + // B300: low-latency + balanced (the 4-GPU GB300 node fits the ~381 GB build). + // GB300: low-latency / balanced / high-throughput measured on a single 4xGB300 + // node — balanced & high-throughput add DP-Attention (dp4); low-latency uses MTP 5-1-6. // ==================================================================== { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: true, env: [], flags: [ - "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", @@ -663,7 +663,6 @@ sgl-eval run aime25 \\ verified: true, env: [], flags: [ - "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", @@ -678,7 +677,6 @@ sgl-eval run aime25 \\ verified: true, env: [], flags: [ - "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", @@ -687,7 +685,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--chunked-prefill-size 8192", - "--mem-fraction-static 0.8", + "--mem-fraction-static 0.85", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -697,12 +695,37 @@ sgl-eval run aime25 \\ verified: true, env: [], flags: [ - "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", + "--dp 4", + "--enable-dp-attention", + // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the + // verify overhead of a long draft outweighs the accept-length gain. + "--speculative-algorithm EAGLE", + "--speculative-num-steps 2", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 3", "--chunked-prefill-size 8192", - "--mem-fraction-static 0.8", + "--mem-fraction-static 0.92", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--quantization modelopt_fp4", + "--dp 4", + "--enable-dp-attention", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.92", + "--max-running-requests 512", "--host {{HOST_IP}}", "--port {{PORT}}", ],