diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx index 562df1039..410b84021 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx @@ -102,4 +102,36 @@ export const benchmarks = [ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" } }, + // ---- B200 + NVFP4 ---- (8-GPU single node, TP8; nvidia/GLM-5.2-NVFP4 via --quantization modelopt_fp4, + // measured on the lmsysorg/sglang:dev-glm52-nvfp4 preview image, flush-cache every run. + // ttft_ms/tpot_ms are P50; tokens_per_sec_per_gpu = output tok/s/GPU. + // balanced & high-throughput add DP-Attention (dp8); low-latency uses MTP 5-1-6, balanced MTP 2-1-3.) + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 295, tpot_ms: 1.85, tokens_per_sec_per_gpu: 58.6 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 2491, tpot_ms: 5.43, tokens_per_sec_per_gpu: 254.3 }, + ], + }, + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 5837, tpot_ms: 12.70, tokens_per_sec_per_gpu: 418.9 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 16736, tpot_ms: 30.00, tokens_per_sec_per_gpu: 593.7 }, + ], + }, + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "dev-glm52-nvfp4", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 130174, tpot_ms: 67.12, tokens_per_sec_per_gpu: 589.4 }, + ], + }, ]; diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index 2915a10ee..4aa913203 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -94,6 +94,7 @@ sgl-eval run aime25 \\ gb300: "lmsysorg/sglang:latest", b300: "lmsysorg/sglang:latest", // NVFP4 needs the dev image with modelopt_fp4 support (per-quant override). + "b200|nvfp4": "lmsysorg/sglang:dev-glm52-nvfp4", "b300|nvfp4": "lmsysorg/sglang:dev-glm52-nvfp4", "gb300|nvfp4": "lmsysorg/sglang:dev-glm52-nvfp4", }, @@ -613,11 +614,71 @@ sgl-eval run aime25 \\ }, // ==================================================================== - // NVFP4 (Blackwell Ultra) — nvidia/GLM-5.2-NVFP4 (Model Optimizer). TP4. - // B300: low-latency + balanced (the 4-GPU GB300 node fits the ~381 GB build). - // GB300: low-latency / balanced / high-throughput measured on a single 4xGB300 - // node — balanced & high-throughput add DP-Attention (dp4); low-latency uses MTP 5-1-6. + // NVFP4 — nvidia/GLM-5.2-NVFP4 (Model Optimizer). + // B200: 8-GPU single node, TP8 (low-latency / balanced / high-throughput); balanced & + // high-throughput add DP-Attention (dp8). low-latency uses MTP 5-1-6, balanced MTP 2-1-3. + // B300/GB300: 4-GPU single node, TP4 (the node fits the ~381 GB build); GB300 adds dp4 on + // balanced & high-throughput. Blackwell NVFP4 measured on the dev-glm52-nvfp4 preview image. // ==================================================================== + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--quantization modelopt_fp4", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.85", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--quantization modelopt_fp4", + "--dp 8", + "--enable-dp-attention", + // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the + // verify overhead of a long draft outweighs the accept-length gain. + "--speculative-algorithm EAGLE", + "--speculative-num-steps 2", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 3", + // Larger chunked-prefill (32768 → ~4096/rank under dp8) is the dominant balanced lever. + "--chunked-prefill-size 32768", + "--mem-fraction-static 0.92", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--quantization modelopt_fp4", + "--dp 8", + "--enable-dp-attention", + "--chunked-prefill-size 32768", + "--mem-fraction-static 0.92", + "--max-running-requests 512", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: true,