diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index 3b2444cfc..5ef4ee9a8 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -57,7 +57,7 @@ import { benchmarks } from "/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx - Speed numbers are measured with `--random-range-ratio 1.0`, `--flush-cache`, on `main @ 09ca4fc`. Spec cells pin the EAGLE acceptance length via the serve env `SGLANG_SIMULATE_ACC_LEN` (low-latency 5-1-6 = 3.5, balanced 2-1-3 = 2); high-throughput has no spec. + Speed numbers are measured with `--random-range-ratio 1.0`, `--flush-cache`, on `main @ 09ca4fc` (H200 FP8 cells: `v0.5.14 @ 49e384ce`). Spec cells pin the EAGLE acceptance length via the serve env `SGLANG_SIMULATE_ACC_LEN` (low-latency 5-1-6 = 3.5; FP8 balanced 1-1-2 = 2; NVFP4 balanced 2-1-3 = 2); high-throughput has no spec. ## Playground diff --git a/docs/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx b/docs/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx index 8d5e6a9a7..0a5d19c72 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx @@ -3,10 +3,41 @@ // Numbers pending: each entry is a bare `match` stub (renders "pending") until measured // end-to-end on the corresponding hardware, then filled with sglang_version + speed/accuracy. export const benchmarks = [ - // ---- H200 + FP8 ---- (serve recipe in glm-5.2.jsx; benchmark pending re-measurement) - { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + // ---- H200 + FP8 ---- (8-GPU single node; serve recipe in glm-5.2.jsx; real weights, + // --random-range-ratio 1.0, flush-cache every run) + { + // EAGLE MTP 5-1-6, mfs 0.8. env SGLANG_SIMULATE_ACC_LEN=3.5 + // (match-expected: 50% accept 3 / 50% accept 4) fixes the acceptance length. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + sglang_version: "v0.5.14 @ 49e384ce", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 668, tpot_ms: 5.05, tokens_per_sec_per_gpu: 197 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 6148, tpot_ms: 16.11, tokens_per_sec_per_gpu: 813 }, + ], + }, + { + // Balanced: DP8 + deepep + mfs 0.85 + chunked-prefill 32768 (÷dp8 = 4096) + max-running 256, + // 1-1-2 EAGLE. env SGLANG_SIMULATE_ACC_LEN=2 (match-expected: accept 2 of 2 draft tokens). + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + sglang_version: "v0.5.14 @ 49e384ce", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 7473, tpot_ms: 23.49, tokens_per_sec_per_gpu: 2343 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 80562, tpot_ms: 28.08, tokens_per_sec_per_gpu: 2391 }, + ], + }, + { + // HT: DP8 + deepep + mfs 0.85 + max-running 256, no spec (so no SIMULATE_ACC_LEN). + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + sglang_version: "v0.5.14 @ 49e384ce", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 553480, tpot_ms: 61.71, tokens_per_sec_per_gpu: 1656 }, + ], + }, // ---- B200 + FP8 ---- (8-GPU single node, TP8; real weights, --random-range-ratio 1.0, flush-cache every run) { // EAGLE MTP 5-1-6, mfs 0.8, no cuda-graph-max-bs. env SGLANG_SIMULATE_ACC_LEN=3.5