diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx index c53cc230b..f3d08c9c1 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -111,6 +111,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; - **Memory.** The FP8 weights are large (MoE total, not active params). Start around `--mem-fraction-static 0.8` on H200 (TP8) and tune up; raise it for the 4-GPU GB300 single-node layout (TP4). - **DP-Attention + DeepEP** for the balanced/high-throughput strategies spreads attention across data-parallel ranks and routes MoE through DeepEP. - **BF16 weights need more GPUs.** The full-precision build (`zai-org/GLM-5.2`, ~1.5 TB) does not fit a single 8×H200 / 8×B200 / 4×GB300 node. It fits single-node on **8×B300** (TP8, ~2.1 TB HBM) — **verified**; on the smaller GPUs it needs a **multi-node** layout (e.g. 2×8×H200 or 2×8×B200 at TP16, 2×4×GB300 at TP8), and those **multi-node BF16 recipes are still proposed/inferred** (`verified: false`). FP8 is the recommended deployment. Use the same DSA / MTP / chunked-prefill guidance as FP8. On B300, BF16 low-latency matches FP8 (the sm103 FP8 path is not yet optimized), but FP8 wins at the balanced/high-throughput points. +- **PD Disaggregation (prefill/decode).** GLM-5.2 is a DSA model and runs under prefill/decode disaggregation — toggle the **PD Disagg** card in the [Playground above](#playground) (pick a Prefill/Decode role + transfer backend, then front the roles with `sglang_router.launch_router --pd-disaggregation`). The Mooncake backend **auto-detects the InfiniBand HCA**, so no device flag is needed by default; only add `--disaggregation-ib-device mlx5_0` (your NIC) if auto-detection picks the wrong device or KV transfer fails to connect. On H200 Docker, expose the IB HCAs to the container (`--privileged --ulimit memlock=-1`, or `--device /dev/infiniband:/dev/infiniband --cap-add IPC_LOCK`) — without IB exposure Mooncake silently falls back to TCP. - **Chunked-prefill size is regime-dependent.** At long input (8K+) the default `--chunked-prefill-size 2048` is too small and leaves the balanced point prefill-bound (queueing dominates TTFT). Raising it to `--chunked-prefill-size 32768` on the balanced recipe gave roughly **+34–78% output throughput and −39–59% TTFT** on 8×H200 and 8×B200 (8K-in / 1K-out) in our testing. It is **neutral for high-throughput** (decode-bound there) — keep the default. `--max-running-requests` tracks KV capacity, not a tuning free-for-all: ~60–90 concurrent 8K+1K FP8 requests fit on a single 8-GPU node, so pin balanced near `--max-running-requests 80` and let high-throughput run wider. - **AMD GPUs (MI300X / MI325X / MI355X).** FP8 (`zai-org/GLM-5.2-FP8`) runs single-node at `tp=8` on all three. BF16 (`zai-org/GLM-5.2`, ~1.51 TB) only fits single-node on **MI325X** (2 TB HBM) and **MI355X** (2.3 TB); **MI300X** (1.5 TB) cannot hold the BF16 weights plus KV cache on one node, so use FP8 there (or a multi-node BF16 layout once validated). Use the DSA tilelang backend (`--dsa-prefill-backend tilelang --dsa-decode-backend tilelang`) and add `--chunked-prefill-size 131072` plus `--watchdog-timeout 1200` (20 min for weight loading). FP8 uses about half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). GLM-5.2 and DeepSeek-V3.2 share the same model structure; for other DSA / HiSparse tips see the [DeepSeek-V3.2 cookbook](../DeepSeek/DeepSeek-V3_2). diff --git a/docs_new/src/snippets/_playground.jsx b/docs_new/src/snippets/_playground.jsx index 34dffa7ba..c3253f935 100644 --- a/docs_new/src/snippets/_playground.jsx +++ b/docs_new/src/snippets/_playground.jsx @@ -39,9 +39,14 @@ export const Playground = ({ config }) => { const STORAGE_KEY = "sglang-deploy-env"; const pgFeatures = config.playgroundFeatures || {}; + // Single-host PD runs prefill + decode as two engines on one box. Each derives + // 5 consecutive ZMQ/dist ports from its --port (port+233, see server_args.py + // ZMQ_TCP_PORT_DELTA), so the serve ports are spaced 100 apart to keep those + // derived ranges from overlapping — no --dist-init-addr needed single-host. + // `dist` is only used by the multi-node renderer (cross-node rendezvous). const PD_PORTS = { prefill: { serve: 30000, dist: 30335 }, - decode: { serve: 30001, dist: 30435 }, + decode: { serve: 30100, dist: 30435 }, }; // ========================================================================== @@ -698,12 +703,10 @@ export const Playground = ({ config }) => { if (value.ibDevice && value.ibDevice !== "auto") { adds.push(`--disaggregation-ib-device ${value.ibDevice}`); } - // Single-host bootstrap port only (multi-node gets --dist-init-addr - // from the renderer). - if (sel.nodes === "single" - && !flags.some((f) => f.startsWith("--dist-init-addr"))) { - adds.push(`--dist-init-addr 127.0.0.1:${PD_PORTS[value.mode].dist}`); - } + // Single-host needs no --dist-init-addr: prefill/decode derive their + // ZMQ/dist ports from the role-specific --port (spaced 100 apart, see + // PD_PORTS), so the ranges don't overlap. Multi-node still gets a + // cross-node --dist-init-addr from the renderer. flags = h.insertBeforeTail(flags, adds); // Role-specific serving port so the router's prefill / decode targets diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index 0c2838417..84965ec1e 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -164,7 +164,48 @@ sgl-eval run aime25 \\ ], }, - // ----- Card 5: "Hierarchical KV Cache" ----- + // ----- Card 5: "PD Disaggregation" ----- + // GLM-5.2 is a DSA model (same family as DeepSeek-V3.2/V4) and supports + // prefill/decode disaggregation. Owns the `--disaggregation-*` flags; the + // engine also pins role-specific serving ports (spaced apart) so prefill + + // decode don't collide on one host. + pdDisagg: { + modes: [ + { id: "off", label: "Off" }, + { id: "prefill", label: "Prefill role" }, + { id: "decode", label: "Decode role" }, + ], + transferBackends: [ + // Mooncake (recommended). The NCCL/MNNVL env is only needed on the + // NVLink-multinode Grace-Blackwell platform (GB300 here). + { id: "mooncake", label: "Mooncake", + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + ], + envWhen: { hw: ["gb300"] } }, + { id: "nixl", label: "NiXL" }, + ], + // No IB-device knob: mooncake auto-detects the HCA. Pass + // --disaggregation-ib-device only if discovery picks the wrong NIC + // (see Configuration Tips). + // Router fronting the prefill + decode roles; substitute /. + router: { + port: 8000, + command: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PREFILL_PORT}} \\ + --decode http://:{{DECODE_PORT}} \\ + --host 0.0.0.0 --port {{ROUTER_PORT}} \\ + --disable-circuit-breaker \\ + --health-check-interval-secs 999999`, + }, + }, + + // ----- Card 6: "Hierarchical KV Cache" ----- hicache: { backends: [ { id: null, label: "Auto" },