From 3c9efaf3e1929c0dd4a2209046fb0adad72e3b8a Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:18:56 -0700 Subject: [PATCH] [docs] Kimi-K3: widen the H200 High-Throughput recipe to 4x8 TP32/EP32 (#32834) Co-authored-by: Claude Opus 5 --- .../autoregressive/Moonshotai/Kimi-K3.mdx | 6 ++-- .../snippets/configs/moonshotai/kimi-k3.jsx | 30 +++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index bbea12ad4..518fc6d0f 100644 --- a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -34,7 +34,7 @@ If you do not want to use a Docker image, reproduce the dependency installation -Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8, GB300 2×4, MI350X/MI355X 1×8), so it is not a separate choice. +Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8), so it is not a separate choice. **PD Mode** — `Unified` serves prefill and decode together. `Prefill` / `Decode` split them into dedicated pools (see [PD disaggregation](#3-4-pd-disaggregation)); `Prefill` ships two strategies on the TP8 platforms, both chunked at 16k: `Default` (TP8) and `Long-Context` (`--pp-size 8 --tp-size 1`, see [Deep PP](#deep-pp-for-long-context-prefill)). @@ -42,7 +42,7 @@ Pick your hardware, then the deployment shape and operating point. Node count fo - **Low-Latency** — plain TP, no DCP. For chat. - **Balanced** — the accuracy-preserving default: TP16/DCP16 on B200/GB200, TP8/DCP8 on B300/GB300, TP8 ROCm/AITER on MI35x. -- **High-Throughput** — the large-scale lane: pick a **Cluster Size** and **Large-Scale Preset** in the Playground ([details](#large-scale-presets)). On H100/H200: Balanced plus `extra_buffer_lazy`. +- **High-Throughput** — the large-scale lane: pick a **Cluster Size** and **Large-Scale Preset** in the Playground ([details](#large-scale-presets)). On H100: Balanced plus `extra_buffer_lazy`; on H200 the cell itself widens to 4×8 TP32/EP32 at `--mem-fraction-static 0.90`. - **Long-Context** — B200 only: TP8/PP2 splits KV and KDA state across two pipeline stages. **Spec Decode** — independent of the strategy: DSPARK layers onto any `pp_size == 1` recipe, proposing 7 draft tokens per step (tune in the Playground); DFLASH has no published draft checkpoint. Its win is largest on short interactive traffic and fades as the prompt grows. @@ -151,7 +151,7 @@ Remaining kernel sources JIT once from the public `flashinfer` wheel (a few minu | GB300 2×4 | TP8/DCP8 | MNNVL transport and cuMem auto-detected | | B200 2×8 | TP16 (+DCP16); Long-Context TP8/PP2, 128K ctx, 8192 chunked prefill | DSPARK off on Long-Context (`pp_size == 1` required) | | GB200 4×4 | TP16/DCP16 | MNNVL auto-detected | -| H200 2×8 | TP16/EP16 + symm-mem | same block on both ranks; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | +| H200 2×8 (4×8 on Unified High-Throughput) | TP16/EP16 + symm-mem, Marlin + FlashMLA; High-Throughput widens to TP32/EP32 over 4 nodes at mem-frac 0.90 with `extra_buffer_lazy` | same block on every node; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | | H100 4×8 | TP32/EP32, Marlin + FlashMLA | SM90a build of the K3 image; pin NCCL/Gloo to the same NIC on all nodes; least post-weight headroom (80 GB) | | MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention, graph bs up to 256; DSPARK supported | diff --git a/docs_new/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs_new/src/snippets/configs/moonshotai/kimi-k3.jsx index b155d682d..f8cd68346 100644 --- a/docs_new/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs_new/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -11,14 +11,16 @@ export const config = { modelName: "Kimi-K3", // B300 (1×8 TP8), GB300 (2×4 TP8 MNNVL), B200 (2×8 TP16, or TP8/PP2 for - // Long-Context), GB200 (4×4 TP16 MNNVL), H200 (2×8 TP16/EP16), H100 - // (4×8 TP32/EP32), and MI350X/MI355X (1×8 TP8) have serving recipes. + // Long-Context), GB200 (4×4 TP16 MNNVL), H200 (2×8 TP16/EP16, or 4×8 TP32/EP32 + // for High-Throughput), H100 (4×8 TP32/EP32), and MI350X/MI355X (1×8 TP8) have + // serving recipes. supportedHardware: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x"], // Single checkpoint and a single shipped quantization (MXFP4), so neither is a // reader-facing axis. Node count is fixed by the hardware recipe (B200 2x8, - // H100 4x8, B300 1x8, H200 2x8, GB200 4x4, GB300 2x4, - // MI350X/MI355X 1x8), so it rides on the cell rather than on a selector. + // H100 4x8, B300 1x8, H200 2x8 — 4x8 on Unified High-Throughput, GB200 4x4, + // GB300 2x4, MI350X/MI355X 1x8), so it rides on the cell rather than on a + // selector. matchDims: [ { id: "pdMode", @@ -1004,26 +1006,35 @@ export const config = { ], }, { - // Throughput-tuned (perf-command: mem-frac 0.90, graph-bs 256, extra_buffer_lazy → max_running 98). + // The one H200 cell that widens past a single pair of nodes. As run — with + // DSPARK and HiCache L1+L2 layered on, at ratio 0.058 — the static + // allocation leaves 12.54 GB free per GPU and 1940352 KV tokens. match: { hw: "h200", pdMode: "unified", strategy: "high-throughput" }, - nnodes: 2, + nnodes: 4, verified: false, verificationStatus: "in-progress", env: [ "NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1", + "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0", + "SGLANG_K3_ATTN_RES_MODE=jit", + "SGLANG_MOE_FUSED_GATE_RADIX=1", + "SGLANG_HOST_IP={{LOCAL_IP}}", + "NCCL_SOCKET_IFNAME={{NETWORK_IFACE}}", + "GLOO_SOCKET_IFNAME={{NETWORK_IFACE}}", ], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", - "--tp-size 16", - "--ep-size 16", + "--tp-size 32", + "--ep-size 32", "--moe-runner-backend marlin", "--decode-attention-backend flashmla", "--enable-symm-mem", "--mem-fraction-static 0.90", "--mamba-radix-cache-strategy extra_buffer_lazy", + "--dist-timeout 3600", "--reasoning-parser kimi_k3", "--tool-call-parser kimi_k3", "--host {{HOST_IP}}", @@ -1912,7 +1923,8 @@ export const config = { "Set This node IP separately on each node; use the same cross-node NIC name on all four nodes.", ], h200: [ - "Multi-node K3 needs the cross-node NIC pinned on BOTH ranks:", + "Low-Latency and Balanced run TP16/EP16 across 2 nodes; Unified High-Throughput widens to TP32/EP32 across 4.", + "Multi-node K3 needs the cross-node NIC pinned on EVERY node:", " GLOO_SOCKET_IFNAME= # e.g. bond0", " NCCL_SOCKET_IFNAME= # force NCCL off kube-ipvs0", " SGLANG_HOST_IP=",