diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index fd57a7fd5..499df1e1a 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -29,7 +29,8 @@ Then run the **Python** output of the command panel below in that environment. ```bash Command -docker pull lmsysorg/sglang:latest +docker pull lmsysorg/sglang:latest # NVIDIA (CUDA) +docker pull lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 # AMD MI350X / MI355X (ROCm) ``` For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. @@ -151,7 +152,7 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request | GB200 4×4 | TP16/DCP16 | MNNVL auto-detected | | H200 2×8 (4×8 on Unified High-Throughput) | TP16/EP16 + symm-mem, Marlin + FlashMLA; High-Throughput widens to TP32/EP32 over 4 nodes at mem-frac 0.90 with `extra_buffer_lazy` | same block on every node; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | | H100 4×8 | TP32/EP32, Marlin + FlashMLA | SM90a build of the K3 image; pin NCCL/Gloo to the same NIC on all nodes; least post-weight headroom (80 GB) | -| MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention, graph bs up to 256, fp8 kvcache; DSPARK supported | +| MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention (`SGLANG_MLA_DECODE_TUNE=1` for gfx950 MLA decode geometry), graph bs up to 256, fp8 kvcache; DSPARK supported | **DCP notes** — the DCP cells are Balanced and High-Throughput on every Blackwell platform, in both the `Unified` and `Decode` roles: diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 72783e08d..601e1508d 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -234,7 +234,9 @@ export const config = { title: "Spec Decode", default: "dspark", options: [ - { id: "none", label: "Non-Spec" }, + { id: "none", label: "Non-Spec", + env: (s) => (["mi350x", "mi355x"].includes(s.hw) ? ["SGLANG_MLA_DECODE_TUNE=1"] : []), + }, { id: "dspark", label: "DSPARK", @@ -411,8 +413,8 @@ export const config = { gb300: "lmsysorg/sglang:kimi-k3", b200: "lmsysorg/sglang:kimi-k3", gb200: "lmsysorg/sglang:kimi-k3", - mi350x: "lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727", - mi355x: "lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727", + mi350x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817", + mi355x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817", // NVFP4 needs a build with sgl-project/sglang#35077; the purpose-built dev // image is cut from that PR's head (CUDA 13). "b300|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4",