diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index e16398d44..c3ac3d27f 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -19,7 +19,7 @@ For all methods and hardware platforms, see the [official SGLang installation gu ```bash Command docker pull lmsysorg/sglang:kimi-k3 # CUDA13 docker pull lmsysorg/sglang:kimi-k3-cu12 # CUDA12 -docker pull lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 # ROCM +docker pull lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260805 # ROCM daily image ``` These tags publish with the public K3 launch; until then, build from the Dockerfiles linked below. @@ -154,7 +154,7 @@ Remaining kernel sources JIT once from the public `flashinfer` wheel (a few minu | GB200 4×4 | TP16/DCP16 | MNNVL auto-detected | | H200 2×8 (4×8 on Unified High-Throughput) | TP16/EP16 + symm-mem, Marlin + FlashMLA; High-Throughput widens to TP32/EP32 over 4 nodes at mem-frac 0.90 with `extra_buffer_lazy` | same block on every node; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | | H100 4×8 | TP32/EP32, Marlin + FlashMLA | SM90a build of the K3 image; pin NCCL/Gloo to the same NIC on all nodes; least post-weight headroom (80 GB) | -| MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention, graph bs up to 256; DSPARK supported | +| MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention, graph bs up to 256, fp8 kvcache; DSPARK supported | **DCP notes** — the DCP cells are Balanced and High-Throughput on every Blackwell platform, in both the `Unified` and `Decode` roles: diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 35bcfff53..2c88f40b4 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -962,6 +962,7 @@ export const config = { "--trust-remote-code", "--tp-size 8", "--attention-backend triton", + "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", "--cuda-graph-max-bs 256", @@ -988,6 +989,7 @@ export const config = { "--trust-remote-code", "--tp-size 8", "--attention-backend triton", + "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", "--cuda-graph-max-bs 256", @@ -1739,6 +1741,7 @@ export const config = { "--trust-remote-code", "--tp-size 8", "--attention-backend triton", + "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", "--cuda-graph-max-bs 256", @@ -1766,6 +1769,7 @@ export const config = { "--trust-remote-code", "--tp-size 8", "--attention-backend triton", + "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", "--cuda-graph-max-bs 256",