From 5f3606c7b2aec653fcfdbe3465379e700c4a5492 Mon Sep 17 00:00:00 2001 From: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:10:44 -0700 Subject: [PATCH] [Cookbook][AMD] Kimi-K3 MI350X/MI355X: pin a ROCm image with the DSPARK graph-capture fix, add measured cell numbers (#39029) --- .../autoregressive/Moonshotai/Kimi-K3.mdx | 2 +- .../configs/moonshotai/kimi-k3-benchmarks.jsx | 27 +++++++++++++++++++ .../snippets/configs/moonshotai/kimi-k3.jsx | 4 +-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index ea3f81d0e..eb80a7e96 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -30,7 +30,7 @@ Then run the **Python** output of the command panel below in that environment. ```bash Command docker pull lmsysorg/sglang:latest # NVIDIA (CUDA) -docker pull lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903 # AMD MI350X / MI355X (ROCm) +docker pull lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 # AMD MI350X / MI355X (ROCm) ``` For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx index e3434ac20..a58a8e3a7 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx @@ -91,4 +91,31 @@ export const benchmarks = [ ttft_ms: 11664, tpot_ms: 22.17, tokens_per_sec_per_gpu: 1946 }, ], }, + { + // DSPARK acceptance pinned to 5 of 8 draft tokens (SGLANG_SIMULATE_ACC_LEN=5 + // SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_RAGGED_VERIFY_MODE=static), so the + // rows are independent of the benchmark's random prompts; measured accept length 5.00. + match: { hw: "mi350x", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "dspark" }, + sglang_version: "v0.5.19 @ 12771786", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 604, tpot_ms: 5.39, tokens_per_sec_per_gpu: 186 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 6035, tpot_ms: 15.32, tokens_per_sec_per_gpu: 864 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 23720, tpot_ms: 31.45, tokens_per_sec_per_gpu: 913 }, + ], + }, + { + match: { hw: "mi350x", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "none" }, + sglang_version: "v0.5.19 @ 12771786", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 590, tpot_ms: 18.55, tokens_per_sec_per_gpu: 58 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 6331, tpot_ms: 32.80, tokens_per_sec_per_gpu: 462 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 18665, tpot_ms: 70.39, tokens_per_sec_per_gpu: 813 }, + ], + }, ]; diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 4bca32497..dc65ce99b 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -470,8 +470,8 @@ export const config = { gb200: "lmsysorg/sglang:kimi-k3", // 20260903 or newer: the AITER SiTU A4W4/A8W4 layout fix (sgl-project/sglang#33838, // merged Sep 3) and the fused gfx950 KDA decode boundary (#34198) first ship here. - mi350x: "lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903", - mi355x: "lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903", + mi350x: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910", + mi355x: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910", // NVFP4 needs a build with sgl-project/sglang#35077; the purpose-built dev // image is cut from that PR's head (CUDA 13). "b300|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4",