minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[Cookbook][AMD] Kimi-K3 MI350X/MI355X: pin a ROCm image with the DSPARK graph-capture fix, add measured cell numbers (#39029)

Browse Source
This commit is contained in:
Kevin Mi
2026-09-11 16:10:44 -07:00
committed by GitHub
parent 6671cfc775
commit 5f3606c7b2
3 changed files with 30 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx
+1 -1
View File
@@ -30,7 +30,7 @@ Then run the **Python** output of the command panel below in that environment.
```bash Command
docker pull lmsysorg/sglang:latest # NVIDIA (CUDA)
docker pull lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903 # AMD MI350X / MI355X (ROCm)
docker pull lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 # AMD MI350X / MI355X (ROCm)
```
For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces.
docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx
+27
View File
@@ -91,4 +91,31 @@ export const benchmarks = [
ttft_ms: 11664, tpot_ms: 22.17, tokens_per_sec_per_gpu: 1946 },
],
},
{
// DSPARK acceptance pinned to 5 of 8 draft tokens (SGLANG_SIMULATE_ACC_LEN=5
// SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_RAGGED_VERIFY_MODE=static), so the
// rows are independent of the benchmark's random prompts; measured accept length 5.00.
match: { hw: "mi350x", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "dspark" },
sglang_version: "v0.5.19 @ 12771786",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 604, tpot_ms: 5.39, tokens_per_sec_per_gpu: 186 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 6035, tpot_ms: 15.32, tokens_per_sec_per_gpu: 864 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 23720, tpot_ms: 31.45, tokens_per_sec_per_gpu: 913 },
],
},
{
match: { hw: "mi350x", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "none" },
sglang_version: "v0.5.19 @ 12771786",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 590, tpot_ms: 18.55, tokens_per_sec_per_gpu: 58 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 6331, tpot_ms: 32.80, tokens_per_sec_per_gpu: 462 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 18665, tpot_ms: 70.39, tokens_per_sec_per_gpu: 813 },
],
},
];
docs/src/snippets/configs/moonshotai/kimi-k3.jsx
+2 -2
View File
@@ -470,8 +470,8 @@ export const config = {
gb200: "lmsysorg/sglang:kimi-k3",
// 20260903 or newer: the AITER SiTU A4W4/A8W4 layout fix (sgl-project/sglang#33838,
// merged Sep 3) and the fused gfx950 KDA decode boundary (#34198) first ship here.
mi350x: "lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903",
mi355x: "lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260903",
mi350x: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910",
mi355x: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910",
// NVFP4 needs a build with sgl-project/sglang#35077; the purpose-built dev
// image is cut from that PR's head (CUDA 13).
"b300|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4",