diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx
index cf2542662..af0ac209f 100644
--- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx
+++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx
@@ -67,6 +67,7 @@ metatags:
**Available Models:**
- **INT4 (native checkpoint)**: [moonshotai/Kimi-K2.7-Code](https://huggingface.co/moonshotai/Kimi-K2.7-Code)
+- **MXFP4**: [amd/Kimi-K2.7-Code-MXFP4](https://huggingface.co/amd/Kimi-K2.7-Code-MXFP4) — validated on AMD MI350X/MI355X.
**License:** Modified MIT for the native checkpoint.
@@ -97,6 +98,7 @@ import { KimiK27CodeDeployment } from '/src/snippets/autoregressive/kimi-k27-cod
- **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs.
- **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls.
- **AMD FP8 KV Cache**: On AMD platforms the generator adds `--kv-cache-dtype fp8_e4m3` by default and sets `--mem-fraction-static 0.8` to fit the INT4 weights plus KV cache. FP8 KV cache trades a small amount of accuracy for memory; omit the flag if you observe accuracy regressions on your workload.
+- **MXFP4 Checkpoint**: The [MXFP4 checkpoint](https://huggingface.co/amd/Kimi-K2.7-Code-MXFP4) is validated on MI350X/MI355X (TP=4) with the `rocm/sgl-dev:v0.5.17-rocm720-mi35x-20260812` image (ROCm 7.2). Select `MXFP4` in the Quantization control to target `amd/Kimi-K2.7-Code-MXFP4`. The MXFP4 quantization is auto-detected from the checkpoint, so no `--quantization` flag is required; the generator emits the AITER/ROCm environment block plus `--attention-backend aiter`, `--mem-fraction-static 0.90`, FP8 KV cache, `--disable-radix-cache`, and `--enable-aiter-allreduce-fusion`. The reasoning and tool-call parsers are auto-detected, so they are not passed explicitly.
## 4. Model Invocation
@@ -502,8 +504,37 @@ sglang serve \
For GB300, use `--tp 4`.
+Deploy the MXFP4 checkpoint on AMD MI350X/MI355X (reasoning and tool-call parsers are auto-detected from the checkpoint):
+
+```shell Command
+SGLANG_USE_AITER=1 \
+HIP_FORCE_DEV_KERNARG=1 \
+SGLANG_EXPERT_PARALLEL_SIZE=1 \
+SGLANG_USE_DYNAMIC_MXFP4_LINEAR=0 \
+TORCH_BLAS_PREFER_HIPBLASLT=1 \
+TENSILE_STREAMK_DYNAMIC_GRID=6 \
+AITER_QUICK_REDUCE_QUANTIZATION=INT4 \
+AITER_USE_FLYDSL_MOE_SORTING=1 \
+AITER_AR_1STAGE_MAX_KB=512 \
+AITER_MXFP4_INTERMEDIATE=1 \
+ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \
+sglang serve \
+ --model-path amd/Kimi-K2.7-Code-MXFP4 \
+ --tp 4 \
+ --trust-remote-code \
+ --attention-backend aiter \
+ --mem-fraction-static 0.90 \
+ --kv-cache-dtype fp8_e4m3 \
+ --disable-radix-cache \
+ --enable-aiter-allreduce-fusion \
+ --host 0.0.0.0 \
+ --port 30000
+```
+
## 5. Benchmark
+### 5.1 Model Card Benchmarks
+
The following results are from the official Kimi-K2.7-Code model card. They were evaluated with thinking mode enabled through Kimi Code CLI at `temperature=1.0`, `top_p=0.95`, and a 262,144-token context length unless otherwise stated.
@@ -554,3 +585,129 @@ The following results are from the official Kimi-K2.7-Code model card. They were
+
+### 5.2 MXFP4 Benchmark
+
+**Test Environment:**
+
+- Hardware: 4× AMD MI355X
+- Model: `amd/Kimi-K2.7-Code-MXFP4` (MXFP4)
+- Tensor Parallelism: 4
+- Docker Image: `rocm/sgl-dev:v0.5.17-rocm720-mi35x-20260812` (ROCm 7.2)
+
+#### 5.2.1 Accuracy Benchmark
+
+GSM8K accuracy of the MXFP4 checkpoint, measured on the deployed server across several client parallelism levels (2,000 questions each):
+
+```shell Command
+python3 benchmark/gsm8k/bench_sglang.py \
+ --num-questions 2000 \
+ --parallel \
+ --port 30000
+```
+
+
+
+
+ | Parallel |
+ GSM8K Accuracy |
+
+
+
+
+ | 100 |
+ 0.951 |
+
+
+ | 300 |
+ 0.953 |
+
+
+ | 600 |
+ 0.942 |
+
+
+ | 1200 |
+ 0.950 |
+
+
+
+
+#### 5.2.2 Latency Benchmark
+
+We use SGLang's built-in `bench_serving` tool with the `random` dataset (input 8192, output 1024, range ratio 0.8). For each concurrency `C`, `--num-prompts` is set to `C × 5`.
+
+```shell Command
+python3 -m sglang.bench_serving \
+ --backend sglang \
+ --model amd/Kimi-K2.7-Code-MXFP4 \
+ --dataset-name random \
+ --random-input-len 8192 \
+ --random-output-len 1024 \
+ --random-range-ratio 0.8 \
+ --max-concurrency \
+ --num-prompts \
+ --host 0.0.0.0 \
+ --port 30000
+```
+
+
+
+
+ | Concurrency |
+ Output Throughput (tok/s) |
+ Total Throughput (tok/s) |
+ Mean TTFT (ms) |
+ Median TTFT (ms) |
+ Mean TPOT (ms) |
+ Mean E2E Latency (ms) |
+
+
+
+
+ | 4 |
+ 416.22 |
+ 3747.97 |
+ 260.35 |
+ 194.14 |
+ 9.03 |
+ 8720.79 |
+
+
+ | 8 |
+ 670.88 |
+ 6086.74 |
+ 364.15 |
+ 199.00 |
+ 11.19 |
+ 10687.52 |
+
+
+ | 16 |
+ 963.47 |
+ 8766.11 |
+ 490.72 |
+ 203.12 |
+ 15.49 |
+ 14680.68 |
+
+
+ | 32 |
+ 1329.55 |
+ 11857.47 |
+ 775.37 |
+ 215.45 |
+ 22.61 |
+ 21831.61 |
+
+
+ | 64 |
+ 1740.46 |
+ 15759.18 |
+ 1343.98 |
+ 300.81 |
+ 34.41 |
+ 33068.86 |
+
+
+
diff --git a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
index e90c74e03..f65430023 100644
--- a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
+++ b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
@@ -14,9 +14,21 @@ export const KimiK27CodeDeployment = () => {
{ id: 'mi355x', label: 'MI355X', default: false },
],
},
+ quantization: {
+ name: 'quantization',
+ title: 'Quantization',
+ getDynamicItems: (values) => {
+ const isMXFP4 = ['mi350x', 'mi355x'].includes(values.hardware);
+ return [
+ { id: 'int4', label: 'INT4', subtitle: 'Base checkpoint', default: !isMXFP4 },
+ { id: 'mxfp4', label: 'MXFP4', subtitle: 'AMD FP4', default: isMXFP4, disabled: !isMXFP4, disabledReason: !isMXFP4 ? 'MXFP4 only on AMD MI350X/MI355X' : '' },
+ ];
+ },
+ },
reasoning: {
name: 'reasoning',
title: 'Reasoning Parser',
+ condition: (values) => values.quantization !== 'mxfp4',
items: [
{ id: 'disabled', label: 'Disabled', default: false },
{ id: 'enabled', label: 'Enabled', default: true },
@@ -25,6 +37,7 @@ export const KimiK27CodeDeployment = () => {
toolcall: {
name: 'toolcall',
title: 'Tool Call Parser',
+ condition: (values) => values.quantization !== 'mxfp4',
items: [
{ id: 'disabled', label: 'Disabled', default: false },
{ id: 'enabled', label: 'Enabled', default: true },
@@ -33,6 +46,7 @@ export const KimiK27CodeDeployment = () => {
dpattention: {
name: 'dpattention',
title: 'DP Attention',
+ condition: (values) => values.quantization !== 'mxfp4',
items: [
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false },
@@ -112,10 +126,42 @@ export const KimiK27CodeDeployment = () => {
};
const generateCommand = () => {
- const { hardware, reasoning, toolcall, dpattention } = values;
+ const { hardware, quantization, reasoning, toolcall, dpattention } = values;
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi350x' || hardware === 'mi355x';
+ const isMXFP4 = quantization === 'mxfp4';
const hwConfig = modelConfigs[hardware];
const tpValue = hwConfig.tp;
+
+ if (isMXFP4) {
+ const mxfp4Env = [
+ 'SGLANG_USE_AITER=1',
+ 'HIP_FORCE_DEV_KERNARG=1',
+ 'SGLANG_EXPERT_PARALLEL_SIZE=1',
+ 'SGLANG_USE_DYNAMIC_MXFP4_LINEAR=0',
+ 'TORCH_BLAS_PREFER_HIPBLASLT=1',
+ 'TENSILE_STREAMK_DYNAMIC_GRID=6',
+ 'AITER_QUICK_REDUCE_QUANTIZATION=INT4',
+ 'AITER_USE_FLYDSL_MOE_SORTING=1',
+ 'AITER_AR_1STAGE_MAX_KB=512',
+ 'AITER_MXFP4_INTERMEDIATE=1',
+ 'ROCM_QUICK_REDUCE_QUANTIZATION=INT4',
+ ].join(' \\\n');
+ return (
+ mxfp4Env + ' \\\n' +
+ 'sglang serve \\\n' +
+ ' --model-path amd/Kimi-K2.7-Code-MXFP4 \\\n' +
+ ' --tp 4 \\\n' +
+ ' --trust-remote-code \\\n' +
+ ' --attention-backend aiter \\\n' +
+ ' --mem-fraction-static 0.90 \\\n' +
+ ' --kv-cache-dtype fp8_e4m3 \\\n' +
+ ' --disable-radix-cache \\\n' +
+ ' --enable-aiter-allreduce-fusion \\\n' +
+ ' --host 0.0.0.0 \\\n' +
+ ' --port 30000'
+ );
+ }
+
const modelName = 'moonshotai/Kimi-K2.7-Code';
let cmd = '';