minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] Add Kimi-K2.7-Code-MXFP4 to cookbook (#36246)

Browse Source 
Co-authored-by: Hung <Emmanuel0612@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Hsiu-Chun, Hung
2026-08-25 17:37:32 +08:00
committed by GitHub
co-authored by Hung Claude
parent 3bc1c580c6
commit a618d4c064
2 changed files with 204 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
+47 -1
View File
@@ -14,9 +14,21 @@ export const KimiK27CodeDeployment = () => {
{ id: 'mi355x', label: 'MI355X', default: false },
],
},
quantization: {
name: 'quantization',
title: 'Quantization',
getDynamicItems: (values) => {
const isMXFP4 = ['mi350x', 'mi355x'].includes(values.hardware);
return [
{ id: 'int4', label: 'INT4', subtitle: 'Base checkpoint', default: !isMXFP4 },
{ id: 'mxfp4', label: 'MXFP4', subtitle: 'AMD FP4', default: isMXFP4, disabled: !isMXFP4, disabledReason: !isMXFP4 ? 'MXFP4 only on AMD MI350X/MI355X' : '' },
];
},
},
reasoning: {
name: 'reasoning',
title: 'Reasoning Parser',
condition: (values) => values.quantization !== 'mxfp4',
items: [
{ id: 'disabled', label: 'Disabled', default: false },
{ id: 'enabled', label: 'Enabled', default: true },
@@ -25,6 +37,7 @@ export const KimiK27CodeDeployment = () => {
toolcall: {
name: 'toolcall',
title: 'Tool Call Parser',
condition: (values) => values.quantization !== 'mxfp4',
items: [
{ id: 'disabled', label: 'Disabled', default: false },
{ id: 'enabled', label: 'Enabled', default: true },
@@ -33,6 +46,7 @@ export const KimiK27CodeDeployment = () => {
dpattention: {
name: 'dpattention',
title: 'DP Attention',
condition: (values) => values.quantization !== 'mxfp4',
items: [
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false },
@@ -112,10 +126,42 @@ export const KimiK27CodeDeployment = () => {
};
const generateCommand = () => {
const { hardware, reasoning, toolcall, dpattention } = values;
const { hardware, quantization, reasoning, toolcall, dpattention } = values;
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi350x' || hardware === 'mi355x';
const isMXFP4 = quantization === 'mxfp4';
const hwConfig = modelConfigs[hardware];
const tpValue = hwConfig.tp;
if (isMXFP4) {
const mxfp4Env = [
'SGLANG_USE_AITER=1',
'HIP_FORCE_DEV_KERNARG=1',
'SGLANG_EXPERT_PARALLEL_SIZE=1',
'SGLANG_USE_DYNAMIC_MXFP4_LINEAR=0',
'TORCH_BLAS_PREFER_HIPBLASLT=1',
'TENSILE_STREAMK_DYNAMIC_GRID=6',
'AITER_QUICK_REDUCE_QUANTIZATION=INT4',
'AITER_USE_FLYDSL_MOE_SORTING=1',
'AITER_AR_1STAGE_MAX_KB=512',
'AITER_MXFP4_INTERMEDIATE=1',
'ROCM_QUICK_REDUCE_QUANTIZATION=INT4',
].join(' \\\n');
return (
mxfp4Env + ' \\\n' +
'sglang serve \\\n' +
' --model-path amd/Kimi-K2.7-Code-MXFP4 \\\n' +
' --tp 4 \\\n' +
' --trust-remote-code \\\n' +
' --attention-backend aiter \\\n' +
' --mem-fraction-static 0.90 \\\n' +
' --kv-cache-dtype fp8_e4m3 \\\n' +
' --disable-radix-cache \\\n' +
' --enable-aiter-allreduce-fusion \\\n' +
' --host 0.0.0.0 \\\n' +
' --port 30000'
);
}
const modelName = 'moonshotai/Kimi-K2.7-Code';
let cmd = '';