[NVIDIA] Update Minimax-M2.5,M2.7 docs with flags for performance (#24465)
This commit is contained in:
@@ -540,6 +540,11 @@ SGLang supports various environment variables that can be used to configure its
|
|||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass)</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass)</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`false`</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`false`</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_USE_FUSED_PARALLEL_QKNORM</code></td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use the fused parallel QK RMSNorm kernel for MiniMax-M2.x on CUDA when attention TP size > 1</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`false`</td>
|
||||||
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|
||||||
|
|||||||
@@ -72,7 +72,13 @@ export const MiniMaxM25Deployment = () => {
|
|||||||
|
|
||||||
const modelName = `${modelFamily}/MiniMax-M2.5`;
|
const modelName = `${modelFamily}/MiniMax-M2.5`;
|
||||||
|
|
||||||
|
const isBlackwell = hardware === 'b200';
|
||||||
|
const useAllreduceFusion = hardware === 'h200' || hardware === 'b200';
|
||||||
|
|
||||||
let cmd = '';
|
let cmd = '';
|
||||||
|
if (useAllreduceFusion) {
|
||||||
|
cmd += 'SGLANG_USE_FUSED_PARALLEL_QKNORM=1 \\\n';
|
||||||
|
}
|
||||||
cmd += 'python -m sglang.launch_server \\\n';
|
cmd += 'python -m sglang.launch_server \\\n';
|
||||||
cmd += ` --model-path ${modelName}`;
|
cmd += ` --model-path ${modelName}`;
|
||||||
|
|
||||||
@@ -102,6 +108,16 @@ export const MiniMaxM25Deployment = () => {
|
|||||||
cmd += ` \\\n --trust-remote-code`;
|
cmd += ` \\\n --trust-remote-code`;
|
||||||
cmd += ` \\\n --mem-fraction-static 0.85`;
|
cmd += ` \\\n --mem-fraction-static 0.85`;
|
||||||
|
|
||||||
|
if (isBlackwell) {
|
||||||
|
cmd += ` \\\n --moe-runner-backend flashinfer_trtllm_routed`;
|
||||||
|
cmd += ` \\\n --fp8-gemm-backend flashinfer_trtllm`;
|
||||||
|
cmd += ` \\\n --dtype bfloat16`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (useAllreduceFusion) {
|
||||||
|
cmd += ` \\\n --enable-flashinfer-allreduce-fusion`;
|
||||||
|
}
|
||||||
|
|
||||||
if (isAMD) {
|
if (isAMD) {
|
||||||
cmd += ` \\\n --kv-cache-dtype fp8_e4m3`;
|
cmd += ` \\\n --kv-cache-dtype fp8_e4m3`;
|
||||||
cmd += ` \\\n --attention-backend triton`;
|
cmd += ` \\\n --attention-backend triton`;
|
||||||
|
|||||||
@@ -38,6 +38,19 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
precision: {
|
||||||
|
name: 'precision',
|
||||||
|
title: 'Precision',
|
||||||
|
getDynamicItems: (values) => {
|
||||||
|
const hw = values.hardware;
|
||||||
|
const isBlackwell = hw === 'b200' || hw === 'gb300';
|
||||||
|
return [
|
||||||
|
{ id: 'fp8', label: 'FP8', default: true, disabled: false },
|
||||||
|
{ id: 'fp4', label: 'FP4', default: false, disabled: !isBlackwell,
|
||||||
|
disabledReason: 'NVFP4 requires Blackwell (B200/GB300)' }
|
||||||
|
];
|
||||||
|
}
|
||||||
|
},
|
||||||
thinking: {
|
thinking: {
|
||||||
name: 'thinking',
|
name: 'thinking',
|
||||||
title: 'Thinking Capabilities',
|
title: 'Thinking Capabilities',
|
||||||
@@ -115,7 +128,7 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
|
|
||||||
// Generate command mirrors sgl-cookbook src/components/autoregressive/MiniMaxM27ConfigGenerator/index.js
|
// Generate command mirrors sgl-cookbook src/components/autoregressive/MiniMaxM27ConfigGenerator/index.js
|
||||||
const generateCommand = () => {
|
const generateCommand = () => {
|
||||||
const { hardware, gpuCount, thinking, toolcall } = values;
|
const { hardware, gpuCount, precision, thinking, toolcall } = values;
|
||||||
|
|
||||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
||||||
const isGB300 = hardware === 'gb300';
|
const isGB300 = hardware === 'gb300';
|
||||||
@@ -126,9 +139,22 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300';
|
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300';
|
||||||
}
|
}
|
||||||
|
|
||||||
const modelName = 'MiniMaxAI/MiniMax-M2.7';
|
const isBlackwell = hardware === 'b200' || hardware === 'gb300';
|
||||||
|
const isFp4 = precision === 'fp4';
|
||||||
|
|
||||||
let cmd = 'sglang serve \\\n';
|
if (isFp4 && !isBlackwell) {
|
||||||
|
return '# NVFP4 requires Blackwell hardware (B200 or GB300)';
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelName = isFp4 ? 'nvidia/MiniMax-M2.7-NVFP4' : 'MiniMaxAI/MiniMax-M2.7';
|
||||||
|
|
||||||
|
const useAllreduceFusion = hardware === 'h200' || hardware === 'b200' || hardware === 'gb300';
|
||||||
|
|
||||||
|
let cmd = '';
|
||||||
|
if (useAllreduceFusion) {
|
||||||
|
cmd += 'SGLANG_USE_FUSED_PARALLEL_QKNORM=1 \\\n';
|
||||||
|
}
|
||||||
|
cmd += 'sglang serve \\\n';
|
||||||
cmd += ` --model-path ${modelName}`;
|
cmd += ` --model-path ${modelName}`;
|
||||||
|
|
||||||
if (isXeon) {
|
if (isXeon) {
|
||||||
@@ -159,6 +185,18 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
cmd += ' \\\n --attention-backend triton';
|
cmd += ' \\\n --attention-backend triton';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isBlackwell) {
|
||||||
|
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm_routed';
|
||||||
|
if (!isFp4) {
|
||||||
|
cmd += ' \\\n --fp8-gemm-backend flashinfer_trtllm';
|
||||||
|
cmd += ' \\\n --dtype bfloat16';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (useAllreduceFusion) {
|
||||||
|
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user