diff --git a/docs_new/docs/references/environment_variables.mdx b/docs_new/docs/references/environment_variables.mdx index 1c03030ae..348a73371 100644 --- a/docs_new/docs/references/environment_variables.mdx +++ b/docs_new/docs/references/environment_variables.mdx @@ -540,6 +540,11 @@ SGLang supports various environment variables that can be used to configure its Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass) `false` + + SGLANG_USE_FUSED_PARALLEL_QKNORM + Use the fused parallel QK RMSNorm kernel for MiniMax-M2.x on CUDA when attention TP size > 1 + `false` + diff --git a/docs_new/src/snippets/autoregressive/minimax-m25-deployment.jsx b/docs_new/src/snippets/autoregressive/minimax-m25-deployment.jsx index a0aa95740..d1aa7d495 100644 --- a/docs_new/src/snippets/autoregressive/minimax-m25-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/minimax-m25-deployment.jsx @@ -72,7 +72,13 @@ export const MiniMaxM25Deployment = () => { const modelName = `${modelFamily}/MiniMax-M2.5`; + const isBlackwell = hardware === 'b200'; + const useAllreduceFusion = hardware === 'h200' || hardware === 'b200'; + let cmd = ''; + if (useAllreduceFusion) { + cmd += 'SGLANG_USE_FUSED_PARALLEL_QKNORM=1 \\\n'; + } cmd += 'python -m sglang.launch_server \\\n'; cmd += ` --model-path ${modelName}`; @@ -102,6 +108,16 @@ export const MiniMaxM25Deployment = () => { cmd += ` \\\n --trust-remote-code`; cmd += ` \\\n --mem-fraction-static 0.85`; + if (isBlackwell) { + cmd += ` \\\n --moe-runner-backend flashinfer_trtllm_routed`; + cmd += ` \\\n --fp8-gemm-backend flashinfer_trtllm`; + cmd += ` \\\n --dtype bfloat16`; + } + + if (useAllreduceFusion) { + cmd += ` \\\n --enable-flashinfer-allreduce-fusion`; + } + if (isAMD) { cmd += ` \\\n --kv-cache-dtype fp8_e4m3`; cmd += ` \\\n --attention-backend triton`; diff --git a/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx b/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx index 640094bfe..2254773b2 100644 --- a/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx @@ -38,6 +38,19 @@ export const MiniMaxM27Deployment = () => { ]; } }, + precision: { + name: 'precision', + title: 'Precision', + getDynamicItems: (values) => { + const hw = values.hardware; + const isBlackwell = hw === 'b200' || hw === 'gb300'; + return [ + { id: 'fp8', label: 'FP8', default: true, disabled: false }, + { id: 'fp4', label: 'FP4', default: false, disabled: !isBlackwell, + disabledReason: 'NVFP4 requires Blackwell (B200/GB300)' } + ]; + } + }, thinking: { name: 'thinking', title: 'Thinking Capabilities', @@ -115,7 +128,7 @@ export const MiniMaxM27Deployment = () => { // Generate command mirrors sgl-cookbook src/components/autoregressive/MiniMaxM27ConfigGenerator/index.js const generateCommand = () => { - const { hardware, gpuCount, thinking, toolcall } = values; + const { hardware, gpuCount, precision, thinking, toolcall } = values; const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x'; const isGB300 = hardware === 'gb300'; @@ -126,9 +139,22 @@ export const MiniMaxM27Deployment = () => { return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300'; } - const modelName = 'MiniMaxAI/MiniMax-M2.7'; + const isBlackwell = hardware === 'b200' || hardware === 'gb300'; + const isFp4 = precision === 'fp4'; - let cmd = 'sglang serve \\\n'; + if (isFp4 && !isBlackwell) { + return '# NVFP4 requires Blackwell hardware (B200 or GB300)'; + } + + const modelName = isFp4 ? 'nvidia/MiniMax-M2.7-NVFP4' : 'MiniMaxAI/MiniMax-M2.7'; + + const useAllreduceFusion = hardware === 'h200' || hardware === 'b200' || hardware === 'gb300'; + + let cmd = ''; + if (useAllreduceFusion) { + cmd += 'SGLANG_USE_FUSED_PARALLEL_QKNORM=1 \\\n'; + } + cmd += 'sglang serve \\\n'; cmd += ` --model-path ${modelName}`; if (isXeon) { @@ -159,6 +185,18 @@ export const MiniMaxM27Deployment = () => { cmd += ' \\\n --attention-backend triton'; } + if (isBlackwell) { + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm_routed'; + if (!isFp4) { + cmd += ' \\\n --fp8-gemm-backend flashinfer_trtllm'; + cmd += ' \\\n --dtype bfloat16'; + } + } + + if (useAllreduceFusion) { + cmd += ' \\\n --enable-flashinfer-allreduce-fusion'; + } + return cmd; };