diff --git a/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index d3b6018de..8a1d67b34 100644 --- a/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -88,6 +88,7 @@ import { MiMoV25Deployment } from '/src/snippets/autoregressive/mimo-v25-deploym **MiMo-V2.5 (310B):** - The checkpoint has a TP=4-interleaved fused `qkv_proj`; attention-TP per DP group **must** be 4. Use `--dp = TP / 4`; for TP > 4 this also requires DP-attention. Total GPUs must be a multiple of 4. A bare `--tp 8` without `--dp 2` will fail to load with `MiMoV2 fused qkv_proj checkpoint is TP=4-interleaved; got attention tp_size=8`. - Single-node deployments: H100/H200 8× GPUs (`--tp 8 --dp 2`), B200 4× GPUs (`--tp 4`, dp=1, no DP-attn flag needed), GB300 4× GPUs (`--tp 4`, single NVL4 node). FP8 quantization. +- On Blackwell, pass `--mm-attention-backend fa4` for the V2.5 vision encoder. The checkpoint config requests FlashAttention-3 internally, but SGLang rejects FA3 on Blackwell and expects FA4 for multimodal attention. - `--enable-dp-lm-head` and `--mm-enable-dp-encoder` are required whenever `--enable-dp-attention` is on, to keep LM head and encoder sharding consistent. - EAGLE MTP uses the checkpoint's MTP weights. Enable with `--speculative-algorithm EAGLE` and `--enable-multi-layer-eagle` (both Hopper and Blackwell). - **Multimodal**: Supports image, video, and audio understanding; see Section 4.3 for invocation examples. diff --git a/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx b/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx index bb9b49a38..cc7e98a6d 100644 --- a/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx @@ -11,8 +11,8 @@ export const MiMoV25Deployment = () => { // so attention-TP per DP group must be 4; effective parallelism = TP/DP = 4. // H200 → tp=8, dp=2, single-node, FP8 (verified) // H100 → tp=8, dp=2, single-node, FP8 - // B200 → tp=4, dp=1, single-node, FP8 - // GB300 → tp=4, dp=1, single-node, FP8 + // B200 → tp=4, dp=1, single-node, FP8 (Blackwell: vision fa4) + // GB300 → tp=4, dp=1, single-node, FP8 (Blackwell: vision fa4) // // Optional toggles: // EAGLE MTP — adds --speculative-* flags + SGLANG_ENABLE_SPEC_V2=1. @@ -350,6 +350,7 @@ export const MiMoV25Deployment = () => { flags.push(` --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 64}'`); } } else { + if (blackwell) flags.push(" --mm-attention-backend fa4"); flags.push(" --mem-fraction-static 0.65"); flags.push(" --chunked-prefill-size 16384"); }