diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx index 42709d4973..475f269eaf 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx @@ -1,12 +1,12 @@ --- title: MiMo-V2.6 -description: "Deploy MiMo-V2.6-Flash and Pro with SGLang on NVIDIA B300 GPUs, using MXFP4 MoE weights, BF16 routing, DFlash decoding, and a 1M-token context window." +description: "Deploy MiMo-V2.6-Flash and Pro with SGLang on NVIDIA H200 and B300 GPUs, using MXFP4 MoE weights, BF16 routing, DFlash decoding, and a 1M-token context window." tag: NEW --- ## Deployment -The recipes below target **MiMo-V2.6-Flash on 4× B300** and **MiMo-V2.6-Pro on 8× B300**, each on a single node. Both use MXFP4 MoE weights, a BF16 MoE router, and DFlash speculative decoding. B300 validation was reported by the model team; the launch settings come from [SGLang PR #40448](https://github.com/sgl-project/sglang/pull/40448). +The recipes below target **MiMo-V2.6-Flash on 4× H200 or B300** and **MiMo-V2.6-Pro on 8× H200 or B300**, each on a single node. Both use MXFP4 MoE weights, a BF16 MoE router, and DFlash speculative decoding. @@ -69,7 +69,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; Variant Total parameters Context window - B300 recipe + Recipe (H200 / B300) @@ -97,9 +97,9 @@ import { Playground } from "/src/snippets/_playground.jsx"; ### Parallelism and precision - **Flash:** `--tp 4 --ep 4`; **Pro:** `--tp 8 --ep 8`. Both keep `--dp 1 --pp-size 1`. TP and EP use the same GPU ranks; do not multiply them to determine the GPU count. -- **MoE compute and communication:** Keep `--moe-runner-backend deep_gemm --moe-a2a-backend deepep --deepep-mode auto`, together with `--moe-dense-tp-size 1` and `--enable-dp-lm-head`, as in the PR. +- **MoE compute and communication:** Keep `--moe-runner-backend deep_gemm` on B300 and `--moe-runner-backend marlin` on H200, with `--moe-a2a-backend deepep --deepep-mode auto`, together with `--moe-dense-tp-size 1` and `--enable-dp-lm-head`, as in the PR. - **Precision:** MXFP4 describes the MoE expert weights, not the entire model. BF16 router weights are selected by the checkpoint's `moe_router_dtype`; the support commit computes router output logits in FP32. The PR relies on checkpoint quantization metadata and does not add a `--quantization` override. -- **Attention:** Keep `--attention-backend fa4` and `--mm-attention-backend fa4` on B300. The recipe also enables `--mm-enable-dp-encoder` for multimodal encoding. +- **Attention:** Keep `--attention-backend fa4` and `--mm-attention-backend fa4` on both H200 and B300. The recipe also enables `--mm-enable-dp-encoder` for multimodal encoding. ### Context length and memory diff --git a/docs/src/snippets/configs/XiaomiMiMo/mimo-v2.6.jsx b/docs/src/snippets/configs/XiaomiMiMo/mimo-v2.6.jsx index 9daf23ce75..7d8d842023 100644 --- a/docs/src/snippets/configs/XiaomiMiMo/mimo-v2.6.jsx +++ b/docs/src/snippets/configs/XiaomiMiMo/mimo-v2.6.jsx @@ -1,13 +1,14 @@ // Instantiated from cookbook-add-model/templates/config.jsx.tmpl. // Recipes: https://github.com/sgl-project/sglang/pull/40448 // SGLang: 983e643854f15cf9ef4370a49dfd74b6af54c3e3. -// B300 validation reported by the model team. Public V2.6 checkpoints are +// B300 validation reported by the model team. H200 cells reuse the B300 +// recipes with `--moe-runner-backend marlin` in place of deep_gemm. Public V2.6 checkpoints are // XiaomiMiMo/MiMo-V2.6-{Flash,Pro}-RL (MXFP4 experts, bf16 router, bundled // dflash/ drafter); modelNames are served aliases, and the checkpoint paths // stay editable so a local copy can be used instead. export const config = { modelName: "MiMo-V2.6", - supportedHardware: ["b300"], + supportedHardware: ["h200", "b300"], variants: [ { id: "flash", label: "Flash", subtitle: "309B / 15B active · 4 GPUs" }, { id: "pro", label: "Pro", subtitle: "1.02T / 42B active · 8 GPUs" }, @@ -35,7 +36,7 @@ export const config = { -H 'Content-Type: application/json' \\ -d '{"model":"{{MODEL_NAME}}","messages":[{"role":"user","content":"What is 15% of 240?"}],"chat_template_kwargs":{"enable_thinking":true}}'`, // The MiMo-V2.6 support (PR #40448) is on main, so the nightly tag carries it. - dockerImages: { b300: "lmsysorg/sglang:dev" }, + dockerImages: { h200: "lmsysorg/sglang:dev", b300: "lmsysorg/sglang:dev" }, dockerMounts: ["\"{{MODEL_ROOT}}:/model:ro\""], github: { cookbookModel: "MiMo-V2.6 (Flash / Pro)" }, playgroundFeatures: { @@ -134,5 +135,37 @@ export const config = { "--port {{PORT}}", ], }, + { + match: { hw: "h200", variant: "flash", quant: "mxfp4", strategy: "balanced", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path '{{FLASH_MODEL_PATH}}'", + "--served-model-name {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend marlin", + "--trust-remote-code", + "--reasoning-parser mimo", + "--tool-call-parser mimo", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "pro", quant: "mxfp4", strategy: "balanced", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path '{{PRO_MODEL_PATH}}'", + "--served-model-name {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend marlin", + "--trust-remote-code", + "--reasoning-parser mimo", + "--tool-call-parser mimo", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, ], };