[Doc] Add H200 recipes to MiMo-V2.6 cookbook (#40969)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
542a043f1b
commit
ffac53d779
@@ -1,12 +1,12 @@
|
||||
---
|
||||
title: MiMo-V2.6
|
||||
description: "Deploy MiMo-V2.6-Flash and Pro with SGLang on NVIDIA B300 GPUs, using MXFP4 MoE weights, BF16 routing, DFlash decoding, and a 1M-token context window."
|
||||
description: "Deploy MiMo-V2.6-Flash and Pro with SGLang on NVIDIA H200 and B300 GPUs, using MXFP4 MoE weights, BF16 routing, DFlash decoding, and a 1M-token context window."
|
||||
tag: NEW
|
||||
---
|
||||
|
||||
## Deployment
|
||||
|
||||
The recipes below target **MiMo-V2.6-Flash on 4× B300** and **MiMo-V2.6-Pro on 8× B300**, each on a single node. Both use MXFP4 MoE weights, a BF16 MoE router, and DFlash speculative decoding. B300 validation was reported by the model team; the launch settings come from [SGLang PR #40448](https://github.com/sgl-project/sglang/pull/40448).
|
||||
The recipes below target **MiMo-V2.6-Flash on 4× H200 or B300** and **MiMo-V2.6-Pro on 8× H200 or B300**, each on a single node. Both use MXFP4 MoE weights, a BF16 MoE router, and DFlash speculative decoding.
|
||||
|
||||
<a id="install" />
|
||||
|
||||
@@ -69,7 +69,7 @@ import { Playground } from "/src/snippets/_playground.jsx";
|
||||
<th style={{textAlign: "left", padding: "10px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Variant</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Total parameters</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Context window</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>B300 recipe</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Recipe (H200 / B300)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -97,9 +97,9 @@ import { Playground } from "/src/snippets/_playground.jsx";
|
||||
### Parallelism and precision
|
||||
|
||||
- **Flash:** `--tp 4 --ep 4`; **Pro:** `--tp 8 --ep 8`. Both keep `--dp 1 --pp-size 1`. TP and EP use the same GPU ranks; do not multiply them to determine the GPU count.
|
||||
- **MoE compute and communication:** Keep `--moe-runner-backend deep_gemm --moe-a2a-backend deepep --deepep-mode auto`, together with `--moe-dense-tp-size 1` and `--enable-dp-lm-head`, as in the PR.
|
||||
- **MoE compute and communication:** Keep `--moe-runner-backend deep_gemm` on B300 and `--moe-runner-backend marlin` on H200, with `--moe-a2a-backend deepep --deepep-mode auto`, together with `--moe-dense-tp-size 1` and `--enable-dp-lm-head`, as in the PR.
|
||||
- **Precision:** MXFP4 describes the MoE expert weights, not the entire model. BF16 router weights are selected by the checkpoint's `moe_router_dtype`; the support commit computes router output logits in FP32. The PR relies on checkpoint quantization metadata and does not add a `--quantization` override.
|
||||
- **Attention:** Keep `--attention-backend fa4` and `--mm-attention-backend fa4` on B300. The recipe also enables `--mm-enable-dp-encoder` for multimodal encoding.
|
||||
- **Attention:** Keep `--attention-backend fa4` and `--mm-attention-backend fa4` on both H200 and B300. The recipe also enables `--mm-enable-dp-encoder` for multimodal encoding.
|
||||
|
||||
### Context length and memory
|
||||
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
// Instantiated from cookbook-add-model/templates/config.jsx.tmpl.
|
||||
// Recipes: https://github.com/sgl-project/sglang/pull/40448
|
||||
// SGLang: 983e643854f15cf9ef4370a49dfd74b6af54c3e3.
|
||||
// B300 validation reported by the model team. Public V2.6 checkpoints are
|
||||
// B300 validation reported by the model team. H200 cells reuse the B300
|
||||
// recipes with `--moe-runner-backend marlin` in place of deep_gemm. Public V2.6 checkpoints are
|
||||
// XiaomiMiMo/MiMo-V2.6-{Flash,Pro}-RL (MXFP4 experts, bf16 router, bundled
|
||||
// dflash/ drafter); modelNames are served aliases, and the checkpoint paths
|
||||
// stay editable so a local copy can be used instead.
|
||||
export const config = {
|
||||
modelName: "MiMo-V2.6",
|
||||
supportedHardware: ["b300"],
|
||||
supportedHardware: ["h200", "b300"],
|
||||
variants: [
|
||||
{ id: "flash", label: "Flash", subtitle: "309B / 15B active · 4 GPUs" },
|
||||
{ id: "pro", label: "Pro", subtitle: "1.02T / 42B active · 8 GPUs" },
|
||||
@@ -35,7 +36,7 @@ export const config = {
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{"model":"{{MODEL_NAME}}","messages":[{"role":"user","content":"What is 15% of 240?"}],"chat_template_kwargs":{"enable_thinking":true}}'`,
|
||||
// The MiMo-V2.6 support (PR #40448) is on main, so the nightly tag carries it.
|
||||
dockerImages: { b300: "lmsysorg/sglang:dev" },
|
||||
dockerImages: { h200: "lmsysorg/sglang:dev", b300: "lmsysorg/sglang:dev" },
|
||||
dockerMounts: ["\"{{MODEL_ROOT}}:/model:ro\""],
|
||||
github: { cookbookModel: "MiMo-V2.6 (Flash / Pro)" },
|
||||
playgroundFeatures: {
|
||||
@@ -134,5 +135,37 @@ export const config = {
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "mxfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: false,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path '{{FLASH_MODEL_PATH}}'",
|
||||
"--served-model-name {{MODEL_NAME}}",
|
||||
"--tp 4",
|
||||
"--moe-runner-backend marlin",
|
||||
"--trust-remote-code",
|
||||
"--reasoning-parser mimo",
|
||||
"--tool-call-parser mimo",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "pro", quant: "mxfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: false,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path '{{PRO_MODEL_PATH}}'",
|
||||
"--served-model-name {{MODEL_NAME}}",
|
||||
"--tp 8",
|
||||
"--moe-runner-backend marlin",
|
||||
"--trust-remote-code",
|
||||
"--reasoning-parser mimo",
|
||||
"--tool-call-parser mimo",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user