From dba617f2ec2bc957f4a7a443d669dcd78e693fcd Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:14:35 +0100 Subject: [PATCH] doc: update docs for new model (#28060) --- .../autoregressive/MiniMax/MiniMax-M2.7.mdx | 1 - .../autoregressive/MiniMax/MiniMax-M3.mdx | 502 ++++++++++++++++++ docs_new/cookbook/autoregressive/intro.mdx | 2 +- docs_new/docs.json | 1 + .../MiniMaxAI/minimax-m3-benchmarks.jsx | 93 ++++ .../snippets/configs/MiniMaxAI/minimax-m3.jsx | 370 +++++++++++++ 6 files changed, 967 insertions(+), 2 deletions(-) create mode 100644 docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx create mode 100644 docs_new/src/snippets/configs/MiniMaxAI/minimax-m3-benchmarks.jsx create mode 100644 docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx index 5141361c5..aa357327a 100644 --- a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx @@ -2,7 +2,6 @@ title: MiniMax-M2.7 metatags: description: "Deploy MiniMax-M2.7 with SGLang on NVIDIA GPUs, AMD GPUs, and Intel Xeon CPUs — model self-evolution, professional software engineering, and native agent teams." -tag: NEW --- ## 1. Model Introduction diff --git a/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx new file mode 100644 index 000000000..68a1ae1ee --- /dev/null +++ b/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx @@ -0,0 +1,502 @@ +--- +title: MiniMax-M3 +description: "Deploy MiniMax-M3 with SGLang — a ~428B-param (23B activated) multimodal Mixture-of-Experts reasoning model with MiniMax Sparse Attention and 1M context, MXFP8 on NVIDIA Blackwell & AMD Instinct, bf16 on Hopper." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install -U uv +uv venv --python 3.12 && source .venv/bin/activate + +# MiniMax-M3 ships in SGLang PR #27944, not yet in a tagged release — install from +# the PR head. The serving runtime is in the base dependencies, so no extra is needed: +git clone https://github.com/sgl-project/sglang.git +cd sglang +git fetch origin pull/27944/head && git checkout FETCH_HEAD +uv pip install -e python +``` + +Then run the **Python** output of the command panel below in that environment. The **Docker** tab is simpler — its image bundles the CUDA-13 runtime and the #27944 code. Once [PR #27944](https://github.com/sgl-project/sglang/pull/27944) is merged and released, `uv pip install sglang` will pull M3 support directly. + + + + + +```bash Command +# Pull the M3 image the command panel selects for your platform, e.g.: +docker pull lmsysorg/sglang:dev-cu13-minimax-m3 +``` + +The command panel below fills in the right tag per platform: `dev-cu13-minimax-m3` (CUDA 13 — B300, GB200, GB300), `dev-cu12-minimax-m3` (CUDA 12 — Hopper H200), or `dev-minimax-m3` (default). On AMD Instinct it uses the matching ROCm image (MI300X/MI325X → `…-rocm700-mi30x`, MI350X/MI355X → `…-rocm720-mi35x`). For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker), substituting the inner `sglang serve ...` with what the command generator produces. + + +These images do not yet bundle MiniMax's MSA sparse-attention kernel (bundling it by default is in progress). Blackwell users who want the recommended fast path install MSA manually after pulling — see **§2.1**. Without it, the same recipe still serves on the built-in Triton sparse path. + + + + + + + + +Pick your hardware + recipe to generate the launch command. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/MiniMaxAI/minimax-m3.jsx"; +import { benchmarks } from "/src/snippets/configs/MiniMaxAI/minimax-m3-benchmarks.jsx"; + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +[MiniMax-M3](https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8) is MiniMax's native-multimodal Mixture-of-Experts reasoning model: **~428B total parameters with ~23B activated per token** (128 experts, 4 active per token), 60 layers, and a **1M-token context** over text, image, and video. Its defining feature is **MiniMax Sparse Attention (MSA)** — a block-sparse "lightning indexer" attention that keeps long-context cost low (MiniMax reports ~9× prefill / ~15× decode speedup over M2 at 1M context). This page serves the **MXFP8** variant (`MiniMaxAI/MiniMax-M3-MXFP8`, ~440 GB) on NVIDIA Blackwell and AMD Instinct; on NVIDIA Hopper (H200), use the full-precision **bfloat16** build [`MiniMaxAI/MiniMax-M3`](https://huggingface.co/MiniMaxAI/MiniMax-M3) (§2.4). Released under the **MiniMax Community License**. + +Key characteristics as served by SGLang: + +- **Multimodal (vision + text)**: accepts interleaved text and images through the OpenAI-compatible chat API (loaded as `MiniMaxM3SparseForConditionalGeneration`). Image input via URL and base64 is validated; video input has not been tested here. +- **Reasoning model**: emits its chain of thought wrapped in `...`. Always launch with **`--reasoning-parser auto`** — it auto-detects the right parser from the chat template, and SGLang then strips the tags and returns the trace separately in `message.reasoning_content`. +- **Native tool calling**: a custom namespace-token XML format, parsed into standard OpenAI `tool_calls`. Always launch with **`--tool-call-parser auto`** — it auto-detects the right parser from the chat template. Single, parallel, and nested (object / array) arguments are supported. +- **Sparse attention**: most layers use M3's "lightning indexer" block-sparse attention (top-k 128-token blocks), which keeps decode cost roughly flat in context length. On Blackwell, MiniMax's open-source [MSA kernel](https://github.com/MiniMax-AI/MSA) accelerates this path further (§2.1). +- **MXFP8 quantization across vendors**: the MXFP8 MoE weights run natively on NVIDIA Blackwell (B200 / B300 / GB200 / GB300) and on AMD Instinct MI350X/MI355X (gfx950 / CDNA4), both of which have hardware MX-scaled matmul. On AMD MI300X/MI325X (gfx942 / CDNA3) — no hardware MX — SGLang converts the weights to block-fp8 `[128,128]` at load and serves them on the tuned ROCm kernels (§2.3). The vision tower stays unquantized. + +**Recommended generation** (from the model card): `temperature` 1.0, `top_p` 0.95, `top_k` 40 — SGLang applies these automatically from the model's `generation_config.json`. + +**Resources:** [HuggingFace](https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8) · [MSA kernel](https://github.com/MiniMax-AI/MSA) + +## 2. Configuration Tips + +### 2.1 MSA sparse-attention fast path (recommended for Blackwell users) + +[MiniMax MSA](https://github.com/MiniMax-AI/MSA) (`fmha_sm100`, MIT-licensed) is the recommended Blackwell kernel for M3's main sparse-attention step — faster and more memory-efficient than the built-in Triton fallback. It is purely additive — install it and the recipe above engages it automatically; without it the same recipe still serves on the built-in Triton path. The swap is numerically equivalent (cosine ≥ 0.99999 vs Triton), decode stays CUDA-graph-capturable, prefill TTFT drops ~9–12% at 8K–64K context, and the MSA path survives memory configurations where the Triton path OOMs. + +**Requirements** (from the [MSA README](https://github.com/MiniMax-AI/MSA#requirements)): + +- **GPU**: NVIDIA SM100 family — sm_100 (B200 / GB200) and sm_103 (B300 / GB300). +- **Toolchain**: CUDA Toolkit with `nvcc` ≥ 12.x on `PATH` (or `CUDA_HOME` set) — the kernels are JIT-compiled at first import. +- **Python**: ≥ 3.10; **OS**: Linux — works on both **x86_64 and aarch64 (Grace, e.g. GB200 / GB300)**; the aarch64 build needs no source edits. + + + +```bash Command +# --recursive pulls the CUTLASS submodule required for JIT compilation +git clone --recursive https://github.com/MiniMax-AI/MSA.git msa +cd msa && pip install . +# Verify the SGLang gate (True -> MSA engaged on this device; False -> Triton fallback): +python -c "from sglang.srt.layers.attention.minimax_sparse_ops.msa import msa_available; print(msa_available())" +``` + + + + +The first import JIT-compiles the kernels, which can take 30 s to a few minutes on a cold `nvcc` cache — this is normal, not a hang. Subsequent server starts hit the JIT cache. + + + +**Warm the JIT cache before a multi-GPU launch.** On a *cold* cache, several tensor-parallel ranks racing to JIT-compile MSA's plan kernel can leave one rank loading a half-linked module (`AttributeError: Module has no function 'plan'` at CUDA-graph capture). Run the gate-check `python -c "..."` (or any single-process `fmha_sm100_plan` call) once before launching the server — that compiles the kernel single-process, and every rank then hits the warm cache. + + +The gate requires `--attention-backend fa4 --page-size 128` (already part of the Blackwell recipe above; on current `main` these are also the auto-selected M3 defaults on SM100 GPUs). Force the Triton path at any time with the env var `SGLANG_DISABLE_MSA=1`. MSA is a Blackwell (SM100) kernel and does not apply to the AMD ROCm paths. + + +For multimodal (image) serving, keep the same text recipe above — `--attention-backend fa4 --page-size 128` (MSA) is unchanged — and add `--mm-attention-backend flashinfer_cudnn` for the vision tower. The text and vision-tower attention backends are independent knobs; MSA only touches the language-model sparse attention, not image handling. + + +### 2.2 Memory and workload tuning + +The NVIDIA Blackwell recipe is the validated single-node **4-GPU (`--tp 4`)** config, which is also the GB200 / GB300 single-node ceiling. It runs identically on B200 (sm_100), B300 (sm_103), and GB300 (sm_103, aarch64); GB200 (sm_100, aarch64) is inferred-supported — both of its axes are validated above — but not directly benchmarked. The AMD recipes use **8-GPU (`--tp 8`)**. + +- **Memory**: `--mem-fraction-static` trades KV-pool capacity against prefill **activation headroom** — `0.75` is the safe default on NVIDIA (`0.80` on AMD). A higher value is fine at low concurrency but OOMs under high concurrency or long context, so raise it only for interactive single-stream serving. +- **Long context (32K+)**: keep `--mem-fraction-static` at the platform default and raise `--chunked-prefill-size` to `16384`. Decode TPOT stays roughly flat in context length thanks to sparse attention; 1K–128K prompts are validated. +- **8-GPU nodes**: B200 / B300 hosts with 8 GPUs can use `--tp 8` for more throughput / KV headroom; tp4 is documented as the NVIDIA cross-family common denominator. +- **Expert parallelism**: to trade latency for throughput add `--ep` (see [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism)). On AMD, set `--ep` equal to `--tp`. Shared-experts fusion is automatically disabled when EP > 1; on AMD standard EP the server also disables `--enable-aiter-allreduce-fusion` automatically to preserve accuracy. +- `--trust-remote-code` is required to load the MiniMax config / processor classes. + +### 2.3 AMD Instinct (ROCm) + +MiniMax-M3 runs on AMD Instinct GPUs through two code paths, by architecture — both selected automatically; you still pass `--quantization mxfp8` either way: + +- **MI350X / MI355X (gfx950, CDNA4)** has hardware MX-scaled matmul, so the **MXFP8 weights are served natively**. SGLang auto-detects the checkpoint, selects the Triton MiniMax-M3 MoE path with the packaged tuned MXFP8 configs, and enables AITER fused all-reduce for single-node tensor parallelism. The launch command is the NVIDIA recipe minus the Blackwell-only backend flags. +- **MI300X / MI325X (gfx942, CDNA3)** has **no** hardware MX matmul. SGLang transparently **converts the MXFP8 weights to block-fp8 `[128,128]` at load time**, then serves them with the tuned ROCm block-fp8 kernels (`--attention-backend aiter`, `--moe-runner-backend triton`; the `aiter` runner also works and scores marginally higher). On a cold start the first generation can JIT-compile AITER configs and exceed the default warmup/HTTP timeout, so the recipe adds `--watchdog-timeout 3600 --skip-server-warmup`. The block-fp8 step adds only a small relative error over MXFP8's native `1×32` scaling — negligible on GSM8K (see the benchmark card). + +Select an MI300X/MI325X or MI350X/MI355X tile in the command panel above to get the exact launch command for each path. + + +The AMD recipes are validated end-to-end on **text** workloads — chat, reasoning separation, and tool calling. The vision tower was not exercised on ROCm; for image input on AMD, omit the Blackwell `--mm-attention-backend flashinfer_cudnn` flag and let the encoder use the ROCm default backend, and treat vision as unvalidated on that path. + + +### 2.4 Serving on Hopper (H200) with the bf16 build + +The MXFP8 kernels are Blackwell-only, so Hopper (H200) serves the full-precision bfloat16 build [`MiniMaxAI/MiniMax-M3`](https://huggingface.co/MiniMaxAI/MiniMax-M3). Select **H200 + BF16** in the Deploy panel above for the exact command — it runs at `--tp 8` (the bf16 weights need a full 8-GPU node). SGLang picks the right backends for Hopper automatically, so the recipe stays minimal: + +- **MoE runner**: Triton, auto-selected for bf16 weights. +- **Attention**: FlashAttention-3 with page size 1. MSA (§2.1) is a Blackwell kernel, so M3's sparse step runs on the built-in Triton path here. +- **CUDA graph**: on, with full decode-graph capture. + +Validated on 8×H200 — reasoning and tool-call auto-detection plus long-context generation. For prefill/decode disaggregation on Hopper, see §3.4. + +## 3. Advanced Usage + +### 3.1 Reasoning + +Launch with `--reasoning-parser auto` (or toggle **Reasoning Parser** in the **Parsers** card of the [Playground above](#playground)). The `` trace then lands in `message.reasoning_content`, separate from the final answer in `message.content` — no client-side tag stripping needed. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="MiniMaxAI/MiniMax-M3-MXFP8", + messages=[{"role": "user", "content": "What is 15% of 240? Explain briefly."}], + max_tokens=2048, +) + +message = response.choices[0].message +print("=============== Reasoning ===============") +print(message.reasoning_content) +print("=============== Answer ==================") +print(message.content) +``` + + + + + +```text Output +=============== Reasoning =============== +15% of 240. 15% = 0.15. 240 * 0.15 = 36. Quick check: 10% is 24, 5% is 12, 24 + 12 = 36. +=============== Answer ================== +15% of 240 is **36**. +(10% of 240 = 24, and 5% of 240 = 12; 24 + 12 = 36.) +``` + + + +When streaming, the trace arrives on `delta.reasoning_content` and the answer on `delta.content`, so the two sections can be rendered separately in real time: + + + +```python Example +response = client.chat.completions.create( + model="MiniMaxAI/MiniMax-M3-MXFP8", + messages=[{"role": "user", "content": "Solve step by step: what is 15% of 240?"}], + max_tokens=2048, + stream=True, +) + +for chunk in response: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if getattr(delta, "reasoning_content", None): + print(delta.reasoning_content, end="", flush=True) # thinking stream + if delta.content: + print(delta.content, end="", flush=True) # answer stream +print() +``` + +**Output Example:** + +```text Output +[delta.reasoning_content — thinking stream] +Let me solve this step by step. + +15% of 240 += 0.15 × 240 += 36 + +Let me verify: 10% of 240 = 24, 5% of 240 = 12, so 15% = 24 + 12 = 36. ✓ + +[delta.content — answer stream] +# Solving 15% of 240 +## Step 1: Convert the percentage to a decimal +15% = 15/100 = 0.15 +## Step 2: Multiply by 240 +0.15 × 240 = 36 +## Answer +**15% of 240 = 36** +``` + + + +### 3.2 Tool Calling + +Launch with `--tool-call-parser auto` (or toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) — it auto-detects M3's tool-call parser from the chat template. M3 emits tool calls in a custom namespace-token XML format: + +```text Raw model output +]<]minimax[>[ +]<]minimax[>[]<]minimax[>[Beijing]<]minimax[>[]<]minimax[>[ +]<]minimax[>[ +``` + +The parser converts that into the standard OpenAI `tool_calls` structure: + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] + +response = client.chat.completions.create( + model="MiniMaxAI/MiniMax-M3-MXFP8", + messages=[{"role": "user", "content": "What's the weather in Beijing?"}], + tools=tools, +) + +message = response.choices[0].message +if message.tool_calls: + for call in message.tool_calls: + print(f"Tool: {call.function.name}") + print(f"Args: {call.function.arguments}") +``` + + + + + +```text Output +Tool: get_weather +Args: {"location": "Beijing"} +``` + + + +Beyond a single flat call, the parser also supports: + +- **Parallel calls** — multiple `` blocks inside the single `` wrapper, surfaced as multiple `message.tool_calls` entries. +- **Nested object arguments** — an `object`-typed parameter is emitted as nested XML tags and reconstructed into a JSON object. +- **Array arguments** — an `array`-typed parameter uses repeated `` children and is reconstructed into a JSON list. + +For example, a tool with object and array parameters round-trips cleanly: + +```text Output +create_event {"title": "Design sync", "attendees": ["alice", "bob"], "location": {"room": "R2", "floor": 3}} +``` + +To return a tool result, append the assistant's `tool_calls` turn plus a matching `tool` message and ask the model to continue — the follow-up answer may place text in `reasoning_content` as well as `content`, so print both. + +### 3.3 Multimodal (Vision) Input + +Images go through the standard OpenAI `image_url` content type. The vision tower is always loaded; for image serving add `--mm-attention-backend flashinfer_cudnn` (the vision-tower backend) to the Blackwell deployment recipe — the text `--attention-backend` is unchanged (§2.1 note). On AMD, omit `--mm-attention-backend` and let the encoder use the ROCm default (vision is unvalidated on ROCm — §2.3). + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="MiniMaxAI/MiniMax-M3-MXFP8", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png" + }, + }, + {"type": "text", "text": "Describe this image in detail."}, + ], + } + ], + max_tokens=1024, +) +print(response.choices[0].message.content) +``` + +**Output Example:** + +```text Output +This image captures a striking and unusual urban scene on what appears to be a busy New York City street. + +**Main Subject:** +A man stands on the rear bumper of a yellow taxi cab (an SUV-style cab, likely a Ford Escape hybrid), operating a full-sized ironing board set up across the back of the vehicle. He is wearing a bright yellow long-sleeved shirt and dark pants, and is actively ironing a blue garment, holding an iron in his right hand. + +**Vehicles:** +- The yellow SUV taxi on the right is stationary, its rear hatch serving as the ironing platform. +- A second yellow taxi (a sedan) drives past on the left, captured with motion blur. + +**Setting:** +Tall city buildings with classic urban architecture, an American flag, and white lane markings — a bustling downtown area, possibly Midtown Manhattan. +``` + + + +Notes: + +- If the server cannot fetch external URLs, embed the image as a base64 `data:image/png;base64,...` URI — SGLang decodes it server-side. +- Multiple images per message are supported; add more `image_url` entries to the `content` list. +- Reasoning and tool calling work the same way for multimodal requests — a vision prompt can still produce a `` trace and/or tool calls. + +### 3.4 Prefill-Decode (PD) Disaggregation + +[PD disaggregation](../../../docs/advanced_features/pd_disaggregation) runs prefill and decode on **separate** SGLang servers linked by an RDMA KV-transfer fabric (mooncake or NIXL), fronted by the PD router. M3 needs one thing beyond a dense model: alongside the main KV cache, every sparse "lightning-indexer" layer keeps a **K-only index buffer**, and that buffer must reach the decode server too — otherwise sparse attention reads stale state. SGLang transfers it alongside the main KV — reusing the same page mapping — so M3 disaggregates correctly with no extra flags. + +**Supported topology** (the released MiniMax-M3, whose sparse layers are all K-only): + +- **Equal tensor parallelism** — the prefill and decode servers run the same `--tp`. +- **Single pipeline stage** — PP = 1 (the default). +- **mooncake or NIXL** transfer backend over RDMA / InfiniBand. + +Launch the prefill server, then the decode server — the same recipe with `--disaggregation-mode decode` and no bootstrap port. Pick your hardware: + + + + + +On Blackwell the MXFP8 recipe — fa4, page size 128, deep_gemm MoE, and the MSA fast path (§2.1) — is auto-selected, so each role adds only the `--disaggregation-*` flags. This is the validated **2 × 4×B200** setup (TP4 prefill on node A, TP4 decode on node B); point `--disaggregation-ib-device` at your RDMA NIC(s). + +```bash Prefill server (node A) +sglang serve \ + --model-path MiniMaxAI/MiniMax-M3-MXFP8 \ + --trust-remote-code \ + --reasoning-parser auto \ + --tool-call-parser auto \ + --tp 4 \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend nixl \ + --disaggregation-ib-device mlx5_0 \ + --host 0.0.0.0 --port 30000 \ + --disaggregation-bootstrap-port 8998 +``` + +```bash Decode server (node B) +sglang serve \ + --model-path MiniMaxAI/MiniMax-M3-MXFP8 \ + --trust-remote-code \ + --reasoning-parser auto \ + --tool-call-parser auto \ + --tp 4 \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend nixl \ + --disaggregation-ib-device mlx5_0 \ + --host 0.0.0.0 --port 30001 +``` + + + + + +On Hopper (H200) M3 runs the bf16 build (§2.4) with Triton MoE and the built-in Triton sparse path, pinned to `--page-size 128` so both roles share the page layout the sparse-index transfer relies on. This is the validated **2 × 8×H200** setup (TP8 each). + +```bash Prefill server (node A) +sglang serve \ + --model-path MiniMaxAI/MiniMax-M3 \ + --trust-remote-code \ + --reasoning-parser auto \ + --tool-call-parser auto \ + --tp 8 \ + --attention-backend triton \ + --moe-runner-backend triton \ + --page-size 128 \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend mooncake \ + --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 \ + --host 0.0.0.0 --port 30000 \ + --disaggregation-bootstrap-port 8998 +``` + +```bash Decode server (node B) +sglang serve \ + --model-path MiniMaxAI/MiniMax-M3 \ + --trust-remote-code \ + --reasoning-parser auto \ + --tool-call-parser auto \ + --tp 8 \ + --attention-backend triton \ + --moe-runner-backend triton \ + --page-size 128 \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend mooncake \ + --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 \ + --host 0.0.0.0 --port 30001 +``` + + + + + +Then start the PD router, pointing it at the prefill bootstrap (URL plus its `--disaggregation-bootstrap-port`) and the decode endpoint: + +```bash PD router +python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://:30000 8998 \ + --decode http://:30001 \ + --policy round_robin \ + --host 0.0.0.0 --port 8000 +``` + +Clients hit the router exactly like a single server — it splits each request across the two stages transparently: + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://:8000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="MiniMaxAI/MiniMax-M3-MXFP8", + messages=[{"role": "user", "content": "What is 2 + 2?"}], + max_tokens=64, +) +print(response.choices[0].message.content) +``` + +**Output Example:** + +```text Output +2 + 2 = 4 +``` + + + +**Validation.** PD disaggregation preserves output quality — the K-only sparse index transfers arrive intact and disaggregated output matches non-disaggregated serving. GSM8K is scored with the single sgl-eval harness used by the benchmark card above (full 1319-question split, chat with `--thinking`); see that card for per-platform single-node accuracy. + +- **2 × 4×B200** (TP4+TP4, MXFP8, NIXL over InfiniBand) — output matches single-node serving. The 2-node PD serving benchmark (512-token input, 256-token output, 16 concurrent — a different workload from the card's single-node `random` isl=2048 / osl=256 / conc=64 row, so the throughput figures are not directly comparable) measured mean TTFT 1.1 s and TPOT 16.6 ms (≈ 60 tok/s per stream, ≈ 2.3k tok/s aggregate). +- **2 × 8×H200** (TP8+TP8, bf16, mooncake) — output matches single-node serving. diff --git a/docs_new/cookbook/autoregressive/intro.mdx b/docs_new/cookbook/autoregressive/intro.mdx index c4abd04a5..b1cc8b897 100644 --- a/docs_new/cookbook/autoregressive/intro.mdx +++ b/docs_new/cookbook/autoregressive/intro.mdx @@ -58,7 +58,7 @@ metatags: block-fp8) was +// accuracy-only. MI350X / MI325X inherit their same-arch sibling's recipe +// (stubs). (sgl-eval does NOT measure serving throughput — TTFT/TPOT/tok-s come +// from sglang.bench_serving.) +// +// GSM8K — unified on a SINGLE harness: sgl-eval (github.com/sgl-project/sgl-eval) +// `run gsm8k`, full 1319-question test split, chat endpoint with --thinking +// (M3's reasoning path) + M3's recommended sampling (temp 1.0 / top_p 0.95 / +// top_k 40), symbolic grading. This is the config's Reproduce command. B200 +// (MSA path) and H200 (bf16, built-in Triton sparse) are measured on PR #27944; +// a 3-run mean±std and the matching #27944 bench_serving speed are in progress. +// Per-platform re-measurement under sgl-eval is in progress; rows still pending +// show `gsm8k_pct: null` (no GSM8K row rendered) with the legacy-harness number +// kept in a comment. Legacy harnesses were NOT comparable across platforms +// (NVIDIA: few_shot_gsm8k --num-questions 200; AMD: run_eval gsm8k 1319 examples) — +// which is exactly why we re-measure on one harness. +export const benchmarks = [ + { + match: { hw: "b200", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + sglang_version: "PR #27944", + speed: [ + // bench_serving --flush-cache, MSA path; run-1 (3-run mean pending). + { workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64, num_prompts: 128 }, + ttft_ms: 2410, tpot_ms: 148.4, tokens_per_sec_per_gpu: 124 }, + ], + accuracy: { gsm8k_pct: 94.4 }, // #27944, sgl-eval --thinking, full 1319, recommended sampling (temp 1.0/top_p 0.95/top_k 40), MSA path; run-1 94.39% (greedy 94.16%); --no-thinking 88.6% + }, + { + // Hopper H200: bf16 build (MXFP8 is Blackwell-only) at tp8, built-in Triton + // sparse path (MSA is Blackwell-only). GSM8K re-measured on #27944. + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + sglang_version: "PR #27944", + speed: [ + // bench_serving --flush-cache, bf16 Triton path; run-1 (3-run mean pending). + { workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64, num_prompts: 128 }, + ttft_ms: 1068, tpot_ms: 78.0, tokens_per_sec_per_gpu: 105 }, + ], + accuracy: { gsm8k_pct: 97.0 }, // #27944, sgl-eval --thinking, full 1319, recommended sampling (temp 1.0/top_p 0.95/top_k 40); run-1 97.04% + }, + { + match: { hw: "b300", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + sglang_version: "main (2026-06-11)", + speed: [ + { workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64 }, + ttft_ms: null, tpot_ms: 32.8, tokens_per_sec_per_gpu: 365 }, + ], + accuracy: { gsm8k_pct: null }, // TODO: pending sgl-eval re-measure on B300 (legacy few_shot 200: 87.5) + }, + // GB200: inferred-supported, not directly benchmarked. + { match: { hw: "gb200", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" } }, + { + match: { hw: "gb300", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + sglang_version: "main (2026-06-11)", + speed: [ + { workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64 }, + ttft_ms: 4746, tpot_ms: 39.3, tokens_per_sec_per_gpu: 277 }, + { workload: { dataset: "random", isl: 8192, osl: 256, max_concurrency: 24 }, + ttft_ms: 3324, tpot_ms: 32.9, tokens_per_sec_per_gpu: 131 }, + ], + accuracy: { gsm8k_pct: null }, // TODO: pending sgl-eval re-measure on GB300 (legacy few_shot 200: 87.5) + }, + // MI355X (gfx950): native MXFP8. Speed: bench_serving 1024/1024 @ conc 64, tp8 + // -> 1678 output tok/s (3355 total incl. input); 1678 / 8 = ~210 tokens/sec/GPU. + // No TTFT/TPOT reported for this run. + { + match: { hw: "mi355x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + sglang_version: "main (2026-06-11)", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 640 }, + ttft_ms: null, tpot_ms: null, tokens_per_sec_per_gpu: 210 }, + ], + accuracy: { gsm8k_pct: null }, // TODO: pending sgl-eval re-measure on MI355X (legacy run_eval 1319: 92.2) + }, + // MI350X (gfx950): inferred-supported from MI355X, not separately benchmarked. + { match: { hw: "mi350x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" } }, + // MI300X (gfx942): MXFP8 -> block-fp8 [128,128]. + { + match: { hw: "mi300x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + sglang_version: "main (2026-06-11)", + accuracy: { gsm8k_pct: null }, // TODO: pending sgl-eval re-measure on MI300X (legacy run_eval 1319: 92.0, triton 0.917-0.929 / aiter ~0.929) + }, + // MI325X (gfx942): inferred-supported from MI300X, not separately benchmarked. + { match: { hw: "mi325x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" } }, +]; diff --git a/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx b/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx new file mode 100644 index 000000000..6d2bdfdc7 --- /dev/null +++ b/docs_new/src/snippets/configs/MiniMaxAI/minimax-m3.jsx @@ -0,0 +1,370 @@ +// MiniMax-M3 cookbook config. Consumed by _deployment.jsx + _playground.jsx; +// see _deployment.jsx header for the field contract. +// +// MXFP8 MoE: validated single-node tp4 on NVIDIA Blackwell — B200 (sm_100), +// B300 (sm_103), GB300 (sm_103, aarch64 Grace); GB200 (sm_100, aarch64) is +// inferred-supported (both axes validated above) but not directly benchmarked. +// AMD: validated single-node tp8 — MI355X (gfx950, CDNA4) serves MXFP8 +// natively; MI300X (gfx942, CDNA3) auto-converts MXFP8 -> block-fp8 [128,128] +// at load and serves it with the tuned ROCm kernels. MI350X (gfx950) and +// MI325X (gfx942) are inferred-supported from their same-arch siblings. +// Hopper (H200) cannot run the MXFP8 kernels, so it serves the bf16 build +// (MiniMaxAI/MiniMax-M3) at tp8 — validated on 8xH200. See §2.4 on the page. + +export const config = { + modelName: "MiniMax-M3", + + supportedHardware: ["b200", "b300", "gb200", "gb300", "mi300x", "mi325x", "mi350x", "mi355x", "h200"], + + variants: [ + { id: "default", label: "Default" }, + ], + quantizations: [ + { id: "mxfp8", label: "MXFP8" }, + { id: "bf16", label: "BF16" }, + ], + strategies: [ + { id: "balanced", label: "Balanced" }, + ], + nodesOptions: [ + { id: "single", label: "Single Node" }, + ], + + modelNames: { + "default|mxfp8": "MiniMaxAI/MiniMax-M3-MXFP8", + "default|bf16": "MiniMaxAI/MiniMax-M3", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, + accuracy: { + gsm8k_pct: +`pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --model {{MODEL_NAME}} \\ + --temperature 1.0 --top-p 0.95 \\ + --thinking`, + }, + numPromptsByConc: { 24: 24, 64: 128 }, + }, + + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ], + + dockerImages: { + // M3-specific dev images (multi-arch amd64+arm64). cu13 carries the sm_103 + // (B300/GB300) + Grace arm64 builds; cu12 is the Hopper/CUDA-12 build; + // dev-minimax-m3 is the rolling default. + b200: "lmsysorg/sglang:dev-minimax-m3", + b300: "lmsysorg/sglang:dev-cu13-minimax-m3", + gb200: "lmsysorg/sglang:dev-cu13-minimax-m3", + gb300: "lmsysorg/sglang:dev-cu13-minimax-m3", + h200: "lmsysorg/sglang:dev-cu12-minimax-m3", + // AMD ROCm images — pin the exact tag from the validated build (see Configuration Tips). + mi300x: "lmsysorg/sglang:-rocm700-mi30x", + mi325x: "lmsysorg/sglang:-rocm700-mi30x", + mi350x: "lmsysorg/sglang:-rocm720-mi35x", + mi355x: "lmsysorg/sglang:-rocm720-mi35x", + }, + + github: { + cookbookModel: "MiniMaxAI/MiniMax-M3-MXFP8", + }, + + playgroundFeatures: { + + // ----- Attention Parallelism ----- + attention: { + knobs: [ + { id: "tp", label: "TP", values: [null, 1, 2, 4, 8] }, + { id: "cp", label: "CP", values: [null, 1, 2, 4] }, + { id: "dpAttn", label: "DP-Attention", + values: [null, false, 1, 2, 4, 8], + labels: { "auto": "Auto", "false": "Off" } }, + ], + }, + + // ----- MoE Parallelism ----- + moe: { + backend: { + options: [ + { id: null, label: "Inherited" }, + { id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] }, + ], + }, + ep: { label: "EP", values: [null, 2, 4, 8] }, + }, + + // ----- Parsers ----- + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser auto" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser auto" }, + ], + }, + + // ----- PD Disaggregation ----- + pdDisagg: { + modes: [ + { id: "off", label: "Off" }, + { id: "prefill", label: "Prefill role" }, + { id: "decode", label: "Decode role" }, + ], + transferBackends: [ + { id: "mooncake", label: "Mooncake", + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + ], + envWhen: { hw: ["gb200", "gb300"] } }, + { id: "nixl", label: "NiXL" }, + ], + ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"], + router: { + port: 8000, + command: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PREFILL_PORT}} \\ + --decode http://:{{DECODE_PORT}} \\ + --policy round_robin \\ + --host 0.0.0.0 --port {{ROUTER_PORT}}`, + }, + }, + + // ----- Hierarchical KV Cache ----- + hicache: { + backends: [ + { id: null, label: "Auto" }, + { id: "file", label: "File" }, + { id: "mooncake", label: "Mooncake" }, + { id: "hf3fs", label: "HF3FS" }, + { id: "nixl", label: "NiXL" }, + ], + writePolicies: [ + { id: "auto", label: "Auto" }, + { id: "write_through", label: "Write-through" }, + { id: "write_back", label: "Write-back" }, + { id: "write_through_selective", label: "Write-through (selective)" }, + ], + }, + }, + + // NVIDIA Blackwell: one validated single-node tp4 recipe per family. fa4 + + // page 128 + deep_gemm are the M3 SM100 auto-defaults on current main, so this + // is also the bare-launch behavior; they engage MiniMax's MSA sparse-attention + // kernel when fmha_sm100 is installed (see Configuration Tips), Triton otherwise. + // AMD: tp8. MI350X/MI355X (gfx950) serve MXFP8 natively (backends auto). MI300X/ + // MI325X (gfx942) need --attention-backend aiter + --moe-runner-backend triton, + // and the MXFP8 weights are auto-converted to block-fp8 at load; the cold-start + // AITER JIT can exceed the default warmup window, hence the watchdog/skip flags. + cells: [ + { + match: { hw: "b200", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 4", + "--attention-backend fa4", + "--page-size 128", + "--moe-runner-backend deep_gemm", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.75", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 4", + "--attention-backend fa4", + "--page-size 128", + "--moe-runner-backend deep_gemm", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.75", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB200 (sm_100 + aarch64): inferred-supported (both axes validated on + // B200 and GB300), not directly benchmarked. Same recipe as the others. + match: { hw: "gb200", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 4", + "--attention-backend fa4", + "--page-size 128", + "--moe-runner-backend deep_gemm", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.75", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 4", + "--attention-backend fa4", + "--page-size 128", + "--moe-runner-backend deep_gemm", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.75", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI355X (gfx950, CDNA4): native MXFP8, backends auto-selected. + match: { hw: "mi355x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: ["SGLANG_USE_AITER=1"], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 8", + "--quantization mxfp8", + "--dtype bfloat16", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI350X (gfx950, CDNA4): inferred-supported from MI355X (same arch), + // not directly benchmarked. Same native-MXFP8 recipe. + match: { hw: "mi350x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + env: ["SGLANG_USE_AITER=1"], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 8", + "--quantization mxfp8", + "--dtype bfloat16", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI300X (gfx942, CDNA3): no hardware MX matmul — SGLang converts MXFP8 -> + // block-fp8 [128,128] at load. aiter attention + triton MoE runner are the + // validated backends; watchdog/skip-warmup ride out the cold-start AITER JIT. + match: { hw: "mi300x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: ["SGLANG_USE_AITER=1"], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 8", + "--quantization mxfp8", + "--dtype bfloat16", + "--attention-backend aiter", + "--moe-runner-backend triton", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.80", + "--watchdog-timeout 3600", + "--skip-server-warmup", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI325X (gfx942, CDNA3): inferred-supported from MI300X (same arch), + // not directly benchmarked. Same MXFP8 -> block-fp8 recipe. + match: { hw: "mi325x", variant: "default", quant: "mxfp8", strategy: "balanced", nodes: "single" }, + env: ["SGLANG_USE_AITER=1"], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 8", + "--quantization mxfp8", + "--dtype bfloat16", + "--attention-backend aiter", + "--moe-runner-backend triton", + "--chunked-prefill-size 8192", + "--mem-fraction-static 0.80", + "--watchdog-timeout 3600", + "--skip-server-warmup", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // Hopper (H200): MXFP8 MoE kernels are Blackwell-only, so Hopper serves the + // full-precision bf16 build (MiniMaxAI/MiniMax-M3) at tp8 — bf16 weights don't + // fit a single 4-GPU node. Everything else auto-resolves for Hopper: fa3 + // attention, page_size 1, MoE auto-pins to Triton (the bf16 deep_gemm path is + // not used), decode keeps full CUDA graph; MSA (§2.1) is Blackwell-only so the + // sparse step runs on the built-in Triton fallback. See §2.4. + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--reasoning-parser auto", + "--tool-call-parser auto", + "--tp 8", + "--mem-fraction-static 0.75", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};