From cf2d4fd679623b4aa0cad17b8ae18e9229d3bcff Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 8 Aug 2026 19:23:00 +0800 Subject: [PATCH] docs: clarify K3 VLM feature transport (#34099) --- .../autoregressive/Moonshotai/Kimi-K3.mdx | 48 ++++++------------- .../snippets/configs/moonshotai/kimi-k3.jsx | 36 ++++++++++++++ 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 32c1d402d..a3be9cf21 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -288,37 +288,22 @@ Clients then send requests to the router (`:8000`) instead of an individual role The open-source K3 serving contract currently supports **image input only** — its processor rejects video and audio input. -#### Recommended high-speed VLM +#### VLM feature transport -The command panel now opens on the **B300 · Unified · Balanced** -recipe below. It makes the VLM-specific performance choices explicit: +Use **VLM Transport** in the command picker. `Auto` is a topology-aware starting +point, not a claim that one configuration is fastest for every workload. -```bash Command -sglang serve \ - --trust-remote-code \ - --model-path moonshotai/Kimi-K3 \ - --tp-size 8 \ - --dcp-size 8 \ - --mem-fraction-static 0.85 \ - --mm-feature-transport cuda_ipc \ - --mm-processor-worker-num 2 \ - --mm-io-worker-num 16 \ - --reasoning-parser kimi_k3 \ - --tool-call-parser kimi_k3 \ - --host 0.0.0.0 \ - --port 30000 -``` +| Picker selection | Processor-to-scheduler feature path | +|---|---| +| Auto · single-node Unified CUDA | CUDA IPC | +| Auto · Unified GB200/GB300 | CUDA VMM when IMEX is available; CPU otherwise | +| Auto · PD or other topologies | CPU | +| CPU | CPU, with no GPU feature pool | -- `--mm-feature-transport cuda_ipc` — single-node only: skips the CPU round - trip and uses a bounded GPU pool. -- On multi-node GB200/GB300, SGLang automatically uses CUDA VMM with CUDA - FABRIC handles when an IMEX channel is available; otherwise it uses CPU. - Pass `--mm-feature-transport cpu` to opt out. -- CUDA IPC and CUDA VMM share the `SGLANG_MM_FEATURE_CACHE_MB` HBM budget - (1 GiB by default) and fall back to CPU per tensor when the pool is full. -- 2 processor / 16 I/O workers are the measured defaults; more adds contention. -- Leave `--mm-attention-backend` unset — auto-selected, with a correctness fallback. -- Don't add `--mm-enable-dp-encoder`; K3 already shards images across TP ranks. +CUDA IPC and CUDA VMM reserve up to `SGLANG_MM_FEATURE_CACHE_MB` (1 GiB by +default) on the base GPU and fall back to CPU per tensor when full. This setting +does not control EPD encoder output or PD KV/KDA transfer. K3 already defaults to +2 processor workers and 16 I/O workers; leave those flags unset unless tuning. #### VLM compatibility @@ -327,7 +312,7 @@ sglang serve \ | PD | Supported. Image processing and ViT run on prefill; the PD transfer then moves both paged MLA KV and KDA recurrent state as described in [PD disaggregation](#pd-disaggregation). | | EPD | Supported on the public `kimi-k3` branch. Use an `--encoder-only` vision role and a `--language-only` prefill role; add the normal decode role for full EPD. See the [EPD guide](../../../docs/advanced_features/epd_disaggregation). | | MM encoder DP | Built in. K3 shards complete images across TP ranks, so leave `--mm-enable-dp-encoder` unset in unified, PD-prefill, and encoder-only roles. | -| MM feature transport | CUDA IPC is used on a single node. CUDA VMM with FABRIC handles is auto-selected for multi-node GB200/GB300 when IMEX is available. Both use a bounded HBM pool and are separate from EPD's `--encoder-transfer-backend` and the PD KV/KDA transfer. | +| MM feature transport | Processor-to-scheduler features only. EPD encoder output and PD KV/KDA transfer use their own backends. | | ViT BCG | Compatible with unified and encoder-only roles, but recommended only for repeated encoder shapes after measuring the HBM trade-off below. | #### Should ViT BCG be enabled? @@ -341,11 +326,10 @@ Keep ViT BCG **off** for general serving; enable `SGLANG_VIT_ENABLE_CUDA_GRAPH=1 #### Low-HBM VLM Use this profile when keeping HBM headroom matters more than peak concurrency. -It removes the 1 GiB CUDA IPC pool, keeps ViT BCG disabled, halves the context +It removes the GPU feature pool, keeps ViT BCG disabled, halves the context window, caps concurrency, and lowers the static-memory target: ```bash Command -SGLANG_VIT_ENABLE_CUDA_GRAPH=0 \ sglang serve \ --trust-remote-code \ --model-path moonshotai/Kimi-K3 \ @@ -354,8 +338,6 @@ sglang serve \ --enable-symm-mem \ --mem-fraction-static 0.82 \ --mm-feature-transport cpu \ - --mm-processor-worker-num 2 \ - --mm-io-worker-num 16 \ --reasoning-parser kimi_k3 \ --tool-call-parser kimi_k3 \ --host 0.0.0.0 \ diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 2c88f40b4..6c3fa6e49 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -160,6 +160,42 @@ export const config = { // Orthogonal to the cell grid: the picked option layers flags onto whichever // cell is showing, so turning speculation on does not triple the cell count. overlayDims: [ + { + id: "mmTransport", + title: "VLM Transport", + default: "auto", + showWhen: (s) => s.pdMode !== "decode", + options: [ + { + id: "auto", + label: "Auto (topology-aware)", + hints: (s) => { + if (s.pdMode !== "unified") { + return [ + "VLM transport: Auto -> CPU for PD; KV/KDA transfer is separate.", + ]; + } + if (s.hw === "b300") { + return [ + "VLM transport: Auto -> CUDA IPC (up to 1 GiB HBM; CPU fallback when full).", + ]; + } + if (["gb200", "gb300"].includes(s.hw)) { + return [ + "VLM transport: Auto -> CUDA VMM with IMEX, otherwise CPU (up to 1 GiB HBM).", + ]; + } + return ["VLM transport: Auto -> CPU on this topology."]; + }, + }, + { + id: "cpu", + label: "CPU (save HBM)", + flags: ["--mm-feature-transport cpu"], + hints: ["VLM transport: CPU; no GPU feature pool."], + }, + ], + }, { id: "spec", title: "Spec Decode",