diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx index d298d527b..613af7517 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -42,7 +42,7 @@ Qwen3.5 features a Gated Delta Networks combined with sparse Mixture-of-Experts Qwen3.5-397B-A17B [Qwen/Qwen3.5-397B-A17B](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) [Qwen/Qwen3.5-397B-A17B-FP8](https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8) - NVIDIA NVFP4: [nvidia/Qwen3.5-397B-A17B-NVFP4-V2](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4-V2)
AMD MXFP4: [amd/Qwen3.5-397B-A17B-MXFP4](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4) + NVIDIA NVFP4: [nvidia/Qwen3.5-397B-A17B-NVFP4-V2](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4-V2)
AMD MXFP4: [amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2) Qwen3.5-122B-A10B @@ -128,7 +128,7 @@ This section provides deployment configurations optimized for different hardware - Speculative decoding (MTP) can significantly reduce latency for interactive use cases. - **H100 / B200 / B300 FP8:** Add `--enable-symm-mem` to enable NCCL symmetric memory for faster collectives and better performance under multi-GPU settings. It only helps when tp > 1, so on B200/B300 it applies to the 397B-A17B FP8 recipe (tp=4); the 122B-A10B and 35B-A3B FP8 recipes are single-GPU there. - **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `SGLANG_USE_AITER_UNIFIED_ATTN=1` with `--attention-backend aiter`, which requires `--page-size 16` and can also enable `--enable-aiter-allreduce-fusion`. Additionally set `AITER_FLYDSL_FORCE=1` to force the AITER FlyDSL MoE kernels and `SGLANG_MAMBA_SSM_DTYPE=bfloat16` to store the Mamba SSM state in bfloat16 (instead of the default float32). For the **MXFP4 checkpoint on MI355X**, set `ROCM_QUICK_REDUCE_QUANTIZATION=INT4` to route multi-GPU collectives through INT4-quantized ROCm quick all-reduce, and drop `--enable-aiter-allreduce-fusion` (the two are mutually exclusive; quick all-reduce is preferred for this recipe). -- **KV cache offloading (MXFP4 on MI355X):** Selecting **Host DRAM (HiCache)** adds a host-memory tier below the device KV cache, which lets long-context agentic workloads keep more prefix cached and reach higher concurrency than the device KV pool alone allows. It emits `--enable-hierarchical-cache --hicache-ratio 1.5 --hicache-write-policy write_through --hicache-io-backend kernel --hicache-mem-layout page_first`. HiCache extends the radix cache, so the generated command drops `--disable-radix-cache` — the two options are mutually exclusive and SGLang rejects them together at startup. `--hicache-ratio 1.5` sizes the host tier at 1.5x the device KV cache, so make sure the node has that much free CPU DRAM per server on top of the model weights. Leave this **Disabled** for fixed-length throughput serving, where the extra host traffic buys nothing. +- **KV cache offloading (MXFP4 on MI355X):** Selecting **Host DRAM (HiCache)** adds a host-memory tier below the device KV cache, which lets long-context agentic workloads keep more prefix cached and reach higher concurrency than the device KV pool alone allows. It emits `--enable-hierarchical-cache --hicache-ratio 1.5 --hicache-write-policy write_through --hicache-io-backend direct --hicache-mem-layout page_first_direct`. HiCache extends the radix cache, so the generated command drops `--disable-radix-cache` — the two options are mutually exclusive and SGLang rejects them together at startup. `--hicache-ratio 1.5` sizes the host tier at 1.5x the device KV cache, so make sure the node has that much free CPU DRAM per server on top of the model weights. Leave this **Disabled** for fixed-length throughput serving, where the extra host traffic buys nothing. - **Watchdog timeout:** Increase `--watchdog-timeout` to `1200` or higher for this large model, as weight loading can take significant time. - **Mamba Radix Cache**: Qwen3.5's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-radix-cache-strategy`: - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. Required for AMD MI GPUs. @@ -159,7 +159,7 @@ This section provides deployment configurations optimized for different hardware - **MI300X (192GB)** runs with tp=4. - **MI325X (256GB)** runs with tp=2. - **MI355X (288GB)** runs with tp=2. - - **FP4**: The FP4 quantized model requires ~250GB for weights, cutting memory by almost 4x. NVFP4 ([nvidia/Qwen3.5-397B-A17B-NVFP4-V2](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4-V2)) requires B200/B300 (Blackwell architecture); AMD provides an MXFP4 checkpoint ([amd/Qwen3.5-397B-A17B-MXFP4](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4)) for MI355X. + - **FP4**: The FP4 quantized model requires ~250GB for weights, cutting memory by almost 4x. NVFP4 ([nvidia/Qwen3.5-397B-A17B-NVFP4-V2](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4-V2)) requires B200/B300 (Blackwell architecture); AMD provides an MXFP4 checkpoint with FP8 attention ([amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2)) for MI355X. - **B200 (183GB)** runs with tp=4 (tp=2 with expert parallelism 2 when MTP is enabled). (NVFP4) - **B300 (275GB)** runs with tp=2. (NVFP4) - **MI355X (288GB)** runs with tp=2 (use tp=4 for low concurrency). (MXFP4) diff --git a/docs/src/snippets/autoregressive/qwen35-deployment.jsx b/docs/src/snippets/autoregressive/qwen35-deployment.jsx index 4d7f6764d..aafab8055 100644 --- a/docs/src/snippets/autoregressive/qwen35-deployment.jsx +++ b/docs/src/snippets/autoregressive/qwen35-deployment.jsx @@ -334,9 +334,9 @@ export const Qwen35Deployment = () => { let modelName; if (quantization === 'fp4') { - // AMD MI355X uses the MXFP4 checkpoint; Blackwell uses NVFP4-V2. + // AMD MI355X uses MXFP4-AttnFP8-V2 (MXFP4 MoE, FP8 attention); Blackwell uses NVFP4-V2. modelName = hardware === 'mi355x' - ? 'amd/Qwen3.5-397B-A17B-MXFP4' + ? 'amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2' : 'nvidia/Qwen3.5-397B-A17B-NVFP4-V2'; } else { const suffix = MODEL_SUFFIX[model]; @@ -517,8 +517,8 @@ export const Qwen35Deployment = () => { cmd += ' \\\n --enable-hierarchical-cache'; cmd += ' \\\n --hicache-ratio 1.5'; cmd += ' \\\n --hicache-write-policy write_through'; - cmd += ' \\\n --hicache-io-backend kernel'; - cmd += ' \\\n --hicache-mem-layout page_first'; + cmd += ' \\\n --hicache-io-backend direct'; + cmd += ' \\\n --hicache-mem-layout page_first_direct'; } else { cmd += ' \\\n --disable-radix-cache'; }