diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx index c82ce26f5..9644b83aa 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -42,7 +42,7 @@ Qwen3.5 features a Gated Delta Networks combined with sparse Mixture-of-Experts Qwen3.5-397B-A17B [Qwen/Qwen3.5-397B-A17B](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) [Qwen/Qwen3.5-397B-A17B-FP8](https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8) - [nvidia/Qwen3.5-397B-A17B-NVFP4](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4) + NVIDIA NVFP4: [nvidia/Qwen3.5-397B-A17B-NVFP4](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4)
AMD MXFP4: [amd/Qwen3.5-397B-A17B-MXFP4](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4) Qwen3.5-122B-A10B @@ -103,10 +103,10 @@ uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=pytho docker pull lmsysorg/sglang:latest # Or use Docker (AMD MI300X/MI325X) -docker pull lmsysorg/sglang:v0.5.9-rocm720-mi30x +docker pull lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi30x-20260604 # Or use Docker (AMD MI355X) -docker pull lmsysorg/sglang:v0.5.9-rocm720-mi35x +docker pull lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260604 ``` For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). @@ -127,7 +127,7 @@ This section provides deployment configurations optimized for different hardware - Speculative decoding (MTP) can significantly reduce latency for interactive use cases. - **H100 FP8:** Add `--enable-symm-mem` to enable NCCL symmetric memory for faster collectives and better performance under multi-GPU settings. -- **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `--attention-backend triton`. Both the full attention layers and the Gated Delta Net (linear attention) layers use Triton-based kernels on ROCm. Example: `SGLANG_USE_AITER=1 python3 -m sglang.launch_server --model-path Qwen/Qwen3.5-397B-A17B --tp 8 --attention-backend triton --trust-remote-code`. See [AMD's Day-0 support article](https://www.amd.com/en/developer/resources/technical-articles/2026/day-0-support-for-qwen-3-5-on-amd-instinct-gpus.html) for details. +- **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `SGLANG_USE_AITER_UNIFIED_ATTN=1` with `--attention-backend aiter`, which requires `--page-size 16` and can also enable `--enable-aiter-allreduce-fusion`. - **Watchdog timeout:** Increase `--watchdog-timeout` to `1200` or higher for this large model, as weight loading can take significant time. - **Mamba Radix Cache**: Qwen3.5's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-scheduler-strategy`: - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. Required for AMD MI GPUs. @@ -156,9 +156,10 @@ This section provides deployment configurations optimized for different hardware - **MI300X (192GB)** runs with tp=4. - **MI325X (256GB)** runs with tp=2. - **MI355X (288GB)** runs with tp=2. - - **FP4**: The FP4 quantized model requires ~250GB for weights, cutting memory by almost 4x. Only compatible with B200/B300 (Blackwell architecture). - - **B200 (183GB)** runs with tp=4. - - **B300 (275GB)** runs with tp=2. + - **FP4**: The FP4 quantized model requires ~250GB for weights, cutting memory by almost 4x. NVFP4 ([nvidia/Qwen3.5-397B-A17B-NVFP4](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4)) requires B200/B300 (Blackwell architecture); AMD provides an MXFP4 checkpoint ([amd/Qwen3.5-397B-A17B-MXFP4](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4)) for MI355X. + - **B200 (183GB)** runs with tp=4. (NVFP4) + - **B300 (275GB)** runs with tp=2. (NVFP4) + - **MI355X (288GB)** runs with tp=2 (use tp=4 for low concurrency). (MXFP4) @@ -218,7 +219,7 @@ This section provides deployment configurations optimized for different hardware - +
288GB 4 2N/A2 / 4 (MXFP4)
@@ -255,17 +256,24 @@ sglang serve \ Deploy Qwen3.5-397B-A17B with the following command (MI300X/MI325X/MI355X): ```shell Command -sglang serve \ +SGLANG_USE_AITER=1 \ +SGLANG_USE_AITER_UNIFIED_ATTN=1 \ +python3 -m sglang.launch_server \ --model-path Qwen/Qwen3.5-397B-A17B \ --tp 8 \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder \ + --trust-remote-code \ + --page-size 16 \ --mem-fraction-static 0.8 \ - --attention-backend triton \ + --attention-backend aiter \ + --enable-aiter-allreduce-fusion \ + --model-loader-extra-config '{"enable_multithread_load": true}' \ + --watchdog-timeout 1200 \ --host 0.0.0.0 \ --port 30000 ``` -> **Note:** TP8 works on all MI GPUs. For MI325X/MI355X, you can use --tp 4 as the minimum requirement. +> **Note:** To enable MTP (speculative decoding), append `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. ### 4.1 Basic Usage diff --git a/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx index 0266b7b13..a86ffd9d2 100644 --- a/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx @@ -20,7 +20,7 @@ export const Qwen35Deployment = () => { // 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1 // 27B: tp=1 on all hardware (including MI300X, MI325X, MI355X) // - // FP4 (397B only, Blackwell required): B200 tp=4, B300 tp=2 + // FP4 (397B only): NVFP4 on Blackwell B200 tp=4, B300 tp=2; AMD MXFP4 on MI355X tp=2 const MOE_MODELS = new Set(['397b', '122b', '35b']); const FP8_MODELS = new Set(['397b', '122b', '35b', '27b']); @@ -64,7 +64,7 @@ export const Qwen35Deployment = () => { { id: 'b300', label: 'B300', default: isNvfp4, disabled: false }, { id: 'mi300x', label: 'MI300X', default: false, disabled: isNvfp4 }, { id: 'mi325x', label: 'MI325X', default: false, disabled: isNvfp4 }, - { id: 'mi355x', label: 'MI355X', default: false, disabled: isNvfp4 }, + { id: 'mi355x', label: 'MI355X', default: false, disabled: false }, { id: 'xeon', label: 'XEON', default: false, disabled: isNvfp4 } ]; } @@ -152,7 +152,7 @@ export const Qwen35Deployment = () => { b300: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } }, mi300x: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 } }, mi325x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } }, - mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } }, + mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } }, xeon: { bf16: { tp: 6 }, fp8: { tp: 6 } } }, '122b': { @@ -318,7 +318,10 @@ export const Qwen35Deployment = () => { let modelName; if (quantization === 'fp4') { - modelName = 'nvidia/Qwen3.5-397B-A17B-NVFP4'; + // AMD MI355X uses the MXFP4 checkpoint; Blackwell uses NVFP4. + modelName = hardware === 'mi355x' + ? 'amd/Qwen3.5-397B-A17B-MXFP4' + : 'nvidia/Qwen3.5-397B-A17B-NVFP4'; } else { const suffix = MODEL_SUFFIX[model]; const quantSuffix = quantization === 'fp8' ? '-FP8' : ''; @@ -403,8 +406,11 @@ export const Qwen35Deployment = () => { cmd += ` \\\n --tokenizer-worker-num 6`; } - // Enable allreduce fusion for all Qwen3.5 configs (skip for FP4: benchmark only enables this for TP>=8). - if (quantization !== 'fp4' && hardware !== 'xeon') { + // Enable FlashInfer allreduce fusion for NVIDIA Qwen3.5 configs (skip for FP4: + // benchmark only enables this for TP>=8). AMD MI GPUs use the AITER allreduce + // fusion flag instead, handled in the AMD backend block below. + const amdGpu = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x'; + if (quantization !== 'fp4' && hardware !== 'xeon' && !amdGpu) { cmd += ` \\\n --enable-flashinfer-allreduce-fusion`; } @@ -421,9 +427,17 @@ export const Qwen35Deployment = () => { cmd += ` \\\n --attention-backend trtllm_mha`; } - // Append AMD GPU-specific backend configurations - if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') { - cmd += ` \\\n --attention-backend triton`; + // Append AMD GPU-specific backend configurations. + // All AMD MI GPUs use the AITER unified-attention backend (pair with + // SGLANG_USE_AITER=1 and SGLANG_USE_AITER_UNIFIED_ATTN=1; see cookbook prose), + // which requires --page-size 16. Enable AITER allreduce fusion for multi-GPU. + if (amdGpu) { + cmd = "SGLANG_USE_AITER=1 \\\nSGLANG_USE_AITER_UNIFIED_ATTN=1 \\\n" + cmd; + cmd += " \\\n --attention-backend aiter"; + cmd += " \\\n --page-size 16"; + if (hwConfig.tp > 1) { + cmd += " \\\n --enable-aiter-allreduce-fusion"; + } } // Tokenizer workers for H200 and B200/B300 @@ -442,15 +456,26 @@ export const Qwen35Deployment = () => { // FP4-specific backend settings if (quantization === 'fp4') { - cmd += ' \\\n --quantization modelopt_fp4'; - cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass'; - cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; - cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; - cmd += ' \\\n --chunked-prefill-size 32768'; - cmd += ' \\\n --max-prefill-tokens 32768'; - cmd += ' \\\n --max-running-requests 128'; - cmd += ' \\\n --stream-interval 30'; - cmd += ' \\\n --disable-radix-cache'; + if (hardware === 'mi355x') { + // AMD MXFP4 on MI355X: backend / --page-size 16 / AITER allreduce fusion + // are emitted by the AMD backend block above. Add the FP4-specific flags here. + cmd += ' \\\n --disable-radix-cache'; + // Cap concurrency under MTP to avoid OOM at tp=2. + if (speculative === 'enabled') { + cmd += ' \\\n --max-running-requests 128'; + } + } else { + // NVIDIA NVFP4 on Blackwell (B200 / B300). + cmd += ' \\\n --quantization modelopt_fp4'; + cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass'; + cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + cmd += ' \\\n --chunked-prefill-size 32768'; + cmd += ' \\\n --max-prefill-tokens 32768'; + cmd += ' \\\n --max-running-requests 128'; + cmd += ' \\\n --stream-interval 30'; + cmd += ' \\\n --disable-radix-cache'; + } } // Add memory fraction last