diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx index 92063dd59..a4ae06377 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx @@ -47,6 +47,11 @@ Both variants share the same hybrid reasoning, tool-calling, and multimodal inte MoE 35B / 3B active [Qwen/Qwen3.6-35B-A3B-FP8](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8) + + Qwen3.6-35B-A3B (NVFP4) + MoE 35B / 3B active (Blackwell) + [nvidia/Qwen3.6-35B-A3B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4) + Qwen3.6-27B (BF16) Dense 27B @@ -78,7 +83,7 @@ uv pip install sglang # Or install from source uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=python' -# Or use Docker (NVIDIA GPUs; also serves the NVFP4 variant nvidia/Qwen3.6-27B-NVFP4) +# Or use Docker (NVIDIA GPUs; also serves the NVFP4 variants) docker pull lmsysorg/sglang:latest ``` @@ -106,15 +111,17 @@ This section provides deployment configurations optimized for different hardware - The `--mem-fraction-static` flag is recommended for optimal memory utilization, adjust it based on your hardware and workload. - Context length defaults to 262,144 tokens. If you encounter OOM errors, consider reducing it, but maintain at least 128K to preserve thinking capabilities. - **CUDA IPC Transport**: Add `SGLANG_USE_CUDA_IPC_TRANSPORT=1` as an environment variable to use CUDA IPC for transferring multimodal features, significantly improving TTFT (Time To First Token). Note: this consumes additional memory proportional to image size, so you may need to lower `--mem-fraction-static` or `--max-running-requests`. -- **Multimodal Attention Backend**: Use `--mm-attention-backend fa3` on H100/H200 for better vision performance, or `--mm-attention-backend fa4` on B200. +- **Multimodal Attention Backend**: Use `--mm-attention-backend fa3` on H100/H200 for better vision performance, or `--mm-attention-backend fa4` on B200/B300. - For processing large images or videos, you may need to lower `--mem-fraction-static` to leave room for image feature tensors. - Hardware requirements: - **35B-A3B BF16**: ~70GB for weights. TP=1 fits on all supported hardware. - **35B-A3B FP8**: ~35GB for weights. TP=1 fits on all supported hardware. + - **35B-A3B NVFP4**: ~23GB for weights. TP=1 fits on B200/B300. - **27B BF16**: ~54GB for weights. TP=1 fits on all supported hardware. - **27B FP8**: ~27GB for weights. TP=1 fits on all supported hardware. + - **27B NVFP4**: ~22GB for weights. TP=1 fits on B200/B300. -All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU at both precisions: +All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU. NVFP4 is available on B200/B300: @@ -123,6 +130,7 @@ All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU a + @@ -131,18 +139,28 @@ All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU a + + + + + + + + + +
Memory BF16 TP FP8 TPNVFP4 TP
80GB 1 1—
H200 141GB 1 1—
B200 183GB 1 11
B300275GB111
diff --git a/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx index 427ee8e06..cf9402f03 100644 --- a/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx @@ -23,14 +23,13 @@ export const Qwen36Deployment = () => { quantization: { name: 'quantization', title: 'Quantization', - // NVFP4 is a Blackwell-only, 27B-only checkpoint (nvidia/Qwen3.6-27B-NVFP4); - // only surface it when both conditions hold so we never emit an unrunnable command. + // NVFP4 checkpoints are available for both model sizes on Blackwell (B200/B300). getDynamicItems: (values) => { const items = [ { id: 'fp8', label: 'FP8', default: true }, { id: 'bf16', label: 'BF16', default: false }, ]; - const nvfp4Supported = values.modelSize === '27b' && (values.hardware === 'b200' || values.hardware === 'b300'); + const nvfp4Supported = values.hardware === 'b200' || values.hardware === 'b300'; if (nvfp4Supported) { items.push({ id: 'nvfp4', label: 'NVFP4', default: false }); } @@ -94,8 +93,8 @@ export const Qwen36Deployment = () => { baseName: '35B-A3B', h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, - b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, - b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, + b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 }, nvfp4: { tp: 1 } }, + b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 }, nvfp4: { tp: 1 } }, xeon: { bf16: { tp: 3 }, fp8: { tp: 3 } }, }, '27b': { @@ -175,7 +174,7 @@ export const Qwen36Deployment = () => { mambaCache: speculative === 'enabled' ? 'v2' : values.mambaCache, }; - // NVFP4: nvidia/Qwen3.6-27B-NVFP4 on Blackwell (B200/B300). Follows the exact command + // NVFP4: nvidia/Qwen3.6-{35B-A3B,27B}-NVFP4 on Blackwell (B200/B300). Follows the exact command // shape from the checkpoint's docs — explicit --tp-size 1, --attention-backend trtllm_mha, // new-style --mamba-radix-cache-strategy, and explicit --host/--port (no // --mem-fraction-static). Reasoning / tool-call parsers still follow their toggles.