diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx index f7d788b3f..2545de6ee 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -128,6 +128,7 @@ This section provides deployment configurations optimized for different hardware - Speculative decoding (MTP) can significantly reduce latency for interactive use cases. - **H100 FP8:** Add `--enable-symm-mem` to enable NCCL symmetric memory for faster collectives and better performance under multi-GPU settings. - **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `SGLANG_USE_AITER_UNIFIED_ATTN=1` with `--attention-backend aiter`, which requires `--page-size 16` and can also enable `--enable-aiter-allreduce-fusion`. Additionally set `AITER_FLYDSL_FORCE=1` to force the AITER FlyDSL MoE kernels and `SGLANG_MAMBA_SSM_DTYPE=bfloat16` to store the Mamba SSM state in bfloat16 (instead of the default float32). For the **MXFP4 checkpoint on MI355X**, set `ROCM_QUICK_REDUCE_QUANTIZATION=INT8` to route multi-GPU collectives through INT8-quantized ROCm quick all-reduce, and drop `--enable-aiter-allreduce-fusion` (the two are mutually exclusive; quick all-reduce is preferred for this recipe). +- **KV cache offloading (MXFP4 on MI355X):** Selecting **Host DRAM (HiCache)** adds a host-memory tier below the device KV cache, which lets long-context agentic workloads keep more prefix cached and reach higher concurrency than the device KV pool alone allows. It emits `--enable-hierarchical-cache --hicache-ratio 1.5 --hicache-write-policy write_through --hicache-io-backend direct --hicache-mem-layout page_first_direct`. HiCache extends the radix cache, so the generated command drops `--disable-radix-cache` — the two options are mutually exclusive and SGLang rejects them together at startup. `--hicache-ratio 1.5` sizes the host tier at 1.5x the device KV cache, so make sure the node has that much free CPU DRAM per server on top of the model weights. Leave this **Disabled** for fixed-length throughput serving, where the extra host traffic buys nothing. - **Watchdog timeout:** Increase `--watchdog-timeout` to `1200` or higher for this large model, as weight loading can take significant time. - **Mamba Radix Cache**: Qwen3.5's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-radix-cache-strategy`: - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. Required for AMD MI GPUs. @@ -225,7 +226,7 @@ This section provides deployment configurations optimized for different hardware -**FP8 KV Cache**: `--kv-cache-dtype fp8_e4m3` quantizes the KV cache to FP8 at runtime. Since these FP8 model checkpoints do not include pre-calibrated KV cache scaling factors, SGLang defaults to a scale of 1.0, which may cause noticeable accuracy degradation on reasoning-heavy tasks. It is not included in the generated commands above; add it manually only if memory constraints require the trade-off. +**FP8 KV Cache**: `--kv-cache-dtype fp8_e4m3` quantizes the KV cache to FP8 at runtime. Since these FP8 model checkpoints do not include pre-calibrated KV cache scaling factors, SGLang defaults to a scale of 1.0, which may cause noticeable accuracy degradation on reasoning-heavy tasks. It is emitted only for the MXFP4-on-MI355X recipe, where it is part of the tuned configuration; for every other combination the generated commands leave it out, so add it manually only if memory constraints require the trade-off. - **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. diff --git a/docs/src/snippets/autoregressive/qwen35-deployment.jsx b/docs/src/snippets/autoregressive/qwen35-deployment.jsx index f71c52f15..0085283a9 100644 --- a/docs/src/snippets/autoregressive/qwen35-deployment.jsx +++ b/docs/src/snippets/autoregressive/qwen35-deployment.jsx @@ -110,6 +110,17 @@ export const Qwen35Deployment = () => { { id: 'enabled', label: 'Enabled', default: true } ] }, + kvOffload: { + name: 'kvOffload', + title: 'KV Cache Offloading', + // HiCache adds a host-DRAM tier below the device KV cache. Only wired up + // for the MI355X MXFP4 recipe, which is the arm it is tuned on. + condition: (values) => values.hardware === 'mi355x' && values.quantization === 'fp4', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'hicache', label: 'Host DRAM (HiCache)', default: false } + ] + }, mambaCache: { name: 'mambaCache', title: 'Mamba Radix Cache', @@ -296,7 +307,7 @@ export const Qwen35Deployment = () => { // Generate command — must produce byte-identical output to sgl-cookbook's // config.generateCommand(values) for every valid combination. const generateCommand = () => { - const { model, hardware, quantization, speculative, mambaCache } = values; + const { model, hardware, quantization, speculative, mambaCache, kvOffload } = values; let hwConfig = modelConfigs[model]?.[hardware]?.[quantization]; if (!hwConfig) { @@ -479,7 +490,18 @@ export const Qwen35Deployment = () => { // ROCm quick all-reduce env are emitted by the AMD backend block above // (this recipe uses quick all-reduce instead of AITER allreduce fusion). // Add the FP4-specific flags here. - cmd += ' \\\n --disable-radix-cache'; + if (kvOffload === 'hicache') { + // HiCache keeps a host-DRAM tier below the device KV cache, so the + // radix cache has to stay on: --enable-hierarchical-cache and + // --disable-radix-cache are rejected together at startup. + cmd += ' \\\n --enable-hierarchical-cache'; + cmd += ' \\\n --hicache-ratio 1.5'; + cmd += ' \\\n --hicache-write-policy write_through'; + cmd += ' \\\n --hicache-io-backend direct'; + cmd += ' \\\n --hicache-mem-layout page_first_direct'; + } else { + cmd += ' \\\n --disable-radix-cache'; + } cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; // Cap concurrency under MTP to avoid OOM at tp=2. if (speculative === 'enabled') {