From aafa706f8f2d400495f232d9f5c00273bc99a5ed Mon Sep 17 00:00:00 2001
From: Thomas Wang <thomawan@amd.com>
Date: Wed, 15 Jul 2026 10:59:12 +0800
Subject: [PATCH] [AMD] Update qwen3.5 cookbook (#31258)

---
 docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx          | 4 +++-
 docs_new/src/snippets/autoregressive/qwen35-deployment.jsx | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx
index 9644b83aa..17f81a90c 100644
--- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx
+++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.5.mdx
@@ -127,7 +127,7 @@ This section provides deployment configurations optimized for different hardware
 
 - Speculative decoding (MTP) can significantly reduce latency for interactive use cases.
 - **H100 FP8:** Add `--enable-symm-mem` to enable NCCL symmetric memory for faster collectives and better performance under multi-GPU settings.
-- **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `SGLANG_USE_AITER_UNIFIED_ATTN=1` with `--attention-backend aiter`, which requires `--page-size 16` and can also enable `--enable-aiter-allreduce-fusion`.
+- **AMD GPUs (MI300X / MI325X / MI355X):** Use `SGLANG_USE_AITER=1` and `SGLANG_USE_AITER_UNIFIED_ATTN=1` with `--attention-backend aiter`, which requires `--page-size 16` and can also enable `--enable-aiter-allreduce-fusion`. Additionally set `AITER_FLYDSL_FORCE=1` to force the AITER FlyDSL MoE kernels and `SGLANG_MAMBA_SSM_DTYPE=bfloat16` to store the Mamba SSM state in bfloat16 (instead of the default float32).
 - **Watchdog timeout:** Increase `--watchdog-timeout` to `1200` or higher for this large model, as weight loading can take significant time.
 - **Mamba Radix Cache**: Qwen3.5's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-scheduler-strategy`:
   - **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage. Required for AMD MI GPUs.
@@ -258,6 +258,8 @@ Deploy Qwen3.5-397B-A17B with the following command (MI300X/MI325X/MI355X):
 ```shell Command
 SGLANG_USE_AITER=1 \
 SGLANG_USE_AITER_UNIFIED_ATTN=1 \
+AITER_FLYDSL_FORCE=1 \
+SGLANG_MAMBA_SSM_DTYPE=bfloat16 \
 python3 -m sglang.launch_server \
   --model-path Qwen/Qwen3.5-397B-A17B \
   --tp 8 \
diff --git a/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx
index 7c0f3d70d..879dcab4a 100644
--- a/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx
+++ b/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx
@@ -435,7 +435,11 @@ export const Qwen35Deployment = () => {
     // SGLANG_USE_AITER=1 and SGLANG_USE_AITER_UNIFIED_ATTN=1; see cookbook prose),
     // which requires --page-size 16. Enable AITER allreduce fusion for multi-GPU.
     if (amdGpu) {
-      cmd = "SGLANG_USE_AITER=1 \\\nSGLANG_USE_AITER_UNIFIED_ATTN=1 \\\n" + cmd;
+      let amdEnv = "SGLANG_USE_AITER=1 \\\nSGLANG_USE_AITER_UNIFIED_ATTN=1 \\\nAITER_FLYDSL_FORCE=1 \\\n";
+      if (MOE_MODELS.has(model)) {
+        amdEnv += "SGLANG_MAMBA_SSM_DTYPE=bfloat16 \\\n";
+      }
+      cmd = amdEnv + cmd;
       cmd += " \\\n  --attention-backend aiter";
       cmd += " \\\n  --page-size 16";
       if (hwConfig.tp > 1) {