[AMD] cookbook: serve Qwen3.5 MXFP4 on MI355X with an fp8_e4m3 KV cache (#35445)

2026-08-19 18:49:32 +08:00
parent 3e5ce26c2d
commit 574274660f
1 changed files with 1 additions and 0 deletions
@@ -480,6 +480,7 @@ export const Qwen35Deployment = () => {
        // (this recipe uses quick all-reduce instead of AITER allreduce fusion).
        // Add the FP4-specific flags here.
        cmd += ' \\\n  --disable-radix-cache';
+        cmd += ' \\\n  --kv-cache-dtype fp8_e4m3';
        // Cap concurrency under MTP to avoid OOM at tp=2.
        if (speculative === 'enabled') {
          cmd += ' \\\n  --max-running-requests 128';