From 95f5ecd3d26665423d3e6577a2a00c04f5cde733 Mon Sep 17 00:00:00 2001 From: Thomas Wang Date: Mon, 24 Aug 2026 04:26:39 +0800 Subject: [PATCH] [AMD] Update amd deepseek v4 cookbook 0822 (#35854) --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 8 +-- .../configs/deepseek-ai/deepseek-v4.jsx | 61 ++++++++++++------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index f0ae0012c..47d1f5c47 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -52,11 +52,11 @@ docker run --gpus all \ AMD uses the daily-updated `lmsysorg/sglang-rocm` images. You can find the latest images on [Docker Hub](https://hub.docker.com/r/lmsysorg/sglang-rocm/tags). We recommend the ROCm 7.2 version. For example: -- **MI355X** → `lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710` -- **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623` +- **MI355X** → `lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260822` +- **MI300X** → `lmsysorg/sglang-rocm:v0.5.17-rocm720-mi30x-20260822` ```bash Command -docker pull lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 +docker pull lmsysorg/sglang-rocm:v0.5.17-rocm720-{mi35x,mi30x}-20260822 docker run \ --device=/dev/kfd --device=/dev/dri \ @@ -66,7 +66,7 @@ docker run \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=" \ - lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 \ + lmsysorg/sglang-rocm:v0.5.17-rocm720-{mi35x,mi30x}-20260822 \ sglang serve ``` diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 37e9ddd20..f12d86575 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -183,8 +183,8 @@ sgl-eval run aime25 \\ gb300: "lmsysorg/sglang:latest", // AMD daily-updated lmsysorg/sglang-rocm images. Bump the dated tag when you // re-verify on a newer build. - mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623", - mi355x: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710", + mi300x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi30x-20260822", + mi355x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260822", }, // Pre-selects the issue template's `model` dropdown on "Submit verified cell". @@ -1643,7 +1643,7 @@ sgl-eval run aime25 \\ // DSpark requires CUDA; EAGLE binds a head that accepts nothing on 0813 -> target-only. match: { hw: "mi355x", variant: "pro-official", quant: "fp4", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0"], + env: ["SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true"], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", @@ -1652,7 +1652,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 8192", "--host {{HOST_IP}}", @@ -1663,7 +1663,7 @@ sgl-eval run aime25 \\ // DSpark requires CUDA; EAGLE binds a head that accepts nothing on 0813 -> target-only. match: { hw: "mi355x", variant: "pro-official", quant: "fp4", strategy: "balanced", nodes: "single" }, verified: false, - env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0"], + env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true"], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", @@ -1675,7 +1675,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--host {{HOST_IP}}", @@ -1686,7 +1686,7 @@ sgl-eval run aime25 \\ // DSpark requires CUDA; EAGLE binds a head that accepts nothing on 0813 -> target-only. match: { hw: "mi355x", variant: "pro-official", quant: "fp4", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0"], + env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true"], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", @@ -1698,7 +1698,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--host {{HOST_IP}}", @@ -2331,6 +2331,7 @@ sgl-eval run aime25 \\ "SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2340,7 +2341,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 8192", "--host {{HOST_IP}}", @@ -2354,6 +2355,7 @@ sgl-eval run aime25 \\ "SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2363,7 +2365,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 8192", "--speculative-algorithm EAGLE", @@ -2385,6 +2387,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2397,7 +2400,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--host {{HOST_IP}}", @@ -2415,6 +2418,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2427,7 +2431,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2449,6 +2453,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2461,7 +2466,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--host {{HOST_IP}}", @@ -2479,6 +2484,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2491,7 +2497,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2511,6 +2517,7 @@ sgl-eval run aime25 \\ "SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2520,7 +2527,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 8192", "--speculative-algorithm EAGLE", @@ -2542,6 +2549,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2554,7 +2562,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2576,6 +2584,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2588,7 +2597,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2608,6 +2617,7 @@ sgl-eval run aime25 \\ "SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2617,7 +2627,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 8192", "--speculative-algorithm EAGLE", @@ -2639,6 +2649,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2651,7 +2662,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2673,6 +2684,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2685,7 +2697,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2705,6 +2717,7 @@ sgl-eval run aime25 \\ "SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2714,7 +2727,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 8192", "--speculative-algorithm EAGLE", @@ -2736,6 +2749,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2748,7 +2762,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE", @@ -2770,6 +2784,7 @@ sgl-eval run aime25 \\ "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true", ], flags: [ "--trust-remote-code", @@ -2782,7 +2797,7 @@ sgl-eval run aime25 \\ "--page-size 256", "--mem-fraction-static 0.90", "--swa-full-tokens-ratio 0.15", - "--disable-shared-experts-fusion", + "--enforce-shared-experts-fusion", "--kv-cache-dtype fp8_e4m3", "--chunked-prefill-size 65536", "--speculative-algorithm EAGLE",