minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[AMD] Update amd deepseek v4 cookbook 0822 (#35854)

Browse Source
This commit is contained in:
Thomas Wang
2026-08-23 13:26:39 -07:00
committed by GitHub
parent d1af3c8923
commit 95f5ecd3d2
2 changed files with 42 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx
+4 -4
View File
@@ -52,11 +52,11 @@ docker run --gpus all \
AMD uses the daily-updated `lmsysorg/sglang-rocm` images. You can find the latest images on [Docker Hub](https://hub.docker.com/r/lmsysorg/sglang-rocm/tags). We recommend the ROCm 7.2 version.
For example:
- **MI355X** → `lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710`
- **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623`
- **MI355X** → `lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260822`
- **MI300X** → `lmsysorg/sglang-rocm:v0.5.17-rocm720-mi30x-20260822`
```bash Command
docker pull lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710
docker pull lmsysorg/sglang-rocm:v0.5.17-rocm720-{mi35x,mi30x}-20260822
docker run \
--device=/dev/kfd --device=/dev/dri \
@@ -66,7 +66,7 @@ docker run \
-p 30000:30000 \
-v ~/.cache/huggingface:/root/.cache/huggingface \
--env "HF_TOKEN=<your-hf-token>" \
lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 \
lmsysorg/sglang-rocm:v0.5.17-rocm720-{mi35x,mi30x}-20260822 \
sglang serve <use args below>
```
docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx
+38 -23
View File
@@ -183,8 +183,8 @@ sgl-eval run aime25 \\
gb300: "lmsysorg/sglang:latest",
// AMD daily-updated lmsysorg/sglang-rocm images. Bump the dated tag when you
// re-verify on a newer build.
mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623",
mi355x: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710",
mi300x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi30x-20260822",
mi355x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260822",
},
// Pre-selects the issue template's `model` dropdown on "Submit verified cell".
@@ -1643,7 +1643,7 @@ sgl-eval run aime25 \\
// DSpark requires CUDA; EAGLE binds a head that accepts nothing on 0813 -> target-only.
match: { hw: "mi355x", variant: "pro-official", quant: "fp4", strategy: "low-latency", nodes: "single" },
verified: false,
env: ["SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0"],
env: ["SGLANG_USE_ROCM700A=0", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true"],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
@@ -1652,7 +1652,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 8192",
"--host {{HOST_IP}}",
@@ -1663,7 +1663,7 @@ sgl-eval run aime25 \\
// DSpark requires CUDA; EAGLE binds a head that accepts nothing on 0813 -> target-only.
match: { hw: "mi355x", variant: "pro-official", quant: "fp4", strategy: "balanced", nodes: "single" },
verified: false,
env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0"],
env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true"],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
@@ -1675,7 +1675,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--host {{HOST_IP}}",
@@ -1686,7 +1686,7 @@ sgl-eval run aime25 \\
// DSpark requires CUDA; EAGLE binds a head that accepts nothing on 0813 -> target-only.
match: { hw: "mi355x", variant: "pro-official", quant: "fp4", strategy: "high-throughput", nodes: "single" },
verified: false,
env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0"],
env: ["SGLANG_USE_ROCM700A=0", "SGLANG_SHARED_EXPERT_TP1=1", "SGLANG_DP_SHARED_EXPERT_LOCAL=1", "SGLANG_DP_USE_GATHERV=1", "SGLANG_DP_USE_REDUCE_SCATTER=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", "AITER_BF16_FP8_MOE_BOUND=0", "SGLANG_OPT_USE_AITER_BATCHED_GEMM=true"],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
@@ -1698,7 +1698,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--host {{HOST_IP}}",
@@ -2331,6 +2331,7 @@ sgl-eval run aime25 \\
"SGLANG_USE_ROCM700A=0",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2340,7 +2341,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 8192",
"--host {{HOST_IP}}",
@@ -2354,6 +2355,7 @@ sgl-eval run aime25 \\
"SGLANG_USE_ROCM700A=0",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2363,7 +2365,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 8192",
"--speculative-algorithm EAGLE",
@@ -2385,6 +2387,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2397,7 +2400,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--host {{HOST_IP}}",
@@ -2415,6 +2418,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2427,7 +2431,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2449,6 +2453,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2461,7 +2466,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--host {{HOST_IP}}",
@@ -2479,6 +2484,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2491,7 +2497,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2511,6 +2517,7 @@ sgl-eval run aime25 \\
"SGLANG_USE_ROCM700A=0",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2520,7 +2527,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 8192",
"--speculative-algorithm EAGLE",
@@ -2542,6 +2549,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2554,7 +2562,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2576,6 +2584,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2588,7 +2597,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2608,6 +2617,7 @@ sgl-eval run aime25 \\
"SGLANG_USE_ROCM700A=0",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2617,7 +2627,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 8192",
"--speculative-algorithm EAGLE",
@@ -2639,6 +2649,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2651,7 +2662,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2673,6 +2684,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2685,7 +2697,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2705,6 +2717,7 @@ sgl-eval run aime25 \\
"SGLANG_USE_ROCM700A=0",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2714,7 +2727,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 8192",
"--speculative-algorithm EAGLE",
@@ -2736,6 +2749,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2748,7 +2762,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",
@@ -2770,6 +2784,7 @@ sgl-eval run aime25 \\
"SGLANG_DP_USE_REDUCE_SCATTER=1",
"SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton",
"AITER_BF16_FP8_MOE_BOUND=0",
"SGLANG_OPT_USE_AITER_BATCHED_GEMM=true",
],
flags: [
"--trust-remote-code",
@@ -2782,7 +2797,7 @@ sgl-eval run aime25 \\
"--page-size 256",
"--mem-fraction-static 0.90",
"--swa-full-tokens-ratio 0.15",
"--disable-shared-experts-fusion",
"--enforce-shared-experts-fusion",
"--kv-cache-dtype fp8_e4m3",
"--chunked-prefill-size 65536",
"--speculative-algorithm EAGLE",