From 0d651e653b9843bd90c80e7415f5e55be4259c5c Mon Sep 17 00:00:00 2001 From: Thomas Wang Date: Wed, 17 Jun 2026 09:15:16 +0800 Subject: [PATCH] [AMD] Update v4 amd cookbook (#28423) --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 38 +- docs_new/src/snippets/_deployment.jsx | 22 +- .../deepseek-ai/deepseek-v4-benchmarks.jsx | 21 + .../configs/deepseek-ai/deepseek-v4.jsx | 724 ++++++++++++++++++ 4 files changed, 799 insertions(+), 6 deletions(-) diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index e11bd2f9e..f3002061e 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -28,15 +28,15 @@ Then run the **Python** output of the command panel below in that environment. +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). A minimal example (substitute the inner `sglang serve ...` with whatever the command generator below produces): + +**NVIDIA GPUs** + A single image — `lmsysorg/sglang:latest` — covers the **datacenter GPUs** in this cookbook (B200 / B300 / GB200 / GB300 / H100 / H200). For **RTX PRO 6000 (SM120)**, use the nightly `lmsysorg/sglang:dev` instead — SM120 support isn't in `:latest` yet (see the RTX PRO 6000 note below). ```bash Command docker pull lmsysorg/sglang:latest -``` -For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). A minimal example (substitute the inner `sglang serve ...` with whatever the command generator below produces): - -```bash Command docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ @@ -47,6 +47,28 @@ docker run --gpus all \ sglang serve ``` +**AMD GPUs (ROCm)** + +AMD uses the daily-updated `lmsysorg/sglang-rocm` images: + +- **MI355X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615` +- **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260615` + +```bash Command +docker pull lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615 + +docker run \ + --device=/dev/kfd --device=/dev/dri \ + --group-add video \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --shm-size 32g --ipc=host \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615 \ + sglang serve +``` + @@ -233,6 +255,14 @@ RTX PRO 6000 (96 GB) runs **Flash only** — V4-Pro doesn't fit on 8× 96 GB. It `--mem-fraction-static 0.70`; the Deploy panel greys out the other recipes for this card. HiCache and MegaMoE are **not** supported on RTX PRO 6000. For Docker, use the nightly `lmsysorg/sglang:dev` image — SM120 support isn't in `lmsysorg/sglang:latest` yet (the Deploy panel's Docker mode already points this card at `:dev`). +**AMD (MI300X / MI355X) note** + +- **Model checkpoints** — for correct accuracy, the FP4 model uses the stock `deepseek-ai/DeepSeek-V4-{Flash,Pro}`, and the FP8 model uses the repackaged `sgl-project/DeepSeek-V4-{Flash,Pro}-FP8`. +- **Supported models** — **MI300X** supports DeepSeek-V4-Flash in FP8; **MI355X** supports DeepSeek-V4-Flash / Pro in both FP4 and FP8. All recipes run single-node. +- **TP / DP setting** — both TP=4 and TP=8 are supported. At low concurrency we recommend **TP-only**; at high concurrency use **TP + DP**, which additionally needs `--dp 8 --enable-dp-attention --enable-prefill-delayer --prefill-delayer-max-delay-ms 5000`. +- **MTP** — speculative decoding is supported; add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. +- **Kernels** — uses the Unified KV attention and the flydsl MoE. + **MegaMoE** MegaMoE fuses expert dispatch + GEMM into a single kernel for higher throughput diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index dc67a9b40..de8a915d4 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -504,9 +504,27 @@ export const Deployment = ({ config, benchmarks }) => { const image = (config.dockerImages && config.dockerImages[sel.hw]) || "lmsysorg/sglang:dev"; const portFlag = flags.find((x) => x.split(/[\s=]/)[0] === "--port"); const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}"; + const vendorOf = (hwId) => { + for (const [vendor, list] of Object.entries(HARDWARE_CATALOG)) { + if (list.some((h) => h.id === hwId)) return vendor; + } + const extra = (config.hardware || []).find((h) => h.id === hwId); + return (extra && extra.vendor) || "nvidia"; + }; + const gpuAccessLines = vendorOf(sel.hw) === "amd" + ? [ + "docker run", + " --device=/dev/kfd --device=/dev/dri", + " --group-add video", + " --cap-add=SYS_PTRACE --security-opt seccomp=unconfined", + " --shm-size 32g", + ] + : [ + "docker run --gpus all", + " --shm-size 32g", + ]; const dockerLines = [ - "docker run --gpus all", - " --shm-size 32g", + ...gpuAccessLines, // Multi-node needs host networking so the cross-node rendezvous port // (--dist-init-addr) and NCCL/GLOO traffic are reachable; single-node // just maps the serve port. diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx index 347831bc0..b345ada0e 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx @@ -261,4 +261,25 @@ export const benchmarks = [ { match: { hw: "h100", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "multi-2" }, }, + // ==================================================================== + // MI300X + FP8 (Flash) + { match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" } }, + { match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + // MI355X + FP4 (Flash) + { match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" } }, + { match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" } }, + // MI355X + FP8 (Flash) + { match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" } }, + { match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + // MI355X + FP4 (Pro) + { match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" } }, + { match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" } }, + // MI355X + FP8 (Pro) + { match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "balanced", nodes: "single" } }, + { match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, ]; diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 1118d57dd..caebb215f 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -7,6 +7,8 @@ export const config = { supportedHardware: [ "h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000", + // AMD ROCm — MI300X (Flash FP8) + MI355X (Flash/Pro, FP4/FP8). + "mi300x", "mi355x", ], // Model-specific GPUs the shared HARDWARE_CATALOG doesn't carry — the engine @@ -43,6 +45,10 @@ export const config = { // H200 FP8 needs the sgl-project repackaging (Hopper can't run FP4-mixed Instruct). "h200|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8", "h200|pro|fp8": "sgl-project/DeepSeek-V4-Pro-FP8", + // AMD FP8 uses the sgl-project repackaging. + "mi300x|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8", + "mi355x|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8", + "mi355x|pro|fp8": "sgl-project/DeepSeek-V4-Pro-FP8", }, placeholders: { @@ -147,6 +153,10 @@ sgl-eval run aime25 \\ b300: "lmsysorg/sglang:latest", gb200: "lmsysorg/sglang:latest", gb300: "lmsysorg/sglang:latest", + // AMD daily-updated lmsysorg/sglang-rocm images. Bump the dated tag when you + // re-verify on a newer build. + mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260615", + mi355x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615", }, // Pre-selects the issue template's `model` dropdown on "Submit verified cell". @@ -1200,5 +1210,719 @@ sgl-eval run aime25 \\ "--port {{PORT}}", ], }, + + // ==================================================================== + // AMD ROCm (MI300X / MI355X) + // -------------------------------------------------------------------- + + // ---------- MI300X (192GB) — Flash FP8 ---------- + { + match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 8192", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ---------- MI355X (288GB) — Flash FP4 ---------- + { + match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 8192", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ---------- MI355X (288GB) — Flash FP8 ---------- + { + match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 8192", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ---------- MI355X (288GB) — Pro FP4 ---------- + { + match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 8192", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ---------- MI355X (288GB) — Pro FP8 ---------- + { + match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 8192", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [ + "SGLANG_DEFAULT_THINKING=1", + "SGLANG_DSV4_REASONING_EFFORT=max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", + "SGLANG_USE_AITER=1", + "SGLANG_USE_ROCM700A=0", + "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", + "SGLANG_OPT_FP8_WO_A_GEMM=false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", + "SGLANG_OPT_USE_TOPK_V2=false", + "SGLANG_OPT_USE_AITER_INDEXER=true", + "SGLANG_OPT_USE_TILELANG_INDEXER=false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", + "SGLANG_OPT_USE_TILELANG_MHC_POST=false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", + "SGLANG_ROCM_USE_MULTI_STREAM=false", + "AITER_BF16_FP8_MOE_BOUND=0", + "SGLANG_EAGER_INPUT_NO_COPY=true", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--enable-prefill-delayer", + "--prefill-delayer-max-delay-ms 5000", + "--attention-backend dsv4", + "--page-size 256", + "--mem-fraction-static 0.90", + "--swa-full-tokens-ratio 0.1", + "--disable-shared-experts-fusion", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 65536", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, ], };