diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index acde60f04..87bb3a6a5 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -49,13 +49,14 @@ docker run --gpus all \ **AMD GPUs (ROCm)** -AMD uses the daily-updated `lmsysorg/sglang-rocm` images: +AMD uses the daily-updated `lmsysorg/sglang-rocm` images. You can find the latest images on [Docker Hub](https://hub.docker.com/r/lmsysorg/sglang-rocm/tags). We recommend the ROCm 7.2 version. -- **MI355X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615` -- **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260615` +For example: +- **MI355X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623` +- **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623` ```bash Command -docker pull lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615 +docker pull lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623 docker run \ --device=/dev/kfd --device=/dev/dri \ @@ -65,7 +66,7 @@ docker run \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=" \ - lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615 \ + lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623 \ sglang serve ``` diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index d16b6b689..7a0789e6d 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -158,8 +158,8 @@ sgl-eval run aime25 \\ gb300: "lmsysorg/sglang:latest", // AMD daily-updated lmsysorg/sglang-rocm images. Bump the dated tag when you // re-verify on a newer build. - mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260615", - mi355x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260615", + mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623", + mi355x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623", }, // Pre-selects the issue template's `model` dropdown on "Submit verified cell". @@ -1400,26 +1400,9 @@ sgl-eval run aime25 \\ match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1444,26 +1427,10 @@ sgl-eval run aime25 \\ match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1492,26 +1459,10 @@ sgl-eval run aime25 \\ match: { hw: "mi300x", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1542,26 +1493,9 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1586,26 +1520,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1634,26 +1552,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1684,26 +1586,9 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1728,26 +1613,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1776,26 +1645,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1826,26 +1679,9 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1870,26 +1706,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1918,26 +1738,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -1968,26 +1772,9 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -2012,26 +1799,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "balanced", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code", @@ -2060,26 +1831,10 @@ sgl-eval run aime25 \\ match: { hw: "mi355x", variant: "pro", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, env: [ - "SGLANG_DEFAULT_THINKING=1", - "SGLANG_DSV4_REASONING_EFFORT=max", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM=false", - "SGLANG_USE_AITER=1", "SGLANG_USE_ROCM700A=0", - "SGLANG_OPT_USE_FUSED_COMPRESS=true", + "SGLANG_DP_USE_GATHERV=1", "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton", - "SGLANG_OPT_FP8_WO_A_GEMM=false", - "SGLANG_OPT_USE_JIT_INDEXER_METADATA=false", - "SGLANG_OPT_USE_TOPK_V2=false", - "SGLANG_OPT_USE_AITER_INDEXER=true", - "SGLANG_OPT_USE_TILELANG_INDEXER=false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE=false", - "SGLANG_OPT_USE_TILELANG_MHC_POST=false", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1", - "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true", - "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false", - "SGLANG_ROCM_USE_MULTI_STREAM=false", "AITER_BF16_FP8_MOE_BOUND=0", - "SGLANG_EAGER_INPUT_NO_COPY=true", ], flags: [ "--trust-remote-code",