From dabd4cfcfd349eb8c3c3a15210fd74978335a812 Mon Sep 17 00:00:00 2001 From: Bingxu Chen Date: Tue, 7 Jul 2026 15:33:05 +0800 Subject: [PATCH] [AMD] Cap DSV4 Flash max_total_num_tokens (#30313) Co-authored-by: YC Yen-Ching Tseng --- scripts/ci/slurm/launch_mi355x.sh | 2 ++ .../recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml | 1 + .../ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml | 1 + .../ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml | 1 + scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml | 1 + .../recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml | 1 + .../ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml | 1 + .../ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml | 1 + scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml | 1 + 9 files changed, 10 insertions(+) diff --git a/scripts/ci/slurm/launch_mi355x.sh b/scripts/ci/slurm/launch_mi355x.sh index 0635b71b4..ae6b7cbb7 100755 --- a/scripts/ci/slurm/launch_mi355x.sh +++ b/scripts/ci/slurm/launch_mi355x.sh @@ -111,6 +111,7 @@ emit("LBPORT", rt["lb_port"]) emit("MEMFRAC", rt["mem_fraction_static"]) emit("PAGE", rt["page_size"]) emit("MAXREQ", rt["max_running_requests"]) +emit("MAXTOK", rt.get("max_total_tokens", "")) emit("CHUNK", rt["chunked_prefill_size"]) # swa is DSV4-specific; emit empty when a model omits it so the flag is dropped. emit("SWA", rt.get("swa_full_tokens_ratio", "")) @@ -239,6 +240,7 @@ PY EXTRA_FLAGS="" (( PDP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --enable-dp-attention --dp-size $PDP" (( PEP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --ep-size $PEP" +[[ -n "$MAXTOK" ]] && EXTRA_FLAGS="$EXTRA_FLAGS --max-total-tokens $MAXTOK" if [[ "$MTP_ENABLED" == "1" ]]; then EXTRA_FLAGS="$EXTRA_FLAGS --speculative-algorithm $MTP_ALGO \ --speculative-num-steps $MTP_STEPS --speculative-eagle-topk $MTP_TOPK \ diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml index da710c6aa..5c52f17a1 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml index 01a12bfec..02678af98 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml index ec7c8a93c..25c2ad899 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml index 3116ce6fd..7da8a1d64 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml @@ -35,6 +35,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml index 826b1782a..7a3623e31 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml index 82b52e8df..f20abe3ef 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml index 5546a1fcb..899013770 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml index 8539748ae..6e35d1527 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml @@ -33,6 +33,7 @@ runtime: mem_fraction_static: 0.90 page_size: 256 max_running_requests: 256 + max_total_tokens: 8551168 chunked_prefill_size: 8192 swa_full_tokens_ratio: 0.1