minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[AMD] Cap DSV4 Flash max_total_num_tokens (#30313)

Browse Source 
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
Bingxu Chen
2026-07-07 15:33:05 +08:00
committed by GitHub
co-authored by YC Yen-Ching Tseng
parent 99db3b0fa5
commit dabd4cfcfd
9 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/slurm/launch_mi355x.sh
+2
View File
@@ -111,6 +111,7 @@ emit("LBPORT", rt["lb_port"])
emit("MEMFRAC", rt["mem_fraction_static"]) emit("MEMFRAC", rt["mem_fraction_static"])
emit("PAGE", rt["page_size"]) emit("PAGE", rt["page_size"])
emit("MAXREQ", rt["max_running_requests"]) emit("MAXREQ", rt["max_running_requests"])
emit("MAXTOK", rt.get("max_total_tokens", ""))
emit("CHUNK", rt["chunked_prefill_size"]) emit("CHUNK", rt["chunked_prefill_size"])
# swa is DSV4-specific; emit empty when a model omits it so the flag is dropped. # swa is DSV4-specific; emit empty when a model omits it so the flag is dropped.
emit("SWA", rt.get("swa_full_tokens_ratio", "")) emit("SWA", rt.get("swa_full_tokens_ratio", ""))
@@ -239,6 +240,7 @@ PY
EXTRA_FLAGS="" EXTRA_FLAGS=""
(( PDP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --enable-dp-attention --dp-size $PDP" (( PDP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --enable-dp-attention --dp-size $PDP"
(( PEP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --ep-size $PEP" (( PEP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --ep-size $PEP"
[[ -n "$MAXTOK" ]] && EXTRA_FLAGS="$EXTRA_FLAGS --max-total-tokens $MAXTOK"
if [[ "$MTP_ENABLED" == "1" ]]; then if [[ "$MTP_ENABLED" == "1" ]]; then
EXTRA_FLAGS="$EXTRA_FLAGS --speculative-algorithm $MTP_ALGO \ EXTRA_FLAGS="$EXTRA_FLAGS --speculative-algorithm $MTP_ALGO \
--speculative-num-steps $MTP_STEPS --speculative-eagle-topk $MTP_TOPK \ --speculative-num-steps $MTP_STEPS --speculative-eagle-topk $MTP_TOPK \
scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml
+1
View File
@@ -35,6 +35,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1
scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml
+1
View File
@@ -33,6 +33,7 @@ runtime:
mem_fraction_static: 0.90 mem_fraction_static: 0.90
page_size: 256 page_size: 256
max_running_requests: 256 max_running_requests: 256
max_total_tokens: 8551168
chunked_prefill_size: 8192 chunked_prefill_size: 8192
swa_full_tokens_ratio: 0.1 swa_full_tokens_ratio: 0.1