[AMD][DI][CI] 1/N: MI355X disaggregation nightly benchmark (#29084)
Co-authored-by: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Co-authored-by: bingxche <bingxche@amd.com> Co-authored-by: yctseng0211 <yctseng@amd.com>
This commit is contained in:
Zhaoyi Li
co-authored by
Michael
bingxche
yctseng0211
Michael
bingxche
yctseng0211
parent
cfc0a0e0e0
commit
413aeac0c9
@@ -0,0 +1,54 @@
|
||||
# MI355X DeepSeek-V4-Flash (FP4) 2-node 1P1D disaggregation recipe.
|
||||
#
|
||||
# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
|
||||
#
|
||||
# Consumed by:
|
||||
# * scripts/ci/slurm/process_result.py reads `resources` and
|
||||
# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
|
||||
# * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
|
||||
|
||||
resources:
|
||||
prefill_workers: 1
|
||||
decode_workers: 1
|
||||
|
||||
backend:
|
||||
sglang_config:
|
||||
prefill:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
decode:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
|
||||
runtime:
|
||||
image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
|
||||
attention_backend: dsv4
|
||||
# RoCE HCAs MORI uses for cross-node KV transfer.
|
||||
ib_devices: rdma0,rdma1,rdma2,rdma3
|
||||
prefill_port: 30025
|
||||
decode_port: 30026
|
||||
prefill_bootstrap_port: 8998
|
||||
decode_bootstrap_port: 9001
|
||||
lb_port: 8000
|
||||
mem_fraction_static: 0.90
|
||||
page_size: 256
|
||||
max_running_requests: 256
|
||||
chunked_prefill_size: 8192
|
||||
swa_full_tokens_ratio: 0.1
|
||||
|
||||
bench:
|
||||
# bench_serving --max-concurrency sweep; one result JSON per concurrency.
|
||||
concurrencies: [1, 8, 16, 32, 64, 128, 256]
|
||||
num_prompts_factor: 4 # num-prompts = concurrency * factor
|
||||
random_range_ratio: 1.0
|
||||
|
||||
# Correctness gate run through the PD path before the perf sweep
|
||||
# (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
|
||||
# even when throughput looks fine ("fast but wrong").
|
||||
accuracy:
|
||||
enabled: true
|
||||
num_shots: 8
|
||||
num_questions: 1319 # full GSM8K test set
|
||||
threshold: 0.91
|
||||
@@ -0,0 +1,56 @@
|
||||
# MI355X DeepSeek-V4-Pro (FP4) 2-node 1P1D disaggregation recipe.
|
||||
#
|
||||
# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
|
||||
# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
|
||||
# larger, so mem_fraction_static / max_running_requests may need tuning.
|
||||
#
|
||||
# Consumed by:
|
||||
# * scripts/ci/slurm/process_result.py reads `resources` and
|
||||
# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
|
||||
# * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
|
||||
|
||||
resources:
|
||||
prefill_workers: 1
|
||||
decode_workers: 1
|
||||
|
||||
backend:
|
||||
sglang_config:
|
||||
prefill:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
decode:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
|
||||
runtime:
|
||||
image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
|
||||
attention_backend: dsv4
|
||||
# RoCE HCAs MORI uses for cross-node KV transfer.
|
||||
ib_devices: rdma0,rdma1,rdma2,rdma3
|
||||
prefill_port: 30025
|
||||
decode_port: 30026
|
||||
prefill_bootstrap_port: 8998
|
||||
decode_bootstrap_port: 9001
|
||||
lb_port: 8000
|
||||
mem_fraction_static: 0.90
|
||||
page_size: 256
|
||||
max_running_requests: 256
|
||||
chunked_prefill_size: 8192
|
||||
swa_full_tokens_ratio: 0.1
|
||||
|
||||
bench:
|
||||
# bench_serving --max-concurrency sweep; one result JSON per concurrency.
|
||||
concurrencies: [1, 8, 16, 32, 64, 128, 256]
|
||||
num_prompts_factor: 4 # num-prompts = concurrency * factor
|
||||
random_range_ratio: 1.0
|
||||
|
||||
# Correctness gate run through the PD path before the perf sweep
|
||||
# (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
|
||||
# even when throughput looks fine ("fast but wrong").
|
||||
accuracy:
|
||||
enabled: true
|
||||
num_shots: 8
|
||||
num_questions: 1319 # full GSM8K test set
|
||||
threshold: 0.91
|
||||
@@ -0,0 +1,53 @@
|
||||
# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe.
|
||||
#
|
||||
# Consumed by:
|
||||
# * scripts/ci/slurm/process_result.py reads `resources` and
|
||||
# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
|
||||
# * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
|
||||
|
||||
resources:
|
||||
prefill_workers: 1
|
||||
decode_workers: 1
|
||||
|
||||
backend:
|
||||
sglang_config:
|
||||
prefill:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
decode:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
|
||||
runtime:
|
||||
image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
|
||||
attention_backend: dsv4
|
||||
# RoCE HCAs MORI uses for cross-node KV transfer.
|
||||
ib_devices: rdma0,rdma1,rdma2,rdma3
|
||||
prefill_port: 30025
|
||||
decode_port: 30026
|
||||
prefill_bootstrap_port: 8998
|
||||
decode_bootstrap_port: 9001
|
||||
lb_port: 8000
|
||||
mem_fraction_static: 0.90
|
||||
page_size: 256
|
||||
max_running_requests: 256
|
||||
chunked_prefill_size: 8192
|
||||
swa_full_tokens_ratio: 0.1
|
||||
|
||||
bench:
|
||||
# bench_serving --max-concurrency sweep; one result JSON per concurrency.
|
||||
concurrencies: [1, 8, 16, 32, 64, 128, 256]
|
||||
num_prompts_factor: 4 # num-prompts = concurrency * factor
|
||||
random_range_ratio: 1.0
|
||||
|
||||
# Correctness gate run through the PD path after the perf sweep. Mirrors the
|
||||
# single-node registered test test/registered/amd/test_deepseek_v4_flash_fp8.py
|
||||
# (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
|
||||
# even when throughput looks fine ("fast but wrong").
|
||||
accuracy:
|
||||
enabled: true
|
||||
num_shots: 8
|
||||
num_questions: 1319 # full GSM8K test set
|
||||
threshold: 0.91
|
||||
@@ -0,0 +1,55 @@
|
||||
# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe.
|
||||
#
|
||||
# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
|
||||
# larger, so mem_fraction_static / max_running_requests may need tuning.
|
||||
#
|
||||
# Consumed by:
|
||||
# * scripts/ci/slurm/process_result.py reads `resources` and
|
||||
# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
|
||||
# * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
|
||||
|
||||
resources:
|
||||
prefill_workers: 1
|
||||
decode_workers: 1
|
||||
|
||||
backend:
|
||||
sglang_config:
|
||||
prefill:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
decode:
|
||||
tensor-parallel-size: 8
|
||||
expert-parallel-size: 1
|
||||
data-parallel-size: 1
|
||||
|
||||
runtime:
|
||||
image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
|
||||
attention_backend: dsv4
|
||||
# RoCE HCAs MORI uses for cross-node KV transfer.
|
||||
ib_devices: rdma0,rdma1,rdma2,rdma3
|
||||
prefill_port: 30025
|
||||
decode_port: 30026
|
||||
prefill_bootstrap_port: 8998
|
||||
decode_bootstrap_port: 9001
|
||||
lb_port: 8000
|
||||
mem_fraction_static: 0.90
|
||||
page_size: 256
|
||||
max_running_requests: 256
|
||||
chunked_prefill_size: 8192
|
||||
swa_full_tokens_ratio: 0.1
|
||||
|
||||
bench:
|
||||
# bench_serving --max-concurrency sweep; one result JSON per concurrency.
|
||||
concurrencies: [1, 8, 16, 32, 64, 128, 256]
|
||||
num_prompts_factor: 4 # num-prompts = concurrency * factor
|
||||
random_range_ratio: 1.0
|
||||
|
||||
# Correctness gate run through the PD path before the perf sweep
|
||||
# (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
|
||||
# even when throughput looks fine ("fast but wrong").
|
||||
accuracy:
|
||||
enabled: true
|
||||
num_shots: 8
|
||||
num_questions: 1319 # full GSM8K test set
|
||||
threshold: 0.91
|
||||
Reference in New Issue
Block a user