From 48dcadc7703b4e209353116aea71f2128ff92601 Mon Sep 17 00:00:00 2001 From: Zhaoyi Li <36555117+Lzy17@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:08:00 -0500 Subject: [PATCH] [AMD][DI][CI] 5/N Add DSV4 wide-EP16 4-node 2P1D nightly recipes (#31500) Co-authored-by: Chen --- scripts/ci/slurm/nightly-configs.yaml | 143 ++++++++++++++++++ .../dsv4flash/1k1k/2p1d-ep16-mtp.yaml | 107 +++++++++++++ .../mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml | 95 ++++++++++++ .../dsv4pro/1k1k/2p1d-ep16-mtp.yaml | 107 +++++++++++++ .../mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml | 95 ++++++++++++ .../dsv4flash/1k1k/2p1d-ep16-mtp.yaml | 107 +++++++++++++ .../mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml | 95 ++++++++++++ .../dsv4pro/1k1k/2p1d-ep16-mtp.yaml | 107 +++++++++++++ .../mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml | 95 ++++++++++++ 9 files changed, 951 insertions(+) create mode 100644 scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml diff --git a/scripts/ci/slurm/nightly-configs.yaml b/scripts/ci/slurm/nightly-configs.yaml index 071e86348..585d10bb2 100644 --- a/scripts/ci/slurm/nightly-configs.yaml +++ b/scripts/ci/slurm/nightly-configs.yaml @@ -357,6 +357,149 @@ kimik26-fp8-mi355x-mtp-sglang: - conc-list: [1, 8, 16, 32, 64, 128, 256] config_file: scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml +# AMD 4-node disaggregation with narrow-prefill EP8 + WIDE-decode EP16 (Oren's +# 2P1D config: two single-node prefill engines EP8 that the router fans across + +# one decode engine EP16 spanning 2 nodes; 4 nodes total). Runs on the `mi355x` +# amd-sglang cluster (bnxt RoCE), not spur: spur's ionic fabric could not cross- +# rail the MORI MoE all-to-all, so EP16 was brought up and validated on mi355x +# (job 13221, DSV4-Pro-FP4, GSM8K 0.927). Each recipe sets +# runtime.moe_a2a_backend=mori + runtime.kv_transfer_backend=mori + +# runtime.ib_devices=rdma0..7 + runtime.dist_socket_ifname=eno0; launch_mi355x.sh +# derives nodes-per-engine = ceil(TP/8) (prefill 8->1, decode 16->2) and emits the +# cross-node --nnodes/--node-rank/--dist-init-addr args for the decode engine. +# DSV4-Pro MTP drops conc256 (SWA retract->get_cpu_copy NotImplementedError). +dsv4flash-fp8-mi355x-ep16-sglang: + model: sgl-project/DeepSeek-V4-Flash-FP8 + model-prefix: dsv4flash + model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8 + runner: mi355x + precision: fp8 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml + +dsv4flash-fp8-mi355x-ep16-mtp-sglang: + model: sgl-project/DeepSeek-V4-Flash-FP8 + model-prefix: dsv4flash + model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8 + runner: mi355x + precision: fp8 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml + +dsv4pro-fp8-mi355x-ep16-sglang: + model: sgl-project/DeepSeek-V4-Pro-FP8 + model-prefix: dsv4pro + model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8 + runner: mi355x + precision: fp8 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml + +dsv4pro-fp8-mi355x-ep16-mtp-sglang: + model: sgl-project/DeepSeek-V4-Pro-FP8 + model-prefix: dsv4pro + model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8 + runner: mi355x + precision: fp8 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + # conc256 excluded: disagg-decode SWA hybrid pool retract->get_cpu_copy + # is an upstream NotImplementedError (crashes decode). See recipe. + - conc-list: [1, 8, 16, 32, 64, 128] + config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml + +dsv4flash-fp4-mi355x-ep16-sglang: + model: deepseek-ai/DeepSeek-V4-Flash + model-prefix: dsv4flash + model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml + +dsv4flash-fp4-mi355x-ep16-mtp-sglang: + model: deepseek-ai/DeepSeek-V4-Flash + model-prefix: dsv4flash + model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml + +dsv4pro-fp4-mi355x-ep16-sglang: + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4pro + model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml + +dsv4pro-fp4-mi355x-ep16-mtp-sglang: + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4pro + model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + # conc256 excluded: disagg-decode SWA hybrid pool retract->get_cpu_copy + # is an upstream NotImplementedError (crashes decode). See recipe. + - conc-list: [1, 8, 16, 32, 64, 128] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml + # Kimi-K2.6 MXFP4 wide-EP16 2P1D: aiter MoE path, needs only the wide-EP launcher, not #32048. kimik26-mxfp4-mi355x-ep16-sglang: model: amd/Kimi-K2.6-MXFP4 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml new file mode 100644 index 000000000..bdf327cc9 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml @@ -0,0 +1,107 @@ +# MI355X DeepSeek-V4-Flash FP4 4-node 2P1D disaggregation recipe — narrow-prefill EP8 + MTP +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, + # well under the ceiling and below every MTP leg's natural pool). + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + # MTP decode capture drives the MoE dispatch to cuda_graph_bs * draft_tokens + # tokens/rank, so the decode buffers must clear that; 512/2048/1024 validated + # on mi355x (undersized values abort during decode cuda-graph capture). + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 512 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 2048 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 1024 + +# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to +# both prefill and decode. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml new file mode 100644 index 000000000..89ee62d9a --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml @@ -0,0 +1,95 @@ +# MI355X DeepSeek-V4-Flash FP4 4-node 2P1D disaggregation recipe — narrow-prefill EP8 +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 64 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 332 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 128 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml new file mode 100644 index 000000000..7765bd3c4 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml @@ -0,0 +1,107 @@ +# MI355X DeepSeek-V4-Pro FP4 4-node 2P1D disaggregation recipe — narrow-prefill EP8 + MTP +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, + # well under the ceiling and below every MTP leg's natural pool). + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + # MTP decode capture drives the MoE dispatch to cuda_graph_bs * draft_tokens + # tokens/rank, so the decode buffers must clear that; 512/2048/1024 validated + # on mi355x (undersized values abort during decode cuda-graph capture). + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 512 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 2048 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 1024 + +# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to +# both prefill and decode. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml new file mode 100644 index 000000000..8b086b1d1 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml @@ -0,0 +1,95 @@ +# MI355X DeepSeek-V4-Pro FP4 4-node 2P1D disaggregation recipe — narrow-prefill EP8 +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 64 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 332 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 128 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml new file mode 100644 index 000000000..653c86eeb --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml @@ -0,0 +1,107 @@ +# MI355X DeepSeek-V4-Flash FP8 4-node 2P1D disaggregation recipe — narrow-prefill EP8 + MTP +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, + # well under the ceiling and below every MTP leg's natural pool). + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + # MTP decode capture drives the MoE dispatch to cuda_graph_bs * draft_tokens + # tokens/rank, so the decode buffers must clear that; 512/2048/1024 validated + # on mi355x (undersized values abort during decode cuda-graph capture). + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 512 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 2048 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 1024 + +# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to +# both prefill and decode. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml new file mode 100644 index 000000000..a71da7fc1 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml @@ -0,0 +1,95 @@ +# MI355X DeepSeek-V4-Flash FP8 4-node 2P1D disaggregation recipe — narrow-prefill EP8 +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 64 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 332 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 128 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml new file mode 100644 index 000000000..f5928008a --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml @@ -0,0 +1,107 @@ +# MI355X DeepSeek-V4-Pro FP8 4-node 2P1D disaggregation recipe — narrow-prefill EP8 + MTP +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.85 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, + # well under the ceiling and below every MTP leg's natural pool). + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + # MTP decode capture drives the MoE dispatch to cuda_graph_bs * draft_tokens + # tokens/rank, so the decode buffers must clear that; 512/2048/1024 validated + # on mi355x (undersized values abort during decode cuda-graph capture). + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 512 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 2048 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 1024 + +# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to +# both prefill and decode. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml new file mode 100644 index 000000000..c0591de6f --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml @@ -0,0 +1,95 @@ +# MI355X DeepSeek-V4-Pro FP8 4-node 2P1D disaggregation recipe — narrow-prefill EP8 +# + wide-decode EP16 (Oren's config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each; the router fans requests across both) + +# one decode engine (EP16) spanning 2 nodes. Still one logical P/D pair per role +# group, 4 nodes total. nodes-per-engine = ceil(TP/8): prefill 8->1, decode 16->2, +# so the launcher emits cross-node --nnodes/--node-rank/--dist-init-addr for the +# decode engine only. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI, known +# good); decode gets wide EP16 across nodes. Mismatched-TP KV (prefill TP8 -> +# decode TP16) is layout-compatible for DeepSeek MLA (KV is a replicated per-token +# latent, not head-sharded), and is carried over mori. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260715 + attention_backend: dsv4 + # RoCE HCAs (8/node) for mori MoE all-to-all AND the P->D KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + # Wide-EP MoE all-to-all backend (cross-node expert dispatch/combine). + moe_a2a_backend: mori + # KV P->D transfer backend (mori for both a2a and KV on this cluster). + kv_transfer_backend: mori + # Cross-node torch-distributed NIC for the wide decode engine's dist init. + dist_socket_ifname: eno0 + # rocm720 0715 image needs the ROCm-7.0.0-alpha path OFF (validated). + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + # Base defaults; the wide_ep block overrides mem-fraction / max-req per role. + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + # Per-role wide-EP tuning (validated 2P1D run, job 13196). Prefill and decode + # differ, so these split what the single-node EP<=8 recipes leave symmetric. + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + # Cap the decode KV pool: the ionic NIC rejects any single RDMA memory + # region over 4 GiB and mori registers each KV buffer as one region + # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the + # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 64 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 332 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 128 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91