From 9deb6952afa483e38f96385a375b96f463da5303 Mon Sep 17 00:00:00 2001 From: Zhaoyi Li <36555117+Lzy17@users.noreply.github.com> Date: Wed, 12 Aug 2026 05:53:14 -0500 Subject: [PATCH] [AMD][DI][CI] Add GLM-5.2 MXFP4 wide-EP16 2P1D nightly recipes (#34476) --- scripts/ci/slurm/nightly-configs.yaml | 32 +++++++ .../mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml | 90 +++++++++++++++++++ .../mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml | 82 +++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml create mode 100644 scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml diff --git a/scripts/ci/slurm/nightly-configs.yaml b/scripts/ci/slurm/nightly-configs.yaml index 068b1b1ec..829020635 100644 --- a/scripts/ci/slurm/nightly-configs.yaml +++ b/scripts/ci/slurm/nightly-configs.yaml @@ -604,3 +604,35 @@ glm52-fp4-mi355x-dp8ep8-mtp-sglang: search-space: - conc-list: [1, 8, 16, 32, 64, 128, 256] config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8-mtp.yaml + +glm52-fp4-mi355x-ep16-sglang: + model: amd/GLM-5.2-MXFP4 + model-prefix: glm52 + model_path: /it-share/model_coverage/models--amd--GLM-5.1-MXFP4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml + +glm52-fp4-mi355x-ep16-mtp-sglang: + model: amd/GLM-5.2-MXFP4 + model-prefix: glm52 + model_path: /it-share/model_coverage/models--amd--GLM-5.1-MXFP4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml new file mode 100644 index 000000000..25bc577d3 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml @@ -0,0 +1,90 @@ +# MI355X GLM-5.2 (MXFP4) 4-node 2P1D disaggregation recipe -- narrow-prefill EP8 + MTP +# + wide-decode EP16 (mirrors the DSV4-Pro / Kimi Oren config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each) + one decode engine (EP16) spanning 2 +# nodes, 4 nodes total. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI); decode +# gets wide EP16 across nodes over mori. KV (prefill TP8 -> decode TP16) over mori. +# +# GLM-specific bits vs the Kimi EP16 recipe: DSA attention is auto-selected for +# GlmMoeDsaForCausalLM (no explicit --attention-backend), GLM parsers (glm45), +# and --disable-shared-experts-fusion (GLM has a shared expert). + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +model: + env: + SGLANG_USE_AITER: 1 + server_args: + - --reasoning-parser + - glm45 + - --tool-call-parser + - glm45 + - --disable-shared-experts-fusion + +runtime: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260810 + # DSA attention auto-selected for GlmMoeDsaForCausalLM (no --attention-backend). + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + moe_a2a_backend: mori + kv_transfer_backend: mori + dist_socket_ifname: eno0 + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.85 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --dist-timeout 3600" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + decode_extra_flags: "--disable-cuda-graph --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 512 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 2048 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 1024 + +# MTP / EAGLE speculative decoding (built-in NextN head from the base model). +# Applied to both prefill and decode; no external draft checkpoint. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 + random_range_ratio: 1.0 + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml new file mode 100644 index 000000000..4d1759d4a --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml @@ -0,0 +1,82 @@ +# MI355X GLM-5.2 (MXFP4) 4-node 2P1D disaggregation recipe -- narrow-prefill EP8 +# + wide-decode EP16 (mirrors the DSV4-Pro / Kimi Oren config: wide EP only helps decode). +# +# Two prefill engines (EP8, one node each) + one decode engine (EP16) spanning 2 +# nodes, 4 nodes total. Prefill EP8 keeps MoE all-to-all INTRA-node (XGMI); decode +# gets wide EP16 across nodes over mori. KV (prefill TP8 -> decode TP16) over mori. +# +# GLM-specific bits vs the Kimi EP16 recipe: DSA attention is auto-selected for +# GlmMoeDsaForCausalLM (no explicit --attention-backend), GLM parsers (glm45), +# and --disable-shared-experts-fusion (GLM has a shared expert). + +resources: + prefill_workers: 2 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + +model: + env: + SGLANG_USE_AITER: 1 + server_args: + - --reasoning-parser + - glm45 + - --tool-call-parser + - glm45 + - --disable-shared-experts-fusion + +runtime: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260810 + # DSA attention auto-selected for GlmMoeDsaForCausalLM (no --attention-backend). + ib_devices: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + moe_a2a_backend: mori + kv_transfer_backend: mori + dist_socket_ifname: eno0 + rocm700a: 0 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 131072 + swa_full_tokens_ratio: 0.1 + wide_ep: + kv_cache_dtype: fp8_e4m3 + prefill_mem_fraction_static: 0.8 + decode_mem_fraction_static: 0.85 + prefill_chunked_prefill_size: 131072 + prefill_max_running_requests: 1024 + decode_max_running_requests: 1024 + common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --dist-timeout 3600" + prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" + decode_extra_flags: "--disable-cuda-graph --prefill-round-robin-balance" + prefill_extra_env: + MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 + MORI_MAX_DISPATCH_TOKENS_DECODE: 256 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 16384 + decode_extra_env: + MORI_MAX_DISPATCH_TOKENS_DECODE: 64 + MORI_MOE_MAX_INPUT_TOKENS_DECODE: 332 + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: 128 + +bench: + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 + random_range_ratio: 1.0 + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 + threshold: 0.91