diff --git a/scripts/ci/slurm/nightly-configs.yaml b/scripts/ci/slurm/nightly-configs.yaml index 585d10bb2..068b1b1ec 100644 --- a/scripts/ci/slurm/nightly-configs.yaml +++ b/scripts/ci/slurm/nightly-configs.yaml @@ -532,3 +532,75 @@ kimik26-mxfp4-mi355x-ep16-mtp-sglang: search-space: - conc-list: [1, 8, 16, 32, 64, 128, 256] config_file: scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mtp-mxfp4.yaml + +# AMD MI355X 2-node 1P1D disaggregation for GLM-5.2 (MXFP4) over MORI. GLM uses +# GlmMoeDsaForCausalLM (DeepSeek Sparse Attention, auto-selected) with a built-in +# NextN MTP head. Four variants mirroring the DeepSeek-V4 basic tier: +# * (base) : TP8, no MTP +# * -mtp : TP8 + NextN MTP +# * -dp8ep8 : DP-attention 8 + narrow within-node EP8 +# * -dp8ep8-mtp : DP8 + narrow EP8 + NextN MTP +# NOTE: checkpoint on disk is amd/GLM-5.1-MXFP4 (only GLM MXFP4 build mirrored). +glm52-fp4-mi355x-sglang: + model: amd/GLM-5.2-MXFP4 + model-prefix: glm52 + model_path: /it-share/model_coverage/models--amd--GLM-5.1-MXFP4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d.yaml + +glm52-fp4-mi355x-mtp-sglang: + model: amd/GLM-5.2-MXFP4 + model-prefix: glm52 + model_path: /it-share/model_coverage/models--amd--GLM-5.1-MXFP4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-mtp.yaml + +glm52-fp4-mi355x-dp8ep8-sglang: + model: amd/GLM-5.2-MXFP4 + model-prefix: glm52 + model_path: /it-share/model_coverage/models--amd--GLM-5.1-MXFP4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8.yaml + +glm52-fp4-mi355x-dp8ep8-mtp-sglang: + model: amd/GLM-5.2-MXFP4 + model-prefix: glm52 + model_path: /it-share/model_coverage/models--amd--GLM-5.1-MXFP4 + runner: mi355x + precision: fp4 + framework: sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [1, 8, 16, 32, 64, 128, 256] + config_file: scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8-mtp.yaml diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8-mtp.yaml new file mode 100644 index 000000000..5ff8b2efa --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8-mtp.yaml @@ -0,0 +1,80 @@ +# MI355X GLM-5.2 (MXFP4) 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP. +# +# DSA attention is auto-selected (see 1p1d.yaml). DP-attention 8 + within-node +# EP8 + built-in NextN MTP head (no external draft). +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 1 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + +# Model-specific docker env + sglang server args (generic launcher path; keeps +# GLM off the hardcoded DeepSeek-V4 parser branch in launch_mi355x.sh). DSA +# attention is auto-selected for GlmMoeDsaForCausalLM, so no --attention-backend +# is set. GLM has a shared expert (n_shared_experts=1); shared-experts-fusion is +# disabled to mirror the DeepSeek-V4 path. +model: + env: + SGLANG_USE_AITER: 1 + server_args: + - --reasoning-parser + - glm45 + - --tool-call-parser + - glm45 + - --disable-shared-experts-fusion + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260722 + # attention_backend intentionally unset: DSA is auto-selected for + # GlmMoeDsaForCausalLM. + # RoCE HCAs MORI uses for cross-node KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 8192 + swa_full_tokens_ratio: 0.1 + +# MTP / EAGLE speculative decoding (built-in NextN head from the base model). +# Applied to both prefill and decode. No draft_model_path: the NextN head lives +# in the base checkpoint. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8.yaml new file mode 100644 index 000000000..05bf89ad9 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-dp8ep8.yaml @@ -0,0 +1,71 @@ +# MI355X GLM-5.2 (MXFP4) 2-node 1P1D disaggregation recipe — DP8 + narrow EP8. +# +# DSA attention is auto-selected (see 1p1d.yaml). DP-attention 8 + within-node +# EP8, same topology as the DeepSeek-V4 dp8ep8 leg. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 1 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + decode: + tensor-parallel-size: 8 + expert-parallel-size: 8 + data-parallel-size: 8 + +# Model-specific docker env + sglang server args (generic launcher path; keeps +# GLM off the hardcoded DeepSeek-V4 parser branch in launch_mi355x.sh). DSA +# attention is auto-selected for GlmMoeDsaForCausalLM, so no --attention-backend +# is set. GLM has a shared expert (n_shared_experts=1); shared-experts-fusion is +# disabled to mirror the DeepSeek-V4 path. +model: + env: + SGLANG_USE_AITER: 1 + server_args: + - --reasoning-parser + - glm45 + - --tool-call-parser + - glm45 + - --disable-shared-experts-fusion + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260722 + # attention_backend intentionally unset: DSA is auto-selected for + # GlmMoeDsaForCausalLM. + # RoCE HCAs MORI uses for cross-node KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 8192 + swa_full_tokens_ratio: 0.1 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-mtp.yaml new file mode 100644 index 000000000..5526f1906 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d-mtp.yaml @@ -0,0 +1,81 @@ +# MI355X GLM-5.2 (MXFP4) 2-node 1P1D disaggregation recipe — TP8 + MTP. +# +# GLM-5.2 ships a built-in NextN MTP head (num_nextn_predict_layers=1), same +# mechanism as DeepSeek-V4 — no external draft checkpoint. DSA attention is +# auto-selected (see 1p1d.yaml). +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`. + +resources: + prefill_workers: 1 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 1 + data-parallel-size: 1 + decode: + tensor-parallel-size: 8 + expert-parallel-size: 1 + data-parallel-size: 1 + +# Model-specific docker env + sglang server args (generic launcher path; keeps +# GLM off the hardcoded DeepSeek-V4 parser branch in launch_mi355x.sh). DSA +# attention is auto-selected for GlmMoeDsaForCausalLM, so no --attention-backend +# is set. GLM has a shared expert (n_shared_experts=1); shared-experts-fusion is +# disabled to mirror the DeepSeek-V4 path. +model: + env: + SGLANG_USE_AITER: 1 + server_args: + - --reasoning-parser + - glm45 + - --tool-call-parser + - glm45 + - --disable-shared-experts-fusion + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260722 + # attention_backend intentionally unset: DSA is auto-selected for + # GlmMoeDsaForCausalLM. + # RoCE HCAs MORI uses for cross-node KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 8192 + swa_full_tokens_ratio: 0.1 + +# MTP / EAGLE speculative decoding (built-in NextN head from the base model). +# Applied to both prefill and decode. No draft_model_path: the NextN head lives +# in the base checkpoint. +mtp: + enabled: true + num_steps: 3 + eagle_topk: 1 + num_draft_tokens: 4 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep (full GSM8K, + # 8-shot, accuracy > 0.91). A regression here fails the nightly even when + # throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d.yaml new file mode 100644 index 000000000..af452b5d2 --- /dev/null +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/1p1d.yaml @@ -0,0 +1,78 @@ +# MI355X GLM-5.2 (MXFP4) 2-node 1P1D disaggregation recipe. +# +# GLM-5.2 uses GlmMoeDsaForCausalLM (DeepSeek Sparse Attention). sglang +# auto-selects the DSA attention backend for this architecture, so this recipe +# leaves `attention_backend` unset (empty) and lets the server pick DSA. MXFP4 +# enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION), +# same as the DeepSeek-V4 FP4 path. +# +# NOTE: the checkpoint on disk is amd/GLM-5.1-MXFP4 (only GLM MXFP4 build +# currently mirrored on /it-share). model_path in nightly-configs.yaml points at +# it; the recipe naming tracks the model line we are wiring CI for. +# +# Consumed by: +# * scripts/ci/slurm/process_result.py reads `resources` and +# `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table. +# * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`. + +resources: + prefill_workers: 1 + decode_workers: 1 + +backend: + sglang_config: + prefill: + tensor-parallel-size: 8 + expert-parallel-size: 1 + data-parallel-size: 1 + decode: + tensor-parallel-size: 8 + expert-parallel-size: 1 + data-parallel-size: 1 + +# Model-specific docker env + sglang server args (generic launcher path; keeps +# GLM off the hardcoded DeepSeek-V4 parser branch in launch_mi355x.sh). DSA +# attention is auto-selected for GlmMoeDsaForCausalLM, so no --attention-backend +# is set. GLM has a shared expert (n_shared_experts=1); shared-experts-fusion is +# disabled to mirror the DeepSeek-V4 path. +model: + env: + SGLANG_USE_AITER: 1 + server_args: + - --reasoning-parser + - glm45 + - --tool-call-parser + - glm45 + - --disable-shared-experts-fusion + +runtime: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260722 + # attention_backend intentionally unset: DSA is auto-selected for + # GlmMoeDsaForCausalLM. + # RoCE HCAs MORI uses for cross-node KV transfer. + ib_devices: rdma0,rdma1,rdma2,rdma3 + prefill_port: 30025 + decode_port: 30026 + prefill_bootstrap_port: 8998 + decode_bootstrap_port: 9001 + lb_port: 8000 + mem_fraction_static: 0.90 + page_size: 256 + max_running_requests: 256 + chunked_prefill_size: 8192 + swa_full_tokens_ratio: 0.1 + +bench: + # bench_serving --max-concurrency sweep; one result JSON per concurrency. + concurrencies: [1, 8, 16, 32, 64, 128, 256] + num_prompts_factor: 4 # num-prompts = concurrency * factor + random_range_ratio: 1.0 + + # Correctness gate run through the PD path before the perf sweep + # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly + # even when throughput looks fine ("fast but wrong"). + accuracy: + enabled: true + num_shots: 8 + num_questions: 1319 # full GSM8K test set + threshold: 0.91