diff --git a/.agents/skills/cookbook-add-model/references/authoring-reference.md b/.agents/skills/cookbook-add-model/references/authoring-reference.md index d98d2d4fed..1e86fcf09d 100644 --- a/.agents/skills/cookbook-add-model/references/authoring-reference.md +++ b/.agents/skills/cookbook-add-model/references/authoring-reference.md @@ -134,7 +134,7 @@ Each cell describes one verified (or auto-estimated) launch recipe. - DO NOT include `--host` / `--port` literally — use `{{HOST_IP}}` / `{{PORT}}` placeholders so users can override through the Env modal. - Order flags as: `--model-path` first (after any `--trust-remote-code`), - then parallelism (`--tp`, `--dp`, `--enable-dp-attention`), then MoE + then parallelism (`--tp`, `--attn-dp-size`), then MoE flags, then tuning knobs, with `--host` / `--port` last. The playground engine assumes this ordering when inserting overrides (its anchors target `--model-path` / `--tp` / etc., and inserts before the `--host` tail). @@ -171,7 +171,7 @@ schemas (full reference in the `_playground.jsx` header): | Axis key | Widget | Use when | |---|---|---| -| `attention` | TP / CP / DP-Attention sub-knobs (DP-Attention is a combined knob: its value is the DP degree AND toggles `--enable-dp-attention`) | Model exposes parallelism knobs in its cells (§2.2) and you want users to override them. | +| `attention` | TP / CP / DP-Attention sub-knobs (DP-Attention is a single knob: its value is the attention DP size, emitted as `--attn-dp-size N`) | Model exposes parallelism knobs in its cells (§2.2) and you want users to override them. | | `moe` | Backend select (incl. MegaMoE) + EP knob; picking the MegaMoE backend reveals a Quantization sub-select (W4A8/W4A4) | Model is MoE and supports multiple `--moe-*-backend` choices. For Blackwell MoE kernel-fusion, give the `megamoe` backend option a `requiresHw` (and optional `excludesStrategy`) gate, then add a sibling `megamoeQuant` block (`{stripEnv, options}`): W4A8 = `NUM_MAX` only, W4A4 adds the FP4-activations env vars; both strip the DeepEP dispatch env. | | `parsers` | Multi-toggle | Model has reasoning / tool-call parsers. | | `speculative` | Single-select chip group | Model has spec-decoding presets you want to expose. | @@ -208,7 +208,7 @@ schemas (full reference in the `_playground.jsx` header): knobs/values that are disabled under the live facts, so stale picks never emit a blocked combination. Interleave prefill-CP + DP-Attention is deliberately NOT grayed (combined support is planned upstream even though - current releases assert `dp_size == 1` for interleave) — the engine shows + current releases assert `attn_dp_size == 1` for interleave) — the engine shows a warning hint below the command box instead. - The CP knob emits `--attn-cp-size N --enable-prefill-cp --cp-strategy S`, where S is: an optional `{ id: "cpStrategy", values: [null, "interleave", diff --git a/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl b/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl index dfb5e537ad..83baf0a717 100644 --- a/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -176,7 +176,7 @@ sgl-eval run gsm8k \\ playgroundFeatures: { // ----- Card: "Attention Parallelism" ----- KEEP if the model exposes TP/CP/DP - // knobs. DP-Attention is a combined knob: value = DP degree AND toggles `--enable-dp-attention`. + // knobs. DP-Attention is a single knob: value = attention DP size, emitted as `--attn-dp-size N`. attention: { knobs: [ { id: "tp", label: "TP", values: [ @@ -357,8 +357,7 @@ sgl-eval run gsm8k \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -403,8 +402,7 @@ sgl-eval run gsm8k \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--host {{HOST_IP}}", "--port {{PORT}}", diff --git a/.agents/skills/cookbook-migrate-model/SKILL.md b/.agents/skills/cookbook-migrate-model/SKILL.md index ae9b55dad9..26c5314894 100644 --- a/.agents/skills/cookbook-migrate-model/SKILL.md +++ b/.agents/skills/cookbook-migrate-model/SKILL.md @@ -118,7 +118,7 @@ that **changes other parts of the command** (TP, mem) must (the Playground can't do coupled changes), and so does a toggle the legacy page itself labels with operating-point words — e.g. a `dpattention` radio whose options are subtitled "Low Latency" / "High Throughput" (GLM-5.1 / Kimi-K2.6 pattern) — -even when its flags are uncoupled (`--dp N --enable-dp-attention` is a pure +even when its flags are uncoupled (`--attn-dp-size N` is a pure flag add). Any other toggle that only adds/removes its own flags becomes a Playground axis with the flags baked into cells when the legacy default was ON — EXCEPT parsers: diff --git a/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md b/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md index 3e34ae7903..90d35ffc8c 100644 --- a/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md +++ b/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md @@ -12,7 +12,7 @@ this file is about the *mapping decisions*. | hardware radio | `match.hw` | Catalog ids as-is. Off-catalog hardware → `config.hardware` entry — e.g. A100 `{id:"a100", label:"A100", vram:"80GB", vendor:"nvidia"}` (merges into the NVIDIA row), Xeon `{id:"xeon", label:"Xeon", vram:"host RAM", vendor:"intel"}` (engine renders a new INTEL row; any vendor key works). A merged chip like GLM-5's "MI300X/MI325X" splits into two ids with duplicated cells (cells are denormalized by design). | | model-size / model-name radio | `variants` | One variant per deployable checkpoint family; single `{id:"default"}` when there's no variant axis (then `modelNames` keys drop the variant half). | | quantization radio | `quantizations` | Real precision ids (`bf16`/`fp8`/`fp4`/`int4`/…). One `fp4` id even when checkpoints differ per vendor — route via `hw\|variant\|quant` triple keys in `modelNames` (NVFP4 on Blackwell vs AMD MXFP4 is the precedent); per-hw greying falls out of which cells exist. | -| toggle that **couples** with other parts of the command (changes TP/mem/EP), OR one the legacy page labels with **operating-point words** | `strategies` | The Playground applies pure flag diffs — it cannot do coupled changes. Example: Qwen3.5's MTP toggle bumps TP on three H100 combos → strategies `low-latency` (MTP on) / `high-throughput` (MTP off). **Naming counts like coupling**: GLM-5.1's / Kimi-K2.6's `dpattention` adds only `--dp N --enable-dp-attention` (uncoupled), but its options are subtitled "Low Latency" / "High Throughput" — the page's own named operating-point split → strategies; a flag-only spec toggle riding alongside it stays a Playground axis and bakes per its legacy default. GPU-count radios (GLM-4.7, MiniMax-M2.5/2.7) → budget-tier strategies with the legacy SUPPORT matrix preserved by which cells exist. Strategy count follows the page's operating points: 1 → `balanced`, 2 → `low-latency`+`high-throughput`, 3 → the full trio (§4). | +| toggle that **couples** with other parts of the command (changes TP/mem/EP), OR one the legacy page labels with **operating-point words** | `strategies` | The Playground applies pure flag diffs — it cannot do coupled changes. Example: Qwen3.5's MTP toggle bumps TP on three H100 combos → strategies `low-latency` (MTP on) / `high-throughput` (MTP off). **Naming counts like coupling**: GLM-5.1's / Kimi-K2.6's `dpattention` adds only `--attn-dp-size N` (uncoupled), but its options are subtitled "Low Latency" / "High Throughput" — the page's own named operating-point split → strategies; a flag-only spec toggle riding alongside it stays a Playground axis and bakes per its legacy default. GPU-count radios (GLM-4.7, MiniMax-M2.5/2.7) → budget-tier strategies with the legacy SUPPORT matrix preserved by which cells exist. Strategy count follows the page's operating points: 1 → `balanced`, 2 → `low-latency`+`high-throughput`, 3 → the full trio (§4). | | toggle that only adds/removes its own flags | Playground axis (+ bake, EXCEPT parsers and accuracy-degrading flags) | **Parsers (`--reasoning-parser` / `--tool-call-parser`) are NEVER baked into cells** — Deployment commands ship without them regardless of the legacy default or the measured command; the `parsers` axis adds them on top (DSv4 convention; cells mirror the legacy generator's parsers-OFF output). Accuracy-degrading toggles are never baked either — §2 caveats (axis-only, accuracy-safe cells). Other flag-only toggles: legacy default ON → bake into cells AND declare the axis so users can strip (red strikethrough); default OFF → keep cells clean, axis preset only. MTP/EAGLE presets → `speculative` axis; dp-attention → a strategy when the legacy page labels it as the operating-point split or when coupled (see the row above), else `attention.dpAttn`. **EVERY legacy control survives as an interactive control** (a dimension or a Playground axis), never a tips-only mention — but a model-specific control is **config DATA, not engine code**: the axis handler reads options/flags/env/gating straight from `config.playgroundFeatures` (MegaMoE W4A4 is entirely DSv4 config data on the existing `moe` axis — no per-model engine edit). A control that fits an existing axis's data schema is therefore pure config, full stop. A **titled single-select that strips a flag family** (e.g. Nemotron3's "KV Cache DType" `--kv-cache-dtype`) is covered by the merged generic **`flagSelects`** axis → **config-only**: declare a `flagSelects` list of `{ id, title, stripPrefixes, options }` (see the Qwen3.5 mamba example), **no engine PR**. Only a control whose *shape* `flagSelects` still can't express would need a new ONE-TIME generic primitive (never a model-named handler) on a prior engine PR; the backward-compat reasoning (opt-in per key, not in the opt-out set) is in engine-axis.md. | | per-combo hidden option (e.g. spec hidden on Xeon) | absent cells | Don't create cells for combos the legacy widget couldn't produce; the engine greys them automatically. `# Error:` pseudo-commands → no cell + explanation in §2 tips and/or a chip `disable`/`disableReason`. | | coupled secondary knob (e.g. mamba cache V1/V2) | cells + Playground axis | Bake the correct value per cell following the legacy coupling (Qwen3.5: MTP ⇒ `--mamba-radix-cache-strategy extra_buffer` on NVIDIA; AMD/Xeon ⇒ V1/no flag) and document the coupling in §2 tips — AND surface the knob as a Playground axis like every other legacy feature (row above; add the axis when none fits). Baking alone is NOT enough — the every-feature rule supersedes the pilot's cells+prose-only treatment of Qwen3.5's mamba knob (retrofit pending). The mamba knob is the same single-select shape as KV Cache DType, so it rides the merged generic **`flagSelects`** axis — Qwen3.6 / Qwen3-Coder-Next declare it purely in config (a `flagSelects` block), **no engine PR**. | @@ -30,7 +30,7 @@ this file is about the *mapping decisions*. | (absent) | append `--host {{HOST_IP}}`, `--port {{PORT}}` to every cell | | `--nnodes N --node-rank … --dist-init-addr …` literals | delete; `match.nodes: "multi-N"` + `nodesOptions` entry — the engine injects the trio after the last parallelism anchor plus the multi-node header comment | | env-var command prefixes | verbatim into `cell.env[]` (never drop/normalize) | -| flag order as emitted | re-sort to canonical: `--trust-remote-code` → `--model-path` → parallelism (`--tp`/`--dp`/`--enable-dp-attention`/EP) → MoE → tuning → `--host`/`--port` (Playground insert anchors assume this). Keep the legacy relative order within the tuning span so commands stay eyeball-diffable. | +| flag order as emitted | re-sort to canonical: `--trust-remote-code` → `--model-path` → parallelism (`--tp`/`--attn-dp-size`/EP) → MoE → tuning → `--host`/`--port` (Playground insert anchors assume this). Keep the legacy relative order within the tuning span so commands stay eyeball-diffable. | Caveats discovered in the pilot: - The Playground `moe.ep` knob only understands `--ep` — normalize a legacy diff --git a/.agents/skills/sglang-runtime-context/SKILL.md b/.agents/skills/sglang-runtime-context/SKILL.md index 6733281bc2..ea02a3086c 100644 --- a/.agents/skills/sglang-runtime-context/SKILL.md +++ b/.agents/skills/sglang-runtime-context/SKILL.md @@ -227,7 +227,7 @@ slack (`>= len(...) - 1`) or compared key names instead of value sources. read-through over the canonical getters, so they answer with the live process groups. Everything else — `tp_size`, `pp_size`, `attn_cp_size`, `dcp_size`, `moe_dp_size` included, alongside config-only leaves like `nccl_port`, -`enable_dp_attention`, `dp_size`, `ep_size`, `dwdp_size` — is answered from the +`dp_size`, `ep_size`, `dwdp_size` — is answered from the published `parallel` bag. Reading a leaf before publish raises a `ValueError` naming the namespace; an unknown name is an `AttributeError`. diff --git a/benchmark/deepseek_v3/README.md b/benchmark/deepseek_v3/README.md index 14c67c8cb0..141e9d6036 100644 --- a/benchmark/deepseek_v3/README.md +++ b/benchmark/deepseek_v3/README.md @@ -47,7 +47,7 @@ Add [performance optimization options](#performance-optimization-options) as nee [MLA optimizations](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) are enabled by default. Here are some optional optimizations can be enabled as needed. -- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--enable-dp-attention` argument to boost throughput. +- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--attn-dp-size ` argument (e.g. equal to `--tp`) to boost throughput. - [Torch.compile Optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#torchcompile-latency-optimizations): Add `--enable-torch-compile` argument to enable it. This will take some time while server starts. The maximum batch size for torch.compile optimization can be controlled with `--torch-compile-max-bs`. It's recommended to set it between `1` and `8`. (e.g., `--torch-compile-max-bs 8`) ### Usage: Chat with DeepSeek diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx index a89618b870..2ee3ff173e 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx @@ -52,7 +52,7 @@ This section provides deployment configurations optimized for different hardware **DP Attention:** - Enable DP attention for high-throughput scenarios -- The `--dp` value commonly matches the `--tp` value +- The `--attn-dp-size` value commonly matches the `--tp` value - Trade-off: Higher throughput at the cost of slightly increased latency ## 4. Model Invocation diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx index 3aa56939e2..e80e8de413 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx @@ -192,8 +192,6 @@ sglang serve \ --enable-multimodal \ --tp 1 \ --ep 1 \ - --dp 1 \ - --enable-dp-attention \ --host 0.0.0.0 \ --port 30000 ``` diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx index ef8bc0caab..f999fa73be 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx @@ -184,8 +184,6 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-OCR \ --tp 1 \ --ep 1 \ - --dp 1 \ - --enable-dp-attention \ --dtype float16 \ --host 0.0.0.0 \ --port 8000 diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 436c7add6b..3b6b3583e0 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -95,7 +95,7 @@ DeepSeek-R1 shares the same MoE architecture as DeepSeek-V3, so the same hardwar python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-R1 --tp 8 --trust-remote-code ``` -**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. +**Data Parallelism Attention (`--attn-dp-size`):** Recommended for high-throughput scenarios. Use `--tp 8 --attn-dp-size 8` on a single 8-GPU node. **NCCL timeout:** If model loading is slow, increase: `--dist-timeout 3600`. diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index a69839f1e1..b0f22db90b 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -75,7 +75,7 @@ python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --tru ``` DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGLANG_ENABLE_JIT_DEEPGEMM=0`. -**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. +**Data Parallelism Attention (`--attn-dp-size`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--tp 8 --attn-dp-size 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. **NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. @@ -454,8 +454,7 @@ We use SGLang's built-in benchmarking tool to conduct performance evaluation on python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ @@ -529,8 +528,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ --ep 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --host 0.0.0.0 \ --port 8000 ``` diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 0dcad01acd..67e4bcdd94 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -73,7 +73,7 @@ python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3.1 --tp 8 --t ``` DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGLANG_ENABLE_JIT_DEEPGEMM=0`. -**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. +**Data Parallelism Attention (`--attn-dp-size`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--tp 8 --attn-dp-size 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. **NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index f609d2705a..6860fd1a0f 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -173,7 +173,7 @@ DeepSeek-V3.2 and DeepSeek-V3.2-Exp support tool calling capabilities. But they python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3.2-Speciale \ --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tp-size 8 --attn-dp-size 8 \ --reasoning-parser deepseek-v3 ``` @@ -357,8 +357,7 @@ SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE **With DP Attention:** ```bash Command -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 \ - --enable-dp-attention \ +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --attn-dp-size 8 \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 ``` @@ -394,8 +393,7 @@ python -m sglang.launch_server \ --host $LOCAL_IP \ --port $PORT \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --dist-init-addr ${HOST}:${DIST_PORT} \ --trust-remote-code \ --disaggregation-bootstrap-port 8998 \ @@ -411,8 +409,7 @@ python -m sglang.launch_server \ --host $LOCAL_IP \ --port $PORT \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --dist-init-addr ${HOST}:${DIST_PORT} \ --trust-remote-code \ --mem-fraction-static 0.9 @@ -435,7 +432,7 @@ For production deployments (RBG / LWS-based, DeepEP EP parallelism), see [multi_ Enable prefill context parallelism (CP) on CUDA with `--enable-prefill-cp --cp-strategy interleave` for long-sequence workloads. -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for DeepSeek V3.2, GLM-5, GLM-5.1, GLM-5.2, and GLM-5.3. Use `interleave` for these models and keep `--dp 1`; interleave DSA CP does not support `--dp` greater than 1. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for DeepSeek V3.2, GLM-5, GLM-5.1, GLM-5.2, and GLM-5.3. Use `interleave` for these models and keep `--attn-dp-size 1`; interleave DSA CP does not support `--attn-dp-size` greater than 1. **Interleave** (`--cp-strategy interleave`): Distributes tokens by `token_idx % cp_size`. It supports fused MoE, FP8 KV cache, and multi-batch prefill. @@ -462,7 +459,7 @@ export SGLANG_SET_CPU_AFFINITY=1 sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 16 \ - --dp 2 --enable-dp-attention --page-size 64 \ + --attn-dp-size 2 --page-size 64 \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ --context-length 65536 --max-running-requests 128 \ @@ -478,7 +475,7 @@ sglang_args=($sglang_args) sglang "${sglang_args[@]}" 2>&1 | tee $LOG_DIR/$RANK.log ``` -**CP + PP + EP + DP** +**CP + PP + EP** `CP` is currently enabled with `PP=2` on Hopper platform and we can reduce TP=16 to TP=8 from standalone deployment: @@ -486,7 +483,7 @@ sglang "${sglang_args[@]}" 2>&1 | tee $LOG_DIR/$RANK.log # verified on Hopper platform sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ - --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 8 --pp-size 2 --dp 1 --enable-dp-attention \ + --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 8 --pp-size 2 \ --moe-a2a-backend deepep --ep-size 16 \ --page-size 128 \ --chunked-prefill-size 16384 \ @@ -502,7 +499,7 @@ sglang_args=$(echo serve \ --context-length 65536 \ --allow-auto-truncate --enable-metrics \ --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 \ - --served-model-name DeepSeek-V3.2-dsa-pp-cp-ep-dp + --served-model-name DeepSeek-V3.2-dsa-pp-cp-ep ) sglang_args=($sglang_args) @@ -512,14 +509,10 @@ sglang "${sglang_args[@]}" 2>&1 | tee $LOG_DIR/$RANK.log **fp8 KV + CP + PP** -FP8 KV reduces the memory footprint. For DeepSeek V3.2, combine it with interleave CP and PP while keeping `--dp 1`: +FP8 KV reduces the memory footprint. For DeepSeek V3.2, combine it with interleave CP and PP, without attention DP: ```shell Command # verified in Hopper platform -dp_config=" \ - --dp 1 --enable-dp-attention \ -" - cp_config=" \ --enable-prefill-cp \ --cp-strategy interleave \ @@ -529,7 +522,6 @@ cp_config=" \ sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 8 --pp-size 2 --pp-async-batch-depth 1 \ - $dp_config \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ --context-length 65536 --max-running-requests 128 \ @@ -641,8 +633,7 @@ sglang serve \ --model-path deepseek-ai/DeepSeek-V3.2-Exp \ --tp 8 \ --ep 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --host 0.0.0.0 \ --port 30000 ``` @@ -875,7 +866,7 @@ export SGLANG_SET_CPU_AFFINITY=1 sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 16 \ - --dp 2 --enable-dp-attention --page-size 64 \ + --attn-dp-size 2 --page-size 64 \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ --context-length 65536 --max-running-requests 128 \ diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 4297e86b75..6702a46bc1 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -390,7 +390,7 @@ HiCache and MegaMoE are **not** supported on RTX PRO 6000. - **Model checkpoints** — for correct accuracy, the FP4 model uses the stock `deepseek-ai/DeepSeek-V4-{Flash,Pro}`, and the FP8 model uses the repackaged `sgl-project/DeepSeek-V4-{Flash,Pro}-FP8`. - **Supported models** — **MI300X** supports DeepSeek-V4-Flash in FP8; **MI355X** supports DeepSeek-V4-Flash / Pro in both FP4 and FP8. All recipes run single-node. -- **TP / DP setting (MI355X)** — both TP=4 and TP=8 are supported. At low concurrency we recommend **TP-only**; at high concurrency use **TP + DP** (balanced / high-throughput recipes). The verified MI355X DP recipes additionally set `--dp 8 --enable-dp-attention --enable-dp-attention-local-control-broadcast --tokenizer-worker-num 8 --stream-interval 20 --prefill-decode-interval 10`. +- **TP / DP setting (MI355X)** — both TP=4 and TP=8 are supported. At low concurrency we recommend **TP-only**; at high concurrency use **TP + DP** (balanced / high-throughput recipes). The verified MI355X DP recipes additionally set `--attn-dp-size 8 --enable-dp-attention-local-control-broadcast --tokenizer-worker-num 8 --stream-interval 20 --prefill-decode-interval 10`. - **MTP** — speculative decoding on the original Flash / Pro checkpoints; add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The 0813 Pro Official checkpoint keeps its MTP head too, so the same flags work there as a fallback to DSpark. - **DSpark (MI355X Pro Official 0813)** — the 0813 checkpoint bundles the DSpark draft head. Prefer it over EAGLE on 0813 for ordinary serving: enable `--speculative-algorithm DSPARK` (low-latency) and see the [MI355X agentic recipe](#3-7-agentic-long-context-with-hicache-dram-offload-mi355x-fp4-dspark) for the long-context path. DSpark also runs under **PD disaggregation** — see the Pro Official pairs in [§3.8](#3-8-pd-disaggregation-on-mi355x-mori-io). - **Kernels** — uses the Unified KV attention and the flydsl MoE. @@ -408,7 +408,7 @@ export SGLANG_MORI_RECV_BOUND=1 sglang serve \ --model-path {MODEL_PATH} \ - --tp 8 --dp 8 --enable-dp-attention \ + --tp 8 --attn-dp-size 8 \ --ep-size 8 --moe-a2a-backend mori --deepep-mode normal \ ``` @@ -769,8 +769,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --ep-size 8 \ --moe-a2a-backend megamoe \ @@ -792,7 +791,7 @@ python3 -m sglang.launch_server \ --hicache-mem-layout page_first_direct ``` -`--chunked-prefill-size` is a global budget divided by `--dp`, so this keeps 6144 tokens per rank. +`--chunked-prefill-size` is a global budget divided by `--attn-dp-size`, so this keeps 6144 tokens per rank. **Disaggregated 1P1D (DEP8 prefill / DEP8 decode), concurrency 64–128.** Prefill, on one 8-GPU node: ```bash Command @@ -804,8 +803,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --ep-size 8 \ --moe-dense-tp-size 1 \ @@ -841,8 +839,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --ep-size 8 \ --moe-dense-tp-size 1 \ @@ -941,8 +938,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --enable-prefill-delayer \ --enable-dp-attention-local-control-broadcast \ @@ -971,7 +967,7 @@ python3 -m sglang.launch_server \ --watchdog-timeout 3600 ``` -`--chunked-prefill-size` is a global budget divided by `--dp`, so this keeps 8192 tokens per rank. `--enable-dp-lm-head` is required for DSpark under DP attention. Front the DP ranks with `sglang_router --policy cache_aware` so multi-turn sessions land on the rank holding the longest radix/HiCache prefix; above concurrency 160 add `--balance-abs-threshold 32`. +`--chunked-prefill-size` is a global budget divided by `--attn-dp-size`, so this keeps 8192 tokens per rank. `--enable-dp-lm-head` is required for DSpark under DP attention. Front the DP ranks with `sglang_router --policy cache_aware` so multi-turn sessions land on the rank holding the longest radix/HiCache prefix; above concurrency 160 add `--balance-abs-threshold 32`. **DP8 EP8 MegaMoE, concurrency 128–384:** ```bash Command @@ -996,9 +992,8 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 8 \ - --enable-dp-attention \ --enable-dp-lm-head \ --enable-prefill-delayer \ --enable-dp-attention-local-control-broadcast \ diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx index f8a3308efa..c069d84c9b 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -101,7 +101,7 @@ import { GLM51Deployment } from '/src/snippets/autoregressive/glm-51-deployment. - **AMD GPUs**: BF16 and FP8 checkpoints run on MI300X/MI325X/MI355X at tp=8. On MI355X (gfx950), the MXFP4 checkpoint `amd/GLM-5.1-MXFP4` is also supported at tp=4 with `--kv-cache-dtype fp8_e4m3`. All AMD paths pass `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang`, `--chunked-prefill-size 131072`, and `--watchdog-timeout 1200` (20 minutes for weight loading). FP8 uses approximately half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). EAGLE speculative decoding is supported on AMD GPUs: MI300X/MI325X (gfx942) and MI355X (gfx950), but it **requires `--disable-custom-all-reduce`** — the aiter custom all-reduce kernel deadlocks during EAGLE verify at high concurrency, so without this flag the server will hang. - For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5.1 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. -- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.1. Use `--enable-prefill-cp --cp-strategy interleave` with `--dp 1`. +- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.1. Use `--enable-prefill-cp --cp-strategy interleave` with `--attn-dp-size 1`. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to enable the [IndexCache](https://github.com/THUDM/IndexCache) method for GLM-5.1. This can improve serving efficiency with only a small accuracy loss. If you are running rigorous accuracy evaluations, do not enable this feature. diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index eed0bfc2e6..0388ea2bc4 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -255,7 +255,7 @@ For the full setup (streaming, tool-use, count_tokens, persisting env in `~/.cla ### 3.5 Context Parallelism -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.2. For prefill CP on CUDA, use `interleave` with `--dp 1` as shown below. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.2. For prefill CP on CUDA, use `interleave` with `--attn-dp-size 1` as shown below. Prefill context parallelism can help with reduction of TTFT under long context. To enable prefill context parallelism for GLM 5.2, please append the following arguments: diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx index bd6b6760b9..38dfe435eb 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx @@ -243,7 +243,7 @@ For the full setup (streaming, tool-use, count_tokens, persisting env in `~/.cla ### 3.5 Context Parallelism -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.3. For prefill CP on CUDA, use `interleave` with `--dp 1` as shown below. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.3. For prefill CP on CUDA, use `interleave` with `--attn-dp-size 1` as shown below. Prefill context parallelism can help with reduction of TTFT under long context. To enable prefill context parallelism for GLM 5.3, please append the following arguments: diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.mdx index 791ebb4189..9c480d5522 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.mdx @@ -94,7 +94,7 @@ import { GLM5Deployment } from '/src/snippets/autoregressive/glm-5-deployment.js - **AMD GPUs**: Use `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang` for the DSA attention backend. Add `--chunked-prefill-size 131072` and `--watchdog-timeout 1200` (20 minutes for weight loading). EAGLE speculative decoding is not currently supported on AMD for GLM-5. - For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. -- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5. Use `--enable-prefill-cp --cp-strategy interleave` with `--dp 1`. +- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5. Use `--enable-prefill-cp --cp-strategy interleave` with `--attn-dp-size 1`. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` for GLM-5-FP8 if you want to enable the [IndexCache](https://github.com/THUDM/IndexCache) method. This feature is supported through [this PR](https://github.com/sgl-project/sglang/pull/21405) and introduces only a small accuracy loss. However, if you are running rigorous accuracy evaluations, it is not recommended to enable this feature. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx index ecf6f37be6..2eddfd6f77 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx @@ -40,7 +40,7 @@ import { KimiK25Deployment } from '/src/snippets/autoregressive/kimi-k25-deploym - **Memory**: Requires GPUs with >=140GB each. Supported platforms: H200 (8x, TP=8), B300 (8x, TP=8), GB300 (4x, TP=4), MI300X/MI325X (4x, TP=4), MI350X/MI355X (4x, TP=4). Use `--context-length 128000` to conserve memory. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be <= 4 (not 8). Kimi-K2.5 has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. The ROCm 7.2 images (`rocm720`) have an AITER compatibility issue. -- **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. +- **DP Attention**: Enable with `--attn-dp-size ` for production throughput. A common choice is to set `--attn-dp-size` equal to `--tp`, but this is not required. - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx index 696859a101..df3aba605c 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx @@ -94,7 +94,7 @@ import { KimiK26Deployment } from '/src/snippets/autoregressive/kimi-k26-deploym - **NVFP4**: Use `nvidia/Kimi-K2.6-NVFP4` with `--quantization modelopt_fp4` on NVIDIA Blackwell. Use `tp=8` on B300 and `tp=4` on GB300. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be ≤ 4 (not 8). Kimi-K2.6 has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. -- **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. +- **DP Attention**: Enable with `--attn-dp-size ` for production throughput. A common choice is to set `--attn-dp-size` equal to `--tp`, but this is not required. - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. - **AMD FP8 KV Cache**: On AMD platforms the generator adds `--kv-cache-dtype fp8_e4m3` by default and sets `--mem-fraction-static 0.8` to fit the INT4 weights plus KV cache. FP8 KV cache trades a small amount of accuracy for memory; omit the flag if you observe accuracy regressions on your workload. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx index 6d2b93eac7..4c1841b619 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx @@ -94,7 +94,7 @@ import { KimiK27CodeDeployment } from '/src/snippets/autoregressive/kimi-k27-cod - **Transformers Version**: The model card requires `transformers>=4.57.1,<5.0.0`. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be ≤ 4 (not 8). Kimi-K2.7-Code has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. -- **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. +- **DP Attention**: Enable with `--attn-dp-size ` for production throughput. A common choice is to set `--attn-dp-size` equal to `--tp`, but this is not required. - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. - **AMD FP8 KV Cache**: On AMD platforms the generator adds `--kv-cache-dtype fp8_e4m3` by default and sets `--mem-fraction-static 0.8` to fit the INT4 weights plus KV cache. FP8 KV cache trades a small amount of accuracy for memory; omit the flag if you observe accuracy regressions on your workload. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx index af58a3e854..767b283c5e 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx @@ -36,7 +36,7 @@ This section provides a progressive guide from quick deployment to performance o - **Memory**: Requires 8 GPUs with ≥140GB each (H200/B200). Use `--context-length 128000` to conserve memory. - **Expert Parallelism (EP)**: Use `--ep` for better MoE throughput. See [EP docs](../../../docs/advanced_features/expert_parallelism). -- **Data Parallel (DP)**: Enable with `--dp 4 --enable-dp-attention` for production throughput. +- **Data Parallel (DP)**: Enable with `--attn-dp-size 4` for production throughput. - **KV Cache**: Use `--kv-cache-dtype fp8_e4m3` to reduce memory by 50% (CUDA 11.8+). - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` for Kimi-K2-Thinking to separate thinking and content. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. @@ -359,8 +359,7 @@ We use SGLang's built-in benchmarking tool to conduct performance evaluation on python3 -m sglang.launch_server \ --model-path moonshotai/Kimi-K2-Instruct \ --tp 8 \ - --dp 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --trust-remote-code \ --host 0.0.0.0 \ --port 8000 @@ -427,9 +426,8 @@ Max ITL (ms): 19.94 python3 -m sglang.launch_server \ --model-path moonshotai/Kimi-K2-Instruct \ --tp 8 \ - --dp 4 \ + --attn-dp-size 4 \ --ep 4 \ - --enable-dp-attention \ --trust-remote-code \ --host 0.0.0.0 \ --port 8000 diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 6dd8da8b4f..00aa1a1723 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -437,7 +437,7 @@ Two presets come out of this, at `N = 8k` GPUs: | **Peak Capacity (+DCP8)** — `dp = k` + `--dcp-size 8` | Deduplicates the attention-TP group's MLA KV: concurrency ceiling +72% at the same engine throughput, ~1.8× ITL. | Context ≥ ~16K, or per-replica concurrency past 128. | - **Radix cache** is independent of the preset: for prefix-free traffic (offline batch, evals) switch it off (Playground's **Prefix Cache** card) — one state slot per request instead of 4–5. -- The fully data-parallel extreme (`--dp-size` = GPU count, attention-TP 1) — the shape behind the 64-GPU sweep's ~3K tok/s per GPU — is not a preset: 288 GB GPUs only, radix forced off, no head-to-head against the preset shape. +- The fully data-parallel extreme (`--attn-dp-size` = GPU count, attention-TP 1) — the shape behind the 64-GPU sweep's ~3K tok/s per GPU — is not a preset: 288 GB GPUs only, radix forced off, no head-to-head against the preset shape. The Peak Throughput preset at 32 GPUs on B200/B300 (4 nodes × 8; every node runs the same command with its own `--node-rank`). On GB200/GB300 the same 32-GPU shape uses 8 nodes × 4, and the Playground emits `--nnodes 8`: @@ -447,7 +447,7 @@ sglang serve \ --trust-remote-code \ --model-path moonshotai/Kimi-K3 \ --tp-size 32 --ep-size 32 \ - --enable-dp-attention --dp-size 4 --enable-dp-lm-head \ + --attn-dp-size 4 --enable-dp-lm-head \ --nnodes 4 --node-rank --dist-init-addr :20000 \ --moe-a2a-backend megamoe --moe-runner-backend deep_gemm \ --kv-cache-dtype fp8_e4m3 \ @@ -460,7 +460,7 @@ sglang serve \ Scale by holding the per-replica shape fixed and moving only the replica count; pool sizing rides the calculator-driven `--mamba-full-memory-ratio`, which folds in DP, DCP, precision, and speculation: -| GPUs | B200/B300 nodes | GB200/GB300 nodes | `--tp-size` / `--ep-size` | `--dp-size` | +| GPUs | B200/B300 nodes | GB200/GB300 nodes | `--tp-size` / `--ep-size` | `--attn-dp-size` | |---|---|---|---|---| | 16 | 2×8 | 4×4 | 16 | 2 | | 32 | 4×8 | 8×4 | 32 | 4 | diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx index 68af9b0ac8..55a2912027 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx @@ -86,7 +86,7 @@ The generator only emits a runnable command for combinations that NVIDIA / SGLan - **DP attention**: - By default the attention layers are tensor-parallel (sharded across all TP ranks). Enabling DP attention (the toggle above, or `--dp --enable-dp-attention`) instead runs attention as `N` data-parallel groups: each DP rank serves its own slice of the requests with its own KV cache. `--dp` must divide `--tp`. + By default the attention layers are tensor-parallel (sharded across all TP ranks). Enabling DP attention (the toggle above, or `--attn-dp-size `) instead runs attention as `N` data-parallel groups: each DP rank serves its own slice of the requests with its own KV cache. `--attn-dp-size` must divide `--tp`. - **Expert parallel (EP)**: diff --git a/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx b/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx index eeddd0d7cb..62d9eef229 100644 --- a/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx +++ b/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx @@ -84,7 +84,7 @@ import { LagunaXS2Deployment } from '/src/snippets/autoregressive/laguna-xs2-dep - **Quantization**: NVFP4 requires Blackwell (B200 / B300); BF16 and FP8 run on either H200 or B200. FP8's first launch triggers a multi-session DeepGEMM JIT pre-compile (~10-20 min); pre-warm with `python3 -m sglang.compile_deep_gemm --model poolside/Laguna-XS.2-FP8` to avoid that cost on every restart. - **Reasoning parser** (`--reasoning-parser poolside_v1`): Splits `...` segments into `reasoning_content` so `content` holds only the final answer. Disable only if you want the raw `` tags in `content`. - **Tool call parser** (`--tool-call-parser poolside_v1`): Required for OpenAI-compatible tool-call streaming. Disable only for chat-only deployments. -- **DP attention**: For higher-throughput deployments, enable the DP-Attention toggle — it emits `--dp --enable-dp-attention` with `--dp` matching `--tp` (tune independently if needed). +- **DP attention**: For higher-throughput deployments, enable the DP-Attention toggle — it emits `--attn-dp-size ` with `--attn-dp-size` matching `--tp` (tune independently if needed). - **Thinking default**: Thinking is **off by default** at the model level. Opt in per request with `extra_body={"chat_template_kwargs": {"enable_thinking": True}}`. ## 4. Model Invocation diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx index 9a0d8974da..a743f7ba3e 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx @@ -51,7 +51,7 @@ This section provides deployment configurations verified on AMD MI300X, MI325X, **NVIDIA (B200/GB200):** * **GB200 Parallelism**: Use `--tp 4 --ep 4` on GB200. B200 uses the default NVIDIA settings generated above. * **NVFP4 Quantization**: Requires `--quantization modelopt_fp4` and uses a different model path (`nvidia/Qwen3-Coder-...`). -* **DP Attention**: NVFP4 configuration supports `--enable-dp-attention` for improved throughput. +* **DP Attention**: NVFP4 configuration supports DP attention (`--attn-dp-size `) for improved throughput. **Intel Xeon CPU:** * Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index 53067a9a0f..dffd8cda29 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -76,12 +76,12 @@ import { MiMoV25Deployment } from '/src/snippets/autoregressive/mimo-v25-deploym - EAGLE speculative decoding (3 steps, topk=1) typically yields a 2–3× decode speedup. Requires `--enable-multi-layer-eagle` (both Hopper and Blackwell). See §5.4 for acceptance-rate behavior on natural text vs random prompts. **MiMo-V2.5 (310B):** -- The checkpoint has a TP=4-interleaved fused `qkv_proj`; attention-TP per DP group **must** be 4. Use `--dp = TP / 4`; for TP > 4 this also requires DP-attention. Total GPUs must be a multiple of 4. A bare `--tp 8` without `--dp 2` will fail to load with `MiMoV2 fused qkv_proj checkpoint is TP=4-interleaved; got attention tp_size=8`. -- Single-node deployments: H100/H200 8× GPUs (`--tp 8 --dp 2`), B200 4× GPUs (`--tp 4`, dp=1, no DP-attn flag needed), GB300 4× GPUs (`--tp 4`, single NVL4 node). FP8 quantization. +- The checkpoint has a TP=4-interleaved fused `qkv_proj`; attention-TP per DP group **must** be 4. Use `--attn-dp-size = TP / 4`; for TP > 4 this means DP-attention is on. Total GPUs must be a multiple of 4. A bare `--tp 8` without `--attn-dp-size 2` will fail to load with `MiMoV2 fused qkv_proj checkpoint is TP=4-interleaved; got attention tp_size=8`. +- Single-node deployments: H100/H200 8× GPUs (`--tp 8 --attn-dp-size 2`), B200 4× GPUs (`--tp 4`, attention DP size 1, no DP-attn flag needed), GB300 4× GPUs (`--tp 4`, single NVL4 node). FP8 quantization. - On Blackwell, pass `--attention-backend fa4`: MiMoV2's asymmetric KV (`head_dim` 192 / `v_head_dim` 128) fails on the SM100 default `trtllm_mha`, which requires equal K/V widths. - On Blackwell, pass `--mm-attention-backend fa4` for the V2.5 vision encoder. The checkpoint config requests FlashAttention-3 internally, but SGLang rejects FA3 on Blackwell and expects FA4 for multimodal attention. - On Blackwell, pass `--moe-runner-backend flashinfer_trtllm`; the default `auto` falls through to the triton fused-MoE runner, ~12% slower at bs=1 decode. -- `--enable-dp-lm-head` and `--mm-enable-dp-encoder` are required whenever `--enable-dp-attention` is on, to keep LM head and encoder sharding consistent. +- `--enable-dp-lm-head` and `--mm-enable-dp-encoder` are required whenever `--attn-dp-size` is greater than 1, to keep LM head and encoder sharding consistent. - EAGLE MTP uses the checkpoint's MTP weights. Enable with `--speculative-algorithm EAGLE` and `--enable-multi-layer-eagle` (both Hopper and Blackwell). - **Multimodal**: Supports image, video, and audio understanding; see Section 4.3 for invocation examples. @@ -547,7 +547,7 @@ Max ITL (ms): 1600.00 - Hardware: NVIDIA H200 GPU (8×) - Model: `XiaomiMiMo/MiMo-V2.5` (FP8) -- Tensor Parallelism: 8 (DP-attention with `--dp 2`) +- Tensor Parallelism: 8 (DP-attention with `--attn-dp-size 2`) - Recipe: Balanced (DP-attn + EAGLE MTP) - sglang version: `0.0.0.dev1+g7d99af439` (`lmsysorg/sglang:dev-mimo-v2.5`) @@ -777,7 +777,7 @@ Reference numbers from the [day0 enablement PR](https://github.com/sgl-project/s - Hardware: NVIDIA Hopper GPU (2 nodes × 8 GPUs, GPU SKU intentionally not disclosed) - Model: `XiaomiMiMo/MiMo-V2.5-Pro` (FP8) -- Parallelism: `--tp 16 --dp 2 --ep 16 --moe-dense-tp-size 1 --enable-dp-attention` +- Parallelism: `--tp 16 --attn-dp-size 2 --ep 16 --moe-dense-tp-size 1` - Recipe: Hopper EP16 (DeepEP + EAGLE multi-layer MTP) #### 5.5.1 Long-Context Prefill Throughput diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx index 475f269eaf..2ae5966dde 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx @@ -113,7 +113,7 @@ The shared prefill settings are `--chunked-prefill-size 49152 --max-prefill-toke The base recipe uses `--speculative-algorithm DFLASH --speculative-num-draft-tokens 8`. Supply the DFlash checkpoint paired with the selected target model; the draft path is independently editable and does not have to be a subdirectory of the target checkpoint. -At the pinned commit, DFlash on NVIDIA GPUs requires PP=1 and rejects DP-attention. Do not add `--enable-dp-attention` or copy EAGLE-specific settings from the V2.5 cookbook. +At the pinned commit, DFlash on NVIDIA GPUs requires PP=1 and rejects DP-attention. Do not set `--attn-dp-size` above 1 or copy EAGLE-specific settings from the V2.5 cookbook. ## 3. Reasoning and tool calling diff --git a/docs/cookbook/base/reference/server_arguments.mdx b/docs/cookbook/base/reference/server_arguments.mdx index 737a1b35f4..d08442308b 100644 --- a/docs/cookbook/base/reference/server_arguments.mdx +++ b/docs/cookbook/base/reference/server_arguments.mdx @@ -37,10 +37,15 @@ This guide explains the parallelism configuration fields used in SGLang model co `--ep-size`, `--expert-parallel-size`, `--ep` Expert Parallelism - distributes MoE experts + + `attn_dp_size` + `--attn-dp-size`, `--attention-data-parallel-size` + Attention Data Parallelism - data-parallel attention groups inside the TP group, TP for FFN (hybrid) + `enable_dp_attention` `--enable-dp-attention` - DP for attention, TP for FFN (hybrid) + Deprecated. Use --attn-dp-size <dp-size> instead of --dp-size <dp-size> --enable-dp-attention. diff --git a/docs/docs/advanced_features/dcp.mdx b/docs/docs/advanced_features/dcp.mdx index 262b69c4c2..d88bcb588e 100644 --- a/docs/docs/advanced_features/dcp.mdx +++ b/docs/docs/advanced_features/dcp.mdx @@ -17,7 +17,7 @@ On Kimi K3, DCP applies only to MLA layers. Request-indexed KDA state is unchang | `--dcp-size N` | Enables DCP and widens virtual capacity and page size by `N`. Alias: `--decode-context-parallel-size`. | | `--dcp-comm-backend` | Selects the MLA partial-output merge: `ag_rs`, `a2a`, or `fi_a2a`. Unset resolves to `fi_a2a` where the DCP group shares an MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm. | | `--dcp-replicate-q-proj` | Removes the query all-gather for supported MLA weights. Use `--no-dcp-replicate-q-proj` to disable a model-specific default. | -| `--enable-dp-attention` | Composes only when DCP groups nest inside attention TP. | +| `--attn-dp-size N` | Composes only when DCP groups nest inside attention TP. | A DCP group must fit inside one attention-TP group and one attention-DP replica: @@ -26,7 +26,7 @@ attn_tp_size = tp_size / attn_dp_size attn_tp_size % dcp_size == 0 ``` -For example, `TP=64, DP=4, DCP=16` creates four valid 16-rank attention replicas. `DCP=32` would cross replica boundaries and is invalid. +For example, `TP=64, attention DP=4, DCP=16` creates four valid 16-rank attention replicas. `DCP=32` would cross replica boundaries and is invalid. Startup currently checks only `tp_size % dcp_size == 0`, not the stronger condition above. Until that check is fixed, DPA+DCP deployments must enforce containment when choosing the topology. @@ -52,8 +52,7 @@ sglang serve \ --model-path moonshotai/Kimi-K3 \ --tp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ - --dp-size 4 \ + --attn-dp-size 4 \ --dcp-size 8 \ --host 0.0.0.0 \ --port 30000 diff --git a/docs/docs/advanced_features/dp_dpa_smg_guide.mdx b/docs/docs/advanced_features/dp_dpa_smg_guide.mdx index 6fc3fc5fe1..896b533d76 100644 --- a/docs/docs/advanced_features/dp_dpa_smg_guide.mdx +++ b/docs/docs/advanced_features/dp_dpa_smg_guide.mdx @@ -80,14 +80,13 @@ For MoE models like DeepSeek, DPA is **often** paired with Expert Parallelism (E python -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ - --dp-size 8 \ + --attn-dp-size 8 \ --ep 8 \ - --enable-dp-attention \ --moe-a2a-backend deepep \ --moe-runner-backend deep_gemm ``` -> **Note**: `--dp-size` must be explicitly set when using `--enable-dp-attention`. If `dp_size` is 1 (default), DPA will be disabled. +> **Note**: `--attn-dp-size` sets the number of attention data-parallel groups inside the TP group. If `attn_dp_size` is 1 (default), DPA is disabled. For detailed EP configuration (DeepEP, Two-Batch Overlap, EPLB), see [Expert Parallelism](/docs/advanced_features/expert_parallelism). @@ -106,21 +105,28 @@ DPA supports the following model architectures: For models like Llama, with standard GQA, standard DP, or TP is typically recommended. -To enable DPA, add `--enable-dp-attention` to your server launch command. +To enable DPA, add `--attn-dp-size N` (N > 1) to your server launch command. ### Activation Logic -DPA is enabled explicitly via server arguments (CLI or config). You must set both `--dp-size` and `--enable-dp-attention`: +DPA is enabled explicitly via server arguments (CLI or config) with `--attn-dp-size` (alias `--attention-data-parallel-size`). It creates N attention data-parallel groups inside the TP group, one scheduler each, while the FFN stays tensor parallel: ```bash python -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ - --dp-size 8 \ - --enable-dp-attention + --attn-dp-size 8 ``` -**Important**: `--dp-size` must be greater than 1 for DPA to work. When `dp_size == 1` (default), `--enable-dp-attention` is automatically disabled. The constraint `tp_size % dp_size == 0` must also be satisfied. +**Important**: `--attn-dp-size` must be greater than 1 for DPA to work. When `attn_dp_size == 1` (default), DPA is disabled. The constraint `tp_size % attn_dp_size == 0` must also be satisfied. `--dp-size` only sets the number of data-parallel replicas of the model; combining replicas (`--dp-size` > 1) with DPA is not supported. + +> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead. + +**Elastic EP scale joiners.** A rank that joins an existing DPA deployment through elastic EP scale-up runs DPA even when it is launched with `--attn-dp-size 1`: its own attention DP group is one rank wide. + +**DP rank count.** The number of DP ranks (one scheduler each) is `dp_size × attn_dp_size`; one of the two is always 1. +- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Its deprecated `enable_dp_attention` field is always `false`; read `attn_dp_size` instead. +- Wire formats that predate `--attn-dp-size` keep the key `dp_size` for the DP rank count: KV-event descriptors, the PD prefill server info, and load snapshots. ### TP LMHead with all-to-all @@ -135,8 +141,7 @@ python -m sglang.launch_server \ --model-path MODEL_PATH \ --disaggregation-mode decode \ --tp 8 \ - --dp-size 8 \ - --enable-dp-attention + --attn-dp-size 8 ``` Prefill / colocated workers will fallback to TP LMHead with allgather, as well as other parallel strategies. More details can be found in [PR #32313](https://github.com/sgl-project/sglang/pull/32313). @@ -505,7 +510,7 @@ For detailed metrics and monitoring setup, see [SGLang Model Gateway Documentati Cache-aware routing, high availability - DPA (--dp-size N --enable-dp-attention) + DPA (--attn-dp-size N) DeepSeek/MLA models Eliminates KV cache duplication, improved throughput @@ -518,7 +523,7 @@ For detailed metrics and monitoring setup, see [SGLang Model Gateway Documentati **Recommended production setup for DeepSeek:** -1. Enable **DPA** for attention layers (`--dp-size 8 --enable-dp-attention`) +1. Enable **DPA** for attention layers (`--attn-dp-size 8`) 2. Enable **EP** for MoE layers (`--ep 8 --moe-a2a-backend deepep`) 3. Use **SMG** with **cache_aware** policy diff --git a/docs/docs/advanced_features/expert_parallelism.mdx b/docs/docs/advanced_features/expert_parallelism.mdx index 26e38cf017..077884fd9f 100644 --- a/docs/docs/advanced_features/expert_parallelism.mdx +++ b/docs/docs/advanced_features/expert_parallelism.mdx @@ -72,7 +72,7 @@ SGLang's EP integrates diverse, highly efficient backends for different use case DeepEP and Mooncake backends support two modes for token dispatch: `normal` mode (optimized for prefill workloads with high throughput) and `low_latency` mode (optimized for decode workloads with low latency and CUDA Graph compatibility). MORI backend only supports `normal` mode now. NIXL-EP and PPLX currently operate in low-latency mode with CUDA Graph support (PPLX reuses the DeepEP low-latency masked expert-compute path). Users are recommended to set `--deepep-mode auto` to enable automatic dispatch mode switching during runtime. Setting `--deepep-mode normal` or `--deepep-mode low_latency` is useful for debugging or development purposes. -Currently, DeepEP, Mooncake, NIXL-EP, `ascend_fuseep`, `pplx` and MORI only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. Note that `pplx` additionally requires `--enable-dp-attention` with at least 2 DP groups (i.e., `tp_size / attention_tp_size > 1`); otherwise pplx-kernels' AllToAll cannot be constructed. +Currently, DeepEP, Mooncake, NIXL-EP, `ascend_fuseep`, `pplx` and MORI only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. Note that `pplx` additionally requires DP attention with at least 2 attention DP groups (i.e., `--attn-dp-size` > 1, so `tp_size / attention_tp_size > 1`); otherwise pplx-kernels' AllToAll cannot be constructed. ### Backends for MoE Computation diff --git a/docs/docs/advanced_features/hisparse_guide.mdx b/docs/docs/advanced_features/hisparse_guide.mdx index 54c412bafd..2fe612f211 100644 --- a/docs/docs/advanced_features/hisparse_guide.mdx +++ b/docs/docs/advanced_features/hisparse_guide.mdx @@ -137,7 +137,7 @@ python3 -m sglang.launch_server \ --port 8000 --host 0.0.0.0 \ --context-length 81920 \ --chunked-prefill-size 65536 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tp-size 8 --attn-dp-size 8 \ --mem-fraction-static 0.85 \ --disaggregation-mode prefill \ --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ @@ -152,7 +152,7 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --port 8000 --host 0.0.0.0 \ --context-length 81920 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tp-size 8 --attn-dp-size 8 \ --mem-fraction-static 0.85 \ --disable-radix-cache \ --disaggregation-mode decode \ diff --git a/docs/docs/advanced_features/hyperparameter_tuning.mdx b/docs/docs/advanced_features/hyperparameter_tuning.mdx index 327e2efce9..db3952a872 100644 --- a/docs/docs/advanced_features/hyperparameter_tuning.mdx +++ b/docs/docs/advanced_features/hyperparameter_tuning.mdx @@ -78,5 +78,5 @@ Data parallelism is better for throughput. When there is enough GPU memory, alwa - `torch.compile` accelerates small models on small batch sizes. You can enable it with `--enable-torch-compile`. - Try other quantization (e.g. FP8 quantization with `--quantization fp8`) -- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--enable-dp-attention --dp-size 8`). +- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--attn-dp-size 8`). - If the workload has many shared prefixes, try `--schedule-policy lpm`. Here, `lpm` stands for longest prefix match. It reorders requests to encourage more cache hits but introduces more scheduling overhead. diff --git a/docs/docs/advanced_features/lora.mdx b/docs/docs/advanced_features/lora.mdx index 3d318ab3b4..2cfcdc0818 100644 --- a/docs/docs/advanced_features/lora.mdx +++ b/docs/docs/advanced_features/lora.mdx @@ -422,8 +422,7 @@ The following NVIDIA CUDA example uses four attention-DP ranks: python3 -m sglang.launch_server \ --model-path MODEL_PATH \ --tp 4 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --enable-lora \ --lora-backend triton \ --lora-paths '{"lora_name":"ADAPTER_NAME","lora_path":"ADAPTER_PATH","pinned":true}' @@ -431,7 +430,7 @@ python3 -m sglang.launch_server \ Keep these constraints in mind: -- `--dp-size` must equal `--tp-size`, so every attention DP rank owns its whole token batch. The server rejects other layouts at startup. +- `--attn-dp-size` must equal `--tp-size`, so every attention DP rank owns its whole token batch. The server rejects other layouts at startup. - All adapters must be GPU-pinned (`pinned: true`), including dynamically loaded adapters. You can pin at most `--max-loras-per-batch - 1` adapters; the remaining slot is reserved for the base model. - Decode CUDA graphs are supported. LoRA prefill runs eagerly when DPA is enabled. - `--max-loras-per-batch` limits the union of adapters active across all DP ranks in a synchronized forward, not the adapters on one rank alone. diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 7fd878bc94..5ce40a82e4 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -132,8 +132,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # prefill 1 @@ -148,8 +147,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # decode 0 @@ -164,8 +162,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -181,8 +178,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -348,8 +344,7 @@ python -m sglang.launch_server \ --disaggregation-mode decode \ --port 30001 \ --tp 4 \ - --dp 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --trust-remote-code \ --disaggregation-ib-device mlx5_3,mlx5_4 @@ -413,8 +408,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # prefill 1 @@ -429,8 +423,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # decode 0 @@ -445,8 +438,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -462,8 +454,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -706,7 +697,7 @@ do --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ --chunked-prefill-size -1 --max-prefill-tokens 10240 --moe-a2a-backend deepep --deepep-mode normal \ --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 + --attn-dp-size 8 --disable-shared-experts-fusion --dtype bfloat16 NODE_RANK=$i break fi @@ -730,9 +721,9 @@ do export HCCL_SOCKET_IFNAME=data0.3001 export GLOO_SOCKET_IFNAME=data0.3001 python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ + --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --attn-dp-size 16 \ --mem-fraction-static 0.8 --max-running-requests 448 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ + --moe-a2a-backend deepep --deepep-mode low_latency --enable-dp-lm-head \ --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ --load-balance-method round_robin diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 15f0543c4b..8f4a52b5e0 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1220,7 +1220,13 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--data-parallel-size`
`--dp-size` - The data parallelism size. + The number of data-parallel replicas of the model. + `1` + Type: int + + + `--attention-data-parallel-size`
`--attn-dp-size` + The attention data parallelism size: the number of data-parallel attention groups inside the TP group, one scheduler each, while the FFN stays tensor parallel. `1` Type: int @@ -2564,8 +2570,8 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--enable-dp-attention` - Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported. - `False` + Deprecated. Use --attn-dp-size <dp-size> instead of --dp-size <dp-size> --enable-dp-attention. + False bool flag (set to enable) diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx index 727d091bf6..c3dff04502 100644 --- a/docs/docs/advanced_features/speculative_decoding.mdx +++ b/docs/docs/advanced_features/speculative_decoding.mdx @@ -100,21 +100,21 @@ SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, DFlash draft model (linear block verification) Yes --speculative-algorithm DFLASH + --speculative-draft-model-path ... - No --enable-dp-attention; pp_size == 1; disables overlap scheduler & mixed chunked prefill + No DP attention (--attn-dp-size > 1); pp_size == 1; disables overlap scheduler & mixed chunked prefill STANDALONE Smaller draft LLM (token-level) Yes --speculative-algorithm STANDALONE + --speculative-draft-model-path ... - Does not support --enable-dp-attention + Does not support DP attention (--attn-dp-size > 1) NGRAM Ngram cache from previous tokens No --speculative-algorithm NGRAM - CUDA-only; no --enable-dp-attention; disables overlap scheduler & mixed chunked prefill + CUDA-only; no DP attention (--attn-dp-size > 1); disables overlap scheduler & mixed chunked prefill @@ -678,7 +678,7 @@ Relevant parameters: -> **Note:** Standalone speculative decoding currently **does not support** `--enable-dp-attention`. +> **Note:** Standalone speculative decoding currently **does not support** DP attention (`--attn-dp-size` > 1). ```bash Command python3 -m sglang.launch_server \ @@ -815,7 +815,7 @@ Enable it with: Notes: - Ngram speculative decoding **only supports CUDA**. -- It currently **does not support** `--enable-dp-attention`. +- It currently **does not support** DP attention (`--attn-dp-size` > 1). - It disables the overlap scheduler and mixed chunked prefill. - If `--speculative-ngram-max-bfs-breadth > 1` (thus `speculative_eagle_topk > 1`) and `page_size > 1`, use `--attention-backend flashinfer`; otherwise the server will error. - Optional: set `SGLANG_NGRAM_FORCE_GREEDY_VERIFY=True` to force greedy verification. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx index 6c2a8bb081..1767d1fbd0 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx @@ -125,8 +125,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --reasoning-parser deepseek-r1 \ @@ -160,12 +159,11 @@ do --port 8001 \ --nnodes 1 \ --tp-size 16 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.9 \ --max-running-requests 416 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 \ @@ -302,8 +300,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 3000 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -430,8 +427,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -468,12 +464,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -636,8 +631,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -675,12 +669,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 32 \ + --attn-dp-size 32 \ --mem-fraction-static 0.82 \ --max-running-requests 1024 \ --quantization modelslim \ --moe-a2a-backend ascend_fuseep \ - --enable-dp-attention \ --deepep-mode low_latency \ --moe-dense-tp 1 \ --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 \ @@ -838,8 +831,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -876,12 +868,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1040,8 +1031,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -1078,12 +1068,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1241,8 +1230,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -1279,12 +1267,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 8 \ + --attn-dp-size 8 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx index eedc6777bd..b8a4cdb65a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx @@ -159,10 +159,9 @@ do --node-rank $i \ --nnodes 2 \ --tp 32 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 32 \ --moe-dense-tp-size 1 \ - --enable-dp-attention \ --enable-dp-lm-head \ --watchdog-timeout 9000 \ --mem-fraction-static 0.79 \ @@ -356,10 +355,9 @@ do --node-rank $i \ --nnodes 2 \ --tp 32 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 32 \ --moe-dense-tp-size 1 \ - --enable-dp-attention \ --enable-dp-lm-head \ --watchdog-timeout 9000 \ --mem-fraction-static 0.79 \ @@ -552,10 +550,9 @@ do --node-rank $i \ --nnodes 2 \ --tp 32 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 32 \ --moe-dense-tp-size 1 \ - --enable-dp-attention \ --enable-dp-lm-head \ --watchdog-timeout 9000 \ --mem-fraction-static 0.79 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx index 6e8ff2e502..390945be62 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx @@ -131,8 +131,7 @@ do --max-prefill-tokens 70000 \ --chunked-prefill-size -1 \ --max-running-requests 112 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode normal \ --quantization modelslim \ @@ -177,8 +176,7 @@ do --chunked-prefill-size 32768 \ --disaggregation-transfer-backend ascend \ --max-running-requests 896 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ @@ -319,8 +317,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.7 \ --chunked-prefill-size 131072 \ --max-running-requests 64 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ @@ -444,8 +441,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.7 \ --chunked-prefill-size 131072 \ --max-running-requests 160 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx index 276fa9080a..16a12899ed 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx @@ -105,8 +105,7 @@ do --attention-backend ascend \ --device npu \ --tp-size 32 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --chunked-prefill-size 65536 \ --max-prefill-tokens 280000 \ --trust-remote-code \ @@ -294,9 +293,8 @@ do --trust-remote-code \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.85 \ --max-running-requests 32 \ --attention-backend ascend \ @@ -454,8 +452,7 @@ do --disable-shared-experts-fusion \ --disable-cuda-graph \ --dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --load-balance-method round_robin \ --enable-prefill-cp \ --cp-strategy zigzag \ @@ -492,9 +489,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.87 \ --max-running-requests 96 \ --attention-backend ascend \ @@ -702,8 +698,7 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ - --enable-dp-attention \ + --attn-dp-size 32 \ --ep-size 32 \ --mem-fraction-static 0.85 \ --max-running-requests 32 \ @@ -903,8 +898,7 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ - --enable-dp-attention \ + --attn-dp-size 32 \ --ep-size 32 \ --mem-fraction-static 0.85 \ --max-running-requests 32 \ @@ -1062,8 +1056,7 @@ do --disable-shared-experts-fusion \ --disable-cuda-graph \ --dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --load-balance-method round_robin \ --enable-prefill-cp \ --cp-strategy zigzag \ @@ -1100,9 +1093,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.87 \ --max-running-requests 96 \ --attention-backend ascend \ @@ -1308,9 +1300,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.865 \ --max-running-requests 96 \ --attention-backend ascend \ @@ -1516,9 +1507,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.85 \ --max-running-requests 320 \ --attention-backend ascend \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx index f74bfc7454..1d3ceb13e7 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx @@ -159,8 +159,7 @@ do --port 8001 \ --trust-remote-code \ --tp-size 16 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --ep-size 16 \ --mem-fraction-static 0.895 \ --max-running-requests 128 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx index 5419b1792a..9356e08d79 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx @@ -119,8 +119,7 @@ do --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 32 \ + --attn-dp-size 32 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 \ @@ -295,8 +294,6 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 2 \ - --enable-dp-attention \ - --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -493,8 +490,6 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 2 \ - --enable-dp-attention \ - --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -690,8 +685,6 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 16 \ - --enable-dp-attention \ - --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -883,8 +876,7 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 2 \ - --enable-dp-attention \ - --dp-size 2 \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -1019,8 +1011,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 4 8 9 10 11 \ @@ -1129,8 +1120,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 3 4 \ @@ -1244,8 +1234,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 3 4 5 \ @@ -1357,8 +1346,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 4 8 11 12 13 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx index c18f549457..7204747970 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx @@ -82,7 +82,7 @@ python3 -m sglang.launch_server \ --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ --disaggregation-bootstrap-port 8996 \ --disable-piecewise-cuda-graph \ - --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --attn-dp-size 2 --enable-dp-lm-head \ --moe-a2a-backend deepep --deepep-mode normal ``` @@ -110,7 +110,7 @@ python3 -m sglang.launch_server \ --disaggregation-mode decode --disaggregation-transfer-backend ascend \ --disaggregation-bootstrap-port 8996 \ --moe-a2a-backend deepep --deepep-mode low_latency \ - --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --attn-dp-size 2 --enable-dp-lm-head \ --speculative-algorithm DFLASH \ --speculative-num-draft-tokens 8 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx index 8599e2c4f6..377e538991 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx @@ -166,8 +166,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency @@ -350,8 +349,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency @@ -534,8 +532,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency @@ -718,8 +715,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx index 4fdf7a210e..e63a327eac 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx @@ -312,8 +312,7 @@ python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ --host 127.0.0.1 --port 6688 \ --tp-size 16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --mem-fraction-static 0.65 \ --max-running-requests 4 \ --reasoning-parser minimax-append-think \ @@ -422,8 +421,7 @@ python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ --host 127.0.0.1 --port 6688 \ --tp-size 16 \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.53 \ --max-running-requests 96 \ --disable-radix-cache \ @@ -527,8 +525,7 @@ python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ --host 127.0.0.1 --port 6688 \ --tp-size 16 \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 320 \ --disable-radix-cache \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx index 6c3a3f49a1..d877d34ddd 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx @@ -204,8 +204,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ --tp 16 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --enable-dp-lm-head \ --mem-fraction-static 0.8 \ --cuda-graph-bs-decode 1 2 4 8 16 20 24 26 27 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx index d18fb53893..6a821b99ae 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx @@ -105,8 +105,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 8 \ --tp-size 2 \ - --enable-dp-attention \ - --dp-size 2 \ + --attn-dp-size 2 \ --mem-fraction-static 0.85 \ --cuda-graph-bs-decode 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \ --dtype bfloat16 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx index 767fd2f5a9..3848c65480 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx @@ -455,8 +455,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -572,8 +571,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -688,8 +686,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -804,8 +801,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -921,8 +917,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -1038,8 +1033,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -1149,8 +1143,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx index f0f6e8e68a..f875fe1181 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx @@ -228,8 +228,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode auto \ @@ -337,8 +336,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 28672 \ --context-length 81920 \ --max-total-tokens 122304 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx index bffa9c5330..4f4000d7eb 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx @@ -31,7 +31,7 @@ recommended to use v0.5.16 or a later version. | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | -| DP Attention | `--enable-dp-attention` | +| DP Attention | `--attn-dp-size 16` | The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment @@ -218,7 +218,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 80000 \ --chunked-prefill-size 131072 \ --max-running-requests 160 \ - --dp-size 8 --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep --deepep-mode auto \ --quantization modelslim --enable-dp-lm-head \ --kv-cache-dtype bfloat16 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx index 4a18b869c2..7fa82659bf 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx @@ -384,7 +384,7 @@ do --disaggregation-transfer-backend ascend --max-running-requests 64 \ --served-model-name glm-5 --chunked-prefill-size 524288 --max-prefill-tokens 180000 --moe-a2a-backend deepep --deepep-mode normal \ --disable-shared-experts-fusion --disable-cuda-graph --dtype bfloat16 \ - --dp-size 4 --enable-dp-attention \ + --attn-dp-size 4 \ --load-balance-method round_robin \ --enable-dp-lm-head --moe-dense-tp 1 \ --speculative-draft-model-quantization unquant \ @@ -418,9 +418,9 @@ do export SGLANG_NPU_USE_MULTI_STREAM=1 python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8003 --trust-remote-code --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --ep-size 16 \ + --port 8003 --trust-remote-code --nnodes 1 --node-rank $i --tp-size 16 --attn-dp-size 16 --ep-size 16 \ --mem-fraction-static 0.8 --max-running-requests 128 --attention-backend ascend --device npu --quantization modelslim \ - --served-model-name glm-5 --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency \ + --served-model-name glm-5 --moe-a2a-backend deepep --deepep-mode low_latency \ --cuda-graph-max-bs-decode 4 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx index a497850d55..11f47b94cd 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx @@ -30,7 +30,7 @@ v0.5.16 or a later version. | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--enable-multi-layer-eagle` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0` | | DP LM Head | `--enable-dp-lm-head` | -| DP Attention | `--enable-dp-attention` | +| DP Attention | `--attn-dp-size 2` | The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx index 4c398e08c5..dfed0ada17 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx @@ -24,7 +24,7 @@ source. | Feature | Example usage | |-------------------------------|-----------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 64` | -| Data Parallelism | `--enable-dp-attention \`
`--dp-size 4` | +| DP Attention | `--attn-dp-size 4` | | Expert Parallelism | `--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 8192` | @@ -136,7 +136,7 @@ quay.io/ascend/sglang:main-cann9.0.0-a3 Multi-node deployment distributes the model across multiple Ascend A3 Series nodes using tensor parallelism while keeping prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single node can provide. The validated configuration deploys the W4A8 quantized checkpoint on 4 nodes with `--tp-size 64` -(16 dies per node), DP attention (`--dp-size 4`), and DeepEP in `auto` mode. +(16 dies per node), DP attention (`--attn-dp-size 4`), and DeepEP in `auto` mode. Modify the IP addresses of the four nodes, then run the same script on all four nodes. Each node determines its own rank by matching the local IP address against `IPS`. @@ -211,7 +211,7 @@ do --quantization modelslim \ --dtype bfloat16 \ --tp-size 64 \ - --enable-dp-attention --dp-size 4 --enable-dp-lm-head \ + --attn-dp-size 4 --enable-dp-lm-head \ --mem-fraction-static 0.8 \ --chunked-prefill-size 8192 \ --cuda-graph-bs-decode 16 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx b/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx index 2b5c45fa05..0e2ab1f178 100644 --- a/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx @@ -218,8 +218,8 @@ These arguments and environment variables are critical for tuning decode perform - `--dp-size` / `--data-parallel-size` - Sets the data parallelism degree for the decode server. With DP attention enabled, attention layers are sharded across DP ranks while FFN/MoE layers use tensor parallelism. Higher values create more independent decode instances, increasing throughput through parallel request processing. Choose a value that divides evenly into your total card count, with remaining cards used for TP/EP. + `--attn-dp-size` / `--attention-data-parallel-size` + Sets the attention DP size for the decode server: the number of data-parallel attention groups inside the TP group. This is a key optimization for decode throughput — attention is DP-sharded to reduce KV cache duplication, while FFN/MoE layers remain TP-sharded to leverage expert parallelism. Best suited for MoE models where attention is not the compute bottleneck. Higher values create more independent decode instances, increasing throughput through parallel request processing. Choose a value that divides the TP size evenly. `8` @@ -227,11 +227,6 @@ These arguments and environment variables are critical for tuning decode perform Sets the expert parallelism degree. For MoE models, this distributes experts across cards, reducing per-card expert loading overhead and enabling all-to-all dispatch. The code default is `1`; set explicitly for MoE models. The optimal value depends on your model's expert count and architecture. DeepSeek V3.2 with 256 routed experts uses `ep=32`. For models with fewer experts, use a proportionally smaller value. `32` - - `--enable-dp-attention` - Enables data parallelism for attention layers while keeping tensor parallelism for FFN/MoE layers. This is a key optimization for decode throughput — attention is DP-sharded to reduce KV cache duplication, while MoE layers remain TP-sharded to leverage expert parallelism. Best suited for MoE models where attention is not the compute bottleneck. - Enabled - `--enable-dp-lm-head` Enables vocabulary parallelism across the DP attention group, sharding the LM head weight across ranks. Each rank only computes logits for its vocabulary shard, avoiding a costly all-gather of logits across the DP group. This is essential when DP attention is enabled to maintain throughput. @@ -727,9 +722,9 @@ The symbols used are defined as follows: Splits model weights across multiple NPU devices so that large models can be loaded and run cooperatively. Each device holds a shard of every weight tensor and communicates via HCCL all-reduce. This is the primary mechanism for deploying models that exceed single-device memory. -### Data Parallelism (`--dp-size`, `--enable-dp-attention`) +### Data Parallelism (`--dp-size`, `--attn-dp-size`) -Replicates the model across independent device groups to increase throughput by processing multiple requests in parallel. With --enable-dp-attention, the attention layers are replicated across DP ranks while the FFN/MoE layers remain tensor-parallel/expert-parallel. This reduces the communication overhead of attention layers in TP (which is significant during small-batch decode) while keeping the heavy compute and memory demands of MoE layers distributed. +`--dp-size` replicates the model across independent device groups to increase throughput by processing multiple requests in parallel. `--attn-dp-size` instead runs attention data parallel inside the TP group: the attention layers are replicated across attention DP ranks while the FFN/MoE layers remain tensor-parallel/expert-parallel. This reduces the communication overhead of attention layers in TP (which is significant during small-batch decode) while keeping the heavy compute and memory demands of MoE layers distributed. Model replicas (`--dp-size` > 1) cannot be combined with attention DP. ### Expert Parallelism (`--ep-size`) @@ -779,7 +774,7 @@ On Ascend, this is primarily controlled via environment variable SGLANG_ENABLE_O ### DP LM Head (`--enable-dp-lm-head`) -Shards the language model head (the final vocabulary projection layer) across DP ranks to reduce memory consumption and improve decode throughput. Without this, each DP rank holds a full copy of the LM head, which is wasteful for large vocabularies. Typically used together with `--enable-dp-attention`. +Shards the language model head (the final vocabulary projection layer) across DP ranks to reduce memory consumption and improve decode throughput. Without this, each DP rank holds a full copy of the LM head, which is wasteful for large vocabularies. Typically used together with `--attn-dp-size`. ### MLAPO (`SGLANG_NPU_USE_MLAPO=1`) diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx index d66366ba92..e2586903fe 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx @@ -1080,6 +1080,12 @@ click [Server Arguments](../../../advanced_features/server_arguments). Type: int A2/A3 Series + + `--attention-data-parallel-size`
`--attn-dp-size` + `1` + Type: int + A2/A3 Series + `--load-balance-method` `auto` @@ -2190,12 +2196,6 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD bool flag
(set to enable) A2/A3 Series - - `--enable-dp-attention` - `False` - bool flag
(set to enable) - A2/A3 Series - `--enable-dp-attention-local-control-broadcast` `False` diff --git a/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx b/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx index 380b7c5454..5d4135dcce 100644 --- a/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx +++ b/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx @@ -63,8 +63,7 @@ spec: # - --deepep-config # - /home/aiges/tuned/tuned_8sms.json - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size + - --attn-dp-size - "16" - --disable-radix-cache - --moe-a2a-backend @@ -210,8 +209,7 @@ spec: # - --deepep-config # - /home/aiges/tuned/tuned_8sms.json - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size + - --attn-dp-size - "16" - --disable-radix-cache - --moe-a2a-backend @@ -370,9 +368,8 @@ spec: - "262144" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head - - --dp-size + - --attn-dp-size - "16" - --moe-a2a-backend - deepep @@ -503,10 +500,9 @@ spec: - "262144" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head #- --enable-two-batch-overlap - - --dp-size + - --attn-dp-size - "16" - --moe-a2a-backend - deepep diff --git a/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx b/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx index c0fa4228e9..86dad0f357 100644 --- a/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx +++ b/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx @@ -69,8 +69,7 @@ spec: - --eplb-algorithm - deepseek - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size + - --attn-dp-size - "8" - --moe-a2a-backend - deepep @@ -267,9 +266,8 @@ spec: - "29" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head - - --dp-size + - --attn-dp-size - "8" - --moe-a2a-backend - deepep @@ -338,9 +336,8 @@ spec: - "29" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head - - --dp-size + - --attn-dp-size - "32" - --moe-a2a-backend - "deepep" diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index a70ed97ba9..842a335194 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -810,7 +810,8 @@ export const Deployment = ({ config, benchmarks }) => { // Insert the multi-node trio after the last parallelism flag, // falling back to right after --model-path. const PARALLELISM_ANCHORS = new Set([ - "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", + "--attn-dp-size", "--attention-data-parallel-size", + "--dp-size", "--dp", "--tp-size", "--tp", "--sp-degree", "--ulysses-degree", "--ring-degree", ]); let i = flags.reduce( diff --git a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx index 97a0b2baf9..6be4e81f0d 100644 --- a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx +++ b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx @@ -67,7 +67,7 @@ export const KimiK3MambaRatioCalculator = () => { const hasFlag = (name) => flags.some((f) => f.split(/[\s=]/)[0] === name); const tp = Number(flagArg("--tp-size")) || 8; - const dp = hasFlag("--enable-dp-attention") ? Number(flagArg("--dp-size")) || 1 : 1; + const dp = Number(flagArg("--attn-dp-size")) || 1; // KDA state and the replicated MLA KV both live per attention-TP group. // PP needs no term: it splits the layers of both pools equally. const attnTp = Math.max(1, Math.round(tp / dp)); @@ -231,9 +231,9 @@ export const KimiK3MambaRatioCalculator = () => { : `DSPARK (D = ${block + 1})`; const derivedChips = [ // With DP attention on, show the whole topology so a large-scale preset is - // visibly understood: total GPUs = DP replicas x attention-TP group width. + // visibly understood: total GPUs = attention DP size x attention-TP group width. dp > 1 - ? `${tp} GPUs = DP ${dp} × Attention TP ${attnTp}` + ? `${tp} GPUs = Attention DP ${dp} × Attention TP ${attnTp}` : `Attention TP ${attnTp}`, `DCP ${dcp}`, `KV ${kvDtype === "fp8_e4m3" ? "FP8" : "BF16"}`, diff --git a/docs/src/snippets/_playground.jsx b/docs/src/snippets/_playground.jsx index d8188aa95e..6dbe918198 100644 --- a/docs/src/snippets/_playground.jsx +++ b/docs/src/snippets/_playground.jsx @@ -360,6 +360,7 @@ export const Playground = ({ config }) => { const TP_HEADS = ["--tp-size", "--tp", "--tensor-parallel-size"]; const EP_HEADS = ["--ep-size", "--ep", "--expert-parallel-size"]; const DP_HEADS = ["--dp-size", "--dp", "--data-parallel-size"]; + const ATTN_DP_HEADS = ["--attn-dp-size", "--attention-data-parallel-size"]; const parseIntFlagAny = (flags, heads) => { for (const head of heads) { const n = parseIntFlag(flags, head); @@ -370,22 +371,25 @@ export const Playground = ({ config }) => { const flagSpelling = (flags, heads, fallback) => heads.find((head) => (flags || []).some((f) => f.split(/[\s=]/)[0] === head)) || fallback; + // DP-Attention is on when the attention DP size is above 1 (1 = off). + const dpAttnOnIn = (flags) => (parseIntFlagAny(flags, ATTN_DP_HEADS) ?? 1) > 1; // Insertion-anchor sets (priority-ordered; each includes siblings so // insertion still works in partial cells). const ANCHOR_NEAR_MODEL_PATH = ["--model-path"]; const ANCHOR_NEAR_TP = ["--tp-size", "--tp", "--model-path"]; const ANCHOR_NEAR_DP = ["--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; - const ANCHOR_NEAR_DPATTN = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; + const ANCHOR_NEAR_DPATTN = [...ATTN_DP_HEADS, "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; const ANCHOR_NEAR_MOE = ["--moe-a2a-backend", "--moe-runner-backend", - "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; + ...ATTN_DP_HEADS, "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; // Helper bundle passed to every axis handler. const helpers = { matchConstraint, evaluateChip, findEntry, isHidden, stripFlagsByFirstToken, stripEnvByPrefix, insertBeforeTail, insertAfter, parseIntFlag, hasFlag, findFlagArg, - TP_HEADS, EP_HEADS, DP_HEADS, parseIntFlagAny, flagSpelling, + TP_HEADS, EP_HEADS, DP_HEADS, ATTN_DP_HEADS, parseIntFlagAny, flagSpelling, + dpAttnOnIn, ANCHOR_NEAR_MODEL_PATH, ANCHOR_NEAR_TP, ANCHOR_NEAR_DP, ANCHOR_NEAR_DPATTN, ANCHOR_NEAR_MOE, }; @@ -452,27 +456,23 @@ export const Playground = ({ config }) => { const AXIS_HANDLERS = { // ---- Axis: Attention Parallelism ---------------------------------------- - // TP / CP / DP-Attention sub-knobs; `null` = inherit. DP-Attention is - // combined: a numeric value emits `--dp N --enable-dp-attention`, `false` - // strips both. An optional `cpStrategy` knob (values from --cp-strategy: - // "zigzag" / "interleave") picks the CP layout; without it the strategy - // baked in the base is preserved, defaulting to "interleave" (the legacy - // knob's round-robin-split). + // TP / CP / DP-Attention sub-knobs; `null` = inherit. DP-Attention's + // value is the attention DP size: a numeric value emits + // `--attn-dp-size N`, `false` strips it. An optional `cpStrategy` knob + // (values from --cp-strategy: "zigzag" / "interleave") picks the CP + // layout; without it the strategy baked in the base is preserved, + // defaulting to "interleave" (the legacy knob's round-robin-split). attention: { initState: () => ({ tp: null, cp: null, cpStrategy: null, dpAttn: null }), - // DP-Attention: `--dp N --enable-dp-attention` → N; neither → false; - // bare `--enable-dp-attention` → 1. CP: any enable spelling → - // `--attn-cp-size N` (bare enable → 2, the legacy convention), plus the - // baked strategy (legacy mode flags mapped to zigzag/interleave). + // DP-Attention: `--attn-dp-size N` → N; absent → false. CP: any enable + // spelling → `--attn-cp-size N` (bare enable → 2, the legacy + // convention), plus the baked strategy (legacy mode flags mapped to + // zigzag/interleave). deriveFromBase: (cell, fc, h) => { const flags = (cell && cell.flags) || []; - const dpVal = h.parseIntFlagAny(flags, h.DP_HEADS); - const hasDpAttn = h.hasFlag(flags, "--enable-dp-attention"); - let dpAttn; - if (dpVal !== null) dpAttn = dpVal; - else if (hasDpAttn) dpAttn = 1; - else dpAttn = false; + const dpVal = h.parseIntFlagAny(flags, h.ATTN_DP_HEADS); + const dpAttn = dpVal !== null ? dpVal : false; const cpSize = h.parseIntFlag(flags, "--attn-cp-size"); return { tp: h.parseIntFlagAny(flags, h.TP_HEADS), @@ -488,7 +488,7 @@ export const Playground = ({ config }) => { const knobEntry = (id) => (fc.knobs || []).find((k) => k.id === id) || {}; const factsNow = () => ({ ...(sel || {}), - dpAttnOn: h.hasFlag(flags, "--enable-dp-attention"), + dpAttnOn: h.dpAttnOnIn(flags), cpOn: cpEnabledIn(flags), cpStrategy: bakedCpStrategy(flags) || "interleave", effTp: h.parseIntFlagAny(flags, h.TP_HEADS), @@ -503,8 +503,7 @@ export const Playground = ({ config }) => { if (knobEntry("cp").freeSize) return null; const dpIntent = (value.dpAttn !== null && value.dpAttn !== undefined) ? value.dpAttn - : (h.hasFlag(flags, "--enable-dp-attention") - ? (h.parseIntFlagAny(flags, h.DP_HEADS) ?? 1) : false); + : (h.parseIntFlagAny(flags, h.ATTN_DP_HEADS) ?? false); if (typeof dpIntent === "number" && dpIntent > 1) return null; return h.parseIntFlagAny(flags, h.TP_HEADS); }; @@ -524,7 +523,7 @@ export const Playground = ({ config }) => { && h.evaluateChip(e, facts).disabled); }; // NOTE: interleave prefill-CP + DP-Attention currently fails the - // runtime's dp_size == 1 assert, but combined support is planned + // runtime's attn_dp_size == 1 assert, but combined support is planned // upstream — the combination is allowed here (with a warning hint // below the command box) rather than banned. @@ -567,7 +566,7 @@ export const Playground = ({ config }) => { (r) => r && h.matchConstraint(factsNow(), r.when)); if (dpForced) { flags = h.stripFlagsByFirstToken(flags, [ - ...h.DP_HEADS, "--enable-dp-attention", + ...h.ATTN_DP_HEADS, "--enable-dp-attention-local-control-broadcast", ]); // The cell's DP-only env would otherwise outlive the flags it tunes. @@ -579,22 +578,21 @@ export const Playground = ({ config }) => { && !blocked("dpAttn", value.dpAttn)) { // Capture the spelling before stripping — the TP/EP handlers do the // same, and a lookup on the stripped array always hits the fallback. - const dpHead = h.flagSpelling(flags, h.DP_HEADS, "--dp-size"); + const dpHead = h.flagSpelling(flags, h.ATTN_DP_HEADS, "--attn-dp-size"); // The local-control-broadcast companion only means anything with DP // attention on, so it has to go down with it — stripping just - // `--enable-dp-attention` would leave it orphaned in the command. + // `--attn-dp-size` would leave it orphaned in the command. // apply re-seeds from the base cell, so this restores it when the // cell had it and the user is only re-sizing DP rather than disabling. const hadLocalBroadcast = h.hasFlag(flags, "--enable-dp-attention-local-control-broadcast"); flags = h.stripFlagsByFirstToken(flags, [ - ...h.DP_HEADS, "--enable-dp-attention", + ...h.ATTN_DP_HEADS, "--enable-dp-attention-local-control-broadcast", ]); if (typeof value.dpAttn === "number" && value.dpAttn > 0) { flags = h.insertAfter(flags, h.ANCHOR_NEAR_TP, [ `${dpHead} ${value.dpAttn}`, - "--enable-dp-attention", ...(hadLocalBroadcast ? ["--enable-dp-attention-local-control-broadcast"] : []), ]); @@ -608,7 +606,7 @@ export const Playground = ({ config }) => { if (!knobs.length) return null; const setKnob = (k, v) => setValue({ ...value, [k]: v }); // Interleave prefill-CP + DP-Attention is deliberately NOT grayed: - // current releases assert dp_size == 1 for interleave, but combined + // current releases assert attn_dp_size == 1 for interleave, but combined // support is planned upstream — a warning hint below the command box // covers it instead. const labelFor = (knob) => (c) => { @@ -930,7 +928,7 @@ export const Playground = ({ config }) => { const picked = (fc.options || []).find((p) => p.id === value); if (picked && h.evaluateChip(picked, { ...sel, - dpAttnOn: h.hasFlag(flags, "--enable-dp-attention"), + dpAttnOn: h.dpAttnOnIn(flags), }).disabled) { return { flags, env }; } @@ -1414,7 +1412,7 @@ export const Playground = ({ config }) => { // flags rather than the Deploy dims — the attention axis runs first. // A role override with `allowTp` is a TP-only shape validated as-is. if (fc.requiresDpAttention && !(roleOverride && roleOverride.allowTp) - && !flags.some((f) => f.split(/[\s=]/)[0] === "--enable-dp-attention")) { + && !h.dpAttnOnIn(flags)) { return { flags, env }; } flags = h.stripFlagsByFirstToken(flags, HICACHE_HEADS); @@ -1517,7 +1515,7 @@ export const Playground = ({ config }) => { apply: ({ flags, env, value, fc, sel, h, derived }) => { const evalBase = { ...(sel || {}), - dpAttnOn: h.hasFlag(flags, "--enable-dp-attention"), + dpAttnOn: h.dpAttnOnIn(flags), pdMode: h.findFlagArg(flags, "--disaggregation-mode") || "off", }; for (const spec of (fc || [])) { @@ -1701,7 +1699,7 @@ export const Playground = ({ config }) => { if (multinode && !f.some((x) => x.startsWith("--nnodes"))) { // Insert the multi-node trio after the last parallelism flag (matches // _deployment.jsx so untouched-base output is byte-identical). - const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp"]; + const PARALLELISM_ANCHORS = [...ATTN_DP_HEADS, "--dp-size", "--dp", "--tp-size", "--tp"]; let at = -1; for (const anchor of PARALLELISM_ANCHORS) { at = f.findIndex((x) => x.split(/[\s=]/)[0] === anchor); @@ -2418,7 +2416,7 @@ export const Playground = ({ config }) => { ? (staleExplicit("dpAttn", attnDelta.dpAttn) ? null : attnDelta.dpAttn) : (attnDerived.dpAttn !== undefined ? attnDerived.dpAttn : null); const dpAttnOn = (effDpAttn === true) - || (typeof effDpAttn === "number" && effDpAttn > 0); + || (typeof effDpAttn === "number" && effDpAttn > 1); // Runtime derivation attn_cp_size = tp/dp: with DP-Attention off, the only // enable-able CP size is TP. With DP-Attention on, sizes are NOT gated — // CP + DP-Attention is an allowed experiment covered by a warning hint @@ -2542,11 +2540,11 @@ export const Playground = ({ config }) => { // Interleave prefill-CP + DP-Attention hint on the EFFECTIVE command: // deliberately allowed (combined support is planned upstream), but current - // releases assert dp_size == 1 for the interleave layout at startup. + // releases assert attn_dp_size == 1 for the interleave layout at startup. const pgCpDpHint = cpEnabledIn(pgFlagsLatest) && (bakedCpStrategy(pgFlagsLatest) || "interleave") === "interleave" - && pgFlagsLatest.some((f) => f.split(/[\s=]/)[0] === "--enable-dp-attention"); + && dpAttnOnIn(pgFlagsLatest); // Submission snippets: proposed cell + existing cell at the same match. const proposedCellSnippet = baseCell @@ -2822,7 +2820,7 @@ export const Playground = ({ config }) => { )} {pgCpDpHint && (
- ⚠️ Interleave prefill-CP together with DP-Attention: current SGLang releases assert dp_size == 1 for the interleave layout, so this command fails at startup. Combined CP + DP-Attention support is planned upstream — keep one of the two off until it lands. + ⚠️ Interleave prefill-CP together with DP-Attention: current SGLang releases assert attn_dp_size == 1 for the interleave layout, so this command fails at startup. Combined CP + DP-Attention support is planned upstream — keep one of the two off until it lands.
)} diff --git a/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx index 95726df1e8..23f24cb6a0 100644 --- a/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx @@ -51,9 +51,9 @@ export const DeepSeekMathV2Deployment = () => { // TP setting cmd += ` \\\n --tp ${tpValue}`; - // DP Attention: --dp matches --tp + // DP Attention: --attn-dp-size matches --tp if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } // EP setting (commonly matches tp for MoE models) diff --git a/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx index e80f979aa3..bdf9e36819 100644 --- a/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx @@ -24,7 +24,6 @@ export const DeepSeekOCRDeployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon', disabledReason: 'Intel Xeon CPUs only support Tensor Parallel (TP)' }, { id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon', disabledReason: 'Intel Xeon CPUs only support Tensor Parallel (TP)' } ] } @@ -117,11 +116,6 @@ export const DeepSeekOCRDeployment = () => { cmd += ` \\\n --tp 1`; } - // DP strategy - if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; - } - // EP strategy if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 1`; diff --git a/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx index 2e7cb06701..dd82ae0e8d 100644 --- a/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx @@ -26,7 +26,6 @@ export const DeepSeekOCR2Deployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon' || v.hardware === 'arc_b', disabledReason: 'Only Tensor Parallel (TP) is supported on this hardware' }, { id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon' || v.hardware === 'arc_b', disabledReason: 'Only Tensor Parallel (TP) is supported on this hardware' } ] }, @@ -52,10 +51,6 @@ export const DeepSeekOCR2Deployment = () => { cmd += ` \\\n --tp 1`; } - if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; - } - if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 1`; } diff --git a/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx index 1d51eaf2fd..3353074f78 100644 --- a/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx @@ -116,7 +116,7 @@ export const DeepSeekR1BasicDeployment = () => { command += isXeon ? ' \\\n --tp 6' : ' \\\n --tp 8'; } if (strategyValues.includes('dp')) { - command += ' \\\n --dp 8 \\\n --enable-dp-attention'; + command += ' \\\n --attn-dp-size 8'; } if (strategyValues.includes('ep')) { command += ' \\\n --ep 8'; diff --git a/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx index 5b97889462..4733499aa7 100644 --- a/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx @@ -151,7 +151,7 @@ export const DeepSeekV3Deployment = () => { cmd += ` --model-path ${modelPath}`; if (strategyArray.includes('tp')) cmd += isXeon ? ' \\\n --tp 6' : ' \\\n --tp 8'; - if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention'; + if (strategyArray.includes('dp')) cmd += ' \\\n --attn-dp-size 8'; if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8'; if (strategyArray.includes('mtp')) { cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4'; diff --git a/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx index 4f75db97cc..a60a3c0dbc 100644 --- a/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx @@ -148,7 +148,7 @@ export const DeepSeekV31Deployment = () => { // TP is mandatory cmd += isXeon ? ` \\\n --tp 6` : ` \\\n --tp 8`; if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; diff --git a/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx index 4e4ece37be..a7617130c7 100644 --- a/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx @@ -219,7 +219,7 @@ export const DeepSeekV32Deployment = () => { const epSize = 8; cmd += ` \\\n --tp ${tpSize}`; if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp ${dpSize} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${dpSize}`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep ${epSize}`; diff --git a/docs/src/snippets/autoregressive/ernie-45-deployment.jsx b/docs/src/snippets/autoregressive/ernie-45-deployment.jsx index 9adf1adf48..b024bb0aa5 100644 --- a/docs/src/snippets/autoregressive/ernie-45-deployment.jsx +++ b/docs/src/snippets/autoregressive/ernie-45-deployment.jsx @@ -55,7 +55,7 @@ export const Ernie45Deployment = () => { } if (strategyArray.includes('dp') && modelsize === '300b') { - cmd += ` \\\n --dp ${dpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${dpValue}`; } if (strategyArray.includes('ep') && modelsize === '300b') { diff --git a/docs/src/snippets/autoregressive/glm-45-deployment.jsx b/docs/src/snippets/autoregressive/glm-45-deployment.jsx index 1e7cdf4450..1ef8dfd009 100644 --- a/docs/src/snippets/autoregressive/glm-45-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-45-deployment.jsx @@ -121,7 +121,7 @@ export const GLM45Deployment = () => { // Strategy-specific parameters if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; diff --git a/docs/src/snippets/autoregressive/glm-46-deployment.jsx b/docs/src/snippets/autoregressive/glm-46-deployment.jsx index f931a5f2e9..fbfd1fd1ff 100644 --- a/docs/src/snippets/autoregressive/glm-46-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-46-deployment.jsx @@ -132,7 +132,7 @@ export const GLM46Deployment = () => { // Strategy-specific parameters if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; if (hardware === 'b300') { cmd += ` \\\n --cuda-graph-max-bs-decode 256`; } diff --git a/docs/src/snippets/autoregressive/glm-47-deployment.jsx b/docs/src/snippets/autoregressive/glm-47-deployment.jsx index 9c6fbf8029..3a39b3a5f4 100644 --- a/docs/src/snippets/autoregressive/glm-47-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-47-deployment.jsx @@ -176,7 +176,7 @@ export const GLM47Deployment = () => { cmd += ` \\\n --max-context-length 8192 \\\n --mem-fraction-static 0.9`; } if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; diff --git a/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx b/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx index 4f5200544b..1f6dceeb3c 100644 --- a/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx @@ -23,7 +23,6 @@ export const GLM47FlashDeployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false }, { id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false } ] }, @@ -113,9 +112,6 @@ export const GLM47FlashDeployment = () => { } // Strategy-specific parameters - if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; - } if (strategyArray.includes('mtp')) { if (hardware === 'b200') { diff --git a/docs/src/snippets/autoregressive/glm-5-deployment.jsx b/docs/src/snippets/autoregressive/glm-5-deployment.jsx index 1346c07c4e..4d0654cac5 100644 --- a/docs/src/snippets/autoregressive/glm-5-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-5-deployment.jsx @@ -199,7 +199,7 @@ export const GLM5Deployment = () => { } if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; if (hardware === 'b300') { cmd += ' \\\n --cuda-graph-max-bs-decode 256'; } diff --git a/docs/src/snippets/autoregressive/glm-51-deployment.jsx b/docs/src/snippets/autoregressive/glm-51-deployment.jsx index 3b65d1a2c5..29a157be13 100644 --- a/docs/src/snippets/autoregressive/glm-51-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-51-deployment.jsx @@ -166,7 +166,7 @@ export const GLM51Deployment = () => { } if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45'; if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47'; diff --git a/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx index 04ccff38f1..e6fe475339 100644 --- a/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx @@ -75,7 +75,7 @@ export const KimiK2Deployment = () => { const strategyArray = Array.isArray(strategy) ? strategy : []; cmd += ` \\\n --tp 8`; if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 4 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 4`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 4`; diff --git a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx index 982fd443de..ad21e8df6a 100644 --- a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx @@ -176,9 +176,9 @@ export const KimiK25Deployment = () => { cmd += ` \\\n --tp ${tpValue}`; cmd += ' \\\n --trust-remote-code'; - // DP Attention: --dp matches --tp + // DP Attention: --attn-dp-size matches --tp if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } // Reasoning parser diff --git a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx index 3e7b998bc0..251155c307 100644 --- a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx @@ -175,7 +175,7 @@ export const KimiK26Deployment = () => { cmd += ' \\\n --trust-remote-code'; if (dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } if (reasoning === 'enabled') { diff --git a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx index f654300232..4d3cff8386 100644 --- a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx @@ -179,7 +179,7 @@ export const KimiK27CodeDeployment = () => { cmd += ' \\\n --trust-remote-code'; if (dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } if (reasoning === 'enabled') { diff --git a/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx b/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx index 47aef99069..048e71f0b0 100644 --- a/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx +++ b/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx @@ -127,8 +127,7 @@ export const LagunaXS2Deployment = () => { if (dpAttention === 'enabled') { lines[lines.length - 1] += ' \\'; - lines.push(` --dp ${tp} \\`); - lines.push(' --enable-dp-attention'); + lines.push(` --attn-dp-size ${tp}`); } if (reasoning === 'enabled') { diff --git a/docs/src/snippets/autoregressive/llama31-deployment.jsx b/docs/src/snippets/autoregressive/llama31-deployment.jsx index 2914e34de9..26b8472ff5 100644 --- a/docs/src/snippets/autoregressive/llama31-deployment.jsx +++ b/docs/src/snippets/autoregressive/llama31-deployment.jsx @@ -235,7 +235,6 @@ export const Llama31Deployment = () => { // NVIDIA-specific optimizations if (!isAMD && !isXeon && hardware !== 'arc_b') { if (optimization === 'throughput') { - args.push(`--enable-dp-attention`); args.push(`--mem-fraction-static 0.85`); } else if (optimization === 'latency') { args.push(`--speculative-algorithm EAGLE3`); diff --git a/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx b/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx index 936bc4a2f1..1fdd810f6d 100644 --- a/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx +++ b/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx @@ -112,7 +112,7 @@ export const MiMoV2FlashDeployment = () => { // DP settings if (!isMI355X && strategyArray.includes('dp')) { - cmd += ` \\\n --dp-size 2 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 2`; } // Performance Optimizations diff --git a/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx b/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx index 5611d2016a..9add5be8f9 100644 --- a/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx +++ b/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx @@ -138,14 +138,14 @@ export const MiMoV25Deployment = () => { const c = {}; // Both checkpoints are TP-interleaved (Pro: 8, base: 4), so attention-TP per // DP group must equal that factor. When the spec carries dp>1 (Pro/Hopper - // tp=16, base tp=8) DP-attention with `--dp = tp/factor` is required; without + // tp=16, base tp=8) DP-attention with `--attn-dp-size = tp/factor` is required; without // it a bare `--tp` gives attn_tp = tp != factor and the loader rejects the // checkpoint. When no dp>1 (Pro/Blackwell tp=8, base tp=4) it's a single // attention group and DP-attention must stay off. if (spec && !jax) { const factor = isPro ? 8 : 4; if (spec.dp > 1) { - c.dpAttention = { force: "enabled", reason: `Checkpoint is TP=${factor}-interleaved; DP-attention is required (--dp = tp/${factor} = ${spec.dp}).` }; + c.dpAttention = { force: "enabled", reason: `Checkpoint is TP=${factor}-interleaved; DP-attention is required (--attn-dp-size = tp/${factor} = ${spec.dp}).` }; } else { c.dpAttention = { force: "disabled", reason: `Single attention group on this hardware (tp=${factor}, no dp-attention).` }; } @@ -243,7 +243,8 @@ export const MiMoV25Deployment = () => { // sgl-jax conventions: // - `--tp-size` is always the total JAX device count; per-DP TP is // derived automatically as tp/dp. - // - No `--enable-dp-attention` flag — DP attention is the default + // - No separate attention-DP flag (SGLang's `--attn-dp-size`) — + // sgl-jax's `--dp-size` is DP attention by default // (FFN layers auto-pick EP-split for MoE, attn-TP-split for dense). const isV7x = hardware === "tpu-v7x"; const useEp = expertParallelism === "enabled"; @@ -309,8 +310,7 @@ export const MiMoV25Deployment = () => { flags.push(` --tp ${tp}`); if (useDpAttn) { - flags.push(` --dp ${dpSize}`); - flags.push(" --enable-dp-attention"); + flags.push(` --attn-dp-size ${dpSize}`); if (!isPro) { flags.push(" --enable-dp-lm-head"); flags.push(" --mm-enable-dp-encoder"); diff --git a/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx b/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx index 537f2b54c4..b8f444d786 100644 --- a/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx +++ b/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx @@ -30,7 +30,7 @@ export const Nemotron3UltraDeployment = () => { [...new Set(VERIFIED_CONFIGS.filter((c) => c.model === model && c.hardware === hardware).map((c) => c.tp))]; // DP attention is verified at dp=2 for BF16, and dp in {2,4,8} for NVFP4. SGLang - // requires tp_size % dp_size == 0, so dp is capped at both the selected TP and the + // requires tp_size % attn_dp_size == 0, so dp is capped at both the selected TP and the // max verified TP for this model+hardware (whichever is smaller). const dpCandidatesForModel = (model) => (model === 'bf16' ? ['2'] : ['2', '4', '8']); @@ -128,12 +128,12 @@ export const Nemotron3UltraDeployment = () => { }; }); }, - // dp_size must divide tp_size; only emit when the selected DP is valid for the current TP. + // attn_dp_size must divide tp_size; only emit when the selected DP is valid for the current TP. commandRule: (value, state) => value && value !== 'disabled' && dpCandidatesForModel(state.model).includes(value) && Number(value) <= Number(state.tp) - ? `--dp ${value} \\\n --enable-dp-attention` + ? `--attn-dp-size ${value}` : null }, mtp: { diff --git a/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx b/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx index 9ba7e28876..6aa6e8e028 100644 --- a/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx +++ b/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx @@ -119,11 +119,6 @@ export const Qwen3CoderDeployment = () => { cmd += ` \\\n --ep 2`; } - // DP attention setting - if (quantization === 'nvfp4') { - cmd += ` \\\n --enable-dp-attention`; - } - // MOE runner backend for NVIDIA if (isNvidia) { if (quantization === 'nvfp4') { diff --git a/docs/src/snippets/configs/Qwen/qwen3.8.jsx b/docs/src/snippets/configs/Qwen/qwen3.8.jsx index b0f0ce4fc0..337e890d00 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8.jsx @@ -367,9 +367,8 @@ export const config = { "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp-size 16", - "--dp-size 4", + "--attn-dp-size 4", "--ep-size 16", - "--enable-dp-attention", "--enable-dp-lm-head", "--enable-dp-attention-local-control-broadcast", "--moe-dense-tp-size 1", @@ -452,9 +451,8 @@ export const config = { "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp-size 16", - "--dp-size 4", + "--attn-dp-size 4", "--ep-size 16", - "--enable-dp-attention", "--enable-dp-lm-head", "--enable-dp-attention-local-control-broadcast", "--moe-dense-tp-size 1", @@ -540,8 +538,7 @@ export const config = { "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp-size 16", - "--dp-size 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--enable-dp-lm-head", "--enable-dp-attention-local-control-broadcast", "--moe-dense-tp-size 1", diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index a985eff0c4..e237814df0 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -235,11 +235,11 @@ sgl-eval run mmmu_pro \\ playgroundFeatures: { // ----- Card 1: "Attention Parallelism" ----- - // DP-Attention is a combined knob: value is the DP degree AND toggles `--enable-dp-attention`. + // DP-Attention is a single knob: value is the attention DP size, emitted as `--attn-dp-size N`. // CP sizes auto-gate in the engine to the runtime derivation // attn_cp_size = tp/dp (a user-passed --attn-cp-size is overridden). // CP is single-machine only (tp_size <= 8). Interleave CP + DP-Attention - // currently fails the runtime's dp_size == 1 assert but is allowed here + // currently fails the runtime's attn_dp_size == 1 assert but is allowed here // with a warning (combined support is planned upstream). No `cpStrategy` // knob: DeepSeek-V4 supports only interleave (the runtime rejects zigzag). attention: { @@ -263,7 +263,7 @@ sgl-eval run mmmu_pro \\ // The low-latency and balanced PD roles run TP-only. That is what lets // their decode ladders run to the full ceiling (8, 32 on Pro Official // low-latency, and 96): the ceiling - // is server-wide and floor-divided by attn_dp_size, so only at dp_size + // is server-wide and floor-divided by attn_dp_size, so only at attn_dp_size // 1 is it also the per-rank batch. Forced rather than left to the // reader, because switching DP on would cut the slots per rank without // changing either flag in the command — the ladder would still read 96 @@ -889,8 +889,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -905,8 +904,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -927,8 +925,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -944,8 +941,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -982,8 +978,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--chunked-prefill-size 32768", "--swa-full-tokens-ratio 0.1", @@ -1007,8 +1002,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -1063,8 +1057,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1079,8 +1072,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1101,8 +1093,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1118,8 +1109,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1154,8 +1144,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-runner-backend flashinfer_mxfp4", "--disable-flashinfer-autotune", "--chunked-prefill-size 32768", @@ -1181,8 +1170,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -1408,8 +1396,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1424,8 +1411,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1446,8 +1432,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1463,8 +1448,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1507,8 +1491,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1533,8 +1516,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.78", "--cuda-graph-max-bs-decode 64", @@ -1673,8 +1655,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1689,8 +1670,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1711,8 +1691,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1728,8 +1707,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1764,8 +1742,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1789,8 +1766,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.9", "--cuda-graph-max-bs-decode 128", @@ -1833,8 +1809,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1855,8 +1830,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.9", "--cuda-graph-max-bs-decode 128", @@ -1898,8 +1872,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--chunked-prefill-size 32768", "--swa-full-tokens-ratio 0.1", @@ -1917,8 +1890,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -1962,8 +1934,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-runner-backend flashinfer_mxfp4", "--disable-flashinfer-autotune", "--chunked-prefill-size 32768", @@ -1983,8 +1954,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -2028,8 +1998,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.78", "--cuda-graph-max-bs-decode 64", @@ -2046,8 +2015,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.78", "--cuda-graph-max-bs-decode 64", @@ -2195,8 +2163,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -2223,8 +2190,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -2356,8 +2322,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -2381,8 +2346,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--cuda-graph-max-bs-decode 128", "--max-running-requests 256", @@ -2402,8 +2366,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--cuda-graph-max-bs-decode 8", "--max-running-requests 32", @@ -2427,8 +2390,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -2452,8 +2414,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--mem-fraction-static 0.88", "--cuda-graph-max-bs-decode 128", @@ -2857,8 +2818,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-prefill-delayer", "--prefill-delayer-max-delay-ms 5000", "--attention-backend dsv4", @@ -2890,8 +2850,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-prefill-delayer", "--prefill-delayer-max-delay-ms 5000", "--attention-backend dsv4", @@ -2983,8 +2942,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3019,8 +2977,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3059,8 +3016,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3095,8 +3051,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3166,8 +3121,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3206,8 +3160,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3278,8 +3231,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3318,8 +3270,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3389,8 +3340,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3429,8 +3379,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3606,8 +3555,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3625,8 +3573,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -3661,8 +3608,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3681,8 +3627,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -3713,8 +3658,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3733,8 +3677,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -3763,8 +3706,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3782,8 +3724,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 3585e5d716..0c821ee348 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -364,7 +364,7 @@ export const config = { config.isNpuHw(s) || ["h100", "h200", "mi350x", "mi355x"].includes(s.hw) || config.isPipelined(s) || - !!config.flagOf(config.cellFor(s), "--enable-dp-attention"), + config.sizeOf(config.cellFor(s), "--attn-dp-size") > 1, disableReason: (s) => config.isNpuHw(s) ? "Only DSPARK is supported on this recipe." @@ -536,7 +536,7 @@ export const config = { playgroundFeatures: { // ----- Card: "Attention Parallelism" ----- - // DP-Attention is a combined knob: value = DP degree AND toggles `--enable-dp-attention`. + // DP-Attention is a single knob: value = attention DP size, emitted as `--attn-dp-size N`. // K3's MLA latent KV is TP-replicated, so DP-attention RAISES per-GPU KV pressure — // dp=8/attn_tp=1 OOMs on a single node; use dp=2/attn_tp=4. No CP knob: K3 uses // decode context parallel (`--dcp-size`), a different lever from prefill `--attn-cp-size`. @@ -602,12 +602,6 @@ export const config = { { id: "dpAttn", label: "DP-Attention", values: [ null, - { - // 950PR/DT Series only: the recipe enables DP-Attention at dp=1 - // (attn-TP 32), unlike A3's dp=4. - value: 1, - hide: { hw: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x", "a3"] }, - }, { value: false, get disable() { return [ @@ -615,10 +609,6 @@ export const config = { when: { hw: ["a3"] }, reason: "Only DP-Attention=4 (2 on Prefill) is supported on this recipe.", }, - { - when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", - }, ]; }, }, { @@ -632,7 +622,7 @@ export const config = { }, { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, ]; }, }, @@ -641,7 +631,7 @@ export const config = { get disable() { return [ { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, { when: { hw: ["a3"], pdMode: ["prefill"] }, @@ -658,7 +648,7 @@ export const config = { }, { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, { when: { hw: ["b300", "gb300"] }, @@ -680,7 +670,7 @@ export const config = { }, { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, { when: { hw: ["b300", "gb300"] }, @@ -1058,7 +1048,7 @@ export const config = { stripPrefixes: [ "--tp-size", "--tp", "--tensor-parallel-size", "--ep-size", "--ep", "--expert-parallel-size", - "--enable-dp-attention", "--dp-size", "--enable-dp-lm-head", + "--attn-dp-size", "--enable-dp-lm-head", "--dcp-size", "--dcp-comm-backend", // Every B200 Unified cell carries --pp-size 2; left standing it // multiplies against the preset's --tp-size for a world size the @@ -1083,7 +1073,7 @@ export const config = { const nnodes = n / gpusPerNode; return [ `--tp-size ${n}`, `--ep-size ${n}`, - ...(dp > 1 ? ["--enable-dp-attention", `--dp-size ${dp}`, "--enable-dp-lm-head"] : []), + ...(dp > 1 ? [`--attn-dp-size ${dp}`, "--enable-dp-lm-head"] : []), ...(nnodes > 1 ? [`--nnodes ${nnodes}`, "--node-rank {{NODE_RANK}}", "--dist-init-addr {{NODE0_IP}}:20000"] : []), "--moe-a2a-backend megamoe", "--moe-runner-backend deep_gemm", "--kv-cache-dtype fp8_e4m3", "--mamba-ssm-dtype bfloat16", @@ -1115,7 +1105,7 @@ export const config = { const nnodes = n / gpusPerNode; return [ `--tp-size ${n}`, `--ep-size ${n}`, - ...(dp > 1 ? ["--enable-dp-attention", `--dp-size ${dp}`, "--enable-dp-lm-head"] : []), + ...(dp > 1 ? [`--attn-dp-size ${dp}`, "--enable-dp-lm-head"] : []), "--dcp-size 8", ...(nnodes > 1 ? [`--nnodes ${nnodes}`, "--node-rank {{NODE_RANK}}", "--dist-init-addr {{NODE0_IP}}:20000"] : []), "--moe-a2a-backend megamoe", "--moe-runner-backend deep_gemm", @@ -2452,8 +2442,7 @@ export const config = { "--model-path {{MODEL_NAME}}", "--tp-size 16", "--dcp-size 8", - "--dp-size 2", - "--enable-dp-attention", + "--attn-dp-size 2", "--ep-size 16", "--mem-fraction-static 0.85", "--disaggregation-decode-extra-slots 16", @@ -2507,8 +2496,7 @@ export const config = { "--quantization modelslim", "--dtype bfloat16", "--tp-size 64", - "--enable-dp-attention", - "--dp-size 4", + "--attn-dp-size 4", "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", "--enable-dense-mlp-attn-tp", @@ -2578,8 +2566,7 @@ export const config = { "--dtype bfloat16", "--tp-size 16", "--pp-size 2", - "--dp-size 2", - "--enable-dp-attention", + "--attn-dp-size 2", "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", "--enable-dense-mlp-attn-tp", @@ -2646,8 +2633,7 @@ export const config = { "--dtype bfloat16", "--tp-size 64", "--pp-size 1", - "--dp-size 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", "--enable-dense-mlp-attn-tp", @@ -2672,8 +2658,8 @@ export const config = { // Ascend 950PR/DT Series: 4 nodes × 8 cards, one rank per card (TP32). // Unified PD, Balanced, DSPARK-only, with the product-line kernels armed // per env: FIAS V2 BSND for the DSpark target-verify/draft attention - // paths and the fine-grained dual-stream MoE overlap. DP-attention runs - // at dp=1 (attn-TP 32), and the shared experts / dense MLP shard across + // paths and the fine-grained dual-stream MoE overlap. It runs without + // attention DP (attn-TP 32), and the shared experts / dense MLP shard across // attention-TP through the server flags (--shared-experts-tp-size 4). // Checkpoint: the official Moonshot MXFP4 build (moonshotai/Kimi-K3, // fetched from ModelScope by SGLANG_USE_MODELSCOPE=1 above). Its routed @@ -2729,7 +2715,6 @@ export const config = { "--device npu", "--dtype bfloat16", "--tp-size 32", - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static 0.9", "--chunked-prefill-size 8192", @@ -2800,7 +2785,6 @@ export const config = { "--device npu", "--dtype bfloat16", "--tp-size 32", - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static 0.85", "--chunked-prefill-size 8192", @@ -2872,7 +2856,6 @@ export const config = { "--device npu", "--dtype bfloat16", "--tp-size 32", - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static 0.85", "--chunked-prefill-size 8192", diff --git a/docs/src/snippets/configs/rednote/dots3-note.jsx b/docs/src/snippets/configs/rednote/dots3-note.jsx index 8e85f19cd2..3c6d9f3edf 100644 --- a/docs/src/snippets/configs/rednote/dots3-note.jsx +++ b/docs/src/snippets/configs/rednote/dots3-note.jsx @@ -135,9 +135,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", @@ -184,9 +183,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", @@ -233,9 +231,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", @@ -282,9 +279,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", diff --git a/docs/src/snippets/configs/zai-org/glm-5.2.jsx b/docs/src/snippets/configs/zai-org/glm-5.2.jsx index 44cfb82ed0..9fa84902fe 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.2.jsx @@ -114,7 +114,7 @@ sgl-eval run aime25 \\ // CP sizes auto-gate in the engine to the runtime derivation // attn_cp_size = tp/dp (a user-passed --attn-cp-size is overridden). // CP is single-machine only (tp_size <= 8). Interleave CP + DP-Attention - // currently fails the runtime's dp_size == 1 assert but is allowed here + // currently fails the runtime's attn_dp_size == 1 assert but is allowed here // with a warning (combined support is planned upstream). // Strategy knob: interleave (ex round-robin-split) is the layout verified // here and the default; zigzag (ex in-seq-split) is exposed as an @@ -280,8 +280,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -303,8 +302,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--max-running-requests 256", @@ -339,8 +337,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -362,8 +359,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--max-running-requests 256", @@ -401,8 +397,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -425,8 +420,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -462,8 +456,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -483,8 +476,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--max-running-requests 256", @@ -728,8 +720,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the // verify overhead of a long draft outweighs the accept-length gain. "--speculative-algorithm EAGLE", @@ -752,8 +743,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--chunked-prefill-size 32768", "--mem-fraction-static 0.92", "--max-running-requests 512", @@ -789,8 +779,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--quantization modelopt_fp4", // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the // verify overhead of a long draft outweighs the accept-length gain. @@ -816,8 +805,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--quantization modelopt_fp4", "--max-running-requests 1024", "--chunked-prefill-size 8192", @@ -855,8 +843,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the // verify overhead of a long draft outweighs the accept-length gain. "--speculative-algorithm EAGLE", @@ -878,8 +865,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--chunked-prefill-size 8192", "--mem-fraction-static 0.92", "--max-running-requests 512", diff --git a/docs/src/snippets/configs/zai-org/glm-5.3.jsx b/docs/src/snippets/configs/zai-org/glm-5.3.jsx index 0806213357..02c8d3bf5e 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3.jsx @@ -143,7 +143,7 @@ sgl-eval run aime25 \\ // CP sizes auto-gate in the engine to the runtime derivation // attn_cp_size = tp/dp (a user-passed --attn-cp-size is overridden). // CP is single-machine only (tp_size <= 8). Interleave CP + DP-Attention - // currently fails the runtime's dp_size == 1 assert but is allowed here + // currently fails the runtime's attn_dp_size == 1 assert but is allowed here // with a warning (combined support is planned upstream). // Strategy knob: interleave (ex round-robin-split) is the default; // zigzag (ex in-seq-split) is exposed as an @@ -317,8 +317,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -360,8 +359,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -405,8 +403,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -448,8 +445,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -698,8 +694,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--speculative-algorithm EAGLE", "--speculative-num-steps 2", "--speculative-eagle-topk 1", @@ -719,8 +714,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--chunked-prefill-size 32768", "--mem-fraction-static 0.92", "--max-running-requests 512", @@ -757,8 +751,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--speculative-algorithm EAGLE", "--speculative-num-steps 2", "--speculative-eagle-topk 1", @@ -799,8 +792,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--speculative-algorithm EAGLE", "--speculative-num-steps 2", "--speculative-eagle-topk 1", diff --git a/python/sglang/compile_deep_gemm.py b/python/sglang/compile_deep_gemm.py index 65d809442d..8a8ae9cd92 100644 --- a/python/sglang/compile_deep_gemm.py +++ b/python/sglang/compile_deep_gemm.py @@ -65,7 +65,7 @@ async def warm_up_compile( disaggregation_mode: str, tokenizer_manager: TokenizerManager ): print("\nGenerate warm up request for compiling DeepGEMM...\n") - dp_size = get_parallel().dp_size + dp_size = get_parallel().num_dp_ranks base_ids = [0, 1, 2, 3] sampling_params = { "temperature": 0.0, diff --git a/python/sglang/srt/arg_groups/argparse_actions.py b/python/sglang/srt/arg_groups/argparse_actions.py index 6b2b31e800..c63052444e 100644 --- a/python/sglang/srt/arg_groups/argparse_actions.py +++ b/python/sglang/srt/arg_groups/argparse_actions.py @@ -42,11 +42,11 @@ def print_deprecated_warning(message: str): # * an old valued flag replaced by a renamed valued flag # -> `DeprecatedAliasStoreAction`. # -# Only the second has a registration today (`--disable-cuda-graph`). The other -# three are kept because the shapes recur -- this package has retired flags of -# every one of them -- and the fiddly parts (`nargs=0` on a boolean, where the -# const goes, warn-and-continue versus `parser.error`) are what a -# reimplementation gets wrong. Pass `new_flag=` so the warning tells the +# Only the second has registrations today (`--disable-cuda-graph`, +# `--enable-dp-attention`). The other three are kept because the shapes recur +# -- this package has retired flags of every one of them -- and the fiddly +# parts (`nargs=0` on a boolean, where the const goes, warn-and-continue versus +# `parser.error`) are what a reimplementation gets wrong. Pass `new_flag=` so the warning tells the # operator what to switch to; that pointer is the whole point. diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 1aeee2839b..0d3f354024 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -23,7 +23,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( with_phase, ) from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import ( is_cpu, is_mps, @@ -186,7 +186,7 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any): "model-arch blacklist", lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model, ), - ("DP attention", lambda: resolved_view(server_args).enable_dp_attention), + ("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))), ("full torch.compile mode", lambda: cfg.enable_torch_compile), ("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1), ( diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index b109483f78..9a83ee1f22 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -11,7 +11,7 @@ from sglang.srt.arg_groups.overrides import ( run_post_process_pass, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of from sglang.srt.utils.common import is_gfx95_supported, is_npu if TYPE_CHECKING: @@ -123,10 +123,18 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: "reindexes with interleave order." ) + # DeepSeek-V4 CP runs data-parallel groups as attention DP. + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + attn_dp_size = cfg.attn_dp_size * cfg.dp_size declare_resolution( server_args, "validate_deepseek_v4_cp", - enable_dp_attention=True, + attn_dp_size=attn_dp_size, + dp_size=1, ) declare_resolution( server_args, @@ -136,10 +144,10 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: declare_resolution( server_args, "validate_deepseek_v4_cp", - attn_cp_size=cfg.tp_size // cfg.dp_size, + attn_cp_size=cfg.tp_size // attn_dp_size, ) if not is_npu(): - assert cfg.dp_size == 1, ( + assert attn_dp_size == 1, ( "For round-robin split mode, dp attention is not supported." ) assert cfg.tp_size <= 8, ( @@ -161,7 +169,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: logger.warning( f"Enable Context Parallel for DeepSeekV4, " f"strategy={cfg.cp_strategy}, " - f"dp_size={cfg.dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, " + f"attn_dp_size={attn_dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, " f"attn_cp_size={cfg.attn_cp_size}, ep_size={cfg.ep_size}, tp_size={cfg.tp_size}" ) @@ -193,7 +201,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "prefill CUDA graphs", cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, ), - ("DP attention", cfg.enable_dp_attention), + ("DP attention", attn_dp_enabled_of(cfg)), ("context parallelism", cfg.attn_cp_size > 1), ("external cache linker", cfg.enable_unified_cache_external_linker), ("unified memory", cfg.enable_unified_memory), @@ -246,8 +254,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: if ( read_ragged_verify_mode() is not RaggedVerifyMode.STATIC or cfg.disaggregation_transfer_backend != "mooncake" - or cfg.dp_size != 1 - or cfg.enable_dp_attention + or num_dp_ranks_of(cfg) != 1 + or attn_dp_enabled_of(cfg) or cfg.attn_cp_size != 1 or cfg.dcp_size != 1 ): @@ -285,7 +293,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, ), # input_ids_global is a DP-wide gather, so the tail slice cannot apply. - ("DP attention", cfg.enable_dp_attention), + ("DP attention", attn_dp_enabled_of(cfg)), ) for feature, enabled in incompatible: if enabled: diff --git a/python/sglang/srt/arg_groups/expert_pack_hook.py b/python/sglang/srt/arg_groups/expert_pack_hook.py index ecb4aa5c90..f6a10e6664 100644 --- a/python/sglang/srt/arg_groups/expert_pack_hook.py +++ b/python/sglang/srt/arg_groups/expert_pack_hook.py @@ -40,6 +40,7 @@ def handle_expert_pack(server_args: Any) -> None: parallelism = ( ("tensor", "--tp-size", cfg.tp_size), ("data", "--dp-size", cfg.dp_size), + ("attention data", "--attn-dp-size", cfg.attn_dp_size), ("expert", "--ep-size", cfg.ep_size), ) for label, option, size in parallelism: diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 20c35da680..d38e398395 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -79,6 +79,7 @@ POSITIONAL_FIELD_ORDER = ( "pp_max_micro_batch_size", "pp_async_batch_depth", "dp_size", + "attn_dp_size", "load_balance_method", "attn_cp_size", "moe_dp_size", diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index 3e0e28e6d4..edf20e3d41 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -73,8 +73,19 @@ class Parallel(msgspec.Struct): dp_size: A[ int, Arg( - help="The data parallelism size.", + help="The number of data-parallel replicas of the model.", aliases=["--data-parallel-size"], + resolvable=True, + ), + ] = 1 + attn_dp_size: A[ + int, + Arg( + help="The attention data parallelism size: the number of " + "data-parallel attention groups inside the TP group, one scheduler " + "each, while the FFN stays tensor parallel.", + aliases=["--attention-data-parallel-size"], + resolvable=True, ), ] = 1 load_balance_method: A[ @@ -159,14 +170,9 @@ class Parallel(msgspec.Struct): bool, "Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.", ] = False - # DP attention - enable_dp_attention: A[ - bool, - Arg( - help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.", - resolvable=True, - ), - ] = False + # Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention` + # resolves to `attn_dp_size = N`, `dp_size = 1`. + enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False enable_dp_attention_local_control_broadcast: A[ bool, "With DP-attention, send control messages to every DP group leader and broadcast within attn_tp_group instead of the full tp_group. Eliminates a costly all-ranks gloo sync on every scheduler iteration.", @@ -184,7 +190,7 @@ class Parallel(msgspec.Struct): help="Use all-to-all instead of TP all-gather followed by DP scatter " "for the TP-sharded LM head under DP attention. By default this is " "enabled only on decode-only PD nodes with pure DP attention " - "(tp_size == dp_size > 1 and attn_cp_size == 1), and disabled on " + "(tp_size == attn_dp_size > 1 and attn_cp_size == 1), and disabled on " "prefill-only and colocated nodes. Pass " "--no-enable-tp-lm-head-all-to-all to opt out. The path is " "incompatible with --enable-dp-lm-head; batches without an equal " @@ -282,10 +288,18 @@ class Parallel(msgspec.Struct): doc="Attention tensor-parallel width: `tp_size` divided by the " "attention-DP and attention-CP dimensions.", ) - attn_dp_size = Derived( - fn="sglang.srt.runtime_context.attn_dp_size_of", - doc="Attention data-parallel width: `dp_size` when DP attention is " - "on, otherwise one.", + attn_dp_enabled = Derived( + fn="sglang.srt.runtime_context.attn_dp_enabled_of", + doc="Whether attention runs data parallel: `attn_dp_size` is wider " + "than one, or this process is an elastic EP scale joiner, which joins " + "an attention-DP deployment with a group of its own that may be one " + "rank wide.", + ) + num_dp_ranks = Derived( + fn="sglang.srt.runtime_context.num_dp_ranks_of", + doc="How many data-parallel ranks the deployment serves with, one " + "scheduler each: `dp_size * attn_dp_size`. Elastic EP scale-up widens " + "it as ranks join.", ) attn_dcp_size = Derived( fn="sglang.srt.runtime_context.attn_dcp_size_of", diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 5db8194b9d..196d8fdfd9 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -18,7 +18,7 @@ from sglang.srt.arg_groups.overrides import ( from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform logger = logging.getLogger(__name__) @@ -450,7 +450,7 @@ def handle_cache_compatibility(server_args: Any) -> None: raise NotImplementedError( "LMCacheUnifiedRadixCache does not yet support speculative decoding" ) - if cfg.enable_dp_attention: + if attn_dp_enabled_of(cfg): raise NotImplementedError( "LMCacheUnifiedRadixCache does not yet support DP attention" ) diff --git a/python/sglang/srt/arg_groups/layernorm_sp_hook.py b/python/sglang/srt/arg_groups/layernorm_sp_hook.py index ddc14ddbaf..025a642c80 100644 --- a/python/sglang/srt/arg_groups/layernorm_sp_hook.py +++ b/python/sglang/srt/arg_groups/layernorm_sp_hook.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs from sglang.srt.arg_groups.overrides import model_config_of, resolving_view +from sglang.srt.runtime_context import attn_dp_enabled_of logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None: validate_layernorm_sp( architecture=architectures[0] if architectures else None, tp_size=cfg.tp_size, - enable_dp_attention=cfg.enable_dp_attention, + attn_dp_enabled=attn_dp_enabled_of(cfg), speculative_algorithm=cfg.speculative_algorithm, ) @@ -33,7 +34,7 @@ def validate_layernorm_sp( *, architecture: Optional[str], tp_size: int, - enable_dp_attention: bool, + attn_dp_enabled: bool, speculative_algorithm: Optional[str], ) -> None: """Fail loud for unsupported / incompatible configs. Callers gate on the flag.""" @@ -49,9 +50,9 @@ def validate_layernorm_sp( "--enable-layernorm-sp requires tp_size > 1: there is no sequence to " "shard across a single TP rank." ) - if enable_dp_attention: + if attn_dp_enabled: raise ValueError( - "--enable-layernorm-sp is not compatible with --enable-dp-attention: " + "--enable-layernorm-sp is not compatible with attention DP: " "SP shards the sequence across the full TP group, which under DP " "attention spans data-parallel groups holding different sequences." ) diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index d45ae3caff..654c956bf3 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -13,7 +13,7 @@ from sglang.srt.arg_groups.overrides import ( ) from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import parse_connector_type logger = logging.getLogger(__name__) @@ -143,11 +143,11 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> local_chunked_prefill_size = ( cfg.chunked_prefill_size + token_partition_size - 1 ) // token_partition_size - elif cfg.enable_dp_attention: - token_partition_size = cfg.dp_size - token_partition_name = "dp_size" + elif attn_dp_enabled_of(cfg): + token_partition_size = cfg.attn_dp_size + token_partition_name = "attn_dp_size" token_alignment = max( - cfg.tp_size // cfg.dp_size // cfg.attn_cp_size, + cfg.tp_size // cfg.attn_dp_size // cfg.attn_cp_size, 1, ) local_chunked_prefill_size = cfg.chunked_prefill_size // token_partition_size diff --git a/python/sglang/srt/arg_groups/memory_hook.py b/python/sglang/srt/arg_groups/memory_hook.py index aed841a572..ae2ac83e84 100644 --- a/python/sglang/srt/arg_groups/memory_hook.py +++ b/python/sglang/srt/arg_groups/memory_hook.py @@ -17,7 +17,7 @@ from sglang.srt.arg_groups.overrides import ( ) from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import get_device_memory_capacity logger = logging.getLogger(__name__) @@ -361,14 +361,14 @@ def reserve_for_graph_mb(server_args: Any) -> float: reserved_mem += decode_cuda_graph_config.max_bs * 2 if ( - resolved_view(server_args).enable_dp_attention + attn_dp_enabled_of(resolved_view(server_args)) and cfg.disaggregation_mode != "prefill" ): # DP attention needs more padding for some operations, and much more for large # cuda graph max bs (torch allocator / implementation inefficiencies). - reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 3 + reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 3 if decode_cuda_graph_config.max_bs > 300: - reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 1.5 + reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 1.5 if ( cfg.disaggregation_mode != "decode" diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index da31f4a1d2..1fe46d0467 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -37,7 +37,11 @@ from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.hardware_backend.mlx.runtime import use_mlx from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase -from sglang.srt.runtime_context import derive_attention_widths, get_platform +from sglang.srt.runtime_context import ( + attn_dp_enabled_of, + derive_attn_tp_size, + get_platform, +) from sglang.srt.utils.common import ( get_quantization_config, is_mps, @@ -290,15 +294,14 @@ def handle_model_specific_adjustments(server_args: Any): ) else: # Pure TP and partial DP Attention mode is active for DSA, logging a warning - if cfg.dp_size < cfg.tp_size: - _, attn_tp_size = derive_attention_widths( - tp_size=cfg.tp_size, - attn_cp_size=cfg.attn_cp_size, - dp_size=cfg.dp_size, - enable_dp_attention=cfg.enable_dp_attention, - ) + attn_tp_size = derive_attn_tp_size( + tp_size=cfg.tp_size, + attn_cp_size=cfg.attn_cp_size, + attn_dp_size=cfg.attn_dp_size, + ) + if attn_tp_size > 1: logger.warning( - f"DSA with TP mode is active, dp_size={cfg.dp_size}, tp_size={cfg.tp_size}, " + f"DSA with TP mode is active, attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size}, " f"attn_tp_size={attn_tp_size}, attention weights will be sharded across {attn_tp_size} ranks." ) @@ -410,7 +413,7 @@ def handle_model_specific_adjustments(server_args: Any): if is_deepseek_dsa(hf_config) and not envs.SGLANG_OPT_USE_TOPK_V2.is_set(): # Prefer HIP top-k by default while honoring an explicit selection. envs.SGLANG_OPT_USE_TOPK_V2.set(False) - if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1: + if not attn_dp_enabled_of(resolved_view(server_args)) and cfg.nnodes == 1: # TODO (Hubert): Put this back later # server_args.enable_aiter_allreduce_fusion = True @@ -514,7 +517,7 @@ def handle_model_specific_adjustments(server_args: Any): quant_method = get_quantization_config(hf_config) is_mxfp4_quant_format = quant_method == "mxfp4" if ( - not resolved_view(server_args).enable_dp_attention + not attn_dp_enabled_of(resolved_view(server_args)) and cfg.nnodes == 1 and get_platform().is_hip ): @@ -543,8 +546,11 @@ def handle_model_specific_adjustments(server_args: Any): if model_arch == "MiMoV2ForCausalLM" and not cfg.encoder_only: expected_attn_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(hf_config) view = resolved_view(server_args) - attn_dp_size = cfg.dp_size if view.enable_dp_attention else 1 - effective_attn_tp_size = cfg.tp_size // attn_dp_size // view.attn_cp_size + effective_attn_tp_size = derive_attn_tp_size( + tp_size=cfg.tp_size, + attn_cp_size=view.attn_cp_size, + attn_dp_size=cfg.attn_dp_size, + ) if ( expected_attn_tp_size is not None and expected_attn_tp_size % effective_attn_tp_size != 0 @@ -555,10 +561,9 @@ def handle_model_specific_adjustments(server_args: Any): "qkv_proj weights are " f"TP={expected_attn_tp_size}-interleaved; got " f"{effective_attn_tp_size} " - f"(tp_size={cfg.tp_size}, dp_size={cfg.dp_size}, " - f"enable_dp_attention={view.enable_dp_attention}, " + f"(tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, " f"attn_cp_size={view.attn_cp_size}). " - "Set --tp, --dp, --enable-dp-attention, and " + "Set --tp, --attn-dp-size, and " "--attention-context-parallel-size so the effective " f"attention TP size is {expected_attn_tp_size}." ) diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 23568e15d1..87b19a5c8d 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -85,7 +85,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: logger.warning( "Context parallel feature is still under experiment. It has only been verified on Hopper platform." ) - overrides["enable_dp_attention"] = True + # DSA CP runs data-parallel groups as attention DP. + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + attn_dp_size = cfg.attn_dp_size * cfg.dp_size + overrides["attn_dp_size"] = attn_dp_size + overrides["dp_size"] = 1 overrides["moe_dense_tp_size"] = 1 if cfg.cp_strategy == "zigzag": overrides["moe_a2a_backend"] = "deepep" @@ -95,7 +103,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: "moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1." ) else: - assert cfg.dp_size == 1, ( + assert attn_dp_size == 1, ( "interleave DSA CP does not support DP attention." ) assert cfg.tp_size <= 8, ( @@ -104,11 +112,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP. # The DSA / MLA CP gather and reduce-scatter # (the dsa_cp_* helpers in adapters/context_parallel.py) assume it. - attn_cp_size = cfg.tp_size // cfg.dp_size + attn_cp_size = cfg.tp_size // attn_dp_size overrides["attn_cp_size"] = attn_cp_size logger.warning( "Enabled DSA context parallel: " - f"strategy={cfg.cp_strategy}, dp_size={cfg.dp_size}, " + f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, " f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, " f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, " f"attn_cp_size={attn_cp_size}, " @@ -158,7 +166,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: "MLA prefill context parallel is still experimental. " "Verified on Hopper with the fa3 backend." ) - overrides["enable_dp_attention"] = True + # MLA CP runs data-parallel groups as attention DP. + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + attn_dp_size = cfg.attn_dp_size * cfg.dp_size + overrides["attn_dp_size"] = attn_dp_size + overrides["dp_size"] = 1 # TODO(kpham-sgl) Supports moe_dense_tp_size != 1. overrides["moe_dense_tp_size"] = 1 overrides["moe_a2a_backend"] = "deepep" @@ -169,11 +185,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP. # The DSA / MLA CP gather and reduce-scatter # (the dsa_cp_* helpers in adapters/context_parallel.py) assume it. - attn_cp_size = cfg.tp_size // cfg.dp_size + attn_cp_size = cfg.tp_size // attn_dp_size overrides["attn_cp_size"] = attn_cp_size logger.warning( f"Enable Context Parallel opt for MLA, " - f"Setting dp_size == {cfg.dp_size} and " + f"Setting attn_dp_size == {attn_dp_size} and " f"attn_cp_size == {attn_cp_size}, " f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, " f"ep_size == {overrides['ep_size']}, " diff --git a/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py b/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py index 7e8f1608e5..b0b9c09ebb 100644 --- a/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py +++ b/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py @@ -5,6 +5,7 @@ from sglang.srt.arg_groups.model_override_base import ( is_attention_backend_not_set, resolving_view, ) +from sglang.srt.runtime_context import attn_dp_enabled_of @_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP") @@ -33,7 +34,7 @@ def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict: ) if cfg.enable_multi_layer_eagle: raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.") - if cfg.enable_dp_attention: + if attn_dp_enabled_of(cfg): raise ValueError("IQuest Q1 does not support DP attention.") if cfg.pp_size > 1: raise ValueError("IQuest Q1 does not support pipeline parallelism.") diff --git a/python/sglang/srt/arg_groups/model_overrides/minicpm.py b/python/sglang/srt/arg_groups/model_overrides/minicpm.py index 3790c30d6e..ed03eb303f 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minicpm.py +++ b/python/sglang/srt/arg_groups/model_overrides/minicpm.py @@ -8,13 +8,13 @@ from sglang.srt.arg_groups.model_override_base import ( resolving_view, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform @_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM") def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: cfg = resolving_view(server_args) - if cfg.enable_dp_attention: + if attn_dp_enabled_of(cfg): raise ValueError("MiniCPM does not support DP attention") has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False) has_hybrid_attention = has_sparse_attention or getattr( diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index 4306ac25eb..13fd2b2a68 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -29,7 +29,7 @@ from sglang.srt.configs.moe_model_registry import ( from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import is_sm100_supported, parse_connector_type logger = logging.getLogger(__name__) @@ -269,8 +269,8 @@ def handle_a2a_moe(server_args: Any): if a2a_backend == "flashinfer_megamoe": validate_flashinfer_megamoe_model(server_args) validate_flashinfer_megamoe_envs() - assert cfg.enable_dp_attention and cfg.dp_size == cfg.tp_size, ( - "FlashInfer MegaMOE is only supported with dp_size == tp_size and --enable-dp-attention" + assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, ( + "FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size" ) if resolved_view(server_args).moe_runner_backend == "auto": declare_resolution( @@ -395,11 +395,8 @@ def handle_a2a_moe(server_args: Any): ) if a2a_now == "flashinfer": - assert ( - resolved_view(server_args).enable_dp_attention - and cfg.dp_size == cfg.tp_size - ), ( - "Flashinfer MoE A2A is only supported with dp_size == tp_size and --enable-dp-attention" + assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, ( + "Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size" ) if cfg.deepep_mode != "auto": logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A") @@ -491,12 +488,12 @@ def handle_a2a_moe(server_args: Any): deepep_mode="low_latency", ) logger.warning("auto set deepep_mode=`low_latency` for PPLX EP") - # pplx-kernels' AllToAll needs numDPGroups (== attention dp_size) > 1; + # pplx-kernels' AllToAll needs numDPGroups (== attn_dp_size) > 1; # without DP attention numDPGroups == 1 and construction fails deep in # the kernel. This also implies ep_size >= 2. - assert resolved_view(server_args).enable_dp_attention and cfg.dp_size >= 2, ( - "moe_a2a_backend='pplx' requires --enable-dp-attention with at " - "least 2 DP groups (--dp-size >= 2)." + assert cfg.attn_dp_size >= 2, ( + "moe_a2a_backend='pplx' requires attention DP with at least 2 " + "groups (--attn-dp-size >= 2)." ) # pplx runs the masked DeepGEMM expert path (sm_90a): reject other # runners and resolve auto -> deep_gemm. Unquantized bf16 pplx needs @@ -581,8 +578,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None: graph_bs = decode_config.max_bs or 0 if view.max_running_requests is not None: - attn_dp_size = view.dp_size if view.enable_dp_attention else 1 - per_rank_pool_bs = max(1, view.max_running_requests // attn_dp_size) + per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size) graph_bs = min(graph_bs, per_rank_pool_bs) tokens_per_req = ( max_speculative_num_draft_tokens(server_args) or 1 diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index a2de0bc293..05eaa3693e 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -62,8 +62,10 @@ logger = logging.getLogger(__name__) from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.runtime_context import ( + attn_dp_enabled_of, get_context, get_platform, + num_dp_ranks_of, ) from sglang.srt.utils.common import ( get_quantization_config, @@ -931,7 +933,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: and not prefer_custom_dsv41 and (get_platform().is_sm90 or get_platform().is_sm100) and view.tp_size > 1 - and not view.enable_dp_attention + and not attn_dp_enabled_of(view) and (view.nnodes == 1 or get_platform().is_sm100) and view.moe_a2a_backend == "none" ): @@ -1304,8 +1306,8 @@ def _page_size_default(view: Any) -> dict: @register_post_process def _data_parallelism_defaults(view: Any) -> dict: - if view.dp_size == 1 and view.ep_join_mode != "scale": - return {"enable_dp_attention": False, "enable_dp_lm_head": False} + if num_dp_ranks_of(view) == 1 and view.ep_join_mode != "scale": + return {"enable_dp_lm_head": False} return {} @@ -1349,9 +1351,8 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict: enable = ( view.disaggregation_mode == "decode" - and view.enable_dp_attention - and view.dp_size > 1 - and view.tp_size == view.dp_size + and view.attn_dp_size > 1 + and view.tp_size == view.attn_dp_size and view.attn_cp_size == 1 and not view.enable_dp_lm_head ) @@ -1362,20 +1363,21 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict: def _dp_lm_head_validation(view: Any) -> dict: """Require DP attention for DP LM head and TP LM-head all-to-all.""" if view.enable_dp_lm_head: - assert view.enable_dp_attention, ( - "Please enable dp attention when setting enable_dp_lm_head. " + assert attn_dp_enabled_of(view), ( + "--enable-dp-lm-head requires attention DP (--attn-dp-size)." ) if view.enable_tp_lm_head_all_to_all: - assert view.enable_dp_attention, ( - "Please enable dp attention when setting enable_tp_lm_head_all_to_all." + assert attn_dp_enabled_of(view), ( + "--enable-tp-lm-head-all-to-all requires attention DP (--attn-dp-size)." ) assert not view.enable_dp_lm_head, ( "--enable-tp-lm-head-all-to-all uses a TP-sharded LM head and is " "incompatible with --enable-dp-lm-head." ) - assert view.tp_size == view.dp_size, ( + assert view.tp_size == view.attn_dp_size, ( "--enable-tp-lm-head-all-to-all currently requires tp_size == " - f"dp_size, got tp_size={view.tp_size}, dp_size={view.dp_size}." + f"attn_dp_size, got tp_size={view.tp_size}, " + f"attn_dp_size={view.attn_dp_size}." ) assert view.attn_cp_size == 1, ( "--enable-tp-lm-head-all-to-all currently requires " diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index 1a396935fb..06101adfd9 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -23,12 +23,40 @@ from sglang.srt.arg_groups.resolution_hooks import run_hook from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import ( + attn_dp_enabled_of, + derive_attn_tp_size, + get_platform, +) from sglang.srt.utils.common import parse_connector_type logger = logging.getLogger(__name__) +def handle_deprecated_dp_attention(server_args: Any): + """Turn the deprecated `--dp-size N --enable-dp-attention` into + `--attn-dp-size N`. + + Runs before any handler reads the DP layout. The old spelling counted + attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and + `dp_size` counts replicas again. + """ + cfg = resolving_view(server_args) + if cfg.enable_dp_attention: + if cfg.attn_dp_size != 1: + raise ValueError( + "--enable-dp-attention is the deprecated spelling of " + "--attn-dp-size; pass --attn-dp-size alone." + ) + declare_resolution( + server_args, + "_handle_deprecated_dp_attention", + attn_dp_size=cfg.dp_size, + dp_size=1, + enable_dp_attention=False, + ) + + def _boundary_parallelism_overrides(cfg, model_type: str) -> dict: """Resolve the token-row modes the model's forward actually enters.""" nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle") @@ -108,8 +136,8 @@ def handle_context_parallelism(server_args: Any): assert cfg.tp_size % view.attn_cp_size == 0, ( "tp_size must be divisible by attn_cp_size" ) - assert cfg.tp_size % (cfg.dp_size * view.attn_cp_size) == 0, ( - "tp_size must be divisible by dp_size * attn_cp_size" + assert cfg.tp_size % (cfg.attn_dp_size * view.attn_cp_size) == 0, ( + "tp_size must be divisible by attn_dp_size * attn_cp_size" ) assert not cfg.enable_aiter_allreduce_fusion, ( @@ -155,14 +183,11 @@ def handle_shared_experts_tp(server_args: Any): if size is None: return - from sglang.srt.runtime_context import derive_attention_widths - view = resolved_view(server_args) - _, attn_tp_size = derive_attention_widths( + attn_tp_size = derive_attn_tp_size( tp_size=cfg.tp_size, attn_cp_size=view.attn_cp_size, - dp_size=cfg.dp_size, - enable_dp_attention=view.enable_dp_attention, + attn_dp_size=cfg.attn_dp_size, ) if size < 1 or attn_tp_size % size != 0: raise ValueError( @@ -224,7 +249,7 @@ def handle_decode_context_parallelism(server_args: Any): def handle_data_parallelism(server_args: Any): - # The dp_size==1 resets moved to the resolution pipeline + # The resets without attention DP live in the resolution pipeline # (arg_groups/overrides.py: _data_parallelism_defaults). from sglang.srt.arg_groups.cuda_graph_hook import ( generate_prefill_cuda_graph_batch_sizes, @@ -232,6 +257,15 @@ def handle_data_parallelism(server_args: Any): cfg = resolving_view(server_args) + if cfg.attn_dp_size < 1: + raise ValueError(f"--attn-dp-size must be positive (got {cfg.attn_dp_size}).") + if cfg.attn_dp_size > 1 and cfg.dp_size > 1: + raise ValueError( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + run_post_process_pass(server_args, _data_parallelism_defaults) if cfg.mm_enable_dp_encoder: @@ -252,18 +286,18 @@ def handle_data_parallelism(server_args: Any): cfg.tp_size, ) - if resolved_view(server_args).enable_dp_attention: + if attn_dp_enabled_of(resolved_view(server_args)): declare_resolution( server_args, "_handle_data_parallelism", schedule_conservativeness=cfg.schedule_conservativeness * 0.3, ) - assert cfg.tp_size % cfg.dp_size == 0 + assert cfg.tp_size % cfg.attn_dp_size == 0 original_chunked_prefill_size = cfg.chunked_prefill_size declare_resolution( server_args, "_handle_data_parallelism", - chunked_prefill_size=cfg.chunked_prefill_size // cfg.dp_size, + chunked_prefill_size=cfg.chunked_prefill_size // cfg.attn_dp_size, ) logger.warning( f"DP attention is enabled. chunked prefill size is adjusted " @@ -367,12 +401,8 @@ def handle_dwdp(server_args: Any): declare_resolution( server_args, "_handle_dwdp", - dp_size=cfg.dwdp_size, - ) - declare_resolution( - server_args, - "_handle_dwdp", - enable_dp_attention=True, + attn_dp_size=cfg.dwdp_size, + dp_size=1, ) declare_resolution( server_args, "_handle_dwdp", enable_dp_attention_local_control_broadcast=True @@ -413,7 +443,7 @@ def handle_dwdp(server_args: Any): logger.info( f"DWDP enabled: dwdp_size={cfg.dwdp_size}, " - f"auto-forced dp_size={cfg.dp_size}, ep_size={cfg.dwdp_size}, " + f"auto-forced attn_dp_size={cfg.attn_dp_size}, ep_size={cfg.dwdp_size}, " f"moe_dense_tp_size=1, moe_a2a_backend=none, " f"dp_attention_local_control_broadcast=True, " f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, " @@ -582,10 +612,10 @@ def handle_elastic_ep(server_args: Any): assert not cfg.enable_pdmux, ( "Elastic EP CUDA graph recapture does not support PDMux." ) - assert resolved.enable_dp_attention, ( - "Elastic EP scale-up requires --enable-dp-attention; without it " - "the TP group is not equivalent to WORLD and the post-scale " - "collective path is invalid." + assert attn_dp_enabled_of(resolved), ( + "Elastic EP scale-up requires attention DP (--attn-dp-size); " + "without it the TP group is not equivalent to WORLD and the " + "post-scale collective path is invalid." ) assert resolved.enable_dp_lm_head, ( "Elastic EP scale-up requires --enable-dp-lm-head so output " @@ -604,9 +634,9 @@ def handle_elastic_ep(server_args: Any): f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP " "and the attention DP group must all coincide with WORLD." ) - assert cfg.dp_size == cfg.tp_size, ( - "Elastic EP scale-up requires dp_size == tp_size " - f"(got dp_size={cfg.dp_size}, tp_size={cfg.tp_size})." + assert cfg.attn_dp_size == cfg.tp_size, ( + "Elastic EP scale-up requires attn_dp_size == tp_size " + f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})." ) assert resolved.moe_a2a_backend == "nixl", ( "Elastic EP scale-up requires --moe-a2a-backend nixl " diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index b15044b252..f7d619d2f3 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -12,6 +12,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.environ import envs +from sglang.srt.runtime_context import attn_dp_enabled_of, num_dp_ranks_of if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -102,7 +103,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: f"(--speculative-algorithm {cfg.speculative_algorithm})" ) - if resolved_view(server_args).enable_dp_attention: + if attn_dp_enabled_of(resolved_view(server_args)): logger.warning( "EXPERIMENTAL: Decode radix cache with DP attention. " "Requires prefix-aware DP rank routing for optimal cache hits." @@ -128,7 +129,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.disaggregation_decode_extra_slots is None: extra_slots = 0 if cfg.max_running_requests is not None: - per_worker = cfg.max_running_requests // max(1, cfg.dp_size) + per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg) if per_worker <= 32: extra_slots = per_worker * 2 declare_resolution( @@ -160,8 +161,8 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.enable_pd_role_switch: view = resolved_view(server_args) unsupported = [] - if view.enable_dp_attention: - unsupported.append("DP attention (--enable-dp-attention)") + if attn_dp_enabled_of(view): + unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})") if view.ep_size > 1: unsupported.append(f"expert parallelism (--ep-size {view.ep_size})") if view.moe_a2a_backend != "none": diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index 3963f548c1..615d48ddb0 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -38,6 +38,11 @@ def run_resolution_pipeline(server_args: Any) -> None: cfg = resolving_view(server_args) + from sglang.srt.arg_groups.parallel_hook import handle_deprecated_dp_attention + + # Before any handler reads the DP layout. + run_hook(handle_deprecated_dp_attention, server_args) + from sglang.srt.arg_groups.mega_moe_hook import handle_mega_moe run_hook(handle_mega_moe, server_args) diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index 9422d8025e..4b0cf78c30 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, FrozenSet, List # to every invocation of that name, including repeated steps. _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( { + "handle_deprecated_dp_attention", "handle_mega_moe", "handle_return_hidden_states_mode", "handle_media_url_security", diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index d2f497396d..18a84bd4cb 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -17,7 +17,7 @@ from sglang.srt.arg_groups.overrides import ( ) from sglang.srt.environ import envs from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -271,7 +271,7 @@ def _handle_dflash(server_args: ServerArgs) -> None: ) # DFLASH + dp attention is validated on NPU only. - if cfg.enable_dp_attention and not cfg.device == "npu": + if attn_dp_enabled_of(cfg) and not cfg.device == "npu": raise ValueError( "Currently DFLASH speculative decoding does not support dp " "attention on non-NPU devices." @@ -538,7 +538,7 @@ def _handle_uno(server_args: ServerArgs) -> None: if (cfg.tp_size, cfg.pp_size) != (1, 1): raise ValueError("UNO requires TP=PP=1.") - if cfg.enable_dp_attention or cfg.attn_cp_size != 1: + if attn_dp_enabled_of(cfg) or cfg.attn_cp_size != 1: raise ValueError("UNO does not support DP attention or context parallelism.") if cfg.enable_lora or cfg.lora_paths: raise ValueError("UNO does not support public Multi-LoRA serving.") @@ -588,8 +588,7 @@ def _handle_dspark(server_args: ServerArgs) -> None: "DSpark speculative decoding only supports CUDA or NPU device." ) - # dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks. - if cfg.enable_dp_attention and cfg.dp_size > 1: + if cfg.attn_dp_size > 1: if not cfg.enable_dp_lm_head: raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.") if not _is_npu and cfg.moe_a2a_backend not in ("none", "megamoe", "mori"): @@ -963,9 +962,8 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool: def _handle_eagle_family(server_args: ServerArgs) -> None: cfg = resolving_view(server_args) - if ( - cfg.speculative_algorithm == "STANDALONE" - and resolved_view(server_args).enable_dp_attention + if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of( + resolved_view(server_args) ): # TODO: support dp attention for standalone speculative decoding raise ValueError( @@ -1254,7 +1252,7 @@ def _handle_ngram(server_args: ServerArgs) -> None: "and produces incorrect results for paged attention backends. " "This combination is only supported for the 'flashinfer' backend." ) - if view.enable_dp_attention: + if attn_dp_enabled_of(view): # TODO: support dp attention for ngram speculative decoding raise ValueError( "Currently ngram speculative decoding does not support dp attention." diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index 388a618675..582329a877 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -18,7 +18,7 @@ from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import parse_ib_device_config, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of from sglang.srt.utils.common import torch_release from sglang.srt.utils.runai_utils import is_runai_obj_uri @@ -72,8 +72,9 @@ def check_pipeline_parallel_compat(cfg: Any) -> None: # Every stage rebuilds the same verify input from the relayed # per-request state, so all stages must see the same batch. # DP attention partitions it per DP rank. - assert not cfg.enable_dp_attention, ( - "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention" + assert not attn_dp_enabled_of(cfg), ( + "SGLANG_ENABLE_PP_SPEC is not compatible with attention DP " + "(--attn-dp-size)" ) else: assert cfg.disaggregation_mode == "prefill", ( @@ -114,8 +115,9 @@ def check_server_args(server_args: Any): if cfg.pp_size > 1: check_pipeline_parallel_compat(cfg) - assert not (cfg.dp_size > 1 and cfg.nnodes != 1 and not cfg.enable_dp_attention), ( - "multi-node data parallel is not supported unless dp attention!" + assert not (cfg.dp_size > 1 and cfg.nnodes != 1), ( + "multi-node data-parallel replicas are not supported; use attention DP " + "(--attn-dp-size) across nodes" ) assert cfg.base_gpu_id >= 0, "base_gpu_id must be non-negative" @@ -375,7 +377,7 @@ def check_load_publish_args(server_args: Any): _, reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, - dp_size=server_cfg.dp_size, + dp_size=num_dp_ranks_of(server_cfg), load_publish_endpoint=mode, ) if reason: @@ -528,10 +530,10 @@ def check_two_batch_overlap(server_args: Any): if ( cfg.enable_two_batch_overlap and cfg.moe_a2a_backend == "none" - and not cfg.enable_dp_attention + and not attn_dp_enabled_of(cfg) ): raise ValueError( "When enabling two batch overlap without an EP a2a backend " - "(moe_a2a_backend='none'), --enable-dp-attention is required " + "(moe_a2a_backend='none'), attention DP (--attn-dp-size) is required " "(DeepSeek-V4 non-EP DP TBO path)." ) diff --git a/python/sglang/srt/debug_utils/dumper.py b/python/sglang/srt/debug_utils/dumper.py index 6bffc95e6e..094a4614da 100644 --- a/python/sglang/srt/debug_utils/dumper.py +++ b/python/sglang/srt/debug_utils/dumper.py @@ -1744,7 +1744,7 @@ class _SGLangPlugin(_FrameworkPlugin): try: parallel = get_parallel() - info["enable_dp_attention"] = self._dp_attn.is_dp_attention_enabled() + info["attn_dp_enabled"] = self._dp_attn.is_dp_attention_enabled() info["attn_tp_rank"] = parallel.attn_tp_rank info["attn_tp_size"] = parallel.attn_tp_size info["attn_dp_rank"] = parallel.attn_dp_rank diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index e56796d5c0..7ff8b3997a 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -200,9 +200,7 @@ class CommonKVManager(BaseKVManager): self.dcp_rank = parallel.attn_dcp_rank self.attn_dp_size = parallel.attn_dp_size self.attn_dp_rank = parallel.attn_dp_rank - self.system_dp_size = ( - 1 if get_parallel().enable_dp_attention else get_parallel().dp_size - ) + self.system_dp_size = get_parallel().dp_size self.system_dp_rank = ( self.kv_args.system_dp_rank if self.kv_args.system_dp_rank else 0 ) @@ -1545,11 +1543,12 @@ class CommonKVSender(BaseKVSender): return self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) - if get_parallel().dp_size > 1 and not req_has_disagg_prefill_dp_rank: + if get_parallel().num_dp_ranks > 1 and not req_has_disagg_prefill_dp_rank: if get_parallel().load_balance_method != "follow_bootstrap_room": self._register_prefill_dp_rank() elif ( - self.kv_mgr.attn_dp_rank != self.bootstrap_room % get_parallel().dp_size + self.kv_mgr.attn_dp_rank + != self.bootstrap_room % get_parallel().num_dp_ranks ): # follow_bootstrap_room was overridden by external routed_dp_rank if envs.SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK.get(): @@ -1560,7 +1559,7 @@ class CommonKVSender(BaseKVSender): f"follow_bootstrap_room conflict: dispatched to dp_rank " f"{self.kv_mgr.attn_dp_rank} but bootstrap_room " f"{self.bootstrap_room} implies dp_rank " - f"{self.bootstrap_room % get_parallel().dp_size}. " + f"{self.bootstrap_room % get_parallel().num_dp_ranks}. " f"Set SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 " f"to allow mixed routing.", ) diff --git a/python/sglang/srt/disaggregation/encoder/http_server.py b/python/sglang/srt/disaggregation/encoder/http_server.py index f38f881b61..29053ef0c4 100644 --- a/python/sglang/srt/disaggregation/encoder/http_server.py +++ b/python/sglang/srt/disaggregation/encoder/http_server.py @@ -209,7 +209,7 @@ def launch_server(server_args: ServerArgs): configure_logger(server_args, prefix=" encode_server") publish(server_args, role="encoder") - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: dp_dispatcher = launch_dp_runtime(server_args) # runtime initializes multiprocess metrics before spawning; # HTTP only exposes their endpoint. diff --git a/python/sglang/srt/disaggregation/encoder/runtime.py b/python/sglang/srt/disaggregation/encoder/runtime.py index 3fab75f432..a45e240523 100644 --- a/python/sglang/srt/disaggregation/encoder/runtime.py +++ b/python/sglang/srt/disaggregation/encoder/runtime.py @@ -1744,10 +1744,10 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime: This function owns backend construction only. HTTP/gRPC middleware, service registration, and network serving remain Transport concerns. """ - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: raise ValueError( - "launch_local_runtime requires --dp-size 1; got " - f"dp_size={get_parallel().dp_size}." + "launch_local_runtime requires a single DP rank; got " + f"num_dp_ranks={get_parallel().num_dp_ranks}." ) # Set up prometheus metrics. @@ -1827,8 +1827,8 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: "Encoder DP mode requires --dp-size > 1 and --tp-size 1; got " f"dp_size={get_parallel().dp_size}, tp_size={get_parallel().tp_size}." ) - dp_size = get_parallel().dp_size - logger.info(f"Launching encoder in DP mode: dp_size={dp_size}") + num_dp_ranks = get_parallel().num_dp_ranks + logger.info(f"Launching encoder in DP mode: num_dp_ranks={num_dp_ranks}") # DP mode: workers (subprocesses) write metrics to the shared multiproc dir; # the main process exposes the aggregated /metrics endpoint. @@ -1837,7 +1837,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: ctx = mp.get_context("spawn") ipc_prefix = random_uuid() - async_zmq_ctx = zmq.asyncio.Context(dp_size + 1) + async_zmq_ctx = zmq.asyncio.Context(num_dp_ranks + 1) result_path = f"ipc:///tmp/{ipc_prefix}_dp_result" result_socket = get_zmq_socket(async_zmq_ctx, zmq.PULL, result_path, True) @@ -1845,13 +1845,13 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: get_zmq_socket( async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_dispatch_{r}", True ) - for r in range(dp_size) + for r in range(num_dp_ranks) ] release_sockets: List[zmq.asyncio.Socket] = [ get_zmq_socket( async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_release_{r}", True ) - for r in range(dp_size) + for r in range(num_dp_ranks) ] worker_processes: List[mp.Process] = [] @@ -1867,7 +1867,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: # exception (atexit holds the list ref and reads it at exit time). atexit.register(_kill_workers) - for dp_rank in range(dp_size): + for dp_rank in range(num_dp_ranks): gpu_id = get_device().base_gpu_id + dp_rank # Pin the device parent-side around spawn (same convention as the # scheduler launcher and DP controller) so the child inherits @@ -1894,7 +1894,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: if get_observability().extra_metric_labels: labels.update(get_observability().extra_metric_labels) return DPDispatcher( - dp_size, + num_dp_ranks, dispatch_sockets, release_sockets, result_socket, diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index 61e0539695..2f025b88ec 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -79,14 +79,11 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]": from sglang.srt.utils import numa_utils parallel = get_parallel() - # With --enable-dp-attention, dp partitions the existing TP group rather - # than spawning additional processes, so dp_size must not be multiplied - # into the process count here (unlike regular DP, where dp_size * tp_size * - # pp_size is the true worker count). - dp_size = 1 if parallel.enable_dp_attention else parallel.dp_size + # Attention DP partitions the existing TP group rather than spawning + # additional processes, so only the replicas multiply the process count. return numa_utils.init_threads_binding( numa_index=get_device().gpu_id, - world_size=dp_size * parallel.tp_size * parallel.pp_size, + world_size=parallel.dp_size * parallel.tp_size * parallel.pp_size, ) diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index af49cbe0cb..d167a2d93d 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -394,15 +394,15 @@ class Engine(EngineScoreMixin, EngineBase): routed_dp_rank = data_parallel_rank if routed_dp_rank is not None: - dp_size = get_parallel().dp_size - if dp_size <= 1 and routed_dp_rank == 0: + num_dp_ranks = get_parallel().num_dp_ranks + if num_dp_ranks <= 1 and routed_dp_rank == 0: logger.debug( - f"routed_dp_rank={routed_dp_rank} is ignored because dp_size={dp_size}" + f"routed_dp_rank={routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}" ) return None - if routed_dp_rank < 0 or routed_dp_rank >= dp_size: + if routed_dp_rank < 0 or routed_dp_rank >= num_dp_ranks: raise ValueError( - f"routed_dp_rank={routed_dp_rank} out of range [0, {dp_size})" + f"routed_dp_rank={routed_dp_rank} out of range [0, {num_dp_ranks})" ) logger.debug(f"routed_dp_rank: {routed_dp_rank}") @@ -886,7 +886,7 @@ class Engine(EngineScoreMixin, EngineBase): """ scheduler_procs = [] use_dp_controller = ( - get_parallel().dp_size > 1 or get_exec().moe.ep_join_mode == "scale" + get_parallel().num_dp_ranks > 1 or get_exec().moe.ep_join_mode == "scale" ) if not use_dp_controller: diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 3258cfb9e9..6dc6e215de 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -2249,7 +2249,7 @@ async def _send_disaggregation_warmup_requests( return await asyncio.gather( *( send_request(session, dp_rank) - for dp_rank in range(get_parallel().dp_size) + for dp_rank in range(get_parallel().num_dp_ranks) ) ) @@ -2313,9 +2313,11 @@ def _execute_server_warmup(server_args: ServerArgs): }, } if get_serving().skip_tokenizer_init: - json_data["input_ids"] = [[10, 11, 12] for _ in range(get_parallel().dp_size)] + json_data["input_ids"] = [ + [10, 11, 12] for _ in range(get_parallel().num_dp_ranks) + ] # TODO Workaround the bug that embedding errors for list of size 1 - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: json_data["input_ids"] = json_data["input_ids"][0] elif ( is_vlm @@ -2359,9 +2361,11 @@ def _execute_server_warmup(server_args: ServerArgs): "temperature": 0.0, } else: - json_data["text"] = ["The capital city of France is"] * get_parallel().dp_size + json_data["text"] = [ + "The capital city of France is" + ] * get_parallel().num_dp_ranks # TODO Workaround the bug that embedding errors for list of size 1 - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: json_data["text"] = json_data["text"][0] # Config debug dumping @@ -2402,7 +2406,7 @@ def _execute_server_warmup(server_args: ServerArgs): if not failed_status_codes: logger.info( "Disaggregation warmup requests completed for all %s DP ranks", - get_parallel().dp_size, + get_parallel().num_dp_ranks, ) logger.info("End of disaggregation warmup") else: diff --git a/python/sglang/srt/entrypoints/v1_loads.py b/python/sglang/srt/entrypoints/v1_loads.py index 1d466e4320..c105f310d3 100644 --- a/python/sglang/srt/entrypoints/v1_loads.py +++ b/python/sglang/srt/entrypoints/v1_loads.py @@ -49,8 +49,8 @@ def _num_accelerators_per_dp_rank() -> int: """ parallel = get_parallel() num_accelerators = parallel.tp_size * parallel.pp_size - if parallel.enable_dp_attention: - num_accelerators //= parallel.dp_size + if parallel.attn_dp_enabled: + num_accelerators //= parallel.num_dp_ranks return num_accelerators diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py index 7ea16d9e2b..abbe52d509 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py @@ -283,7 +283,7 @@ class NPUGraphRunner(DecodeCudaGraphRunner): # Refresh the static DP token buffers bound by the captured # graph (stale values misalign dp-gather segments across # ranks); mirror the capture-side uniform - # [padded_num_tokens] * dp_size. + # [padded_num_tokens] * num_dp_ranks. if self.require_mlp_tp_gather: _padded_num_tokens = bs * self.captured_req_width self.buffers.global_num_tokens_gpu.fill_(_padded_num_tokens) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index a9ae9476b5..349d1cf5ab 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -525,7 +525,7 @@ class AiterAttnBackend(AttentionBackend): 16 if _gathered_num_head < 16 else _gathered_num_head ) - self.enable_dp_attention = is_dp_attention_enabled() + self.attn_dp_enabled = is_dp_attention_enabled() self.qo_indptr_ = torch.zeros( (max_bs + 1,), dtype=torch.int32, device=model_runner.device ) @@ -543,7 +543,7 @@ class AiterAttnBackend(AttentionBackend): # need to fall back to non-persist # only use mla_ps_kernel when fp8 kv_cache # for non-fp8 kv_cache on tp8, use non-persist kernel to avoid performance degradation - # head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or --enable-dp-attention with tp8-dp8) + # head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or tp8 with --attn-dp-size 8) # Native 16-head persist is slow on TP8; keep disabled unless zero-pad # (e.g. Kimi K3 h12 -> qh16) where persist ASM is the fast path. if ( @@ -628,7 +628,7 @@ class AiterAttnBackend(AttentionBackend): nhead = self.mla_kernel_num_head_padded dtype = self.kv_cache_dtype - if self.enable_dp_attention: + if self.attn_dp_enabled: gpu = torch.cuda.current_device() device_properties = torch.cuda.get_device_properties(gpu) cu_num = device_properties.multi_processor_count diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 64562b9de0..a8fceedfcf 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -63,7 +63,7 @@ from sglang.kernels.ops.attention.flash_attention import ( def _should_disable_scheduler_metadata_precompute() -> bool: - return bool(get_parallel().enable_prefill_cp or get_parallel().enable_dp_attention) + return bool(get_parallel().enable_prefill_cp or get_parallel().attn_dp_enabled) @dataclass diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index bf6c1e8f67..521f0d964a 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -23,7 +23,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( from sglang.srt.environ import envs from sglang.srt.runtime_context import ( derive_attention_ranks, - derive_attention_widths, + derive_attn_tp_size, get_device, get_exec, get_flags, @@ -51,13 +51,7 @@ def deployment_attn_dp_size() -> int: the target's replicas. """ parallel = get_parallel() - attn_dp_size, _ = derive_attention_widths( - tp_size=parallel.tp_size, - attn_cp_size=parallel.attn_cp_size, - dp_size=parallel.dp_size, - enable_dp_attention=parallel.enable_dp_attention, - ) - return attn_dp_size + return parallel.num_dp_ranks if parallel.attn_dp_enabled else 1 def dp_gather_width() -> int: @@ -69,7 +63,7 @@ def dp_gather_width() -> int: """ parallel = get_parallel() if world_dp_gather_enabled(): - return parallel.dp_size + return parallel.num_dp_ranks if get_flags().dp.scoped_gather_slot is not None: return deployment_attn_dp_size() return parallel.attn_dp_size @@ -127,7 +121,7 @@ def update_dp_attention_post_scale(new_dp_size: int, new_dp_rank: int): """ get_flags().dp.use_world_group_for_gather = True logger.debug( - "[Elastic EP] dp_attention switched to WORLD: dp_size=%d dp_rank=%d", + "[Elastic EP] dp_attention switched to WORLD: num_dp_ranks=%d dp_rank=%d", new_dp_size, new_dp_rank, ) @@ -154,7 +148,7 @@ class DpPaddingMode(IntEnum): def get_dp_padding_mode( cls, is_extend_in_batch, global_num_tokens: List[int] ) -> DpPaddingMode: - dp_size = dp_gather_width() + gather_width = dp_gather_width() # (trangdough) pplx-kernels a2a is a symmetric collective: every EP rank # must dispatch the same number of tokens or the device-side handshake @@ -169,11 +163,11 @@ class DpPaddingMode(IntEnum): if moe_a2a_backend.is_deepep_v2() and envs.SGLANG_DEEPEP_V2_FORCE_MAX_LEN.get(): return DpPaddingMode.MAX_LEN - # When is_extend_in_batch and dp_size > 1, use SUM_LEN to avoid padding - # overhead from uneven token distribution. - # For dp_size=1, max_len equals sum_len, so prefer MAX_LEN mode + # When is_extend_in_batch and the gather spans several DP ranks, use + # SUM_LEN to avoid padding overhead from uneven token distribution. + # Over one rank, max_len equals sum_len, so prefer MAX_LEN mode # to enable symmetric memory optimization (needed for DSA CP, etc.). - if is_extend_in_batch and dp_size > 1: + if is_extend_in_batch and gather_width > 1: # Hybrid-SSM models materialize idle ranks via the MAX_LEN # fabricated-row conversion; other models keep mainline SUM_LEN. if get_flags().dp.max_len_with_idle and min(global_num_tokens) == 0: @@ -184,7 +178,7 @@ class DpPaddingMode(IntEnum): # prefer MAX_LEN when communication cost is equal to enable symmetric memory max_len = max(global_num_tokens) sum_len = sum(global_num_tokens) - if sum_len * 2 >= max_len * dp_size: + if sum_len * 2 >= max_len * gather_width: return cls.MAX_LEN else: return cls.SUM_LEN @@ -405,25 +399,19 @@ def is_dp_max_padding() -> bool: def compute_dp_attention_world_info( - enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size: int = 1 + tp_rank, tp_size, attn_dp_size, attn_cp_size: int = 1 ): """This rank's place in the attention topology, plus the widths it sits in. - The widths come from `derive_attention_widths`; what this adds is the two - ranks, which are per-process and so are not among the widths + The attention-TP width comes from `derive_attn_tp_size`; what this adds is + the two ranks, which are per-process and so are not among the widths `override_permanently` records. """ - attn_dp_size, attn_tp_size = derive_attention_widths( - tp_size=tp_size, - attn_cp_size=attn_cp_size, - dp_size=dp_size, - enable_dp_attention=enable_dp_attention, + attn_tp_size = derive_attn_tp_size( + tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size ) attn_tp_rank, attn_dp_rank = derive_attention_ranks( - tp_rank=tp_rank, - attn_tp_size=attn_tp_size, - attn_cp_size=attn_cp_size, - enable_dp_attention=enable_dp_attention, + tp_rank=tp_rank, attn_tp_size=attn_tp_size, attn_cp_size=attn_cp_size ) return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size @@ -431,7 +419,7 @@ def compute_dp_attention_world_info( def initialize_dp_attention_flags(server_args: ServerArgs): """Initialize DP runtime flags without changing the worker's placement.""" dp = get_flags().dp - dp.enabled = get_parallel().enable_dp_attention + dp.enabled = get_parallel().attn_dp_enabled if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size: if ep_scale_joiner_of(resolving_view(server_args)): @@ -443,10 +431,9 @@ def initialize_dp_attention(server_args: ServerArgs): initialize_dp_attention_flags(server_args) parallel = get_parallel() _, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info( - parallel.enable_dp_attention, parallel.tp_rank, parallel.tp_size, - parallel.dp_size, + parallel.attn_dp_size, parallel.attn_cp_size, ) parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank) @@ -652,7 +639,7 @@ def _dp_gather_via_all_gather( # Variable-length DP-MoE gather (reference https://github.com/ROCm/ATOM/pull/930): instead of padding every # rank to max_len (all_gather) or all-reducing a sum_len zero-buffer (all_reduce), # gather exactly sum(per-rank tokens) via all_gatherv. Env-gated; only the simple -# tp_size==dp_size (attn_tp_size==1) case is supported for now (e.g. tp8dp8). +# tp_size==attn_dp_size (attn_tp_size==1) case is supported for now (e.g. tp8, attn_dp8). _USE_DP_GATHERV = get_bool_env_var("SGLANG_DP_USE_GATHERV") _DP_GATHER_FP8_GROUP = 128 @@ -725,7 +712,7 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None: """Set MAX_LEN pad rows' (post-translation, local) topk_ids to -1 in place. Under dp-attention MAX_LEN padding the gathered MoE buffer is - [dp_size * max_len, hidden] with rank r's real rows at + [num_dp_ranks * max_len, hidden] with rank r's real rows at [r*max_len, r*max_len + global_num_tokens[r]); the pad rows carry stale hidden values, run the router, and get dispatched into experts whose outputs are then discarded by the post-reorder scatter — pure wasted @@ -794,7 +781,7 @@ def is_dp_gatherv_active() -> bool: """Variable-length DP-MoE gather/scatter (all_gatherv + reduce_scatterv) is enabled and applicable to the CURRENT forward. Requires: - env SGLANG_DP_USE_GATHERV (default off), - - supported layout (attn_tp_size==1, tp_size==dp_size), + - supported layout (attn_tp_size==1, tp_size==attn_dp_size), - SUM_LEN padding mode. The gatherv pair (all_gatherv + reduce_scatterv) is only valid under SUM_LEN; under MAX_LEN the buffer is equal-padded and the gather/combine use all_gather / (aiter) reduce_scatter instead. Reading the @@ -1004,7 +991,7 @@ def can_use_dp_reduce_scatter() -> bool: return True parallel = get_parallel() - return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size + return parallel.tp_size == parallel.num_dp_ranks * parallel.attn_tp_size def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 1b9f0e6958..334b64b7fb 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -175,7 +175,7 @@ class StagePlan: self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) - self._publish_lora_layout = get_parallel().enable_dp_attention and bool( + self._publish_lora_layout = get_parallel().attn_dp_enabled and bool( get_lora().enable_lora ) self._next_input_rows = None diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 39f14bc9f9..0d33b54d34 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -84,7 +84,7 @@ _UNQUANTIZED_LM_HEAD_METHODS = { # None outside a FlashInfer autotune pass; inside one, whether that pass runs the # LM head. Not-None means the forward's output is discarded -- attention backends # read that via get_in_autotune_dummy_run() to skip a cross-node exchange. -# Skipping the LM head skips its [batch * dp_size, vocab] all-gather, which OOMs +# Skipping the LM head skips its [batch * num_dp_ranks, vocab] all-gather, which OOMs # under DP attention with a tight mem_fraction_static. _autotune_run_lm_head: Optional[bool] = None @@ -192,7 +192,7 @@ def should_apply_lm_head_quant_method(lm_head, quant_method) -> bool: # FlashInfer autotune skips the unprofiled LM-head all-gather; its -# [batch * dp_size, vocab] output can OOM under tight DP-attention memory. +# [batch * num_dp_ranks, vocab] output can OOM under tight DP-attention memory. _in_autotune_dummy_run = False diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index efdb3cc360..ff312750af 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -340,9 +340,9 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: # A2A path: bounded by the dispatcher's own workspace limit. max_num_tokens = dispatcher.max_num_tokens * getattr(dispatcher, "ep_size", 1) else: - # Standard allgather path: the MoE sees up to dp_size local forwards - # gathered together, so scale the per-rank forward bound by dp_size. - max_num_tokens = get_parallel().dp_size * cutedsl_moe_max_num_tokens() + # Standard allgather path: the MoE sees up to num_dp_ranks local forwards + # gathered together, so scale the per-rank forward bound by num_dp_ranks. + max_num_tokens = get_parallel().num_dp_ranks * cutedsl_moe_max_num_tokens() top_k = layer.top_k if layer.top_k is not None else layer.moe_runner_config.top_k # inference_mode(False) ensures the wrapper's pre-allocated CUDA-graph # buffers are normal tensors. This call typically happens inside diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index dae118d808..4ef61510f9 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -99,9 +99,9 @@ class LoRAManager: self.enable_lora_overlap_loading: Optional[bool] = ( get_lora().enable_lora_overlap_loading ) - self.enable_dp_attention: bool = get_parallel().enable_dp_attention + self.attn_dp_enabled: bool = get_parallel().attn_dp_enabled if ( - self.enable_dp_attention + self.attn_dp_enabled and self.enable_lora_overlap_loading and not LoRAMemoryPool.supports_dp_attention_overlap_loading ): @@ -112,11 +112,11 @@ class LoRAManager: ) # LoRA routing knows only DP-local and TP-global token layouts. A wider # attention TP/CP group can scatter one DP rank's tokens across ranks. - if self.enable_dp_attention and ( + if self.attn_dp_enabled and ( self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1 ): raise ValueError( - "LoRA with DP attention requires --dp-size equal to --tp-size " + "LoRA with DP attention requires --attn-dp-size equal to --tp-size " f"(got attention TP size {self.attn_tp_size} and attention CP " f"size {get_parallel().attn_cp_size})" ) @@ -138,7 +138,7 @@ class LoRAManager: device=self.device, server_args=server_args, ) - if self.enable_dp_attention and not self.lora_backend.supports_dp_attention: + if self.attn_dp_enabled and not self.lora_backend.supports_dp_attention: raise ValueError( f"LoRA backend {lora_backend!r} does not support DP attention" ) @@ -164,9 +164,9 @@ class LoRAManager: num_tokens_per_req=num_tokens_per_req, ) max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req - if self.enable_dp_attention: + if self.attn_dp_enabled: self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens) - max_moe_tokens *= get_parallel().dp_size + max_moe_tokens *= get_parallel().num_dp_ranks if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens): logger.info( "Right-sized shared MoE LoRA CUDA graph buffers " @@ -211,10 +211,7 @@ class LoRAManager: check_cuda_graph_backend, ) - if ( - self.enable_dp_attention - or not self.lora_backend.supports_prefill_cuda_graph - ): + if self.attn_dp_enabled or not self.lora_backend.supports_prefill_cuda_graph: return False if self.lora_backend.is_moe_lora: return check_cuda_graph_backend( @@ -237,7 +234,7 @@ class LoRAManager: return False # DP attention: per-rank eligibility could diverge across ranks and # desync collectives; keep LoRA prefill eager. - if self.enable_dp_attention: + if self.attn_dp_enabled: return False # Decode-CUDA-graph extend modes (TARGET_VERIFY, DLLM_EXTEND) are # owned by the decode static batch info path. @@ -333,7 +330,7 @@ class LoRAManager: """ Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible. """ - assert not self.enable_dp_attention or lora_ref.pinned, ( + assert not self.attn_dp_enabled or lora_ref.pinned, ( "DP-attention LoRA requires pinned adapters" ) if lora_config.lora_added_tokens_size > 0: @@ -506,7 +503,7 @@ class LoRAManager: ) active_lora_ids = set(forward_batch.lora_ids) - if self.enable_dp_attention: + if self.attn_dp_enabled: get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) gathered_lora_ids = get_parallel().tp_group.all_gather_object( forward_batch.lora_ids @@ -551,7 +548,7 @@ class LoRAManager: self.lora_backend.batch_info.has_active_lora = any( lora_ranks[wi] > 0 for wi in weight_indices ) - if self.enable_dp_attention: + if self.attn_dp_enabled: self.lora_backend.prepare_global_lora_batch(forward_batch) if forward_batch.forward_mode.is_idle(): # Idle ranks must join the global routing collectives, but @@ -563,9 +560,7 @@ class LoRAManager: hasattr(self, "max_bs_in_cuda_graph") and forward_batch.batch_size <= self.max_bs_in_cuda_graph and forward_batch.forward_mode.is_cuda_graph() - and ( - not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph - ) + and (not self.attn_dp_enabled or forward_batch.can_run_decode_cuda_graph) ) def prepare_lora_token_segments( @@ -1194,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers( # num_draft_tokens per request, and the buffers below are per-token, so # they must be sized in tokens rather than requests. max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1) - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: max_tokens *= get_parallel().attn_dp_size max_loras = get_lora().max_loras_per_batch for module in model.modules(): diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py index 506a469c34..7a3e820005 100644 --- a/python/sglang/srt/lora/mem_pool.py +++ b/python/sglang/srt/lora/mem_pool.py @@ -180,7 +180,7 @@ class LoRAMemoryPool: ) # Per-expert MoE weights are sharded by `moe_tp_size`, NOT the outer - # `tp_size`: `moe_tp_size = tp_size // ep_size // dp_size`, so under + # `tp_size`: `moe_tp_size = tp_size // ep_size // moe_dp_size`, so under # e.g. `--tp 4 --ep 4` each rank holds full-width expert weights # (`moe_tp_size == 1`). Sizing per-expert LoRA buffers by `tp_size` # here would yield a 4x-narrower inner dim than the adapter weight @@ -190,7 +190,7 @@ class LoRAMemoryPool: self.moe_tp_size, self.moe_tp_rank = _get_moe_tp_context() # Attention projections shard along the attention TP group, which - # under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size`. + # under attention DP is `attn_tp_size = tp_size // attn_dp_size`. # The corresponding LoRA wrappers slice weights by the base layer's # attn_tp-local rank, so the buffer shapes must match that shard. self.attn_tp_size: int = attn_tp_size @@ -273,7 +273,7 @@ class LoRAMemoryPool: """TP width the module's weights are actually sharded along: routed MoE experts shard by `moe_tp_size` (shared experts by the outer `tp_size` at EP=1), attention projections by `attn_tp_size` (smaller - than the outer `tp_size` under `--enable-dp-attention`), everything + than the outer `tp_size` under attention DP), everything else by the outer `tp_size`.""" if self.is_moe_module(module_name) and not self.is_shared_moe_module( module_name diff --git a/python/sglang/srt/lora/utils.py b/python/sglang/srt/lora/utils.py index f4310e1f00..9d21463ee5 100644 --- a/python/sglang/srt/lora/utils.py +++ b/python/sglang/srt/lora/utils.py @@ -373,7 +373,7 @@ REPLICATED_LINEAR_LORA_NAMES = [ *DSA_INDEXER_LORA_NAMES, ] # Attention-projection LoRA modules shard on the attention-TP group, which -# under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size` rather +# under attention DP is `attn_tp_size = tp_size // attn_dp_size` rather # than the outer TP size. in_proj / in_proj_qkvz (linear-attention hybrids) # belong here too: their layers are built on the attn-TP group (mamba.py and # qwen3_5.py take tp_size/tp_rank from attn_tp when dp attention is enabled). diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 8c1b940c9c..0396ab2c11 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -102,11 +102,11 @@ class LoadBalanceMethod(Enum): class DPBudget: - def __init__(self, dp_size: int): - self.dp_size = dp_size - self.total_requests = [0] * dp_size - self.total_tokens = [0] * dp_size - self.last_timestamp = [0.0] * dp_size + def __init__(self, num_dp_ranks: int): + self.num_dp_ranks = num_dp_ranks + self.total_requests = [0] * num_dp_ranks + self.total_tokens = [0] * num_dp_ranks + self.last_timestamp = [0.0] * num_dp_ranks def update_budget(self, loads): """Update budget from shm snapshots, skipping stale reads.""" @@ -125,7 +125,7 @@ class DPBudget: elif method == LoadBalanceMethod.TOTAL_TOKENS: # Use total_requests as a tie-breaker when total_tokens are equal target_rank = min( - range(self.dp_size), + range(self.num_dp_ranks), key=lambda i: (self.total_tokens[i], self.total_requests[i]), ) else: @@ -155,7 +155,7 @@ class DataParallelController: self.run_scheduler_process_func = run_scheduler_process_func # Init inter-process communication - self.context = zmq.Context(1 + get_parallel().dp_size) + self.context = zmq.Context(1 + get_parallel().num_dp_ranks) if get_parallel().node_rank == 0: self.recv_from_tokenizer = get_zmq_socket( self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False @@ -175,8 +175,10 @@ class DataParallelController: LoadBalanceMethod.TOTAL_TOKENS, ) - self.launch_dp_size: int = get_parallel().dp_size - self.max_dp_size: int = get_parallel().max_ep_size or get_parallel().dp_size + self.launch_dp_size: int = get_parallel().num_dp_ranks + self.max_dp_size: int = ( + get_parallel().max_ep_size or get_parallel().num_dp_ranks + ) assert self.max_dp_size >= self.launch_dp_size, ( f"--max-ep-size ({self.max_dp_size}) must be >= " f"--dp ({self.launch_dp_size})." @@ -186,7 +188,7 @@ class DataParallelController: self.max_dp_size - self.launch_dp_size ) - self.dp_budget = DPBudget(get_parallel().dp_size) + self.dp_budget = DPBudget(get_parallel().num_dp_ranks) self.load_snapshot_reader = create_load_snapshot_reader( port_args, caller="DataParallelController", @@ -203,7 +205,7 @@ class DataParallelController: self._active_workers: list[int] = list(range(self.launch_dp_size)) self._active_count_cache: int = self.launch_dp_size - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: self.launch_dp_attention_schedulers(server_args, port_args) # When local control broadcast is enabled, send control messages to # every DP group leader (attn_tp_rank=0) so each leader broadcasts @@ -374,7 +376,7 @@ class DataParallelController: threads = [] sockets = [] ready_events = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): tmp_port_args = PortArgs.init_new(server_args) tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name @@ -577,7 +579,7 @@ class DataParallelController: bind_count = ( self.max_dp_size if get_exec().moe.elastic_ep_backend is not None - else get_parallel().dp_size + else get_parallel().num_dp_ranks ) for slot in range(bind_count): worker_port, worker_socket = get_zmq_socket_on_host( @@ -607,7 +609,7 @@ class DataParallelController: dp_rank: int | None, worker_ports: list[int] | None = None, ): - if not get_parallel().enable_dp_attention: + if not get_parallel().attn_dp_enabled: logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.") memory_saver_adapter = TorchMemorySaverAdapter.create( @@ -639,13 +641,12 @@ class DataParallelController: for tp_rank in tp_rank_range: rank_port_args = port_args - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: # dp attention has different sharding logic _, _, dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, tp_rank, get_parallel().tp_size, - get_parallel().dp_size, + get_parallel().attn_dp_size, get_parallel().attn_cp_size, ) # compute zmq ports for this dp rank diff --git a/python/sglang/srt/managers/load_snapshot.py b/python/sglang/srt/managers/load_snapshot.py index 968f438f8d..7830129f08 100644 --- a/python/sglang/srt/managers/load_snapshot.py +++ b/python/sglang/srt/managers/load_snapshot.py @@ -71,7 +71,7 @@ def should_use_zmq() -> bool: ``SGLANG_LOAD_SNAPSHOT_USE_ZMQ`` forces zmq mode for testing. """ return ( - get_parallel().enable_dp_attention and get_parallel().nnodes > 1 + get_parallel().attn_dp_enabled and get_parallel().nnodes > 1 ) or envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.get() @@ -120,10 +120,10 @@ def zmq_reader_owner(caller: str) -> bool: return False if caller == "DataParallelController": return ( - get_parallel().dp_size > 1 + get_parallel().num_dp_ranks > 1 and get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS ) - if get_parallel().dp_size > 1 and ( + if get_parallel().num_dp_ranks > 1 and ( get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS ): return False @@ -659,9 +659,9 @@ def create_load_snapshot_reader(port_args, caller: str): ``"MultiTokenizerRouter"`` -- determines who binds the zmq PULL socket when zmq mode is active. """ - dp_size = get_parallel().dp_size + num_dp_ranks = get_parallel().num_dp_ranks if zmq_reader_owner(caller): return ZmqShmLoadSnapshotReader( - _zmq_addr_for(port_args), shm_path_for(port_args.instance_id), dp_size + _zmq_addr_for(port_args), shm_path_for(port_args.instance_id), num_dp_ranks ) - return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), dp_size) + return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), num_dp_ranks) diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index da93dbfaa0..f7d990574d 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -106,9 +106,9 @@ class PrefillDelayer: f"queue_trigger_enabled={self._queue_trigger_enabled}" ) parallel = get_parallel() - self.dp_size = parallel.dp_size - self.enable_dp_attention = parallel.enable_dp_attention - dp_size_dim = self.dp_size if self.enable_dp_attention else 1 + self.num_dp_ranks = parallel.num_dp_ranks + self.attn_dp_enabled = parallel.attn_dp_enabled + dp_size_dim = self.num_dp_ranks if self.attn_dp_enabled else 1 # Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the # env flag is on (or overlap scheduling is disabled), ride the NCCL @@ -238,10 +238,10 @@ class PrefillDelayer: **wait_info, ) - if not self.enable_dp_attention: + if not self.attn_dp_enabled: max_running_requests = ( - max_running_requests + self.dp_size - 1 - ) // self.dp_size + max_running_requests + self.num_dp_ranks - 1 + ) // self.num_dp_ranks global_running_batch_max = int(global_running_batch.max().item()) global_max_prefill_bs_max = int(global_max_prefill_bs.max().item()) @@ -411,10 +411,10 @@ class PrefillDelayerSinglePassExecutor: waiting_queue_len: int, ) -> int: local_max_running_requests = max_running_requests - if not self._prefill_delayer.enable_dp_attention: + if not self._prefill_delayer.attn_dp_enabled: local_max_running_requests = ( - max_running_requests + self._prefill_delayer.dp_size - 1 - ) // self._prefill_delayer.dp_size + max_running_requests + self._prefill_delayer.num_dp_ranks - 1 + ) // self._prefill_delayer.num_dp_ranks # The delayer negotiates before PrefillAdder materializes can_run_list, # so a rejected pass has no exact batch size. This upper bound is exact diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e5b4125f90..76813e8401 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -516,7 +516,7 @@ class Scheduler( self.max_recv_per_poll = envs.SGLANG_SCHEDULER_MAX_RECV_PER_POLL.get() self.max_new_tokens_limit = envs.SGLANG_MAX_NEW_TOKENS_LIMIT.get() self.enable_hisparse = get_memory().enable_hisparse - self.enable_dp_attention = get_parallel().enable_dp_attention + self.attn_dp_enabled = get_parallel().attn_dp_enabled self.enable_unified_memory = get_memory().enable_unified_memory # Init model configs @@ -629,7 +629,7 @@ class Scheduler( token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, tp_group=( self.attn_tp_cpu_group - if self.enable_dp_attention + if self.attn_dp_enabled else self.tp_cpu_group ), tree_cache=self.tree_cache, @@ -823,7 +823,7 @@ class Scheduler( try: self.load_snapshot_writer = create_load_snapshot_writer( port_args, - get_parallel().dp_size, + get_parallel().num_dp_ranks, dp_rank, publish_interval=get_observability().load_snapshot_publish_interval, ) @@ -1216,9 +1216,7 @@ class Scheduler( # the base TP group. Entry rank is the local rank 0 in that group. # Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA # stream/device coupling (#11910). - self.dp_tp_group = ( - self.attn_tp_group if self.enable_dp_attention else self.tp_group - ) + self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group self.dp_tp_cpu_group = self.dp_tp_group.cpu_group self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func() @@ -4646,9 +4644,7 @@ class Scheduler( return ret def _maybe_report_active_ranks(self) -> None: - if not ( - self.enable_dp_attention and get_exec().moe.elastic_ep_backend is not None - ): + if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None): return from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager @@ -5218,7 +5214,6 @@ class Scheduler( # readback reflects values changed via /set_internal_state, not startup. ret = get_context().resolved_server_args_dict() ret["world_size"] = compute_world_size( - enable_dp_attention=get_parallel().enable_dp_attention, dp_size=get_parallel().dp_size, tp_size=get_parallel().tp_size, pp_size=get_parallel().pp_size, diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index ea64aea930..a7c57c4292 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -65,47 +65,47 @@ def _spec_input_cuda_graph_compatible( return spec_info is None or spec_info.cuda_graph_compatible -def _resolve_elastic_world_dp_size( - dp_size: int, +def _resolve_elastic_world_num_dp_ranks( + num_dp_ranks: int, *, group: torch.distributed.ProcessGroup, local_num_tokens: int, local_forward_mode: int, ) -> int: if not world_dp_gather_enabled(): - return dp_size + return num_dp_ranks from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager - live_dp_size = dp_gather_width() + live_num_dp_ranks = dp_gather_width() effective_ep_size = ElasticEPStateManager.get_effective_ep_size() # Query live membership because elastic joins can expand WORLD. world_size = torch.distributed.get_world_size(group) - if live_dp_size != effective_ep_size: + if live_num_dp_ranks != effective_ep_size: raise RuntimeError( - "[Elastic EP] WORLD MLP sync dp_size is out of sync: " + "[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: " f"rank={torch.distributed.get_rank(group)} " - f"live_dp_size={live_dp_size} " + f"live_num_dp_ranks={live_num_dp_ranks} " f"effective_ep_size={effective_ep_size} " - f"world_size={world_size} server_args_dp_size={dp_size} " + f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} " f"local_num_tokens={local_num_tokens} " f"local_forward_mode={local_forward_mode}" ) - if live_dp_size > world_size: + if live_num_dp_ranks > world_size: raise RuntimeError( - "[Elastic EP] WORLD MLP sync dp_size exceeds WORLD size: " + "[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: " f"rank={torch.distributed.get_rank(group)} " - f"live_dp_size={live_dp_size} world_size={world_size} " + f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} " f"effective_ep_size={effective_ep_size}" ) - return live_dp_size + return live_num_dp_ranks @dataclass class MLPSyncBatchInfo: - dp_size: int + num_dp_ranks: int tp_size: int cp_size: int @@ -186,7 +186,7 @@ class MLPSyncBatchInfo: # is a no-op, and the masked fallback writes below would then read and # write the same storage. global_info_tensor = fallback_tensor.repeat( - self.dp_size, self.tp_size * self.cp_size, 1 + self.num_dp_ranks, self.tp_size * self.cp_size, 1 ) if use_all_reduce: @@ -211,7 +211,7 @@ class MLPSyncBatchInfo: ) tp_info = global_info_tensor.view( - self.dp_size * self.tp_size * self.cp_size, info_width + self.num_dp_ranks * self.tp_size * self.cp_size, info_width ) num_ranks_in_tp_info = tp_info.shape[0] if device == "cpu": @@ -296,7 +296,7 @@ def _update_gather_batch( ) -def should_skip_scheduler_all_gather(dp_size: int) -> bool: +def should_skip_scheduler_all_gather(num_dp_ranks: int) -> bool: """Return whether scheduler metadata is already local and rank-invariant. With one attention-DP rank there is no cross-DP state to reconcile. The @@ -307,7 +307,7 @@ def should_skip_scheduler_all_gather(dp_size: int) -> bool: DP1. """ - return dp_size == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get() + return num_dp_ranks == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get() def _local_decode_cuda_graph_vote( @@ -421,7 +421,7 @@ def prepare_mlp_sync_batch_raw( dwdp: bool = False, ): parallel = get_parallel() - dp_size = parallel.dp_size + num_dp_ranks = parallel.num_dp_ranks attn_tp_size = parallel.attn_tp_size tp_group = parallel.tp_group # Check if other DP workers have running batches @@ -498,16 +498,16 @@ def prepare_mlp_sync_batch_raw( local_can_run_tbo, local_forward_mode = tbo_preparer.prepare_all_gather(local_batch) if use_world_group: - dp_size = _resolve_elastic_world_dp_size( - dp_size, + num_dp_ranks = _resolve_elastic_world_num_dp_ranks( + num_dp_ranks, group=group, local_num_tokens=num_tokens, local_forward_mode=local_forward_mode, ) - skip_all_gather = should_skip_scheduler_all_gather(dp_size) + skip_all_gather = should_skip_scheduler_all_gather(num_dp_ranks) mlp_sync_info = MLPSyncBatchInfo( - dp_size=dp_size, + num_dp_ranks=num_dp_ranks, tp_size=attn_tp_size, cp_size=parallel.attn_cp_size, num_tokens=num_tokens, @@ -521,7 +521,7 @@ def prepare_mlp_sync_batch_raw( prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len, ) - if dp_size == 1: + if num_dp_ranks == 1: mlp_sync_info.finalize_local() elif not skip_all_gather: mlp_sync_info.all_gather( @@ -540,7 +540,7 @@ def prepare_mlp_sync_batch_raw( # Decide whether to emit idle batch if skip_all_gather: # Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently) - need_idle_batch = not dwdp and dp_size > 1 + need_idle_batch = not dwdp and num_dp_ranks > 1 else: need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0 diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index 318675488d..ea373c12a5 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -191,8 +191,7 @@ class DynamicChunkSizer: if is_dp_attention_enabled(): # Profiling runs one request on this rank; other DP ranks report 0. - dp_size = get_parallel().attn_dp_size - global_num_tokens = [0] * dp_size + global_num_tokens = [0] * get_parallel().attn_dp_size dp_rank = get_parallel().attn_dp_rank global_num_tokens[dp_rank] = current_seq_len batch.global_num_tokens = global_num_tokens diff --git a/python/sglang/srt/managers/scheduler_components/load_publisher.py b/python/sglang/srt/managers/scheduler_components/load_publisher.py index 50a526a79e..bf3d347183 100644 --- a/python/sglang/srt/managers/scheduler_components/load_publisher.py +++ b/python/sglang/srt/managers/scheduler_components/load_publisher.py @@ -19,7 +19,7 @@ an explicit address); off by default so an upgrade never reserves a port a co-hosted neighbor's KV publisher binds. The port comes from `resolve_load_pub_range` (the same function `/server_info` advertises with, so the two cannot drift). With `auto`, a worker's ZMQ footprint is -`2 * dp_size` ports after its KV base (`2 * dp_size + 1` with the +`2 * num_dp_ranks` ports after its KV base (`2 * num_dp_ranks + 1` with the conventional adjacent replay), so co-hosted workers must space their KV bases that far apart or move the range with an explicit address. """ @@ -164,7 +164,7 @@ class SchedulerLoadPublisher: resolved, reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().num_dp_ranks, load_publish_endpoint=load_publish_endpoint, ) if resolved is None: diff --git a/python/sglang/srt/managers/scheduler_components/profiler_manager.py b/python/sglang/srt/managers/scheduler_components/profiler_manager.py index 2791c3a4b4..27814e104f 100644 --- a/python/sglang/srt/managers/scheduler_components/profiler_manager.py +++ b/python/sglang/srt/managers/scheduler_components/profiler_manager.py @@ -285,7 +285,7 @@ class SchedulerProfilerManager: if get_parallel().tp_rank != 0: return "" - if get_parallel().dp_size > 1 and get_parallel().dp_rank != 0: + if get_parallel().num_dp_ranks > 1 and get_parallel().dp_rank != 0: return "" if get_parallel().pp_size > 1 and get_parallel().pp_rank != 0: return "" @@ -340,7 +340,7 @@ class SchedulerProfilerManager: filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"] # Only add other ranks if parallelism is enabled (size > 1) - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: filename_parts.append(f"DP-{get_parallel().dp_rank}") if get_parallel().pp_size > 1: filename_parts.append(f"PP-{get_parallel().pp_rank}") diff --git a/python/sglang/srt/managers/scheduler_components/recv_skipper.py b/python/sglang/srt/managers/scheduler_components/recv_skipper.py index 364723b6c2..546b56d7f7 100644 --- a/python/sglang/srt/managers/scheduler_components/recv_skipper.py +++ b/python/sglang/srt/managers/scheduler_components/recv_skipper.py @@ -34,7 +34,7 @@ class SchedulerRecvSkipper: return ForwardMode.DECODE def __init__(self): - self._use_synced_mode = get_parallel().enable_dp_attention + self._use_synced_mode = get_parallel().attn_dp_enabled self._counter = 0 self._threshold = get_schedule().scheduler_recv_interval # All can be tuned if needed diff --git a/python/sglang/srt/managers/scheduler_components/request_receiver.py b/python/sglang/srt/managers/scheduler_components/request_receiver.py index 007cdf0ede..f188b547d8 100644 --- a/python/sglang/srt/managers/scheduler_components/request_receiver.py +++ b/python/sglang/srt/managers/scheduler_components/request_receiver.py @@ -173,7 +173,7 @@ class SchedulerRequestReceiver: rank 0 and would overwrite every DP group's aborts but the first. """ local_reqs = local_reqs or [] - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: if get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0: work_reqs, control_reqs = self._split_work_and_control_reqs(recv_reqs) work_reqs.extend(local_reqs) @@ -266,7 +266,7 @@ class SchedulerRequestReceiver: # 1. wait until every rank has opened the shared feature segments parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: if parallel.attn_tp_size > 1: barrier(group=self.attn_tp_cpu_group) if parallel.attn_cp_size > 1: @@ -290,7 +290,7 @@ class SchedulerRequestReceiver: failed[index] = 1 # 3. all ranks reject the same requests before entering model collectives - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: if parallel.attn_tp_size > 1: all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group) if parallel.attn_cp_size > 1: diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index f5051295bd..f62afbae8e 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -186,7 +186,7 @@ class TokenizerControlMixin: mode = spec[2] if len(spec) > 2 else "queueing" comm = FanOutCommunicator( self._dispatch_to_scheduler, - get_parallel().dp_size, + get_parallel().num_dp_ranks, mode, ) setattr(self, f"{name}_communicator", comm) @@ -195,7 +195,7 @@ class TokenizerControlMixin: def update_control_communicator_fan_out(self: TokenizerManager, worker_count: int): primary_group_control = ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and not get_parallel().enable_dp_attention_local_control_broadcast ) if primary_group_control: @@ -444,8 +444,8 @@ class TokenizerControlMixin: request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from distributed" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from distributed" ) results = await self.init_weights_update_group_communicator(obj) @@ -457,8 +457,8 @@ class TokenizerControlMixin: request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for destroy parameter update group" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for destroy parameter update group" ) results = await self.destroy_weights_update_group_communicator(obj) @@ -502,8 +502,8 @@ class TokenizerControlMixin: request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from distributed" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from distributed" ) if obj.abort_all_requests: @@ -535,8 +535,8 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() # TODO: support DP - assert get_parallel().dp_size == 1, ( - "dp_size must be 1 for init_weights_send_group_for_remote_instance" + assert get_parallel().num_dp_ranks == 1, ( + "init_weights_send_group_for_remote_instance requires a single DP rank" ) result = ( await self.init_weights_send_group_for_remote_instance_communicator(obj) @@ -550,8 +550,8 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() # TODO: support DP - assert get_parallel().dp_size == 1, ( - "dp_size must be 1 for send_weights_to_remote_instance" + assert get_parallel().num_dp_ranks == 1, ( + "send_weights_to_remote_instance requires a single DP rank" ) result = (await self.send_weights_to_remote_instance_communicator(obj))[0] return result.success, result.message @@ -562,8 +562,8 @@ class TokenizerControlMixin: request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from tensor" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from tensor" ) if obj.abort_all_requests: @@ -600,8 +600,8 @@ class TokenizerControlMixin: self.auto_create_handle_loop() try: # For now, we only support single data parallel instance - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from IPC" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from IPC" ) logger.info("Starting IPC weight update") @@ -665,8 +665,8 @@ class TokenizerControlMixin: "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for dynamic lora loading" ) logger.info( "Start load Lora adapter. Lora name=%s, path=%s", @@ -751,8 +751,8 @@ class TokenizerControlMixin: "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for dynamic lora loading" ) logger.info( "Start load Lora adapter from tensors. Lora name=%s", @@ -841,8 +841,8 @@ class TokenizerControlMixin: "lora_name must be provided to unload LoRA adapter" ) - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for dynamic lora loading" ) logger.info( "Start unload Lora adapter. Lora name=%s", @@ -862,7 +862,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() results = await self.get_weights_by_name_communicator(obj) all_parameters = [r.parameter for r in results] - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: return all_parameters[0] else: return all_parameters diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 989b7a2faa..797e8a6180 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -488,7 +488,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): self.server_args = server_args assert_published(server_args, role="tokenizer") self.startup_time: Optional[Dict[str, Any]] = None - self.elastic_worker_count = get_parallel().dp_size + self.elastic_worker_count = get_parallel().num_dp_ranks self.elastic_pending_ep_size = None self.elastic_scale_phase = "idle" self.elastic_last_error = None @@ -872,14 +872,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None: - dp_size = self.elastic_worker_count - if dp_size <= 1 and obj.routed_dp_rank == 0: + num_dp_ranks = self.elastic_worker_count + if num_dp_ranks <= 1 and obj.routed_dp_rank == 0: logger.debug( - f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}" + f"routed_dp_rank={obj.routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}" ) - elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size: + elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= num_dp_ranks: raise ValueError( - f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {dp_size})" + f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {num_dp_ranks})" ) self._init_req_state(obj, request) @@ -1670,7 +1670,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return batch_size > 0 and ( get_serving().enable_tokenizer_batch_encode or ( - (not get_parallel().enable_dp_attention) + (not get_parallel().attn_dp_enabled) and (not self._batch_has_text(batch_size, requests)) ) ) diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index ec75dd81de..b03769326a 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -359,7 +359,7 @@ def build_kv_cache( ), is_eagle=spec_algorithm.is_eagle(), tp_cache_group=( - attn_tp_cpu_group if get_parallel().enable_dp_attention else tp_cpu_group + attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group ), attn_cp_cache_group=attn_cp_cpu_group, attn_tp_cache_group=attn_tp_cpu_group, diff --git a/python/sglang/srt/mem_cache/storage/flexkv/README.md b/python/sglang/srt/mem_cache/storage/flexkv/README.md index 23e0daa76f..11b1f326ee 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/README.md +++ b/python/sglang/srt/mem_cache/storage/flexkv/README.md @@ -299,7 +299,7 @@ Supported: `TransferManagerOnRemote` via the same ZMQ channel used for GPU registration. * **CP** (`attn_cp_size > 1`) — sync handled symmetrically with TP. -* **DP attention** (`enable_dp_attention=True`) — the inner +* **DP attention** (`--attn-dp-size` > 1) — the inner `attn_tp_size` is what FlexKV uses for register-side routing. --- diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py index 64141c4bf5..1cec682dd0 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py @@ -2,7 +2,7 @@ The public surface is small (see "Public API" below). The class owns: -* the FlexKV ``KVManager`` (server-client mode when ``dp_size > 1`` or +* the FlexKV ``KVManager`` (server-client mode when ``num_dp_ranks > 1`` or multi-instance; in-process otherwise — handled by FlexKV itself); * the per-rank ``KVTPClient`` that registers this rank's GPU KV cache with the FlexKV TransferManager; diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index 3f055ee49e..bbdfad9231 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -619,7 +619,7 @@ class CPUGraphRunner: self.speculative_algorithm = get_spec().speculative_algorithm self.enable_profile_cuda_graph = get_exec().graph.enable_profile_cuda_graph self.tp_size = get_parallel().tp_size - self.dp_size = get_parallel().dp_size + self.num_dp_ranks = get_parallel().num_dp_ranks self.pp_size = get_parallel().pp_size self.capture_forward_mode = ForwardMode.DECODE @@ -644,7 +644,7 @@ class CPUGraphRunner: "CPUGraphRunner does not support speculative inference yet." ) - assert self.dp_size == 1, "CPUGraphRunner does not support DP yet." + assert self.num_dp_ranks == 1, "CPUGraphRunner does not support DP yet." assert self.pp_size == 1, "CPUGraphRunner does not support PP yet." # Batch sizes to capture diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index f14aa7a9af..a461e6e625 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -530,7 +530,7 @@ def build_decode_registry( require_mlp_tp_gather: bool = False, # Per-bucket attn-TP sharded (SP) predicate; defaults to replicated. attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False, - dp_size: int = 1, + num_dp_ranks: int = 1, register_global_num_tokens: bool = True, share_pool: bool = True, source: Optional[Any] = None, @@ -700,7 +700,7 @@ def build_decode_registry( buf.fill_(ctx.padded_num_tokens) _global_shape = ( - (lambda _bs, _mt: (dp_size,)) + (lambda _bs, _mt: (num_dp_ranks,)) if require_mlp_tp_gather else (lambda _bs, _mt: (1,)) ) @@ -1048,7 +1048,7 @@ def build_eager_registry( register_global_num_tokens=False, require_gathered_buffer=False, require_mlp_tp_gather=False, - dp_size=get_parallel().dp_size, + num_dp_ranks=get_parallel().num_dp_ranks, share_pool=True, source=None, ) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 6733618299..2917e755b4 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1186,7 +1186,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Init lora information if ( model_runner.lora_manager is not None - and not model_runner.lora_manager.enable_dp_attention + and not model_runner.lora_manager.attn_dp_enabled ): # In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool # as a batch, right before running the batch diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index b66e633704..937cd160ae 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -530,7 +530,7 @@ class ModelRunner: new_dp_size=join_effective_ep_size, new_dp_rank=global_ep_rank, ) - get_context().override("elastic_ep.scale_join", dp_size=join_effective_ep_size) + get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size) if self.eplb_manager is not None: self.eplb_manager.disable_rebalance( "EPLB rebalance is disabled while elastic EP scale-up " @@ -965,7 +965,7 @@ class ModelRunner: device=self.device, tp_group=( get_parallel().attn_tp_group.cpu_group - if get_parallel().enable_dp_attention + if get_parallel().attn_dp_enabled else get_parallel().tp_group.cpu_group ), host_to_device_ratio=hisparse_cfg.host_to_device_ratio, @@ -1628,7 +1628,7 @@ class ModelRunner: forward_batch.prepare_mlp_sync_batch(self) else: forward_batch.prepare_attn_tp_scatter_input(self) - if self.lora_manager is not None and self.lora_manager.enable_dp_attention: + if self.lora_manager is not None and self.lora_manager.attn_dp_enabled: self.lora_manager.prepare_lora_batch(forward_batch) # Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is @@ -2217,7 +2217,7 @@ class ModelRunner: new_dp_size=target_size, new_dp_rank=self._elastic_global_rank(), ) - get_context().override("elastic_ep.scale", dp_size=target_size) + get_parallel().override_permanently(num_dp_ranks=target_size) recapture_cuda_graph = self._elastic_cuda_graph_enabled() if recapture_cuda_graph: diff --git a/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py b/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py index 856281b73a..8388da2403 100644 --- a/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py +++ b/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py @@ -82,7 +82,7 @@ class StartupWeightLoadOptions: attn_cp_size: int dcp_size: int pp_size: int - dp_size: int + num_dp_ranks: int ep_size: int cpu_offload_gb: int offload_group_size: int @@ -123,7 +123,7 @@ class StartupWeightLoadOptions: attn_cp_size=get_parallel().attn_cp_size, dcp_size=get_parallel().dcp_size, pp_size=get_parallel().pp_size, - dp_size=get_parallel().dp_size, + num_dp_ranks=get_parallel().num_dp_ranks, ep_size=get_parallel().ep_size, cpu_offload_gb=get_exec().offload.cpu_offload_gb, offload_group_size=get_exec().offload.offload_group_size, @@ -369,7 +369,7 @@ class StartupWeightLoadManager: "decode context parallelism is not supported", ), (options.pp_size != 1, "pipeline parallelism is not supported"), - (options.dp_size != 1, "data parallelism is not supported"), + (options.num_dp_ranks != 1, "data parallelism is not supported"), (options.ep_size != 1, "expert parallelism is not supported"), (options.cpu_offload_gb > 0, "CPU offload is not supported"), ( diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index b7d5ebdefd..f82ed5afb1 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -162,9 +162,11 @@ def _allocate_decode_buffers( encoder_lens = None if require_mlp_tp_gather: - global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32) + global_num_tokens_gpu = torch.zeros( + (parallel.num_dp_ranks,), dtype=torch.int32 + ) global_num_tokens_for_logprob_gpu = torch.zeros( - (parallel.dp_size,), dtype=torch.int32 + (parallel.num_dp_ranks,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) @@ -226,8 +228,8 @@ class BaseRunner(ABC): self.model_runner = model_runner self.device = model_runner.device self.device_module = torch.get_device_module(self.device) - # elastic-EP scale-up rewrites dp_size on the published config - self.dp_size = get_parallel().dp_size + # elastic-EP scale-up widens num_dp_ranks on the published config + self.num_dp_ranks = get_parallel().num_dp_ranks self.pp_size = get_parallel().pp_size self.enable_pdmux = get_disagg().enable_pdmux self.return_hidden_states_mode = ( @@ -613,7 +615,7 @@ class BaseRunner(ABC): assert require_mlp_tp_gather_ or require_attn_tp_gather_ if require_mlp_tp_gather_: - global_num_tokens_cpu = [num_tokens] * get_parallel().dp_size + global_num_tokens_cpu = [num_tokens] * get_parallel().num_dp_ranks elif require_attn_tp_gather_: global_num_tokens_cpu = [num_tokens] else: diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 1846c7b3b2..a89b012f94 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -260,7 +260,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.require_mlp_tp_gather or self.require_attn_tp_gather ) self.require_mlp_sync = ( - get_parallel().enable_dp_attention or self.require_gathered_buffer + get_parallel().attn_dp_enabled or self.require_gathered_buffer ) self.enable_two_batch_overlap = get_exec().overlap.enable_two_batch_overlap self.use_ngram_embedding = model_runner.ngram_embedding_manager.enabled @@ -414,7 +414,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.model_runner.model_config.vocab_size, rows=logits_buffer_rows ), dtype=self.model_runner.model_config.dtype, - dp_size=self.dp_size, + num_dp_ranks=self.num_dp_ranks, pp_size=self.pp_size, is_encoder_decoder=self.is_encoder_decoder, require_mlp_tp_gather=self.require_mlp_tp_gather, @@ -458,7 +458,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): enable_prefill_cp=self.enable_prefill_cp, require_mlp_tp_gather=self.require_mlp_tp_gather, attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded, - dp_size=self.dp_size, + num_dp_ranks=self.num_dp_ranks, source=self.buffers, ) @@ -584,7 +584,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]: if self.require_mlp_tp_gather: - return [num_tokens] * self.dp_size + return [num_tokens] * self.num_dp_ranks if self.require_attn_tp_gather: return [num_tokens] return None @@ -1427,7 +1427,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): ) if ( self.model_runner.lora_manager is not None - and self.model_runner.lora_manager.enable_dp_attention + and self.model_runner.lora_manager.attn_dp_enabled ): self.model_runner.lora_manager.prepare_lora_batch( cast(ForwardBatch, fb_view) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 2def2fd4aa..d0678daa8c 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -311,7 +311,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self.prefill_backend_name = prefill_config.backend self.prefer_eager_mixed_prefill = ( self.prefill_backend_name == Backend.BREAKABLE - and get_parallel().enable_dp_attention + and get_parallel().attn_dp_enabled and getattr( model_runner.attn_backend, "prefer_eager_mixed_prefill_under_dp_attention", @@ -1432,7 +1432,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): return registry.get_slot(name).slice_for(bs, num_tokens) if self.require_mlp_tp_gather: - global_num_tokens_cpu = [num_tokens] * self.dp_size + global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks elif self.require_attn_tp_gather: global_num_tokens_cpu = [num_tokens] else: diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index ac7b53a3ef..e0d173d515 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -136,7 +136,7 @@ class DecodeInputBuffers(ForwardInputBuffers): hidden_size: int, next_token_logits_buffer: torch.Tensor, dtype: torch.dtype, - dp_size: int, + num_dp_ranks: int, pp_size: int, is_encoder_decoder: bool, require_mlp_tp_gather: bool, @@ -207,9 +207,9 @@ class DecodeInputBuffers(ForwardInputBuffers): encoder_lens = None if require_mlp_tp_gather: - global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32) + global_num_tokens_gpu = torch.zeros((num_dp_ranks,), dtype=torch.int32) global_num_tokens_for_logprob_gpu = torch.zeros( - (dp_size,), dtype=torch.int32 + (num_dp_ranks,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 5268254c77..53461b33ec 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3896,9 +3896,9 @@ class DeepseekV4DecoderLayer(nn.Module): # Applies to BOTH prefill and decode: the shared expert is a per-token MLP, # so computing it on this rank's local tokens (M_local rows) is identical to # computing it on the gathered global buffer (M_global rows) and keeping the - # local slice -- but costs 1/dp_size the rows. With a replicated (TP1) shared + # local slice -- but costs 1/attn_dp_size the rows. With a replicated (TP1) shared # expert this cancels the TP1 "full-dim" cost in decode (M_local * dim == - # M_global * dim/tp), so decode no longer pays the ~dp_size x penalty. + # M_global * dim/tp), so decode no longer pays the ~attn_dp_size x penalty. _shared_local = None _do_shared_local = ( _SHARED_EXPERT_LOCAL diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index 539828ef7f..a8fe43d8e8 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -842,7 +842,7 @@ class ZayaAttention(nn.Module): assert attn_tp_size == self.tp_size, ( f"ZAYA1 head-parallel attention requires the attention TP group " f"({attn_tp_size}) to equal the global TP group ({self.tp_size}); " - "DP attention (enable_dp_attention) is not supported for ZAYA1." + "attention DP is not supported for ZAYA1." ) assert self.num_q_heads_full % self.tp_size == 0, ( f"num_attention_heads ({self.num_q_heads_full}) must be divisible " diff --git a/python/sglang/srt/ray/data_parallel_controller.py b/python/sglang/srt/ray/data_parallel_controller.py index 7522f84c54..e2fdfce858 100644 --- a/python/sglang/srt/ray/data_parallel_controller.py +++ b/python/sglang/srt/ray/data_parallel_controller.py @@ -77,7 +77,7 @@ class RayDataParallelController(DataParallelController): sockets = [] dp_port_args_list = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): tmp_port_args = PortArgs.init_new(server_args) tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name @@ -101,7 +101,7 @@ class RayDataParallelController(DataParallelController): sock.close() # Create actors for each DP rank sequentially - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): self._launch_ray_tp_group(server_args, dp_port_args_list[dp_rank], dp_rank) def launch_dp_attention_schedulers( @@ -112,7 +112,7 @@ class RayDataParallelController(DataParallelController): # rank-0 node IP instead of tcp://* to avoid exposing unauthenticated # ZMQ sockets (CVE-2026-3060). worker_ports = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): worker_port, worker_socket = get_zmq_socket_on_host( self.context, zmq.PUSH, host=self.rank0_node_ip ) @@ -158,12 +158,11 @@ class RayDataParallelController(DataParallelController): ) parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: _, _, actual_dp_rank, _ = compute_dp_attention_world_info( - parallel.enable_dp_attention, tp_rank, parallel.tp_size, - parallel.dp_size, + parallel.attn_dp_size, parallel.attn_cp_size, ) rank_port_args = PortArgs.init_new( @@ -230,12 +229,11 @@ class RayDataParallelController(DataParallelController): bundle_idx = bundle_indices[global_rank] - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: _, _, actual_dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, tp_rank, get_parallel().tp_size, - get_parallel().dp_size, + get_parallel().attn_dp_size, get_parallel().attn_cp_size, ) rank_port_args = PortArgs.init_new( diff --git a/python/sglang/srt/ray/engine.py b/python/sglang/srt/ray/engine.py index 0d6b8e3d53..ced74e7172 100644 --- a/python/sglang/srt/ray/engine.py +++ b/python/sglang/srt/ray/engine.py @@ -112,7 +112,6 @@ def _compute_world_size() -> int: will hold the process groups, so there is nothing live to ask. """ return compute_world_size( - enable_dp_attention=get_parallel().enable_dp_attention, dp_size=get_parallel().dp_size, tp_size=get_parallel().tp_size, pp_size=get_parallel().pp_size, @@ -274,10 +273,10 @@ class RayEngine(Engine): ) parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: total_gpus = parallel.tp_size * parallel.pp_size else: - total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size + total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size nnodes = parallel.nnodes gpus_per_node = total_gpus // nnodes @@ -318,7 +317,7 @@ class RayEngine(Engine): rank0_bundle_idx = int(indices_str.split(",")[0]) if indices_str else 0 rank0_node_ip = _get_bundle_node_ip(pg, rank0_bundle_idx) - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: dist_init_addr = f"{rank0_node_ip}:{port_args.nccl_port}" logger.info(f"dist_init_addr: {dist_init_addr}") @@ -448,17 +447,17 @@ class RayEngine(Engine): ) parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: # DP attention folds DP into TP — total GPUs = tp_size * pp_size total_gpus = parallel.tp_size * parallel.pp_size else: - total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size + total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size gpus_per_node = total_gpus // parallel.nnodes logger.info( f"Ray DP cluster: {parallel.nnodes} nodes, " f"{gpus_per_node} GPUs/node, dp_size={parallel.dp_size}, " - f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, " - f"enable_dp_attention={parallel.enable_dp_attention}" + f"attn_dp_size={parallel.attn_dp_size}, " + f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}" ) # Declared on the record itself so `PortArgs.init_new()` can compute diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 5742a0bd7f..d58c8ccb3d 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -70,16 +70,14 @@ def _parallel_fields() -> frozenset: return frozenset(_parallel_config_leaves() | derived) -def derive_attention_widths( - *, tp_size: int, attn_cp_size: int, dp_size: int, enable_dp_attention: bool -) -> tuple: - """Return (attn_dp_size, attn_tp_size) from the configured widths.""" - attn_dp_size = dp_size if enable_dp_attention else 1 - return attn_dp_size, tp_size // attn_dp_size // attn_cp_size +def derive_attn_tp_size(*, tp_size: int, attn_cp_size: int, attn_dp_size: int) -> int: + """Return the attention-TP width: what is left of the TP group after the + attention-DP and attention-CP dimensions.""" + return tp_size // attn_dp_size // attn_cp_size def derive_attention_ranks( - *, tp_rank: int, attn_tp_size: int, attn_cp_size: int, enable_dp_attention: bool + *, tp_rank: int, attn_tp_size: int, attn_cp_size: int ) -> tuple: """Return (attn_tp_rank, attn_dp_rank) for a process at ``tp_rank``. @@ -87,11 +85,11 @@ def derive_attention_ranks( tp_rank = (attn_dp_rank * attn_cp_size + attn_cp_rank) * attn_tp_size + attn_tp_rank + + Without attention DP the TP group is one attention-DP group, so + ``attn_dp_rank`` is zero. """ - attn_tp_rank = tp_rank % attn_tp_size - if not enable_dp_attention: - return attn_tp_rank, 0 - return attn_tp_rank, tp_rank // (attn_tp_size * attn_cp_size) + return tp_rank % attn_tp_size, tp_rank // (attn_tp_size * attn_cp_size) def spawn_world_rank(server_args, *, tp_rank: int, pp_rank: int) -> int: @@ -149,13 +147,9 @@ def derive_parallel_widths( """Derive attention and MoE widths and DCP settings from configuration.""" return { "attn_dp_size": attn_dp_size, - # `attn_dp_size` already accounts for disabled DP attention. - "attn_tp_size": derive_attention_widths( - tp_size=tp_size, - attn_cp_size=attn_cp_size, - dp_size=attn_dp_size, - enable_dp_attention=True, - )[1], + "attn_tp_size": derive_attn_tp_size( + tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size + ), "moe_ep_size": moe_ep_size, "moe_tp_size": tp_size // moe_ep_size // moe_dp_size, "dcp_enabled": dcp_enabled, @@ -165,16 +159,10 @@ def derive_parallel_widths( def parallel_widths_of(cfg: Any) -> dict: """Return derived parallel settings from resolved configuration.""" - attn_dp_size, _ = derive_attention_widths( - tp_size=cfg.tp_size, - attn_cp_size=cfg.attn_cp_size, - dp_size=cfg.dp_size, - enable_dp_attention=cfg.enable_dp_attention, - ) return derive_parallel_widths( tp_size=cfg.tp_size, attn_cp_size=cfg.attn_cp_size, - attn_dp_size=attn_dp_size, + attn_dp_size=cfg.attn_dp_size, moe_ep_size=cfg.ep_size, moe_dp_size=cfg.moe_dp_size, dcp_size=cfg.dcp_size, @@ -200,9 +188,23 @@ def attn_tp_size_of(cfg: Any): return parallel_widths_of(cfg)["attn_tp_size"] -def attn_dp_size_of(cfg: Any): - """`attn_dp_size`, computed at publish. See `parallel_widths_of`.""" - return parallel_widths_of(cfg)["attn_dp_size"] +def attn_dp_enabled_of(cfg: Any) -> bool: + """`attn_dp_enabled`, computed at publish. + + A scale joiner's own group can be one rank wide, but it joins an + attention-DP deployment and runs its DP-attention paths. + """ + from sglang.srt.arg_groups.model_override_base import ep_scale_joiner_of + + return cfg.attn_dp_size > 1 or ep_scale_joiner_of(cfg) + + +def num_dp_ranks_of(cfg: Any) -> int: + """`num_dp_ranks`, computed at publish: replicas times attention-DP groups. + + DP attention does not run inside replicas, so one of the two is one. + """ + return cfg.dp_size * cfg.attn_dp_size def attn_dcp_size_of(cfg: Any): @@ -1560,7 +1562,6 @@ def _attention_ranks(parallel, tp_rank: int) -> dict: tp_rank=tp_rank, attn_tp_size=parallel.attn_tp_size, attn_cp_size=parallel.attn_cp_size, - enable_dp_attention=parallel.enable_dp_attention, ) return {"attn_tp_rank": attn_tp_rank, "attn_dp_rank": attn_dp_rank} @@ -2119,7 +2120,8 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: "endpoint_port_base": port, "topic": cfg.topic, "block_size": kv_event_block_size_of(resolved), - "dp_size": resolved.dp_size, + # The wire key predates attention DP; it counts DP ranks. + "dp_size": num_dp_ranks_of(resolved), } # Load range, from the same resolver SchedulerLoadPublisher binds # with (so the two can't drift). The decline reason is logged once at @@ -2127,7 +2129,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: resolved_range, _reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, - dp_size=resolved.dp_size, + dp_size=num_dp_ranks_of(resolved), load_publish_endpoint=resolved.load_publish_endpoint, ) if resolved_range is not None: diff --git a/python/sglang/srt/rust_server/server.py b/python/sglang/srt/rust_server/server.py index 47c2396b39..06f89f7789 100644 --- a/python/sglang/srt/rust_server/server.py +++ b/python/sglang/srt/rust_server/server.py @@ -120,7 +120,9 @@ class RustServer: os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") # Preserve the DP startup log; ports use node-local offsets. - dp_rank = get_parallel().attn_dp_rank if get_parallel().dp_size > 1 else None + dp_rank = ( + get_parallel().attn_dp_rank if get_parallel().num_dp_ranks > 1 else None + ) if get_exec().moe.is_ep_scale_joiner: # The joining TP group is entirely local to this node. tp_size_per_node = get_parallel().tp_size @@ -180,7 +182,9 @@ class RustServer: # Under DP every rank runs its own server on its own port, so the rank is # what tells two otherwise identical startup lines apart. dp_note = ( - "" if dp_rank is None else f" (DP rank {dp_rank}/{get_parallel().dp_size})" + "" + if dp_rank is None + else f" (DP rank {dp_rank}/{get_parallel().num_dp_ranks})" ) logger.info( "SGLANG_RUST_SERVER enabled, Rust server listen on %s%s", diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index a49b230099..bcd2371510 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -43,7 +43,12 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform, publish +from sglang.srt.runtime_context import ( + attn_dp_enabled_of, + get_platform, + num_dp_ranks_of, + publish, +) from sglang.srt.speculative.decoupled_spec_io import DecoupledSpecIpcConfig from sglang.srt.utils.network import NetworkAddress, get_free_port, wait_port_available @@ -345,6 +350,15 @@ class ServerArgs: new_flag="--cuda-graph-backend-{decode,prefill}=disabled", help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.", ) + # `enable_dp_attention` is `no_cli=True` too; resolution turns it into + # `attn_dp_size`. + parser.add_argument( + "--enable-dp-attention", + action=DeprecatedStoreTrueAction, + new_flag="--attn-dp-size ", + help="Deprecated. Use --attn-dp-size instead of " + "--dp-size --enable-dp-attention.", + ) parser.add_argument( "--enable-flashinfer-allreduce-fusion", action="store_true", @@ -488,16 +502,15 @@ def resolve_encoder_transfer_backend( return "zmq_to_scheduler" -def compute_world_size( - *, enable_dp_attention: bool, dp_size: int, tp_size: int, pp_size: int -) -> int: +def compute_world_size(*, dp_size: int, tp_size: int, pp_size: int) -> int: """Total GPU count across all data-parallel replicas. - Takes the values rather than a config object: the two sizes are the widths + Takes the values rather than a config object: the sizes are the widths the launch asked for, which the Ray driver needs before any process group exists, and passing a context would hand it the live groups instead. + Attention DP runs inside the TP group, so it adds no GPUs. """ - return (1 if enable_dp_attention else dp_size) * tp_size * pp_size + return dp_size * tp_size * pp_size def m3_fp8_attn_gemm_enabled(args) -> bool: @@ -706,7 +719,7 @@ class PortArgs: rank=int(server_args.decoupled_spec_rank), ) - if not cfg.enable_dp_attention: + if not attn_dp_enabled_of(cfg): # Normal case, use IPC within a single node return PortArgs( tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}", @@ -732,12 +745,12 @@ class PortArgs: dist_init_host = na.host dist_init_port = na.port - # Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + dp_size - # rust-path slots); derive from server_args only (never dp_rank) so + # Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + one + # rust-path slot per DP rank); derive from server_args only (never dp_rank) so # every init_new call agrees, decrementing below dist_init_port on # overflow. is_rust_server = envs.SGLANG_RUST_SERVER.get() - NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + cfg.dp_size + NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + num_dp_ranks_of(cfg) if ep_scale_joiner_of(resolving_view(server_args)): port_base = server_args.port + ZMQ_TCP_PORT_DELTA if port_base + NUM_DERIVED_PORTS > 65535: diff --git a/python/sglang/srt/speculative/adaptive_spec_params.py b/python/sglang/srt/speculative/adaptive_spec_params.py index 6d33534996..268ceaeafa 100644 --- a/python/sglang/srt/speculative/adaptive_spec_params.py +++ b/python/sglang/srt/speculative/adaptive_spec_params.py @@ -16,6 +16,7 @@ from sglang.srt.arg_groups.overrides import ( resolved_view, resolving_view, ) +from sglang.srt.runtime_context import attn_dp_enabled_of from sglang.srt.utils import log_info_on_rank0 if TYPE_CHECKING: @@ -66,9 +67,9 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: f"speculative_eagle_topk={cfg.speculative_eagle_topk} " "(only topk=1 is supported)" ) - if resolved_view(server_args).enable_dp_attention: + if attn_dp_enabled_of(resolved_view(server_args)): return ( - "enable_dp_attention=True is not supported " + "attention DP is not supported " "(adaptive tier decisions are not synchronized across DP ranks)" ) if resolved_view(server_args).enable_multi_layer_eagle: diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index eb7655e264..13178752e5 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -403,14 +403,14 @@ class DFlashWorkerV2(BaseSpecWorker): # spec broadcasts must stay within the attn-TP group. self._tp_sync = SpecTpSync( get_parallel().attn_tp_group - if get_parallel().enable_dp_attention + if get_parallel().attn_dp_enabled else get_parallel().tp_group ) # Under dp attention, the draft worker runs on the per-DP attn-TP # group, independent of idle peer DP ranks; it is built and run under # the same placement. - self.draft_owns_attention = get_parallel().enable_dp_attention + self.draft_owns_attention = get_parallel().attn_dp_enabled # Inside the draft scope the context answers the draft's narrowed rank. self._target_tp_rank = get_parallel().tp_rank with draft_pp_context(), draft_tp_context(self.draft_owns_attention): @@ -645,7 +645,7 @@ class DFlashWorkerV2(BaseSpecWorker): "device graph capture.", type(current_platform).__name__, ) - if get_parallel().enable_dp_attention and capture_decode_cuda_graph: + if get_parallel().attn_dp_enabled and capture_decode_cuda_graph: # Idle DP ranks skip the draft step, so they cannot join a # shared graph capture/replay; keep the draft eager under dp # attention. @@ -1295,7 +1295,7 @@ class DFlashWorkerV2(BaseSpecWorker): the full embedding once during init and keep it replicated, making the draft block-id lookup a collective-free on-device F.embedding. """ - if not get_parallel().enable_dp_attention: + if not get_parallel().attn_dp_enabled: return tp_group = get_parallel().tp_group @@ -2335,7 +2335,7 @@ class DFlashWorkerV2(BaseSpecWorker): # verify forward (IDLE mode) so its cross-DP collectives stay in # lockstep with the active DP group; the draft block's # collectives are within-rank and skipped. - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: idle_verify_input = DFlashVerifyInput( draft_token=torch.empty((0,), dtype=torch.long, device=self.device), positions=torch.empty((0,), dtype=torch.int64, device=self.device), @@ -2726,7 +2726,7 @@ class DFlashWorkerV2(BaseSpecWorker): # their DP-gather segment offsets disagree. Fall back to eager verify # whenever any DP rank is idle this round. if ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and verify_forward_batch.original_global_num_tokens_cpu is not None and min(verify_forward_batch.original_global_num_tokens_cpu) == 0 ): @@ -2735,7 +2735,7 @@ class DFlashWorkerV2(BaseSpecWorker): # Mixed-round guard: an extend rank's raw token counts disagree with # the verify batch's spec-scaled counts on the DP-gather layout; run # eager, symmetric with the idle guard above. - if get_parallel().enable_dp_attention and batch.is_extend_in_batch: + if get_parallel().attn_dp_enabled and batch.is_extend_in_batch: verify_forward_batch.can_run_decode_cuda_graph = False target_out = self.target_worker.forward_batch_generation( diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 8a4baf410b..6dad00afa3 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -165,7 +165,7 @@ class DSparkWorkerV2(BaseSpecWorker): self._draft_is_moe = draft_is_deepseek_v4() self._draft_dp_context_enabled = ( - get_parallel().enable_dp_attention and not self._draft_is_moe + get_parallel().attn_dp_enabled and not self._draft_is_moe ) self._is_pd_prefill = get_disagg().disaggregation_mode == "prefill" self._decode_graph_allowed = ( @@ -173,7 +173,7 @@ class DSparkWorkerV2(BaseSpecWorker): and not self._is_pd_prefill ) if ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and self._draft_is_moe and get_parallel().attn_tp_size > 1 ): @@ -232,9 +232,7 @@ class DSparkWorkerV2(BaseSpecWorker): parallel = get_parallel() self._tp_sync = SpecTpSync( - parallel.attn_tp_group - if parallel.enable_dp_attention - else parallel.tp_group + parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group ) self._draft_graph_group = ( parallel.attn_tp_group @@ -294,13 +292,13 @@ class DSparkWorkerV2(BaseSpecWorker): tp_sync=self._tp_sync, ) if ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and not self._draft_is_moe and self._verify_planner.is_compact_mode and self._decode_graph_allowed ): raise ValueError( - "DSpark dense-draft compact verify under --enable-dp-attention does not " + "DSpark dense-draft compact verify under attention DP does not " "yet support cuda graph (idle DP groups cannot join the token-keyed " "compact graph). Re-run with --disable-cuda-graph (eager is lossless), " "or use SGLANG_RAGGED_VERIFY_MODE=static. The dsv4 (MoE) draft supports " @@ -321,7 +319,7 @@ class DSparkWorkerV2(BaseSpecWorker): mask_token_id=self._mask_token_id, draft_block_spec_info=self._draft_block_spec_info, tp_sync=self._tp_sync, - dp_moe_sync=self._draft_is_moe and get_parallel().enable_dp_attention, + dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled, ) self._verify_epilogue = None target_is_dsv41 = ( @@ -336,7 +334,7 @@ class DSparkWorkerV2(BaseSpecWorker): target_is_dsv41 and self._verify_planner.mode_value == "static" and self._draft_is_moe - and not get_parallel().enable_dp_attention + and not get_parallel().attn_dp_enabled and get_parallel().pp_size == 1 ) # ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts @@ -628,7 +626,7 @@ class DSparkWorkerV2(BaseSpecWorker): pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> GenerationBatchResult: if batch.forward_mode.is_idle(): - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: self.target_worker.forward_batch_generation( batch, capture_hidden_mode=CaptureHiddenMode.FULL ) @@ -744,7 +742,7 @@ class DSparkWorkerV2(BaseSpecWorker): def _dp_verify_tier_num_tokens(self, batch: ScheduleBatch) -> Optional[int]: if not ( self._draft_is_moe - and get_parallel().enable_dp_attention + and get_parallel().attn_dp_enabled and batch.global_num_tokens is not None and self._verify_planner.is_compact_mode ): @@ -793,7 +791,7 @@ class DSparkWorkerV2(BaseSpecWorker): global_num_reqs = ( max(batch.global_num_tokens) if self._draft_is_moe - and get_parallel().enable_dp_attention + and get_parallel().attn_dp_enabled and batch.global_num_tokens is not None else None ) @@ -809,7 +807,7 @@ class DSparkWorkerV2(BaseSpecWorker): if batch.forward_mode.is_idle(): self._observers.note_idle_decode_step() - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: if self._draft_is_moe: self._proposer.run_idle_participation(batch) if coordination_plan is not None: diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 4a8973bffc..8da5d34fc1 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -186,8 +186,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): # Use the same attention topology during draft construction and execution. self.draft_owns_attention = ( - get_parallel().enable_dp_attention - and self.speculative_algorithm.is_eagle3() + get_parallel().attn_dp_enabled and self.speculative_algorithm.is_eagle3() ) with ( draft_tp_context(self.draft_owns_attention), diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index 3965f2bee1..def5bcf94a 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -155,7 +155,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): # Fields the parent's capture() reads: self.device = model_runner.device self.device_module = torch.get_device_module(self.device) - self.dp_size = get_parallel().dp_size + self.num_dp_ranks = get_parallel().num_dp_ranks self.pp_size = get_parallel().pp_size self.enable_torch_compile = get_flags().capture.enable_torch_compile self.disable_padding = get_exec().graph.disable_cuda_graph_padding @@ -283,8 +283,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens] if self.require_mlp_tp_gather: - global_num_tokens_cpu = [num_tokens] * self.dp_size - global_num_tokens_for_logprob_cpu = [num_tokens] * self.dp_size + global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks + global_num_tokens_for_logprob_cpu = [num_tokens] * self.num_dp_ranks elif self.require_attn_tp_gather: global_num_tokens_cpu = [num_tokens] # DRAFT_EXTEND_V2 produces logits for all tokens, not bs (see mlp branch above) @@ -678,10 +678,12 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: if self.require_gathered_buffer: if self.require_mlp_tp_gather: - dp_size = runner.dp_size - global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32) + num_dp_ranks = runner.num_dp_ranks + global_num_tokens_gpu = torch.zeros( + (num_dp_ranks,), dtype=torch.int32 + ) global_num_tokens_for_logprob_gpu = torch.zeros( - (dp_size,), dtype=torch.int32 + (num_dp_ranks,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) diff --git a/python/sglang/srt/state_capturer/routed_experts.py b/python/sglang/srt/state_capturer/routed_experts.py index fab5c4d920..ef989b822a 100644 --- a/python/sglang/srt/state_capturer/routed_experts.py +++ b/python/sglang/srt/state_capturer/routed_experts.py @@ -71,14 +71,14 @@ class RoutedExpertsCapturer(BaseTopkCapturer): topk_size = model_config.hf_text_config.num_experts_per_tok num_layers = model_config.hf_text_config.num_hidden_layers - # Scale by dp_size so the buffer covers the full DP-concatenated batch. + # Scale by num_dp_ranks so the buffer covers the full DP-concatenated batch. # _get_local_slice indexes into [attention_dp_rank * cuda_graph_batch, ...) # and otherwise overflows on dp_rank > 0 when max_running_requests > # chunked_prefill_size. # FIXME: spec decoding's num_verify_tokens is still not accounted for. max_batch_size = max( - get_schedule().chunked_prefill_size * get_parallel().dp_size, - max_running_requests * get_parallel().dp_size, + get_schedule().chunked_prefill_size * get_parallel().num_dp_ranks, + max_running_requests * get_parallel().num_dp_ranks, ) super().__init__( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index e2de7af75b..cacf1fb243 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -3985,9 +3985,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): elif not isinstance(moe_a2a_backend, MoeA2ABackend): moe_a2a_backend = MoeA2ABackend(moe_a2a_backend) - # elastic-EP scale-up rewrites dp_size on the published config - if get_parallel().enable_dp_attention: - assert get_parallel().dp_size > 1, "dp_size must be greater than 1" + # elastic-EP scale-up widens num_dp_ranks on the published config + if get_parallel().attn_dp_enabled: + assert get_parallel().num_dp_ranks > 1, ( + "attention DP needs more than one DP rank" + ) if get_exec().moe.elastic_ep_backend is not None: from sglang.srt.elastic_ep.elastic_ep import ( elastic_expanded_world_enabled, @@ -4029,7 +4031,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): else: return ( get_parallel().moe_dense_tp_size - > get_parallel().tp_size // get_parallel().dp_size + > get_parallel().tp_size // get_parallel().num_dp_ranks ) else: return False @@ -4053,8 +4055,8 @@ def require_attn_tp_gather(): not get_moe_a2a_backend().is_none() or get_parallel().moe_dense_tp_size is not None ): - if get_parallel().enable_dp_attention: - return get_parallel().dp_size < get_parallel().tp_size + if get_parallel().attn_dp_enabled: + return get_parallel().num_dp_ranks < get_parallel().tp_size else: return True else: @@ -4067,7 +4069,7 @@ def require_gathered_buffer(): def require_mlp_sync(): - return get_parallel().enable_dp_attention or require_gathered_buffer() + return get_parallel().attn_dp_enabled or require_gathered_buffer() def get_cuda_graph_batch_size_alignment() -> int: diff --git a/python/sglang/srt/utils/cuda_vmm_transport_utils.py b/python/sglang/srt/utils/cuda_vmm_transport_utils.py index 67c5ee1a78..e00d36cb6d 100644 --- a/python/sglang/srt/utils/cuda_vmm_transport_utils.py +++ b/python/sglang/srt/utils/cuda_vmm_transport_utils.py @@ -192,8 +192,8 @@ def _contains_tensor_container(value) -> bool: def get_vmm_feature_consumer_count() -> int: - if get_parallel().enable_dp_attention: - return get_parallel().tp_size // get_parallel().dp_size + if get_parallel().attn_dp_enabled: + return get_parallel().tp_size // get_parallel().num_dp_ranks return get_parallel().tp_size diff --git a/python/sglang/srt/utils/offloader.py b/python/sglang/srt/utils/offloader.py index 100e116398..17bf994d92 100644 --- a/python/sglang/srt/utils/offloader.py +++ b/python/sglang/srt/utils/offloader.py @@ -210,7 +210,7 @@ class OffloaderV2(BaseOffloader): mode: str, ): parallel = get_parallel() - dp_rank, dp_size = parallel.dp_rank, parallel.dp_size + dp_rank, num_dp_ranks = parallel.dp_rank, parallel.num_dp_ranks self.group_size = group_size self.num_in_group = num_in_group self.prefetch_step = prefetch_step @@ -224,7 +224,7 @@ class OffloaderV2(BaseOffloader): set_naive_distributed( NaiveDistributed( rank=dp_rank, - world_size=dp_size, + world_size=num_dp_ranks, rendezvous=f"/tmp/{run_id}", ) ) diff --git a/python/sglang/srt/utils/profile_utils.py b/python/sglang/srt/utils/profile_utils.py index a7d6ab08f3..1e116da443 100644 --- a/python/sglang/srt/utils/profile_utils.py +++ b/python/sglang/srt/utils/profile_utils.py @@ -357,7 +357,7 @@ class _ProfilerTorch(_ProfilerConcreteBase): filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"] # Only add other ranks if parallelism is enabled (size > 1) - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: filename_parts.append(f"DP-{get_parallel().dp_rank}") if get_parallel().pp_size > 1: filename_parts.append(f"PP-{get_parallel().pp_rank}") diff --git a/python/sglang/srt/weight_cache/daemon.py b/python/sglang/srt/weight_cache/daemon.py index 2fe9a16480..a40af383b8 100644 --- a/python/sglang/srt/weight_cache/daemon.py +++ b/python/sglang/srt/weight_cache/daemon.py @@ -162,10 +162,8 @@ class WeightCacheDaemon: self.tp_rank = tp_rank self.pp_size = cfg.pp_size self.pp_rank = pp_rank - self.dp_size = cfg.dp_size self.ep_size = cfg.ep_size self.moe_dp_size = cfg.moe_dp_size - self.enable_dp_attention = cfg.enable_dp_attention self.enable_dp_lm_head = cfg.enable_dp_lm_head self.attn_cp_size = cfg.attn_cp_size self.moe_dense_tp_size = cfg.moe_dense_tp_size @@ -338,12 +336,12 @@ class WeightCacheDaemon: tp_rank=self.tp_rank, pp_size=self.pp_size, pp_rank=self.pp_rank, - dp_size=self.dp_size, + dp_size=get_parallel().dp_size, ep_size=self.ep_size, moe_dp_size=self.moe_dp_size, moe_dp_rank=moe_dp_rank, moe_ep_rank=moe_ep_rank, - enable_dp_attention=self.enable_dp_attention, + attn_dp_size=get_parallel().attn_dp_size, enable_dp_lm_head=self.enable_dp_lm_head, attn_cp_size=self.attn_cp_size, moe_dense_tp_size=self.moe_dense_tp_size, diff --git a/python/sglang/srt/weight_cache/ipc_loader.py b/python/sglang/srt/weight_cache/ipc_loader.py index ef5a09bec0..6bf6a9305b 100644 --- a/python/sglang/srt/weight_cache/ipc_loader.py +++ b/python/sglang/srt/weight_cache/ipc_loader.py @@ -515,8 +515,6 @@ class IpcModelLoader(BaseModelLoader): moe_dp_rank = parallel.moe_dp_rank moe_ep_rank = parallel.moe_ep_rank - dp_size = get_parallel().dp_size - quant_method, quant_config = self._resolve_engine_quant(model_config) engine_config = CacheConfig( @@ -530,12 +528,12 @@ class IpcModelLoader(BaseModelLoader): tp_rank=tp_rank, pp_size=pp_size, pp_rank=pp_rank, - dp_size=dp_size, + dp_size=parallel.dp_size, ep_size=ep_size, moe_dp_size=moe_dp_size, moe_dp_rank=moe_dp_rank, moe_ep_rank=moe_ep_rank, - enable_dp_attention=parallel.enable_dp_attention, + attn_dp_size=parallel.attn_dp_size, enable_dp_lm_head=parallel.enable_dp_lm_head, attn_cp_size=parallel.attn_cp_size, moe_dense_tp_size=parallel.moe_dense_tp_size, diff --git a/python/sglang/srt/weight_cache/protocol.py b/python/sglang/srt/weight_cache/protocol.py index 1ae09eb40e..46cfe588d8 100644 --- a/python/sglang/srt/weight_cache/protocol.py +++ b/python/sglang/srt/weight_cache/protocol.py @@ -39,7 +39,7 @@ class CacheConfig(msgspec.Struct): moe_dp_size: int moe_dp_rank: int moe_ep_rank: int - enable_dp_attention: bool + attn_dp_size: int enable_dp_lm_head: bool attn_cp_size: int moe_dense_tp_size: Optional[int] diff --git a/python/sglang/test/cache_consistency_jitter.py b/python/sglang/test/cache_consistency_jitter.py index 4289c93c47..eed04357a4 100644 --- a/python/sglang/test/cache_consistency_jitter.py +++ b/python/sglang/test/cache_consistency_jitter.py @@ -190,7 +190,7 @@ def get_jitter_engine( engine_kwargs.setdefault("schedule_conservativeness", 0.05) if ( - engine_kwargs.get("enable_dp_attention") + engine_kwargs.get("attn_dp_size", 1) > 1 and "dist_init_addr" not in engine_kwargs ): import portpicker diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py index e08dbb0d1a..3b0f392c5b 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py @@ -367,7 +367,6 @@ class MockModelRunner(ModelRunner): dllm_algorithm=None, dllm_algorithm_config=None, dp_size=1, - enable_dp_attention=False, enable_deterministic_inference=False, enable_mis=False, is_embedding=False, diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index 0458f1f265..336c23698a 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -341,7 +341,6 @@ class DSAMockModelRunner(ModelRunner): dsa_prefill_backend=dsa_prefill_backend, device=device, enable_deterministic_inference=False, - enable_dp_attention=False, enable_prefill_cp=False, enable_mis=False, is_embedding=False, diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py index e07aa633ff..7ddef6953e 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py @@ -364,7 +364,6 @@ class MockDSV4ModelRunner: disaggregation_mode=None, dp_size=1, enable_deterministic_inference=False, - enable_dp_attention=False, enable_mis=False, is_embedding=False, kv_cache_dtype="auto", diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py index ae558bda2e..d535c3d0ac 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py @@ -271,7 +271,6 @@ class MockMLAModelRunner(ModelRunner): dllm_algorithm=None, dllm_algorithm_config=None, dp_size=1, - enable_dp_attention=False, enable_deterministic_inference=False, enable_mis=False, flashinfer_mla_disable_ragged=True, diff --git a/python/sglang/test/kv_canary/consts.py b/python/sglang/test/kv_canary/consts.py index 3a16ca1cf0..3712e812af 100644 --- a/python/sglang/test/kv_canary/consts.py +++ b/python/sglang/test/kv_canary/consts.py @@ -23,9 +23,8 @@ DSV4_POOL_SERVER_ARGS: Final[tuple[str, ...]] = ( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/python/sglang/test/runners.py b/python/sglang/test/runners.py index 363daf9ddc..757880e21b 100644 --- a/python/sglang/test/runners.py +++ b/python/sglang/test/runners.py @@ -581,7 +581,7 @@ class SRTRunner: context_length: Optional[int] = None, max_total_tokens: Optional[int] = None, page_size: Optional[int] = None, - dp_size: int = 1, + attn_dp_size: int = 1, tokenizer_path: Optional[str] = None, mem_fraction_static: float = 0.65, trust_remote_code: bool = False, @@ -607,7 +607,6 @@ class SRTRunner: ): self.model_type = model_type self.is_generation = model_type == "generation" - enable_dp_attention = dp_size > 1 spec_kwargs = {} if speculative_draft_model_path: @@ -645,8 +644,7 @@ class SRTRunner: context_length=context_length, max_total_tokens=max_total_tokens, page_size=page_size, - enable_dp_attention=enable_dp_attention, - dp_size=dp_size, + attn_dp_size=attn_dp_size, tokenizer_path=tokenizer_path, disable_overlap_schedule=disable_overlap_schedule, cuda_graph_max_bs_decode=cuda_graph_max_bs_decode, diff --git a/python/sglang/test/server_fixtures/dsa_mtp_fixture.py b/python/sglang/test/server_fixtures/dsa_mtp_fixture.py index 0a958bc71c..693d980946 100644 --- a/python/sglang/test/server_fixtures/dsa_mtp_fixture.py +++ b/python/sglang/test/server_fixtures/dsa_mtp_fixture.py @@ -3,7 +3,7 @@ Variants combine `DsaMtpServerBase` (server lifecycle) with `DsaMtpEvalConfigDefaults` (shared eval thresholds/params), `GSM8KMixin` and `SpecDecodingMixin`, then set `model` and per-variant -overrides (`enable_dp_attention`, `mem_fraction_static`, `bs_1_speed_thres`). +overrides (`attn_dp_size`, `mem_fraction_static`, `bs_1_speed_thres`). Example: class TestDsv32DP( @@ -13,7 +13,7 @@ Example: SpecDecodingMixin, ): model = "deepseek-ai/DeepSeek-V3.2" - enable_dp_attention = True + attn_dp_size = 8 bs_1_speed_thres = 90 The base itself is NOT a runnable test (no `test_*` methods until a subclass @@ -50,9 +50,8 @@ class DsaMtpServerBase(CustomTestCase): # Subclasses must set `model`; the others have sensible defaults. model: str = "" tp_size: int = 8 - dp_size: int = 8 + attn_dp_size: int = 1 mem_fraction_static: float = 0.7 - enable_dp_attention: bool = False extra_server_args = () # EAGLE MTP config (fixed across DSA-MTP variants). @@ -65,8 +64,8 @@ class DsaMtpServerBase(CustomTestCase): def get_server_args(cls): assert cls.model, f"{cls.__name__} must set `model`" args = ["--trust-remote-code", "--tp", str(cls.tp_size)] - if cls.enable_dp_attention: - args += ["--dp", str(cls.dp_size), "--enable-dp-attention"] + if cls.attn_dp_size > 1: + args += ["--attn-dp-size", str(cls.attn_dp_size)] args += [ "--speculative-algorithm", cls.speculative_algorithm, diff --git a/scripts/ci/cuda/warmup_deep_gemm.py b/scripts/ci/cuda/warmup_deep_gemm.py index edf59679e2..f53b6eff9a 100644 --- a/scripts/ci/cuda/warmup_deep_gemm.py +++ b/scripts/ci/cuda/warmup_deep_gemm.py @@ -43,21 +43,19 @@ FALLBACK_TIMEOUT_SEC = 600 # cache key includes per-rank N/K (depends on tp/dp/ep) — must match each # model's `other_args` in test/registered/ or warmed shapes won't be hit. FALLBACK_ARGS: Dict[str, List[str]] = { - "deepseek-ai/DeepSeek-V3.2": ["--dp", "8", "--enable-dp-attention"], - "zai-org/GLM-5-FP8": ["--dp", "8", "--enable-dp-attention"], + "deepseek-ai/DeepSeek-V3.2": ["--attn-dp-size", "8"], + "zai-org/GLM-5-FP8": ["--attn-dp-size", "8"], "XiaomiMiMo/MiMo-V2-Flash": [ - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--attention-backend", "fa3", ], # --mm-enable-dp-encoder is required: without it DP0 runs the vision # encoder alone and DP1 deadlocks at the next collective. "XiaomiMiMo/MiMo-V2.5": [ - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--mm-enable-dp-encoder", "--attention-backend", "fa3", diff --git a/scripts/ci/slurm/launch_mi355x.sh b/scripts/ci/slurm/launch_mi355x.sh index 9a6488be60..e9231d50cd 100755 --- a/scripts/ci/slurm/launch_mi355x.sh +++ b/scripts/ci/slurm/launch_mi355x.sh @@ -541,10 +541,10 @@ IONIC_EOF # and the generated argv is byte-identical. Oren's wide-EP recipes set decode # EP/DP=16 while prefill stays EP8. PREFILL_DPEP="" -(( PDP > 1 )) && PREFILL_DPEP="$PREFILL_DPEP --enable-dp-attention --dp-size $PDP" +(( PDP > 1 )) && PREFILL_DPEP="$PREFILL_DPEP --attn-dp-size $PDP" (( PEP > 1 )) && PREFILL_DPEP="$PREFILL_DPEP --ep-size $PEP" DECODE_DPEP="" -(( DDP > 1 )) && DECODE_DPEP="$DECODE_DPEP --enable-dp-attention --dp-size $DDP" +(( DDP > 1 )) && DECODE_DPEP="$DECODE_DPEP --attn-dp-size $DDP" (( DEP > 1 )) && DECODE_DPEP="$DECODE_DPEP --ep-size $DEP" # Flags shared by both roles (a2a backend, mtp). --max-total-tokens stays here for # non-wide recipes; wide recipes carry a per-role prefill_max_total via wide_ep. @@ -688,10 +688,10 @@ DENV_ARG=""; [[ -n "$DENV" ]] && DENV_ARG=" $DENV" # port and stays under the 32768 ephemeral floor. These recipes place at most # one single-node engine per node, so two engines cannot collide. # -# Pinning is safe here because --dp-size is only emitted together with -# --enable-dp-attention, and that path gives every DP rank the same nccl port. -# launch_dp_schedulers(), which needs a distinct port per worker, is never -# reached by these recipes. +# Pinning is safe here because these recipes only emit attention DP +# (--attn-dp-size), never --dp-size replicas, and the attention-DP path gives +# every DP rank the same nccl port. launch_dp_schedulers(), which needs a +# distinct port per worker, is never reached by these recipes. PNCCL_ARG="" DNCCL_ARG="" if [[ "$CLUSTER" == "spur" ]]; then diff --git a/scripts/ci/slurm/nightly-configs.yaml b/scripts/ci/slurm/nightly-configs.yaml index 8290206352..5149c42542 100644 --- a/scripts/ci/slurm/nightly-configs.yaml +++ b/scripts/ci/slurm/nightly-configs.yaml @@ -125,7 +125,7 @@ dsv4pro-fp4-mi355x-sglang: # * -dp8ep8 : DP-attention 8 + narrow within-node EP8 # * -dp8ep8-mtp : DP8 + narrow EP8 + EAGLE MTP # launch_mi355x.sh reads expert-/data-parallel-size and the `mtp:` section to -# append --ep-size / --enable-dp-attention --dp-size / --speculative-* flags. +# append --ep-size / --attn-dp-size / --speculative-* flags. dsv4flash-fp8-mi355x-mtp-sglang: model: sgl-project/DeepSeek-V4-Flash-FP8 diff --git a/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs b/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs index 58d1a855b0..f7cc8147c4 100644 --- a/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs +++ b/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs @@ -19,7 +19,7 @@ pub async fn get_dp_info(url: &str, api_key: Option<&str>) -> Result, pub tp_size: Option, pub dp_size: Option, + pub attn_dp_size: Option, pub load_balance_method: Option, pub disaggregation_mode: Option, pub version: Option, @@ -43,6 +44,16 @@ pub struct ServerInfo { pub max_num_reqs: Option, } +impl ServerInfo { + /// How many DP ranks the server runs, one scheduler each: its replicas + /// times its attention-DP groups. A server that predates `attn_dp_size` + /// counts attention-DP groups in `dp_size` and reports no `attn_dp_size`. + pub fn num_dp_ranks(&self) -> Option { + self.dp_size + .map(|dp_size| dp_size * self.attn_dp_size.unwrap_or(1)) + } +} + /// Model information returned from /model_info endpoint. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ModelInfo { @@ -328,8 +339,8 @@ impl StepExecutor for DiscoverMetadataStep { if let Some(tp_size) = server_info.tp_size { labels.insert("tp_size".to_string(), tp_size.to_string()); } - if let Some(dp_size) = server_info.dp_size { - labels.insert("dp_size".to_string(), dp_size.to_string()); + if let Some(num_dp_ranks) = server_info.num_dp_ranks() { + labels.insert("dp_size".to_string(), num_dp_ranks.to_string()); } if let Some(load_balance_method) = server_info.load_balance_method { labels.insert("load_balance_method".to_string(), load_balance_method); @@ -400,3 +411,30 @@ impl StepExecutor for DiscoverMetadataStep { true } } + +#[cfg(test)] +mod server_info_tests { + use super::ServerInfo; + + fn parse(json: &str) -> ServerInfo { + serde_json::from_str(json).expect("valid server info") + } + + #[test] + fn dp_ranks_multiply_replicas_by_attention_dp_groups() { + let info = parse(r#"{"dp_size": 1, "attn_dp_size": 8}"#); + assert_eq!(info.num_dp_ranks(), Some(8)); + } + + #[test] + fn a_server_without_attention_dp_size_counts_dp_size() { + let info = parse(r#"{"dp_size": 4}"#); + assert_eq!(info.num_dp_ranks(), Some(4)); + } + + #[test] + fn no_dp_size_means_no_rank_count() { + let info = parse(r#"{"attn_dp_size": 2}"#); + assert_eq!(info.num_dp_ranks(), None); + } +} diff --git a/test/manual/8-gpu-models/test_dsa_models_basic.py b/test/manual/8-gpu-models/test_dsa_models_basic.py index 672d365c49..e9aaa35b82 100644 --- a/test/manual/8-gpu-models/test_dsa_models_basic.py +++ b/test/manual/8-gpu-models/test_dsa_models_basic.py @@ -26,9 +26,8 @@ class TestDeepseekV32DP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ] @@ -141,9 +140,8 @@ class TestGLM5DP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ] diff --git a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py index 0e1d964915..7eb153674e 100644 --- a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py +++ b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py @@ -36,9 +36,8 @@ class TestMiMoV25ProFP4GraphWithDFlash(GSM8KAscendMixin, CustomTestCase): MIMO_V2_5_PRO_FP4_DFLASH_DRAFT_WEIGHTS_PATH, "--speculative-num-draft-tokens", "8", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", ] diff --git a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py index e710888317..4e54ccf435 100644 --- a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py +++ b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py @@ -38,9 +38,8 @@ class TestMiMoV25W8A8GraphWithMTP(GSM8KAscendMixin, CustomTestCase): "modelslim", "--speculative-draft-model-quantization", "unquant", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", ] diff --git a/test/manual/ascend/test_ascend_deepseek_mtp.py b/test/manual/ascend/test_ascend_deepseek_mtp.py index 482426b375..85ac1a4534 100644 --- a/test/manual/ascend/test_ascend_deepseek_mtp.py +++ b/test/manual/ascend/test_ascend_deepseek_mtp.py @@ -38,9 +38,8 @@ class TestAscendDeepSeekMTP(CustomTestCase): 32768, "--tp-size", 16, - "--dp-size", + "--attn-dp-size", 2, - "--enable-dp-attention", "--speculative-algorithm", "NEXTN", "--speculative-num-steps", diff --git a/test/manual/attention/test_trtllm_mla_backend.py b/test/manual/attention/test_trtllm_mla_backend.py index 193ba89195..966122a0f6 100755 --- a/test/manual/attention/test_trtllm_mla_backend.py +++ b/test/manual/attention/test_trtllm_mla_backend.py @@ -327,7 +327,6 @@ class TestTRTLLMMLA(CustomTestCase): def setUpClass(cls): """Set up global server args for testing.""" server_args = ServerArgs(model_path="dummy") - server_args.enable_dp_attention = False set_global_server_args_for_scheduler(server_args) @classmethod diff --git a/test/manual/chunked_prefill/test_e2e_dp_attention.py b/test/manual/chunked_prefill/test_e2e_dp_attention.py index e5c018d27a..63391c2661 100644 --- a/test/manual/chunked_prefill/test_e2e_dp_attention.py +++ b/test/manual/chunked_prefill/test_e2e_dp_attention.py @@ -13,8 +13,7 @@ class TestChunkedFeatureDPAttention(ChunkedTestBase): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] diff --git a/test/manual/distributed/test_dp_attention_archived.py b/test/manual/distributed/test_dp_attention_archived.py index 40f7ff7b10..1907cbe274 100644 --- a/test/manual/distributed/test_dp_attention_archived.py +++ b/test/manual/distributed/test_dp_attention_archived.py @@ -51,8 +51,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, "--tp-size", "2", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "2", ] if not is_in_amd_ci(): diff --git a/test/manual/distributed/test_dp_attention_large.py b/test/manual/distributed/test_dp_attention_large.py index 7186a71333..4dc7b55dba 100644 --- a/test/manual/distributed/test_dp_attention_large.py +++ b/test/manual/distributed/test_dp_attention_large.py @@ -42,8 +42,7 @@ class TestDPAttentionDP2TP4( other_args=[ "--trust-remote-code", "--tp=4", - "--enable-dp-attention", - "--dp=2", + "--attn-dp-size=2", ], ) @@ -89,8 +88,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( "--speculative-draft-model-path", DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, "--tp-size=4", - "--enable-dp-attention", - "--dp-size=2", + "--attn-dp-size=2", ] if not is_in_amd_ci(): other_args += ["--mem-frac", "0.7"] @@ -151,8 +149,7 @@ class TestDPAttentionDP2TP4VLM(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) diff --git a/test/manual/dsv4/test_b200_flash.py b/test/manual/dsv4/test_b200_flash.py index 30df91e806..81aa095067 100644 --- a/test/manual/dsv4/test_b200_flash.py +++ b/test/manual/dsv4/test_b200_flash.py @@ -43,9 +43,8 @@ class TestB200FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -68,9 +67,8 @@ class TestB200FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_b200_pro.py b/test/manual/dsv4/test_b200_pro.py index f95f29020b..2d01932000 100644 --- a/test/manual/dsv4/test_b200_pro.py +++ b/test/manual/dsv4/test_b200_pro.py @@ -45,9 +45,8 @@ class TestB200ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -76,9 +75,8 @@ class TestB200ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--mem-fraction-static", diff --git a/test/manual/dsv4/test_b300_flash.py b/test/manual/dsv4/test_b300_flash.py index 422299718c..3c2d31f7c2 100644 --- a/test/manual/dsv4/test_b300_flash.py +++ b/test/manual/dsv4/test_b300_flash.py @@ -45,9 +45,8 @@ class TestB300FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -70,9 +69,8 @@ class TestB300FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_b300_pro.py b/test/manual/dsv4/test_b300_pro.py index 0825e85105..f992c99771 100644 --- a/test/manual/dsv4/test_b300_pro.py +++ b/test/manual/dsv4/test_b300_pro.py @@ -47,9 +47,8 @@ class TestB300ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -78,9 +77,8 @@ class TestB300ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--mem-fraction-static", diff --git a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py index 5274621b24..7f9400cf32 100644 --- a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py @@ -61,9 +61,8 @@ class DSV4FlashMTPServerBase(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py b/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py index 1546bbc541..64e5873bb8 100644 --- a/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py @@ -60,9 +60,8 @@ class TestDSV4FlashTP4DP4( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", @@ -98,7 +97,7 @@ class TestDSV4FlashTP4EP( "4", "--ep", "4", - # No --enable-dp-attention by design: covers TP-attn path. + # No --attn-dp-size by design: covers TP-attn path. "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", @@ -132,9 +131,8 @@ class TestDSV4FlashTP4DP4ChunkedPrefillLarge( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--chunked-prefill-size", diff --git a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py index ac84120248..d10c701b0e 100644 --- a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py +++ b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py @@ -54,9 +54,8 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase): "0", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", @@ -96,9 +95,8 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase): "4", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_dsv4_swa_radix_retract.py b/test/manual/dsv4/test_dsv4_swa_radix_retract.py index 7db064d58e..bdc8fa0413 100644 --- a/test/manual/dsv4/test_dsv4_swa_radix_retract.py +++ b/test/manual/dsv4/test_dsv4_swa_radix_retract.py @@ -64,9 +64,8 @@ class TestDSV4FlashSWARadixRetract(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/test/manual/dsv4/test_gb300_flash.py b/test/manual/dsv4/test_gb300_flash.py index e19abae2f2..b3459983fe 100644 --- a/test/manual/dsv4/test_gb300_flash.py +++ b/test/manual/dsv4/test_gb300_flash.py @@ -43,9 +43,8 @@ class TestGB300FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -68,9 +67,8 @@ class TestGB300FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_gb300_pro.py b/test/manual/dsv4/test_gb300_pro.py index 363296c477..11edaaa7b9 100644 --- a/test/manual/dsv4/test_gb300_pro.py +++ b/test/manual/dsv4/test_gb300_pro.py @@ -47,9 +47,8 @@ class TestGB300ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -78,9 +77,8 @@ class TestGB300ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--mem-fraction-static", diff --git a/test/manual/dsv4/test_h200_fp8_flash.py b/test/manual/dsv4/test_h200_fp8_flash.py index 684272b9d0..7051f3276e 100644 --- a/test/manual/dsv4/test_h200_fp8_flash.py +++ b/test/manual/dsv4/test_h200_fp8_flash.py @@ -40,9 +40,8 @@ class TestH200Fp8FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -72,9 +71,8 @@ class TestH200Fp8FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/test/manual/dsv4/test_h200_fp8_pro.py b/test/manual/dsv4/test_h200_fp8_pro.py index 3e724a4b71..ee4b5f5044 100644 --- a/test/manual/dsv4/test_h200_fp8_pro.py +++ b/test/manual/dsv4/test_h200_fp8_pro.py @@ -38,9 +38,8 @@ class TestH200Fp8ProLowLatency(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "16", - "--dp", + "--attn-dp-size", "16", - "--enable-dp-attention", *multinode_args(2), "--moe-a2a-backend", "deepep", @@ -72,9 +71,8 @@ class TestH200Fp8ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "16", - "--dp", + "--attn-dp-size", "16", - "--enable-dp-attention", *multinode_args(2), "--moe-a2a-backend", "deepep", @@ -106,9 +104,8 @@ class TestH200Fp8ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "16", - "--dp", + "--attn-dp-size", "16", - "--enable-dp-attention", *multinode_args(2), "--moe-a2a-backend", "deepep", diff --git a/test/manual/ep/test_elastic_recover.py b/test/manual/ep/test_elastic_recover.py index beab45b92e..4cfefda5e7 100644 --- a/test/manual/ep/test_elastic_recover.py +++ b/test/manual/ep/test_elastic_recover.py @@ -69,7 +69,7 @@ def _server_args(node_rank: int, port: int, recover: bool = False) -> list[str]: "--trust-remote-code", "--tp", str(EP_SIZE), - "--dp", + "--attn-dp-size", str(EP_SIZE), "--nnodes", "2", @@ -79,7 +79,6 @@ def _server_args(node_rank: int, port: int, recover: bool = False) -> list[str]: DIST_INIT_ADDR, "--random-seed", str(RANDOM_SEED), - "--enable-dp-attention", "--enable-dp-lm-head", "--elastic-ep-backend", "mooncake", diff --git a/test/manual/ep/test_elastic_scale.py b/test/manual/ep/test_elastic_scale.py index 092e639bf9..8cc30557d1 100644 --- a/test/manual/ep/test_elastic_scale.py +++ b/test/manual/ep/test_elastic_scale.py @@ -136,9 +136,8 @@ def _scale_up_common_args( "low_latency", "--tp", str(tp_size), - "--dp", + "--attn-dp-size", str(tp_size), - "--enable-dp-attention", "--enable-dp-lm-head", "--elastic-ep-backend", "mooncake", diff --git a/test/manual/ep/test_eplb.py b/test/manual/ep/test_eplb.py index f0c5cce2e4..cce1a29e99 100755 --- a/test/manual/ep/test_eplb.py +++ b/test/manual/ep/test_eplb.py @@ -60,9 +60,8 @@ class _BaseTestDynamicEPLB(CustomTestCase): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", *get_a2a_backend_args(), "--disable-cuda-graph", "--enable-eplb", @@ -117,11 +116,10 @@ class TestStaticEPLB(CustomTestCase): model_path=DEFAULT_MLA_MODEL_NAME_FOR_TEST, trust_remote_code=True, ep_num_redundant_experts=4, - enable_dp_attention=True, disable_cuda_graph=True, expert_distribution_recorder_mode="stat", tp_size=2, - dp_size=2, + attn_dp_size=2, log_level="info", ) engine_kwargs.update(get_a2a_backend_kwargs()) diff --git a/test/manual/ep/test_eplb_mori.py b/test/manual/ep/test_eplb_mori.py index 5710e4d995..88c78e976d 100644 --- a/test/manual/ep/test_eplb_mori.py +++ b/test/manual/ep/test_eplb_mori.py @@ -48,9 +48,8 @@ common_args = [ "8", "--ep-size", "8", - "--dp-size", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "mori", "--trust-remote-code", diff --git a/test/manual/ep/test_moe_deepep.py b/test/manual/ep/test_moe_deepep.py index ce7074b817..d2304aeef5 100644 --- a/test/manual/ep/test_moe_deepep.py +++ b/test/manual/ep/test_moe_deepep.py @@ -64,9 +64,8 @@ class TestDPAttn(unittest.TestCase): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/manual/ep/test_mooncake_expert_backup.py b/test/manual/ep/test_mooncake_expert_backup.py index 5298ed31e4..820adea463 100644 --- a/test/manual/ep/test_mooncake_expert_backup.py +++ b/test/manual/ep/test_mooncake_expert_backup.py @@ -36,8 +36,7 @@ class TestBackup(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--elastic-ep-backend", "mooncake", diff --git a/test/manual/ep/test_nixl_ep.py b/test/manual/ep/test_nixl_ep.py index 6341fe9bc3..81a97cf39b 100644 --- a/test/manual/ep/test_nixl_ep.py +++ b/test/manual/ep/test_nixl_ep.py @@ -33,7 +33,7 @@ NIXL_COMMON = [ "--mem-fraction-static", "0.78", ] -DP_ATTN = ["--dp", "8", "--enable-dp-attention"] +DP_ATTN = ["--attn-dp-size", "8"] ELASTIC_NIXL = [ "--elastic-ep-backend", "nixl", diff --git a/test/manual/ep/test_pplx_small.py b/test/manual/ep/test_pplx_small.py index 3e4c3c4143..ecf14a70d0 100644 --- a/test/manual/ep/test_pplx_small.py +++ b/test/manual/ep/test_pplx_small.py @@ -28,8 +28,7 @@ class TestPureDP(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-a2a-backend", "pplx", @@ -43,7 +42,7 @@ class TestPureDP(CustomTestCase): "0.5", ], # Per-rank dispatch cap must cover the per-rank prefill chunk - # (chunked_prefill_size // dp_size = 8192 // 4 = 2048 on H100). + # (chunked_prefill_size // attn_dp_size = 8192 // 4 = 2048 on H100). env={"SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "4096"}, ) @@ -99,8 +98,7 @@ class TestHybridDPTP(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--moe-a2a-backend", "pplx", @@ -114,7 +112,7 @@ class TestHybridDPTP(CustomTestCase): "0.5", ], # Per-rank dispatch cap must cover the per-rank prefill chunk - # (chunked_prefill_size // dp_size = 8192 // 2 = 4096 on H100). + # (chunked_prefill_size // attn_dp_size = 8192 // 2 = 4096 on H100). env={"SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "4096"}, ) diff --git a/test/manual/hicache/test_npu_kimi_k3_hicache.py b/test/manual/hicache/test_npu_kimi_k3_hicache.py index 444a96e062..86c221cc85 100644 --- a/test/manual/hicache/test_npu_kimi_k3_hicache.py +++ b/test/manual/hicache/test_npu_kimi_k3_hicache.py @@ -28,8 +28,7 @@ class TestKimiK3MixedWithHiCacheL2(GSM8KAscendMixin, CustomTestCase): "bfloat16", "--tp-size", "64", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--moe-a2a-backend", diff --git a/test/manual/nightly/test_deepseek_v32_perf.py b/test/manual/nightly/test_deepseek_v32_perf.py index bb64468c97..85f9ee1d01 100644 --- a/test/manual/nightly/test_deepseek_v32_perf.py +++ b/test/manual/nightly/test_deepseek_v32_perf.py @@ -24,9 +24,8 @@ class TestNightlyDeepseekV32Performance(unittest.TestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true}', ], @@ -37,9 +36,8 @@ class TestNightlyDeepseekV32Performance(unittest.TestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-num-steps", @@ -60,9 +58,8 @@ class TestNightlyDeepseekV32Performance(unittest.TestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--attention-backend", "dsa", "--dsa-prefill-backend", diff --git a/test/manual/quant/test_deepseek_v32_fp4_4gpu.py b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py index 773029805a..d56542bda4 100644 --- a/test/manual/quant/test_deepseek_v32_fp4_4gpu.py +++ b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py @@ -24,9 +24,8 @@ class TestDeepseekV32FP4DP(CustomTestCase): other_args = [ "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-runner-backend", "flashinfer_trtllm", "--quantization", diff --git a/test/manual/test_cross_node_scheduler_info_sync.py b/test/manual/test_cross_node_scheduler_info_sync.py index f6e5a835fb..2704f91b5d 100755 --- a/test/manual/test_cross_node_scheduler_info_sync.py +++ b/test/manual/test_cross_node_scheduler_info_sync.py @@ -44,7 +44,7 @@ from sglang.test.test_utils import ( class TestCase: name: str tp_size: int - dp_size: int + attn_dp_size: int nnodes: int gpus_per_node: int expected_ranks: int @@ -55,7 +55,7 @@ TEST_CASES = { "tp4_nodes2": TestCase( name="tp4_nodes2", tp_size=4, - dp_size=1, + attn_dp_size=1, nnodes=2, gpus_per_node=2, expected_ranks=4, @@ -64,20 +64,20 @@ TEST_CASES = { "dp2_single_node": TestCase( name="dp2_single_node", tp_size=2, - dp_size=2, + attn_dp_size=2, nnodes=1, gpus_per_node=2, expected_ranks=2, - extra_args=["--enable-dp-attention", "--dp", "2", "--attention-backend", "fa3"], + extra_args=["--attn-dp-size", "2", "--attention-backend", "fa3"], ), "dp2_tp2_nodes2": TestCase( name="dp2_tp2_nodes2", tp_size=4, - dp_size=2, + attn_dp_size=2, nnodes=2, gpus_per_node=2, expected_ranks=4, - extra_args=["--enable-dp-attention", "--dp", "2", "--attention-backend", "fa3"], + extra_args=["--attn-dp-size", "2", "--attention-backend", "fa3"], ), } diff --git a/test/manual/test_kv_events.py b/test/manual/test_kv_events.py index 01607eb625..55c5e05783 100644 --- a/test/manual/test_kv_events.py +++ b/test/manual/test_kv_events.py @@ -48,9 +48,6 @@ class TestKvEvents(CustomTestCase): 32, "--cuda-graph-max-bs-decode", 2, - "--enable-dp-attention", - "--dp-size", - 1, ], ) @@ -201,8 +198,7 @@ class TestKvEvents(CustomTestCase): 64, "--cuda-graph-max-bs-decode", 4, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 2, "--tp-size", 2, diff --git a/test/manual/test_ray_engine.py b/test/manual/test_ray_engine.py index e53f343bca..cd98f04287 100644 --- a/test/manual/test_ray_engine.py +++ b/test/manual/test_ray_engine.py @@ -101,12 +101,8 @@ def _create_engine_on_pg( self.engine.shutdown() self.engine = None - enable_dp_attention = (extra_kwargs or {}).get("enable_dp_attention", False) - if enable_dp_attention: - # DP attention folds DP into TP — total GPUs = tp_size * pp_size - total_gpus = tp_size * pp_size - else: - total_gpus = dp_size * tp_size * pp_size + # Attention DP runs inside the TP group, so only replicas add GPUs. + total_gpus = dp_size * tp_size * pp_size pg = placement_group( [{"CPU": 1, "GPU": total_gpus}], strategy="STRICT_PACK", @@ -292,14 +288,14 @@ class TestRayEngineOfflineDP2(unittest.TestCase): # --------------------------------------------------------------------------- -# Tests: Offline DP Attention (dp=2, tp=2) +# Tests: Offline DP Attention (attn_dp_size=2, tp=2) # --------------------------------------------------------------------------- @unittest.skipUnless(_has_ray, "ray is not installed") @unittest.skipUnless(_NUM_GPUS >= 2, "requires at least 2 GPUs") class TestRayEngineOfflineDPAttention(unittest.TestCase): - """Test Ray engine with dp_size=2, tp_size=2, enable_dp_attention=True.""" + """Test Ray engine with attn_dp_size=2, tp_size=2.""" @classmethod def setUpClass(cls): @@ -307,10 +303,9 @@ class TestRayEngineOfflineDPAttention(unittest.TestCase): ray.init(log_to_driver=True, runtime_env=_RAY_RUNTIME_ENV) cls.actor, cls.pg = _create_engine_on_pg( tp_size=2, - dp_size=2, model=_DP_ATTN_MODEL, extra_kwargs={ - "enable_dp_attention": True, + "attn_dp_size": 2, "disable_cuda_graph": True, "port": 31500, }, @@ -630,7 +625,7 @@ class TestRayEnginePlacementGroup(unittest.TestCase): del os.environ["SGLANG_RAY_BUNDLE_INDICES"] def test_custom_pg_dp_attention(self): - """Test custom placement_group with enable_dp_attention=True.""" + """Test custom placement_group with tp_size=2 on the DP-attention model.""" from sglang.srt.ray.engine import RayEngine pg = placement_group([{"GPU": 1}] * 2, strategy="STRICT_PACK") @@ -641,7 +636,6 @@ class TestRayEnginePlacementGroup(unittest.TestCase): tp_size=2, placement_group=pg, use_ray=True, - enable_dp_attention=True, ) result = engine.generate("The capital of France is", _SAMPLING_PARAMS) diff --git a/test/manual/test_two_batch_overlap.py b/test/manual/test_two_batch_overlap.py index 2786aa6f90..5ea72ccdd1 100644 --- a/test/manual/test_two_batch_overlap.py +++ b/test/manual/test_two_batch_overlap.py @@ -34,9 +34,8 @@ class TestTwoBatchOverlap(unittest.TestCase): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", @@ -135,9 +134,8 @@ class TestQwen3TwoBatchOverlap(TestTwoBatchOverlap): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/manual/test_w4a8_deepseek_v3.py b/test/manual/test_w4a8_deepseek_v3.py index e005679854..096f075783 100644 --- a/test/manual/test_w4a8_deepseek_v3.py +++ b/test/manual/test_w4a8_deepseek_v3.py @@ -135,9 +135,8 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase): "deepep", "--deepep-mode", "normal", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-runner-backend", "cutlass", ] @@ -191,9 +190,8 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase): "auto", "--deepep-dispatcher-output-dtype", "bf16", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-runner-backend", "cutlass", "--speculative-algorithm", diff --git a/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py index d2740823cd..a335b8479b 100644 --- a/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py @@ -114,9 +114,8 @@ DEEPSEEK_R1_MODELS = [ other_args=[ "--chunked-prefill-size", "131072", - "--dp-size", + "--attn-dp-size", "8", - "--enable-dp-attention", "--mem-fraction-static", "0.85", "--trust-remote-code", diff --git a/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py index ab1f9b7a14..179d194961 100644 --- a/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py @@ -50,9 +50,8 @@ class TestDeepseekV32DP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--attention-backend", "aiter", "--chunked-prefill-size", diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py index 2ca6033f9a..fd45a55b26 100644 --- a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py @@ -49,9 +49,8 @@ class TestDeepseekV32DP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true}', "--dsa-prefill-backend", diff --git a/test/registered/amd/test_deepseek_v32_basic.py b/test/registered/amd/test_deepseek_v32_basic.py index 71610f4ecf..9c01179a21 100644 --- a/test/registered/amd/test_deepseek_v32_basic.py +++ b/test/registered/amd/test_deepseek_v32_basic.py @@ -30,9 +30,8 @@ class TestDeepseekV32DP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true}', ] diff --git a/test/registered/amd/test_deepseek_v32_mtp.py b/test/registered/amd/test_deepseek_v32_mtp.py index e3a8f4fed0..74cf614ac7 100644 --- a/test/registered/amd/test_deepseek_v32_mtp.py +++ b/test/registered/amd/test_deepseek_v32_mtp.py @@ -36,9 +36,8 @@ class TestDeepseekV32DPMTP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-num-steps", diff --git a/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py b/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py index 331566ff97..88c8a24de7 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py @@ -6,7 +6,7 @@ two-batch-overlap path on MI35x ROCm 7.2. TBO here is the DP-attention TP-MoE variant (moe_a2a_backend='none'): it overlaps one micro-batch's DP all_gatherv (pre-MoE gather) + reduce_scatterv (post-MoE combine) with the other micro-batch's attention + expert compute (prefill only). -Enabled purely via `--enable-dp-attention` + `--enable-two-batch-overlap` (no opt-in +Enabled purely via `--attn-dp-size` + `--enable-two-batch-overlap` (no opt-in env). This test guards that TBO does not regress GSM8K accuracy and that the DP TBO server launches + runs to completion (exercises op_gather/op_moe/op_combine and the event+ref combine-buffer lifetime that fixed the reserved-memory OOM at mem0.9). @@ -93,9 +93,8 @@ class TestDeepseekV4FlashFp8Tbo(CustomTestCase): "8", # DP attention + TBO: non-EP DP TP-MoE two-batch-overlap. DP TBO is # selected because moe_a2a_backend stays 'none'; no opt-in env needed. - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-prefill-delayer", "--enable-two-batch-overlap", "--disable-radix-cache", @@ -113,7 +112,7 @@ class TestDeepseekV4FlashFp8Tbo(CustomTestCase): "0.90", "--swa-full-tokens-ratio", "0.15", - # global chunk; DP-attention divides by dp_size=8 -> 8192/rank. + # global chunk; DP-attention divides by attn_dp_size=8 -> 8192/rank. "--chunked-prefill-size", "65536", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py index cfc73385c6..2077891b24 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py @@ -183,9 +183,8 @@ class TestDeepseekV4DSparkUnifiedKVGSM8K(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-dp-lm-head", "--enable-prefill-delayer", "--disable-radix-cache", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py index 458f70f250..797ede0ba2 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py @@ -6,7 +6,7 @@ two-batch-overlap path on MI35x ROCm 7.2. TBO here is the DP-attention TP-MoE variant (moe_a2a_backend='none'): it overlaps one micro-batch's DP all_gatherv (pre-MoE gather) + reduce_scatterv (post-MoE combine) with the other micro-batch's attention + expert compute (prefill only). -Enabled purely via `--enable-dp-attention` + `--enable-two-batch-overlap` (no opt-in +Enabled purely via `--attn-dp-size` + `--enable-two-batch-overlap` (no opt-in env). This test guards that TBO does not regress GSM8K accuracy and that the DP TBO server launches + runs (exercises op_gather/op_moe/op_combine and the event+ref combine-buffer lifetime that fixed the reserved-memory OOM at mem0.9). @@ -83,9 +83,8 @@ class TestDeepseekV4ProFp4Tbo(CustomTestCase): "8", # DP attention + TBO: non-EP DP TP-MoE two-batch-overlap. DP TBO is # selected because moe_a2a_backend stays 'none'; no opt-in env needed. - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-prefill-delayer", "--enable-two-batch-overlap", "--disable-radix-cache", @@ -103,7 +102,7 @@ class TestDeepseekV4ProFp4Tbo(CustomTestCase): "0.90", "--swa-full-tokens-ratio", "0.15", - # global chunk; DP-attention divides by dp_size=8 -> 8192/rank. + # global chunk; DP-attention divides by attn_dp_size=8 -> 8192/rank. "--chunked-prefill-size", "65536", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py index 97876b7699..ad82b29383 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py @@ -70,9 +70,8 @@ class TestDeepseekV4ProFp4TboMTP(CustomTestCase): "8", # DP attention + TBO: non-EP DP TP-MoE two-batch-overlap. DP TBO is # selected because moe_a2a_backend stays 'none'; no opt-in env needed. - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-prefill-delayer", "--enable-two-batch-overlap", "--disable-radix-cache", @@ -101,7 +100,7 @@ class TestDeepseekV4ProFp4TboMTP(CustomTestCase): "0.90", "--swa-full-tokens-ratio", "0.15", - # global chunk; DP-attention divides by dp_size=8 -> 8192/rank. + # global chunk; DP-attention divides by attn_dp_size=8 -> 8192/rank. "--chunked-prefill-size", "65536", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_moriep_small.py b/test/registered/amd/test_moriep_small.py index 5f407552d9..ab7ec35a47 100644 --- a/test/registered/amd/test_moriep_small.py +++ b/test/registered/amd/test_moriep_small.py @@ -49,9 +49,8 @@ common_args = [ "8", "--ep-size", "8", - "--dp-size", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "mori", "--trust-remote-code", diff --git a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py index aadfefb40a..f6bb9d0459 100644 --- a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py +++ b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py @@ -142,9 +142,8 @@ class TestFlashinferCuteDSLMoeBackendNvFp4OnlineW4A16( "1", "--speculative-num-draft-tokens", "4", - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-a2a-backend", "flashinfer", diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index 13a10ed3c3..b89d74e968 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -170,8 +170,7 @@ class FlashinferTrtllmGenMoeBackendMXFP8A2ABase: other_args=[ "--quantization", "mxfp8", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--tp-size", "4", diff --git a/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py b/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py index 8120531687..34a008401e 100644 --- a/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py +++ b/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py @@ -68,7 +68,6 @@ class TestDSV4FlashFP4B200Balanced_CP_DeepEP( "4", "--attn-cp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -119,7 +118,6 @@ class TestDSV4FlashFP4B200Balanced_CP_Megamoe( "4", "--attn-cp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "megamoe", "--enable-w4a4-mxfp4-megamoe", diff --git a/test/registered/cpu/test_intel_amx_attention_backend_a.py b/test/registered/cpu/test_intel_amx_attention_backend_a.py index 23a27c8269..bb8671f233 100644 --- a/test/registered/cpu/test_intel_amx_attention_backend_a.py +++ b/test/registered/cpu/test_intel_amx_attention_backend_a.py @@ -82,8 +82,7 @@ class TestDPAttention(CustomTestCase): "--disable-overlap-schedule", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] cls.process = popen_launch_server( diff --git a/test/registered/debug_utils/comparator/test_entrypoint.py b/test/registered/debug_utils/comparator/test_entrypoint.py index 422c38cd14..8a034b6c66 100644 --- a/test/registered/debug_utils/comparator/test_entrypoint.py +++ b/test/registered/debug_utils/comparator/test_entrypoint.py @@ -4550,7 +4550,7 @@ class TestEntrypointDpAttentionMissingAlias: "moe_tp_size": 2, "moe_dp_rank": 0, "moe_dp_size": 1, - "enable_dp_attention": True, + "attn_dp_enabled": True, "attn_tp_rank": 0, "attn_tp_size": 1, "attn_dp_rank": tp_rank, diff --git a/test/registered/debug_utils/test_dumper.py b/test/registered/debug_utils/test_dumper.py index 9423c08770..14c65a4532 100644 --- a/test/registered/debug_utils/test_dumper.py +++ b/test/registered/debug_utils/test_dumper.py @@ -2423,7 +2423,7 @@ class TestDumperE2E: "moe_tp_size", "moe_dp_rank", "moe_dp_size", - "enable_dp_attention", + "attn_dp_enabled", "attn_tp_rank", "attn_tp_size", "attn_dp_rank", diff --git a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py index 7615d09574..510a0d6782 100644 --- a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py +++ b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py @@ -197,7 +197,7 @@ class TestSourcePatcherE2ESGLang: _run_e2e_scenario( tmp_path=tmp_path, target_tp=BASELINE_TP, - extra_target_server_args=["--dp", "2", "--enable-dp-attention"], + extra_target_server_args=["--attn-dp-size", "2"], target_patch_config_yaml=PATCH_CONFIG_DP_ATTENTION_YAML, extra_comparator_args=[ "--token-aligner", diff --git a/test/registered/disaggregation/test_disaggregation_dp_attention.py b/test/registered/disaggregation/test_disaggregation_dp_attention.py index 50a8a4f1c5..69d80b3150 100644 --- a/test/registered/disaggregation/test_disaggregation_dp_attention.py +++ b/test/registered/disaggregation/test_disaggregation_dp_attention.py @@ -54,9 +54,8 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase): cls.bootstrap_port, "--tp", str(cls.PREFILL_DP_SIZE), - "--dp", + "--attn-dp-size", str(cls.PREFILL_DP_SIZE), - "--enable-dp-attention", "--load-balance-method", cls.LOAD_BALANCE_METHOD, ] @@ -80,9 +79,8 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase): cls.bootstrap_port, "--tp", str(cls.DECODE_DP_SIZE), - "--dp", + "--attn-dp-size", str(cls.DECODE_DP_SIZE), - "--enable-dp-attention", "--base-gpu-id", str(cls.PREFILL_DP_SIZE), "--load-balance-method", diff --git a/test/registered/disaggregation/test_disaggregation_dsv4.py b/test/registered/disaggregation/test_disaggregation_dsv4.py index 4d5722f549..aa5caa7db3 100644 --- a/test/registered/disaggregation/test_disaggregation_dsv4.py +++ b/test/registered/disaggregation/test_disaggregation_dsv4.py @@ -66,9 +66,8 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8 cls.prefill_nccl_port, "--tp", 4, - "--dp", + "--attn-dp-size", 4, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", @@ -102,9 +101,8 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8 cls.decode_nccl_port, "--tp", 4, - "--dp", + "--attn-dp-size", 4, - "--enable-dp-attention", "--base-gpu-id", 4, "--moe-a2a-backend", diff --git a/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py b/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py index 4ea69b8ea6..6b12303fc2 100644 --- a/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py +++ b/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py @@ -35,9 +35,8 @@ class TestDisaggregationDWDPGptOss(PDDisaggregationServerBase): "0.85", ] extra_decode_args = [ - "--dp", + "--attn-dp-size", str(NUM_DECODE_GPUS), - "--enable-dp-attention", "--disable-flashinfer-autotune", "--mem-fraction-static", "0.85", diff --git a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py index 3a47277b76..bc1a4b190d 100644 --- a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py +++ b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py @@ -210,9 +210,8 @@ class TestDisaggregationHybridAttentionGDNDPDecode(PDDisaggregationServerBase): cls.bootstrap_port, "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--base-gpu-id", "2", diff --git a/test/registered/dp_attn/test_dp_attention.py b/test/registered/dp_attn/test_dp_attention.py index 7e7ea2b6f6..4dbfeb59d3 100644 --- a/test/registered/dp_attn/test_dp_attention.py +++ b/test/registered/dp_attn/test_dp_attention.py @@ -54,8 +54,7 @@ class TestDPAttentionDP2TP2( "4", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) @@ -72,7 +71,7 @@ class TestDPAttentionGatherv( ): """Exercise the variable-length all_gatherv + reduce_scatterv DP-MoE path (SGLANG_DP_USE_GATHERV=1). The path only activates for the - attn_tp_size == 1, tp_size == dp_size layout, which tp2 + dp2 satisfies. + attn_tp_size == 1, tp_size == attn_dp_size layout, which tp2 + attn_dp2 satisfies. Without this test the gatherv/reduce_scatterv code is never exercised by CI (it is gated behind the env var, default off). gsm8k must stay correct since the change is a pure communication reorg, not a numerics change.""" @@ -92,8 +91,7 @@ class TestDPAttentionGatherv( "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--chunked-prefill-size", "256", @@ -124,8 +122,7 @@ class TestDPAttentionMixedChunk( "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--enable-mixed-chunk", "--chunked-prefill-size", @@ -157,8 +154,7 @@ class TestDPRetract( "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--max-total-tokens", "4500", @@ -194,8 +190,7 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) diff --git a/test/registered/dp_attn/test_dp_attention_bcg_kl.py b/test/registered/dp_attn/test_dp_attention_bcg_kl.py index 0b56187464..b16e398949 100644 --- a/test/registered/dp_attn/test_dp_attention_bcg_kl.py +++ b/test/registered/dp_attn/test_dp_attention_bcg_kl.py @@ -220,9 +220,8 @@ class _DPAttentionPrefillCudaGraphKLMixin: "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-deterministic-inference", "--attention-backend", cls.attention_backend, @@ -287,7 +286,7 @@ class _DPAttentionPrefillCudaGraphKLMixin: def test_prefill_and_decode_cache_hit_kl_is_zero(self): server_info = requests.get(self.base_url + "/server_info", timeout=30).json() self.assertFalse(server_info["disable_radix_cache"]) - self.assertTrue(server_info["enable_dp_attention"]) + self.assertEqual(server_info["attn_dp_size"], 2) self.assertTrue(server_info["enable_deterministic_inference"]) self.assertEqual(server_info["attention_backend"], self.attention_backend) self.assertEqual( diff --git a/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py b/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py index 556d986d32..43f058f5ac 100644 --- a/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py +++ b/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py @@ -49,9 +49,8 @@ class TestDPAttentionLocalControlBroadcast(PauseResumeInPlaceMixin, CustomTestCa "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-attention-local-control-broadcast", ], ) diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py index 4958d2f2c3..742ecfdff8 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py @@ -133,9 +133,8 @@ class TestDSV4FlashFP4NonSpecB200( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", @@ -180,9 +179,8 @@ class TestDSV4FlashFP4BreakableCudaGraphB200( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-mixed-chunk", "--cuda-graph-backend-prefill", "breakable", diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py index 80173c0e51..ffa620c113 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py @@ -122,9 +122,8 @@ class TestDSV4FlashFP4BreakableCudaGraphB200Trtllm( "trtllm", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-mixed-chunk", "--cuda-graph-backend-prefill", "breakable", diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py index ac3f47d871..0e565b7478 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py @@ -61,9 +61,8 @@ class TestDSV4FlashFP4B200W4A8MegaMoE( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-a2a-backend", "megamoe", @@ -103,9 +102,8 @@ class TestDSV4FlashFP4B200W4A4MegaMoE( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-a2a-backend", "megamoe", diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py index 4c956f2457..3e0b5a03e0 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py @@ -53,9 +53,8 @@ class TestDSV4FlashFP8H200( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -114,9 +113,8 @@ class TestDSV4FlashFP8H200MegaMoE( "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--moe-runner-backend", @@ -141,10 +139,11 @@ class TestDSV4FlashFP8H200MegaMoE( env={ "SGLANG_DSV4_FP4_EXPERTS": "0", # INVARIANT: this per-rank cap MUST equal - # chunked_prefill_size / dp_size (= 8192 / 4 = 2048), the per-rank - # prefill bound under --enable-dp-attention. If you change - # --chunked-prefill-size or --dp above, update this to match — - # otherwise MegaMoE falls back whenever a prefill chunk exceeds it. + # chunked_prefill_size / attn_dp_size (= 8192 / 4 = 2048), the + # per-rank prefill bound under attention DP. If you change + # --chunked-prefill-size or --attn-dp-size above, update this + # to match — otherwise MegaMoE falls back whenever a prefill + # chunk exceeds it. "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048", }, ) diff --git a/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py b/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py index 85d8a0c8f9..cb8bf2f2b3 100644 --- a/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py +++ b/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py @@ -20,7 +20,7 @@ class TestGLM52DPMTP( ): model = "zai-org/GLM-5.2-FP8" mem_fraction_static = 0.88 - enable_dp_attention = True + attn_dp_size = 8 bs_1_speed_thres = 70 diff --git a/test/registered/e2e/models/test_dsa_glm52_hisparse.py b/test/registered/e2e/models/test_dsa_glm52_hisparse.py index 5cfdd34dd7..bd5add5e6b 100644 --- a/test/registered/e2e/models/test_dsa_glm52_hisparse.py +++ b/test/registered/e2e/models/test_dsa_glm52_hisparse.py @@ -24,9 +24,8 @@ class TestGLM52HiSparse(DefaultServerBase, GSM8KMixin): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--page-size", "64", "--max-running-requests", diff --git a/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py b/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py index db33bda82a..be03a67492 100644 --- a/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py +++ b/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py @@ -20,9 +20,8 @@ class TestGLM52NVFP4DPMTP( ): model = "nvidia/GLM-5.2-NVFP4" tp_size = 4 - dp_size = 4 + attn_dp_size = 4 mem_fraction_static = 0.88 - enable_dp_attention = True bs_1_speed_thres = 180 gsm8k_accuracy_thres = 0.92 extra_server_args = [ diff --git a/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py b/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py index 37611054b1..c4b5ddb91e 100644 --- a/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py +++ b/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py @@ -107,9 +107,8 @@ class TestGLM52DSACacheLayerSplitMixedMoE(TestGLM52DSACacheLayerSplit): gsm8k_num_threads = 32 extra_decode_args = TestGLM52DSACacheLayerSplit.extra_decode_args + [ - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--ep-size", "4", diff --git a/test/registered/e2e/models/test_glm53_flash_b200.py b/test/registered/e2e/models/test_glm53_flash_b200.py index 8dd646ec22..4e93d154ff 100644 --- a/test/registered/e2e/models/test_glm53_flash_b200.py +++ b/test/registered/e2e/models/test_glm53_flash_b200.py @@ -105,8 +105,7 @@ class TestGLM53FlashB200HighThroughput( gsm8k_num_shots = 20 server_args = [ *COMMON_SERVER_ARGS, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--cuda-graph-backend-prefill", "breakable", diff --git a/test/registered/e2e/models/test_glm53_flash_h200.py b/test/registered/e2e/models/test_glm53_flash_h200.py index 1349e8910b..1f9e0747fa 100644 --- a/test/registered/e2e/models/test_glm53_flash_h200.py +++ b/test/registered/e2e/models/test_glm53_flash_h200.py @@ -104,8 +104,7 @@ class TestGLM53FlashH200HighThroughput( gsm8k_num_shots = 20 server_args = [ *COMMON_SERVER_ARGS, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "8", "--mm-enable-dp-encoder", ] diff --git a/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py b/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py index 8ffe7dda98..5582276896 100644 --- a/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py +++ b/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py @@ -32,11 +32,10 @@ class TestGptOss4GpuMxfp4(BaseTestGptOss): other_args=[ "--tp", "4", - "--dp", + "--attn-dp-size", "4", "--ep", "4", - "--enable-dp-attention", "--moe-a2a-backend", "none", "--cuda-graph-max-bs-decode", diff --git a/test/registered/e2e/models/test_mimo_v2.py b/test/registered/e2e/models/test_mimo_v2.py index 2e3ae99f11..46dca802d3 100644 --- a/test/registered/e2e/models/test_mimo_v2.py +++ b/test/registered/e2e/models/test_mimo_v2.py @@ -10,9 +10,8 @@ MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5" MIMO_V2_OTHER_ARGS = [ "--tp", "8", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--mm-enable-dp-encoder", "--attention-backend", "fa3", diff --git a/test/registered/e2e/models/test_mimo_v2_flash.py b/test/registered/e2e/models/test_mimo_v2_flash.py index a4e0704abd..cd823a3000 100644 --- a/test/registered/e2e/models/test_mimo_v2_flash.py +++ b/test/registered/e2e/models/test_mimo_v2_flash.py @@ -16,9 +16,8 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): other_args = [ "--tp", "4", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--trust-remote-code", "--attention-backend", "fa4" if is_blackwell() else "fa3", diff --git a/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py b/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py index 8e6365fec8..94f7004eb0 100644 --- a/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py +++ b/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py @@ -27,7 +27,6 @@ class TestDeepseekV32IndexTopkPattern(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', "--json-model-override-args", diff --git a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py index 26905799a8..7b4ea6f7dc 100644 --- a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py @@ -39,13 +39,12 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase): "1024", "--tp", "4", - "--dp", + "--attn-dp-size", "4", "--ep", "4", "--moe-dense-tp-size", "1", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE), "--quantization", diff --git a/test/registered/e2e/models_large/test_glm52_fp8.py b/test/registered/e2e/models_large/test_glm52_fp8.py index 6130fc4e75..b551d4af2f 100644 --- a/test/registered/e2e/models_large/test_glm52_fp8.py +++ b/test/registered/e2e/models_large/test_glm52_fp8.py @@ -32,7 +32,7 @@ class TestGlm52Fp8(unittest.TestCase): """GLM-5.2 FP8 on H200/B200 (8-GPU, tp=8).""" def test_glm52_fp8(self): - dp_args = ["--dp=8", "--enable-dp-attention"] + dp_args = ["--attn-dp-size=8"] variants = [ ModelLaunchSettings( diff --git a/test/registered/e2e/models_large/test_kimi_k25.py b/test/registered/e2e/models_large/test_kimi_k25.py index e3b6b114dc..439b66b96b 100644 --- a/test/registered/e2e/models_large/test_kimi_k25.py +++ b/test/registered/e2e/models_large/test_kimi_k25.py @@ -31,8 +31,7 @@ class TestKimiK25(unittest.TestCase): ] dp_attn_args = [ - "--dp=8", - "--enable-dp-attention", + "--attn-dp-size=8", ] variants = [ diff --git a/test/registered/e2e/models_large/test_minimax_m25.py b/test/registered/e2e/models_large/test_minimax_m25.py index 900a71b77d..fbcda5b318 100644 --- a/test/registered/e2e/models_large/test_minimax_m25.py +++ b/test/registered/e2e/models_large/test_minimax_m25.py @@ -31,8 +31,7 @@ class TestMiniMaxM25(unittest.TestCase): "--reasoning-parser=minimax-append-think", ] dp_attn_args = base_args + [ - "--enable-dp-attention", - "--dp=8", + "--attn-dp-size=8", ] variants = [ diff --git a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py index 257b62b2f1..43f5a878f9 100644 --- a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py @@ -30,9 +30,8 @@ NEMOTRON_3_SUPER_NVFP4_ARGS = [ ] DP_ATTENTION_EP_ARGS = [ - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--ep-size", "4", diff --git a/test/registered/e2e/models_large/test_qwen35.py b/test/registered/e2e/models_large/test_qwen35.py index 24a2228596..bc7178c058 100644 --- a/test/registered/e2e/models_large/test_qwen35.py +++ b/test/registered/e2e/models_large/test_qwen35.py @@ -31,7 +31,7 @@ class TestQwen35(unittest.TestCase): "--tool-call-parser=qwen3_coder", "--mem-fraction-static=0.8", ] - dp_args = ["--dp=8", "--enable-dp-attention"] + dp_args = ["--attn-dp-size=8"] mtp_args = [ "--speculative-algorithm=EAGLE", "--speculative-num-steps=3", diff --git a/test/registered/e2e/pp/test_pp_spec.py b/test/registered/e2e/pp/test_pp_spec.py index 10e37461e6..2d6bac6222 100644 --- a/test/registered/e2e/pp/test_pp_spec.py +++ b/test/registered/e2e/pp/test_pp_spec.py @@ -198,9 +198,7 @@ class TestPPSpecGate(CustomTestCase): # DP attention partitions the batch per DP rank, so the stages # would no longer rebuild the same verify tree. with self.assertRaises(AssertionError): - self._server_args( - tp_size=2, dp_size=2, enable_dp_attention=True - ).check_server_args() + self._server_args(tp_size=2, attn_dp_size=2).check_server_args() # Adaptive spec changes num_draft_tokens at runtime, which the # relay slices results with. with self.assertRaises(AssertionError): diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index 5acfafa770..051b24fb82 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -34,8 +34,7 @@ class TestDeepseekMTP(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -108,9 +107,8 @@ class TestDeepseekV32TBO(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-two-batch-overlap", "--moe-a2a-backend", "deepep", diff --git a/test/registered/ep/test_deepep_small.py b/test/registered/ep/test_deepep_small.py index cb526f8298..2031624df4 100644 --- a/test/registered/ep/test_deepep_small.py +++ b/test/registered/ep/test_deepep_small.py @@ -32,8 +32,7 @@ class TestPureDP(CustomTestCase): "--trust-remote-code", "--tp-size", "4", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--moe-a2a-backend", "deepep", @@ -121,8 +120,7 @@ class TestTBO(CustomTestCase): "--trust-remote-code", "--tp-size", "4", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -173,8 +171,7 @@ class TestMTPWithTBO(CustomTestCase): other_args=[ "--tp-size", "4", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--enable-two-batch-overlap", "--moe-a2a-backend", diff --git a/test/registered/ep/test_deepep_small_extra.py b/test/registered/ep/test_deepep_small_extra.py index 3b0a2c6b4a..e5c4d3fe5a 100644 --- a/test/registered/ep/test_deepep_small_extra.py +++ b/test/registered/ep/test_deepep_small_extra.py @@ -29,8 +29,7 @@ class TestHybridDPTP(CustomTestCase): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--moe-a2a-backend", "deepep", diff --git a/test/registered/ep/test_eplb_no_a2a.py b/test/registered/ep/test_eplb_no_a2a.py index ea3b385089..2f451901f3 100644 --- a/test/registered/ep/test_eplb_no_a2a.py +++ b/test/registered/ep/test_eplb_no_a2a.py @@ -99,8 +99,7 @@ class TestEPLBNoA2ADPAttention(TestEPLBNoA2A): the expert-weight migration.""" extra_args = [ - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] rebalance_num_iterations = "50" diff --git a/test/registered/ep/test_flashinfer_a2a.py b/test/registered/ep/test_flashinfer_a2a.py index a8bd95cd7e..379cbe00ef 100644 --- a/test/registered/ep/test_flashinfer_a2a.py +++ b/test/registered/ep/test_flashinfer_a2a.py @@ -43,9 +43,8 @@ class TestFlashinferA2ATrtllmRoutedFP4(CustomTestCase): "4", "--ep", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE), "--moe-a2a-backend", @@ -96,9 +95,8 @@ class TestFlashinferA2ACutedslStaticFP4(CustomTestCase): "4", "--ep", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE + 1), "--moe-a2a-backend", @@ -157,9 +155,8 @@ class TestFlashinferA2ATrtllmRoutedFP8(CustomTestCase): "4", "--ep", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE + 2), "--moe-a2a-backend", diff --git a/test/registered/ep/test_mooncake_ep_small.py b/test/registered/ep/test_mooncake_ep_small.py index 26a6916f9c..46e7670244 100644 --- a/test/registered/ep/test_mooncake_ep_small.py +++ b/test/registered/ep/test_mooncake_ep_small.py @@ -93,8 +93,7 @@ class TestTP(CustomTestCase): class TestPureDP(TestTP): extra_args = [ - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", ] @@ -135,8 +134,7 @@ class TestPureDP(TestTP): @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") class TestHybridDPTP(TestPureDP): extra_args = [ - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] diff --git a/test/registered/ep/test_routed_experts_dp_readback.py b/test/registered/ep/test_routed_experts_dp_readback.py index c8a9905324..d946ab4a13 100644 --- a/test/registered/ep/test_routed_experts_dp_readback.py +++ b/test/registered/ep/test_routed_experts_dp_readback.py @@ -71,11 +71,10 @@ class _ReadbackMixin: ), "--tp", "2", - "--dp", + "--attn-dp-size", "2", "--ep", "2", - "--enable-dp-attention", "--enable-return-routed-experts", "--disable-cuda-graph", "--disable-radix-cache", diff --git a/test/registered/ep/test_tbo_shared_experts_fusion.py b/test/registered/ep/test_tbo_shared_experts_fusion.py index dd9cc0a713..52ce20b522 100644 --- a/test/registered/ep/test_tbo_shared_experts_fusion.py +++ b/test/registered/ep/test_tbo_shared_experts_fusion.py @@ -30,8 +30,7 @@ class TestTBOWithSharedExpertsFusion(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py index 873d8c2259..be1cfe563b 100644 --- a/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py @@ -15,9 +15,8 @@ DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":9 BALANCED_ARGS = [ "--trust-remote-code", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py index 800f35cc6e..e154f39cbf 100644 --- a/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py @@ -14,9 +14,8 @@ SERVER_LAUNCH_TIMEOUT = 3600 HIGH_THROUGHPUT_ARGS = [ "--trust-remote-code", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "megamoe", "--mem-fraction-static", diff --git a/test/registered/gb300/test_glm52_nvfp4.py b/test/registered/gb300/test_glm52_nvfp4.py index 0be117022a..358d3ea55d 100644 --- a/test/registered/gb300/test_glm52_nvfp4.py +++ b/test/registered/gb300/test_glm52_nvfp4.py @@ -52,9 +52,7 @@ class TestGlm52Nvfp4(CustomTestCase): ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_MTP_ARGS, + extra_args=COMMON_ARGS + ["--attn-dp-size=4"] + DP_MTP_ARGS, variant="TP4+DP4+DPA+MTP", ), ] diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py index 9554584c04..7469a9d1f7 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py @@ -48,9 +48,7 @@ class TestKimiK25Nvfp4Dp(CustomTestCase): ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_EAGLE_ARGS, + extra_args=COMMON_ARGS + ["--attn-dp-size=4"] + DP_EAGLE_ARGS, variant="TP4+DP4+DPA+EAGLE3", ) ], diff --git a/test/registered/gb300/test_qwen35_fp8_dp.py b/test/registered/gb300/test_qwen35_fp8_dp.py index 380c713f8a..e62234096b 100644 --- a/test/registered/gb300/test_qwen35_fp8_dp.py +++ b/test/registered/gb300/test_qwen35_fp8_dp.py @@ -45,9 +45,7 @@ class TestQwen35Fp8Dp(CustomTestCase): ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_MTP_ARGS, + extra_args=COMMON_ARGS + ["--attn-dp-size=4"] + DP_MTP_ARGS, variant="TP4+DP4+DPA+MTP", ) ], diff --git a/test/registered/kernels/ops/attention/test_dsa_indexer.py b/test/registered/kernels/ops/attention/test_dsa_indexer.py index c5999d23b1..254ad92ed7 100644 --- a/test/registered/kernels/ops/attention/test_dsa_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsa_indexer.py @@ -278,7 +278,6 @@ class TestDSAIndexer(CustomTestCase): def setUpClass(cls): """Set up global server args for testing.""" server_args = ServerArgs(model_path="dummy") - server_args.enable_dp_attention = False server_args.dsa_prefill_backend = "flashmla_sparse" server_args.dsa_decode_backend = "flashmla_sparse" set_global_server_args_for_scheduler(server_args) diff --git a/test/registered/mock_model/test_e2e_tp.py b/test/registered/mock_model/test_e2e_tp.py index 030a6a0163..10f2721a86 100644 --- a/test/registered/mock_model/test_e2e_tp.py +++ b/test/registered/mock_model/test_e2e_tp.py @@ -23,9 +23,8 @@ class TestE2ETensorParallel(CustomTestCase): extra_server_args=[ "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-tp-lm-head-all-to-all", "--cuda-graph-max-bs-decode", "4", diff --git a/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py b/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py index 373a293c22..11ead88f20 100644 --- a/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py +++ b/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py @@ -22,8 +22,7 @@ class TestPrefetchCheckpointsMultiGPU(CustomTestCase): cls.engine = sgl.Engine( model_path="Qwen/Qwen1.5-MoE-A2.7B-Chat", tp_size=4, - dp_size=4, - enable_dp_attention=True, + attn_dp_size=4, disable_radix_cache=True, weight_loader_prefetch_checkpoints=True, cuda_graph_max_bs_decode=1, diff --git a/test/registered/model_loading/test_weight_cache_daemon.py b/test/registered/model_loading/test_weight_cache_daemon.py index 22dbe4726a..7e6bcbd1e9 100644 --- a/test/registered/model_loading/test_weight_cache_daemon.py +++ b/test/registered/model_loading/test_weight_cache_daemon.py @@ -352,11 +352,10 @@ class TestWeightCacheDaemonQwen3MoeDP(TestWeightCacheDaemonTP2): """Qwen3 with static attention DP through the existing IPC fixture.""" daemon_args = [ - "--dp", + "--attn-dp-size", "2", "--ep-size", "1", - "--enable-dp-attention", "--enable-dp-lm-head", "--random-seed", "42", diff --git a/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py b/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py index 7fa1636ef1..253c85fac4 100644 --- a/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py +++ b/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py @@ -46,9 +46,8 @@ class TestCuteDslFlashinferA2A(CustomTestCase): "4", "--ep-size", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-runner-backend", "flashinfer_cutedsl", diff --git a/test/registered/moe/test_flashinfer_a2a_cutlass.py b/test/registered/moe/test_flashinfer_a2a_cutlass.py index 1e4cddc924..9fafc7ffa3 100644 --- a/test/registered/moe/test_flashinfer_a2a_cutlass.py +++ b/test/registered/moe/test_flashinfer_a2a_cutlass.py @@ -50,9 +50,8 @@ class TestFlashinferCutlassFlashinferA2A(CustomTestCase): "4", "--ep-size", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-runner-backend", "flashinfer_cutlass", diff --git a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py index 946891b48b..a3902d7314 100644 --- a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py +++ b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py @@ -67,8 +67,7 @@ class Test01(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", ], ) @@ -104,8 +103,7 @@ class Test02(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", ], ) @@ -178,8 +176,7 @@ class Test04(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -217,8 +214,7 @@ class Test05(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -256,8 +252,7 @@ class Test06(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", ], @@ -294,8 +289,7 @@ class Test07(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", ], @@ -332,8 +326,7 @@ class Test08(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -372,8 +365,7 @@ class Test09(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -450,8 +442,7 @@ class Test11(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-a2a-backend", "deepep", @@ -491,8 +482,7 @@ class Test12(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-a2a-backend", "deepep", @@ -572,8 +562,7 @@ class Test14(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -615,8 +604,7 @@ class Test15(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -658,8 +646,7 @@ class Test16(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -700,8 +687,7 @@ class Test17(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -742,8 +728,7 @@ class Test18(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -786,8 +771,7 @@ class Test19(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -866,8 +850,7 @@ class Test21(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--ep", "8", @@ -905,8 +888,7 @@ class Test22(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--ep", "8", @@ -982,8 +964,7 @@ class Test24(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1023,8 +1004,7 @@ class Test25(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1064,8 +1044,7 @@ class Test26(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--ep", @@ -1104,8 +1083,7 @@ class Test27(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--ep", @@ -1144,8 +1122,7 @@ class Test28(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1186,8 +1163,7 @@ class Test29(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1272,8 +1248,7 @@ class Test31(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--speculative-algo", "EAGLE", @@ -1319,8 +1294,7 @@ class Test32(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--speculative-algo", "EAGLE", @@ -1412,8 +1386,7 @@ class Test34(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1461,8 +1434,7 @@ class Test35(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1510,8 +1482,7 @@ class Test36(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--speculative-algo", @@ -1558,8 +1529,7 @@ class Test37(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--speculative-algo", @@ -1606,8 +1576,7 @@ class Test38(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1656,8 +1625,7 @@ class Test39(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1756,8 +1724,7 @@ class Test41(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-a2a-backend", "deepep", @@ -1809,8 +1776,7 @@ class Test42(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-a2a-backend", "deepep", @@ -1914,8 +1880,7 @@ class Test44(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1969,8 +1934,7 @@ class Test45(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -2024,8 +1988,7 @@ class Test46(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -2078,8 +2041,7 @@ class Test47(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -2132,8 +2094,7 @@ class Test48(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -2188,8 +2149,7 @@ class Test49(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -2290,8 +2250,7 @@ class Test51(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--ep", "8", @@ -2339,8 +2298,7 @@ class Test52(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--ep", "8", @@ -2436,8 +2394,7 @@ class Test54(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -2487,8 +2444,7 @@ class Test55(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -2538,8 +2494,7 @@ class Test56(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--ep", @@ -2588,8 +2543,7 @@ class Test57(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--ep", @@ -2638,8 +2592,7 @@ class Test58(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -2690,8 +2643,7 @@ class Test59(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py index 5549d6a5a0..0445bf4fcd 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py @@ -86,8 +86,7 @@ DEEPSEEK_V4_FLASH_FP8_4P_OTHER_ARGS = [ "auto", "--enable-dp-lm-head", "--disable-radix-cache", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 4, "--reasoning-parser", "deepseek-v4", diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py index de817f8470..f5b869ffb2 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -82,9 +82,8 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ 131072, "--max-running-requests", 192, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index 502e521f9c..87d4952926 100644 --- a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -106,9 +106,8 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ 32, "--nnodes", 2, - "--dp-size", + "--attn-dp-size", 32, - "--enable-dp-attention", "--ep-size", 32, "--mem-fraction-static", diff --git a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py index 1ef75f3b85..76e801b7e2 100644 --- a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py +++ b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py @@ -42,9 +42,8 @@ GLM_5_2_W8A8_16P_TWO_NODE_OTHER_ARGS = [ 32, "--nnodes", 2, - "--dp-size", + "--attn-dp-size", 8, - "--enable-dp-attention", "--chunked-prefill-size", 65536, "--max-prefill-tokens", diff --git a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index 177e68f20c..4a472e322b 100644 --- a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -57,8 +57,7 @@ OTHER_ARGS = [ "ascend_attn", "--sampling-backend", "ascend", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 32, "--moe-a2a-backend", "deepep", diff --git a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py index 9ff5dd16f5..23c7b7ba06 100644 --- a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py +++ b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py @@ -60,8 +60,7 @@ KIMI_K3_W4A8_32P_OTHER_ARGS = [ "bfloat16", "--tp-size", 64, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 4, "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index bd9d193851..e960839ec2 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -34,8 +34,7 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ "--tp-size", 16, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 16, "--mem-fraction-static", 0.75, diff --git a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py index 66d9820f7a..2aef537a1f 100644 --- a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -30,7 +30,6 @@ QWEN3_5_9B_OTHER_ARGS = [ "ascend", "--device", "npu", - "--enable-dp-attention", "--chunked-prefill-size", 4096, "--max-prefill-tokens", diff --git a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 68d57b758e..a75b1c312e 100644 --- a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -62,9 +62,8 @@ QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ 81920, "--max-total-tokens", 122304, - "--dp-size", + "--attn-dp-size", 2, - "--enable-dp-attention", "--enable-dp-lm-head", "--speculative-algorithm", "NEXTN", diff --git a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py index 960fc42065..f93f67d65b 100644 --- a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py +++ b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py @@ -56,8 +56,7 @@ class TestDPAttentionDP2TP2( "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--attention-backend", "ascend", @@ -94,8 +93,7 @@ class TestDPAttentionMixedChunk( "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--enable-mixed-chunk", "--chunked-prefill-size", @@ -128,8 +126,7 @@ class TestDPRetract( "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--max-total-tokens", "4500", @@ -166,8 +163,7 @@ class TestDPAttentionDP2TP2VLM(CustomTestCase): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--attention-backend", "ascend", diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py index 157d0089a4..2fbc3c1f43 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py @@ -40,8 +40,7 @@ class TestDPAttentionRoundBinLoadBalance(CustomTestCase): "--trust-remote-code", "--tp", "16", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--enable-torch-compile", "--torch-compile-max-bs", diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py index ac2e2b2c2e..6b1509e184 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py @@ -70,8 +70,7 @@ class BaseTestNPULoadBalanceMethodDPDisaggregation(TestDisaggregationBase): "prefill", "--tp-size", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--load-balance-method", cls.prefill_load_balance_method, @@ -107,8 +106,7 @@ class BaseTestNPULoadBalanceMethodDPDisaggregation(TestDisaggregationBase): 2, "--tp-size", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--load-balance-method", cls.decode_load_balance_method, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py index 3f45e0a9c1..b6dea5f324 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py @@ -51,9 +51,8 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): "auto", "--tp-size", 16, - "--dp-size", + "--attn-dp-size", 4, - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static", 0.7, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py index 8663a582d4..fe26a763b6 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py @@ -52,9 +52,8 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): "low_latency", "--tp-size", 16, - "--dp-size", + "--attn-dp-size", 4, - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static", 0.7, diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py index 6323f2fd13..3139f87305 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py @@ -109,9 +109,8 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ARGS = [ 131072, "--max-running-requests", 112, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", @@ -157,9 +156,8 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [ "ascend", "--max-running-requests", 896, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 85897ab9b6..b725ae10aa 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -73,9 +73,8 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ 131072, "--max-running-requests", 192, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py index a8577e7080..26c88e0155 100644 --- a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py @@ -106,9 +106,8 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ 32, "--nnodes", 2, - "--dp-size", + "--attn-dp-size", 32, - "--enable-dp-attention", "--ep-size", 32, "--mem-fraction-static", diff --git a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py index 467f085c69..280a3fe638 100644 --- a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py @@ -84,9 +84,8 @@ QWEN3_5_397B_A17B_3K5_1K5_OTHER_ARGS = [ "bfloat16", "--mamba-ssm-dtype", "bfloat16", - "--dp-size", + "--attn-dp-size", 8, - "--enable-dp-attention", "--enable-dp-lm-head", "--speculative-algorithm", "NEXTN", diff --git a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py index 21389a9492..cb494a318d 100644 --- a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py +++ b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py @@ -81,9 +81,8 @@ class TestNPUDeepSeekV4ReturnIndexerTopk(CustomTestCase): "0.72", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--kv-cache-dtype", "auto", diff --git a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py index 3cc9b95527..8caef443fd 100644 --- a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py +++ b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py @@ -56,9 +56,8 @@ class TestNPUGLM52ReturnIndexerTopk(CustomTestCase): "0.56", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--disable-cuda-graph", "--enable-return-indexer-topk", diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index 4013c63bd8..cba73727b5 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -85,7 +85,7 @@ class TestEnableMetrics(CustomTestCase): self.assertIn("1", num_prefill_ranks_values) self._execute_core( - other_args=["--tp", "2", "--dp", "2", "--enable-dp-attention"], + other_args=["--tp", "2", "--attn-dp-size", "2"], verify_metrics_extra=_verify_metrics_extra, expect_mfu_metrics=True, enable_mfu_metrics=True, diff --git a/test/registered/pp/test_pp_parallel_compat.py b/test/registered/pp/test_pp_parallel_compat.py index ceb431b43e..f371549a76 100644 --- a/test/registered/pp/test_pp_parallel_compat.py +++ b/test/registered/pp/test_pp_parallel_compat.py @@ -93,16 +93,15 @@ class TestQwen3MoePPxCP(_Qwen3MoePPCompatMixin, CustomTestCase): class TestQwen3MoePPxDP(_Qwen3MoePPCompatMixin, CustomTestCase): - """PP x DP: pp_size=2 x dp_size=2 attention (tp_size=2).""" + """PP x DP: pp_size=2 x attn_dp_size=2 (tp_size=2).""" parallel_args = [ "--tp-size", "2", "--pp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", ] diff --git a/test/registered/pp/test_pp_single_node.py b/test/registered/pp/test_pp_single_node.py index a652048c8c..ab80d69fb6 100644 --- a/test/registered/pp/test_pp_single_node.py +++ b/test/registered/pp/test_pp_single_node.py @@ -117,8 +117,7 @@ class TestDPAttentionDP2PP2(CustomTestCase): "2", "--pp-size", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py index d70cb6712e..9d458d62b7 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py @@ -54,9 +54,8 @@ class TestUnifiedMiMoHiCacheLoadBackKL(CustomTestCase): "--enable-multimodal", "--tp", "8", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--mm-enable-dp-encoder", "--attention-backend", "fa3", diff --git a/test/registered/rl/test_return_indexer_topk.py b/test/registered/rl/test_return_indexer_topk.py index eb0c24096e..612f3b5e74 100644 --- a/test/registered/rl/test_return_indexer_topk.py +++ b/test/registered/rl/test_return_indexer_topk.py @@ -52,9 +52,8 @@ class TestReturnIndexerTopk(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-return-indexer-topk", # Cap KV pool so the indexer-topk host buffer (488 KB / token for # V3.2) stays bounded; with the default ~600k tokens × 8 procs the diff --git a/test/registered/rl/test_return_routed_experts.py b/test/registered/rl/test_return_routed_experts.py index fc54a43f32..a1664d1746 100644 --- a/test/registered/rl/test_return_routed_experts.py +++ b/test/registered/rl/test_return_routed_experts.py @@ -44,8 +44,8 @@ class TestReturnRoutedExperts(CustomTestCase): under DeepEP a2a + attn_tp_size > 1, across overlap/cuda-graph/radix optimisations. - Both servers run ``--tp 4 --dp 2 --enable-dp-attention --moe-a2a-backend - deepep`` so attn_tp_size=2 and the all-gather hot path in + Both servers run ``--tp 4 --attn-dp-size 2 --moe-a2a-backend deepep`` + so attn_tp_size=2 and the all-gather hot path in RoutedExpertsCapturer.capture is hit on every step. Baseline disables overlap/cuda-graph/radix to give a deterministic ground truth; reference leaves them on. If the gather were skipping a rank or racing against the @@ -59,9 +59,8 @@ class TestReturnRoutedExperts(CustomTestCase): "--enable-deterministic-inference", "--tp", 4, - "--dp", + "--attn-dp-size", 2, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", # Force normal-mode dispatch: deepep auto routes decode through diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index a70286dcb9..34463dd50f 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -207,9 +207,8 @@ class TestServerUpdateWeightsFromDiskMXFP8(UpdateWeightsFromDiskBase, CustomTest "other_args": ( "--tp-size", "4", - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--fp8-gemm-backend", "flashinfer_trtllm", "--moe-runner-backend", @@ -286,9 +285,8 @@ class TestServerUpdateWeightsFromDiskNVFP4W4A16CuteDSL( "other_args": ( "--tp-size", "4", - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--ep-size", "4", "--fp4-gemm-backend", diff --git a/test/registered/rl/test_weight_version_spans.py b/test/registered/rl/test_weight_version_spans.py index f5fa8d4fe3..9afd9291af 100644 --- a/test/registered/rl/test_weight_version_spans.py +++ b/test/registered/rl/test_weight_version_spans.py @@ -48,9 +48,8 @@ class TestWeightVersionSpans(CustomTestCase): "--trust-remote-code", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-draft-model-path", diff --git a/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py b/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py index 8e7fbcb47f..c8e3b19243 100644 --- a/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py +++ b/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py @@ -40,9 +40,8 @@ class TestDSparkDPSpecPrefillCoordination(CustomTestCase): "deepseek-ai/dspark_qwen3_14b_block7", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--attention-backend", "trtllm_mha" if is_sm100_supported() else "fa3", diff --git a/test/registered/spec/dspark/test_dspark_draft_path_default.py b/test/registered/spec/dspark/test_dspark_draft_path_default.py index b0da41feb8..c852e837f8 100644 --- a/test/registered/spec/dspark/test_dspark_draft_path_default.py +++ b/test/registered/spec/dspark/test_dspark_draft_path_default.py @@ -100,9 +100,8 @@ class TestDsparkDpAttentionMoeA2aGate(CustomTestCase): server_args = _make_dspark_server_args( model_path=_BUNDLED_MODEL_PATH, hf_config=_bundled_hf_config() ) - server_args.enable_dp_attention = True server_args.enable_dp_lm_head = True - server_args.dp_size = 2 + server_args.attn_dp_size = 2 server_args.tp_size = 2 server_args.moe_a2a_backend = moe_a2a_backend return server_args diff --git a/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py b/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py index 5bd05c26d8..aca2864e08 100644 --- a/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py +++ b/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py @@ -39,9 +39,8 @@ class TestDPSpecPrefillCoordination(CustomTestCase): DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-dense-tp-size", "1", diff --git a/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py b/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py index bf28b3b8b6..c91ea9bfaa 100644 --- a/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py +++ b/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py @@ -65,9 +65,8 @@ class TestDPSpecPrefillCoordinationMixed(CustomTestCase): DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-dense-tp-size", "1", diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index 1669fd58db..52a0b3e187 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -44,9 +44,8 @@ class TestEAGLE3EngineDPAttention(CustomTestCase): DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-dense-tp-size", "1", diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index 629af34bcf..adea135539 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -307,7 +307,8 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): disaggregation_mode="prefill", disaggregation_decode_host_receive_threshold=0.0, enable_pd_role_switch=True, - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, ep_size=1, moe_a2a_backend="none", pp_size=1, @@ -331,8 +332,8 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): def test_reject_dp_attention(self): with self.assertRaises(ValueError) as ctx: - self._run(self._sa(enable_dp_attention=True)) - self.assertIn("DP attention", str(ctx.exception)) + self._run(self._sa(attn_dp_size=2)) + self.assertIn("attention DP", str(ctx.exception)) def test_reject_expert_parallelism(self): with self.assertRaises(ValueError) as ctx: diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index a834d7b08b..55b0aa785e 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -13,7 +13,7 @@ register_cpu_ci(est_time=3, suite="base-a-test-cpu") @pytest.mark.parametrize( - "nnodes,tp_size,dp_size,ep_join_mode,ranks,expected", + "nnodes,tp_size,attn_dp_size,ep_join_mode,ranks,expected", [ (2, 4, 4, None, (0, 1, 2, 3), [0, 1, 0, 1]), (4, 4, 2, None, (0, 2), [0, 0]), @@ -22,14 +22,13 @@ register_cpu_ci(est_time=3, suite="base-a-test-cpu") ids=["multiple-listeners-per-node", "dp-spans-nodes", "scale-joiner"], ) def test_dp_leaders_reuse_node_local_ports( - nnodes, tp_size, dp_size, ep_join_mode, ranks, expected + nnodes, tp_size, attn_dp_size, ep_join_mode, ranks, expected ): with ( get_context().override_server_args( nnodes=nnodes, tp_size=tp_size, - dp_size=dp_size, - enable_dp_attention=True, + attn_dp_size=attn_dp_size, ep_join_mode=ep_join_mode, host="0.0.0.0", port=30000, @@ -70,8 +69,7 @@ def test_node_listener_placement(pp_size, expected, node_rank): node_rank=node_rank, tp_size=4, pp_size=pp_size, - dp_size=2, - enable_dp_attention=True, + attn_dp_size=2, attn_cp_size=2, ): assert node_hosts_rust_server() == expected[node_rank] @@ -82,8 +80,6 @@ def test_scale_joiner_hosts_listener(): nnodes=2, node_rank=1, tp_size=1, - dp_size=1, - enable_dp_attention=True, ep_join_mode="scale", ): assert node_hosts_rust_server() diff --git a/test/registered/unit/entrypoints/test_v1_loads_aggregate.py b/test/registered/unit/entrypoints/test_v1_loads_aggregate.py index ad3ee9247e..5f9be76017 100644 --- a/test/registered/unit/entrypoints/test_v1_loads_aggregate.py +++ b/test/registered/unit/entrypoints/test_v1_loads_aggregate.py @@ -47,7 +47,6 @@ class _FakeTokenizerManager(TokenizerControlMixin): self.elastic_worker_count = dp_size self.server_args = SimpleNamespace( dp_size=dp_size, - enable_dp_attention=False, nnodes=1, ) @@ -64,7 +63,7 @@ class _FakeHttpTokenizerManager: tp_size=1, dp_size=1, pp_size=1, - enable_dp_attention=False, + attn_dp_size=1, ): from sglang.srt.runtime_context import get_context @@ -78,7 +77,7 @@ class _FakeHttpTokenizerManager: tp_size=tp_size, dp_size=dp_size, pp_size=pp_size, - enable_dp_attention=enable_dp_attention, + attn_dp_size=attn_dp_size, ) self._override.install() @@ -136,8 +135,7 @@ class TestLoadsAcceleratorField(CustomTestCase): manager = _FakeHttpTokenizerManager( [LoadSnapshot(dp_rank=rank) for rank in range(8)], tp_size=8, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) self.addCleanup(manager.restore) diff --git a/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py b/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py index e77137506d..2ace85fb81 100644 --- a/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py +++ b/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py @@ -41,8 +41,8 @@ def _arch(*, hybrid: bool): def _stub_for_initialize( test, *, - dp_size: int, - attn_dp_size: int, + dp_size: int = 1, + attn_dp_size: int = 1, max_running_requests: int = 8, max_mamba_cache_size: int | None = None, pool_size: int = 64, @@ -53,7 +53,7 @@ def _stub_for_initialize( override = get_context().override_server_args( tp_size=attn_dp_size, dp_size=dp_size, - enable_dp_attention=attn_dp_size > 1, + attn_dp_size=attn_dp_size, enable_memory_saver=False, max_running_requests=max_running_requests, max_mamba_cache_size=max_mamba_cache_size, @@ -91,14 +91,14 @@ def _initialize_stub(stub, *, hybrid: bool = False): class TestAttentionDpRequestCapacity(CustomTestCase): def test_pure_dp_replica_retains_full_request_limit(self): stub = _initialize_stub( - _stub_for_initialize(self, dp_size=4, attn_dp_size=1), + _stub_for_initialize(self, dp_size=4), ) self.assertEqual(stub.max_running_requests, 8) self.assertEqual(stub.req_to_token_pool.size, 8) def test_attention_dp_partitions_request_limit(self): stub = _initialize_stub( - _stub_for_initialize(self, dp_size=4, attn_dp_size=4), + _stub_for_initialize(self, attn_dp_size=4), ) self.assertEqual(stub.max_running_requests, 2) self.assertEqual(stub.req_to_token_pool.size, 2) @@ -107,7 +107,6 @@ class TestAttentionDpRequestCapacity(CustomTestCase): stub = _initialize_stub( _stub_for_initialize( self, - dp_size=4, attn_dp_size=4, max_running_requests=8, max_mamba_cache_size=4 * RATIO, @@ -122,7 +121,6 @@ class TestAttentionDpRequestCapacity(CustomTestCase): def test_attention_dp_auxiliary_error_reports_global_cli_units(self): stub = _stub_for_initialize( self, - dp_size=4, attn_dp_size=4, max_running_requests=8, max_mamba_cache_size=4 * RATIO - 1, diff --git a/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py b/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py index 974988edf3..fdd1d43d0f 100644 --- a/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py +++ b/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py @@ -60,8 +60,7 @@ def _published(stub): return get_context().override_server_args( # One attention replica per TP rank gives the stub's attention-DP width. tp_size=stub._attn_dp_size, - dp_size=stub._attn_dp_size, - enable_dp_attention=stub._attn_dp_size > 1, + attn_dp_size=stub._attn_dp_size, max_running_requests=stub._max_running_requests, max_mamba_cache_size=stub._max_mamba_cache_size, disable_radix_cache=stub._disable_radix_cache, diff --git a/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py b/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py index b559d42068..128a7fa46d 100644 --- a/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py +++ b/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py @@ -217,7 +217,9 @@ class TestSchedulerProfilerManagerMPS(unittest.TestCase): torch.mps.profiler, "metal_capture", return_value=capture_ctx ), mock_patch("torch.distributed.barrier"), - get_parallel().override(tp_rank=0, dp_size=1, pp_size=1, moe_ep_size=1), + get_parallel().override( + tp_rank=0, num_dp_ranks=1, pp_size=1, moe_ep_size=1 + ), ): result = mgr._start_profile() self.assertTrue(result.success, result.message) diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 10c7e09911..72a3cfbd7f 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -84,7 +84,7 @@ class TestAttentionCpBoundary(CustomTestCase): tp_rank=cp, attn_dp_size=1, attn_dp_rank=0, - enable_dp_attention=False, + attn_dp_enabled=False, attn_tp_size=1, attn_tp_rank=0, attn_cp_size=CP_SIZE, diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index c244cc9296..968d1f5c57 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -83,7 +83,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): moe_tp_size=attn_dp * attn_cp * attn_tp, moe_dp_size=1, dwdp_size=1, - enable_dp_attention=attn_dp > 1, + attn_dp_enabled=attn_dp > 1, enable_attn_tp_input_scattered=False, tp_group=SimpleNamespace( name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp)) diff --git a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py index ac11493e79..52c18d1144 100644 --- a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py @@ -188,7 +188,7 @@ class TestDpCpGather(CustomTestCase): attn_cp_rank=cp, attn_tp_rank=tp, attn_dp_size=DP_SIZE, - enable_dp_attention=True, + attn_dp_enabled=True, attn_cp_size=CP_SIZE, attn_tp_size=attn_tp_size, tp_size=tp_size, diff --git a/test/registered/unit/layer_boundary/test_layernorm_sp.py b/test/registered/unit/layer_boundary/test_layernorm_sp.py index 468687a0c1..1cd549c634 100644 --- a/test/registered/unit/layer_boundary/test_layernorm_sp.py +++ b/test/registered/unit/layer_boundary/test_layernorm_sp.py @@ -103,7 +103,7 @@ class TestLayerNormSPValidation(CustomTestCase): VALID = dict( architecture="Qwen3ForCausalLM", tp_size=2, - enable_dp_attention=False, + attn_dp_enabled=False, speculative_algorithm=None, ) @@ -119,8 +119,8 @@ class TestLayerNormSPValidation(CustomTestCase): validate_layernorm_sp(**{**self.VALID, "tp_size": 1}) def test_rejects_dp_attention(self): - with self.assertRaisesRegex(ValueError, "dp-attention"): - validate_layernorm_sp(**{**self.VALID, "enable_dp_attention": True}) + with self.assertRaisesRegex(ValueError, "attention DP"): + validate_layernorm_sp(**{**self.VALID, "attn_dp_enabled": True}) def test_rejects_speculative(self): with self.assertRaisesRegex(ValueError, "speculative"): diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py index 11ee2d7511..6a4fea2876 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py @@ -96,9 +96,7 @@ class TestNemotronAuxCapture(CustomTestCase): ) inputs = torch.tensor([[0.3, -0.5, 0.7, 1.1], [-0.4, 0.9, 0.2, -0.6]]) with ( - get_context().override_server_args( - tp_size=tp, enable_dp_attention=dp_enabled - ), + get_context().override_server_args(tp_size=tp), get_flags().dp.override(enabled=dp_enabled), get_parallel().override( attn_tp_group=group, diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py index 0ff3ad10b5..9ad4e97b86 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py @@ -34,9 +34,7 @@ class TestNemotronMTPReduction(CustomTestCase): reduce = Mock(side_effect=lambda x: x * tp) group = SimpleNamespace(all_reduce=reduce) with ( - get_context().override_server_args( - tp_size=tp, enable_dp_attention=True - ), + get_context().override_server_args(tp_size=tp), get_flags().dp.override(enabled=True), get_parallel().override( attn_tp_group=group, diff --git a/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py b/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py index 774484717a..e18cf01dc5 100644 --- a/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py +++ b/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py @@ -68,7 +68,6 @@ def _make_prefill_aware_swa_runner( chunked_prefill_size=8192, disable_radix_cache=False, enable_prefill_cp=False, - enable_dp_attention=False, ) token_to_kv_pool = token_to_kv_pool if token_to_kv_pool is not None else object() token_to_kv_pool_allocator = object() diff --git a/test/registered/unit/lora/test_mem_pool_ep_unit.py b/test/registered/unit/lora/test_mem_pool_ep_unit.py index 7f73273bc9..2848a80648 100644 --- a/test/registered/unit/lora/test_mem_pool_ep_unit.py +++ b/test/registered/unit/lora/test_mem_pool_ep_unit.py @@ -1126,8 +1126,8 @@ class TestMoeBufferShardsByMoeTp(unittest.TestCase): pool.max_loras_per_batch = 2 pool.tp_size = tp_size pool.tp_rank = 0 - # Without --enable-dp-attention the attention TP group equals the - # outer TP group. + # Without attention DP the attention TP group equals the outer TP + # group. pool.attn_tp_size = tp_size pool.moe_ep_size = ep_size pool.moe_ep_rank = ep_rank @@ -1232,8 +1232,8 @@ class TestAttnModulesShardByAttnTp(unittest.TestCase): """Regression: attention-module LoRA buffers must shard by `attn_tp_size`, not the outer `tp_size`. - Under `--enable-dp-attention` attention layers are built on the attn_tp - group (`attn_tp_size = tp_size // dp_size`), so e.g. MLA `o_proj` holds an + Under `--attn-dp-size` attention layers are built on the attn_tp group + (`attn_tp_size = tp_size // attn_dp_size`), so e.g. MLA `o_proj` holds an attn_tp-local input shard. Sizing the LoRA buffer by the outer `tp_size` would make it narrower than the slice produced by `RowParallelLinearWithLoRA.slice_lora_a_weights` (which slices by the base @@ -1437,7 +1437,7 @@ class TestRowParallelLoraAShardsByBaseLinear(unittest.TestCase): """Regression: dense row-parallel LoRA-A buffers must match the base linear's real input shard. - Under `--enable-dp-attention --moe-dense-tp-size 1` the dense-MLP and + Under `--attn-dp-size N --moe-dense-tp-size 1` the dense-MLP and shared-expert `down_proj` are fully replicated (K is the full intermediate size) although the outer `tp_size` is large. Dividing by `tp_size` undersized the buffer and `sgemm_lora_a_fwd` failed its `x.shape[-1] == K` assertion. diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 2d22bf67f3..e6352cf003 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -106,8 +106,8 @@ class _LocalOnlyLoRABackend(BaseLoRABackend): class TestDPAttentionBackendContract(CustomTestCase): def test_manager_requires_global_routing_only_for_dp_attention(self): """DPA must not proceed with local-only routing; non-DPA remains supported.""" - for enable_dp_attention in (False, True): - with self.subTest(enable_dp_attention=enable_dp_attention): + for attn_dp_enabled in (False, True): + with self.subTest(attn_dp_enabled=attn_dp_enabled): backend = _LocalOnlyLoRABackend( max_loras_per_batch=3, device=torch.device("cpu") ) @@ -115,7 +115,7 @@ class TestDPAttentionBackendContract(CustomTestCase): pool.max_loras_per_batch = 3 pool.uid_to_buffer_id = {None: 0, "adapter": 1} manager = LoRAManager.__new__(LoRAManager) - manager.enable_dp_attention = enable_dp_attention + manager.attn_dp_enabled = attn_dp_enabled manager.max_loras_per_batch = 3 manager.num_pinned_loras = 1 manager.lora_backend = backend @@ -146,7 +146,7 @@ class TestDPAttentionBackendContract(CustomTestCase): NotImplementedError, "_LocalOnlyLoRABackend.*prepare_global_lora_batch", ) - if enable_dp_attention + if attn_dp_enabled else nullcontext() ) with ( @@ -190,12 +190,12 @@ def test_manager_rejects_dp_attention_with_multi_rank_attention_groups( ): with ( get_context().override_server_args( - enable_dp_attention=True, enable_lora_overlap_loading=False + tp_size=4, attn_dp_size=2, enable_lora_overlap_loading=False ) as args, get_parallel().override( tp_size=4, attn_dp_size=2, moe_tp_size=4, **attention_widths ), - pytest.raises(ValueError, match="requires --dp-size equal to --tp-size"), + pytest.raises(ValueError, match="requires --attn-dp-size equal to --tp-size"), ): LoRAManager( base_model=torch.nn.Linear(2, 2), @@ -219,7 +219,7 @@ def test_communicator_publishes_layout_at_each_transition( monkeypatch.setattr( module, "get_parallel", - lambda: SimpleNamespace(enable_dp_attention=publish_lora_layout), + lambda: SimpleNamespace(attn_dp_enabled=publish_lora_layout), ) # Start from TP_GLOBAL so an unpublished transition is distinguishable. initial = LoRABatchLayout.TP_GLOBAL @@ -285,7 +285,7 @@ def test_communicator_publishes_layout_at_each_transition( @pytest.mark.parametrize("can_run_decode_cuda_graph", [False, True]) def test_manager_uses_current_dp_cuda_graph_eligibility(can_run_decode_cuda_graph): manager = LoRAManager.__new__(LoRAManager) - manager.enable_dp_attention = True + manager.attn_dp_enabled = True manager.max_bs_in_cuda_graph = 4 tokens = torch.zeros(2, dtype=torch.int64) forward_batch = ForwardBatch( diff --git a/test/registered/unit/lora/test_uno_lora_targets.py b/test/registered/unit/lora/test_uno_lora_targets.py index 9610fb7e02..65c500f036 100644 --- a/test/registered/unit/lora/test_uno_lora_targets.py +++ b/test/registered/unit/lora/test_uno_lora_targets.py @@ -235,7 +235,7 @@ class TestUnoLoRATargets(CustomTestCase): def test_manager_rejects_uno_with_dp_attention(self): with ( get_context().override_server_args( - enable_dp_attention=True, enable_lora_overlap_loading=False + tp_size=2, attn_dp_size=2, enable_lora_overlap_loading=False ) as args, get_parallel().override(attn_tp_size=1), self.assertRaisesRegex( diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index b708820923..461effc9ce 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -18,10 +18,10 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") class TestDPAttnSchedulerMetadata(CustomTestCase): def test_skip_all_gather_policy(self): with envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.override(False): - self.assertTrue(dp_attn.should_skip_scheduler_all_gather(dp_size=1)) - self.assertFalse(dp_attn.should_skip_scheduler_all_gather(dp_size=2)) + self.assertTrue(dp_attn.should_skip_scheduler_all_gather(num_dp_ranks=1)) + self.assertFalse(dp_attn.should_skip_scheduler_all_gather(num_dp_ranks=2)) with envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.override(True): - self.assertTrue(dp_attn.should_skip_scheduler_all_gather(dp_size=2)) + self.assertTrue(dp_attn.should_skip_scheduler_all_gather(num_dp_ranks=2)) def test_dp1_skip_preserves_local_tbo_metadata(self): batch = SimpleNamespace( @@ -44,7 +44,7 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): dp_attn, "get_parallel", return_value=SimpleNamespace( - dp_size=1, + num_dp_ranks=1, attn_tp_size=4, attn_cp_size=1, tp_group=SimpleNamespace( diff --git a/test/registered/unit/managers/test_auxiliary_output.py b/test/registered/unit/managers/test_auxiliary_output.py index c6c12a272e..7545473fa6 100644 --- a/test/registered/unit/managers/test_auxiliary_output.py +++ b/test/registered/unit/managers/test_auxiliary_output.py @@ -532,7 +532,7 @@ def test_pdmux_split_prefill_schedules_auxiliary_output_copy(): scheduler.future_map = object() scheduler._relay_forward_payload = Mock() scheduler.device_module = SimpleNamespace(Event=Mock(return_value=copy_done)) - scheduler.enable_dp_attention = False + scheduler.attn_dp_enabled = False batch = SimpleNamespace( forward_mode=SimpleNamespace( is_prebuilt=lambda: False, diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index c17ad1f712..1e735c7c3d 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -44,14 +44,14 @@ def _load(**overrides) -> LoadSnapshot: return msgspec.structs.replace(_BASE_LOAD, **overrides) -def _make_controller(dp_size: int) -> DataParallelController: +def _make_controller(num_dp_ranks: int) -> DataParallelController: """Bypass __init__; inject only the attrs dispatch methods read.""" ctl = DataParallelController.__new__(DataParallelController) - ctl.workers = [MagicMock(name=f"worker_{i}") for i in range(dp_size)] - ctl.status = [True] * dp_size - ctl._active_workers = list(range(dp_size)) + ctl.workers = [MagicMock(name=f"worker_{i}") for i in range(num_dp_ranks)] + ctl.status = [True] * num_dp_ranks + ctl._active_workers = list(range(num_dp_ranks)) ctl.round_robin_counter = 0 - ctl.dp_budget = DPBudget(dp_size=dp_size) + ctl.dp_budget = DPBudget(num_dp_ranks=num_dp_ranks) return ctl @@ -66,7 +66,7 @@ def _req(routed_dp_rank=None, bootstrap_room=None, input_ids=None): class TestDPBudgetUpdateBudget(CustomTestCase): def test_maps_running_plus_waiting_to_total_requests(self): - budget = DPBudget(dp_size=2) + budget = DPBudget(num_dp_ranks=2) budget.update_budget( [ _load(dp_rank=0, timestamp=1.0, num_running_reqs=3, num_waiting_reqs=2), @@ -76,7 +76,7 @@ class TestDPBudgetUpdateBudget(CustomTestCase): self.assertEqual(budget.total_requests, [5, 6]) def test_maps_num_total_tokens_not_num_used_tokens(self): - budget = DPBudget(dp_size=2) + budget = DPBudget(num_dp_ranks=2) budget.update_budget( [ _load( @@ -90,7 +90,7 @@ class TestDPBudgetUpdateBudget(CustomTestCase): self.assertEqual(budget.total_tokens, [150, 80]) def test_partial_update_only_affects_reported_rank(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.update_budget( [ _load( @@ -123,7 +123,7 @@ class TestDPBudgetDispatch(CustomTestCase): """DPBudget.dispatch picks a rank from current state and updates counters.""" def test_total_requests_dispatch_picks_min_and_increments(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.total_requests = [4, 2, 7] rank = budget.dispatch(LoadBalanceMethod.TOTAL_REQUESTS) self.assertEqual(rank, 1) @@ -134,7 +134,7 @@ class TestDPBudgetDispatch(CustomTestCase): ) def test_total_tokens_dispatch_applies_estimated_tokens(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.total_tokens = [100, 50, 200] budget.total_requests = [0, 0, 0] rank = budget.dispatch(LoadBalanceMethod.TOTAL_TOKENS, estimated_tokens=30) @@ -151,7 +151,7 @@ class TestDPBudgetDispatch(CustomTestCase): ) def test_total_tokens_tie_breaks_on_total_requests(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.total_tokens = [50, 50, 50] budget.total_requests = [4, 2, 7] rank = budget.dispatch(LoadBalanceMethod.TOTAL_TOKENS, estimated_tokens=10) @@ -162,14 +162,14 @@ class TestDPBudgetDispatch(CustomTestCase): def test_dispatch_returns_none_for_methods_not_handled(self): """Round-robin and follow_bootstrap_room dispatch elsewhere; DPBudget only handles the load-aware variants.""" - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) self.assertIsNone(budget.dispatch(LoadBalanceMethod.ROUND_ROBIN)) self.assertIsNone(budget.dispatch(LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM)) class TestRoundRobinScheduler(CustomTestCase): def test_cycles_through_active_workers_in_order(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) for _ in range(8): ctl.round_robin_scheduler(_req()) # 8 reqs across 4 active workers — 2 each, in round-robin order @@ -177,14 +177,14 @@ class TestRoundRobinScheduler(CustomTestCase): self.assertEqual(worker.send_pyobj.call_count, 2, f"worker {i} call count") def test_first_dispatch_picks_worker_zero(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.round_robin_scheduler(_req()) ctl.workers[0].send_pyobj.assert_called_once() for i in (1, 2, 3): ctl.workers[i].send_pyobj.assert_not_called() def test_skips_inactive_workers(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.status[1] = False ctl.status[3] = False for _ in range(6): @@ -197,7 +197,7 @@ class TestRoundRobinScheduler(CustomTestCase): def test_routed_dp_rank_bypasses_counter(self): """External dp-rank routing must not advance the counter.""" - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.round_robin_scheduler(_req(routed_dp_rank=2)) ctl.workers[2].send_pyobj.assert_called_once() self.assertEqual( @@ -212,7 +212,7 @@ class TestRoundRobinScheduler(CustomTestCase): class TestFollowBootstrapRoomScheduler(CustomTestCase): def test_dispatches_by_bootstrap_room_modulo(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) for room, expected_rank in [ (0, 0), (1, 1), @@ -225,12 +225,12 @@ class TestFollowBootstrapRoomScheduler(CustomTestCase): ctl.workers[expected_rank].send_pyobj.assert_called() def test_requires_bootstrap_room(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) with self.assertRaises(AssertionError): ctl.follow_bootstrap_room_scheduler(_req(bootstrap_room=None)) def test_routed_dp_rank_bypasses_bootstrap_room(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.follow_bootstrap_room_scheduler(_req(routed_dp_rank=3, bootstrap_room=1)) ctl.workers[3].send_pyobj.assert_called_once() ctl.workers[1].send_pyobj.assert_not_called() @@ -238,7 +238,7 @@ class TestFollowBootstrapRoomScheduler(CustomTestCase): class TestTotalRequestsScheduler(CustomTestCase): def test_dispatches_to_min_request_worker(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.dp_budget.total_requests = [5, 3, 1, 4] ctl.total_requests_scheduler(_req()) ctl.workers[2].send_pyobj.assert_called_once() @@ -251,7 +251,7 @@ class TestTotalRequestsScheduler(CustomTestCase): ) def test_routed_dp_rank_bypasses_budget(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.dp_budget.total_requests = [5, 3, 1, 4] ctl.total_requests_scheduler(_req(routed_dp_rank=0)) ctl.workers[0].send_pyobj.assert_called_once() @@ -271,7 +271,7 @@ class TestStatusAwarenessInconsistency(CustomTestCase): this test will fail and force a reviewer to confirm intent.""" def test_total_requests_ignores_status(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) # Worker 2 is the global minimum AND marked inactive. ctl.dp_budget.total_requests = [5, 3, 1, 4] ctl.status[2] = False @@ -282,15 +282,15 @@ class TestStatusAwarenessInconsistency(CustomTestCase): class TestRefreshLoadBudgetThrottle(CustomTestCase): @staticmethod - def _controller_with_reader(dp_size, snapshots): - ctl = _make_controller(dp_size) + def _controller_with_reader(num_dp_ranks, snapshots): + ctl = _make_controller(num_dp_ranks) ctl.load_snapshot_reader = MagicMock() ctl.load_snapshot_reader.read_all.return_value = snapshots return ctl def test_throttled_refresh_spreads_a_burst_across_ranks(self): idle = [_load(dp_rank=i, timestamp=1.0, num_total_tokens=0) for i in range(4)] - ctl = self._controller_with_reader(dp_size=4, snapshots=idle) + ctl = self._controller_with_reader(num_dp_ranks=4, snapshots=idle) # A refresh stamp in the future keeps every call inside the window, so # the burst runs entirely on speculative counters. ctl._last_refresh_time = time.perf_counter() + 3600.0 @@ -315,7 +315,7 @@ class TestRefreshLoadBudgetThrottle(CustomTestCase): _load(dp_rank=0, timestamp=2.0, num_total_tokens=10), _load(dp_rank=1, timestamp=2.0, num_total_tokens=20), ] - ctl = self._controller_with_reader(dp_size=2, snapshots=reported) + ctl = self._controller_with_reader(num_dp_ranks=2, snapshots=reported) ctl._last_refresh_time = 0.0 # window has long passed ctl.dp_budget.total_tokens = [999, 999] @@ -331,7 +331,7 @@ class TestRefreshLoadBudgetThrottle(CustomTestCase): def test_unchanged_snapshot_does_not_reset_the_burst(self): frozen = [_load(dp_rank=i, timestamp=1.0, num_total_tokens=0) for i in range(2)] - ctl = self._controller_with_reader(dp_size=2, snapshots=frozen) + ctl = self._controller_with_reader(num_dp_ranks=2, snapshots=frozen) ctl._last_refresh_time = 0.0 ctl.refresh_load_budget() # adopts timestamp 1.0 diff --git a/test/registered/unit/managers/test_load_snapshot_backends.py b/test/registered/unit/managers/test_load_snapshot_backends.py index e93bbfcb8f..0b2bfbf1fd 100644 --- a/test/registered/unit/managers/test_load_snapshot_backends.py +++ b/test/registered/unit/managers/test_load_snapshot_backends.py @@ -7,6 +7,7 @@ import unittest from types import SimpleNamespace from unittest import mock +from sglang.srt.environ import envs from sglang.srt.managers.load_snapshot import ( LoadSnapshot, ShmLoadSnapshotReader, @@ -263,7 +264,7 @@ class TestFactoryFunctions(CustomTestCase): self.addCleanup(override.restore) def test_shm_mode(self): - self._publish(enable_dp_attention=False, nnodes=1, dp_size=1) + self._publish(nnodes=1, dp_size=1) port_args = SimpleNamespace(instance_id="test_shm_factory") writer = create_load_snapshot_writer(port_args, dp_size=1, dp_rank=0) self.assertIsInstance(writer, ShmLoadSnapshotWriter) @@ -278,7 +279,7 @@ class TestFactoryFunctions(CustomTestCase): os.unlink(path) def test_zmq_mode_via_env(self): - self._publish(enable_dp_attention=False, nnodes=1, dp_size=1) + self._publish(nnodes=1, dp_size=1) port_args = SimpleNamespace(instance_id="test_zmq_factory") os.environ["SGLANG_LOAD_SNAPSHOT_USE_ZMQ"] = "1" try: @@ -292,7 +293,7 @@ class TestFactoryFunctions(CustomTestCase): del os.environ["SGLANG_LOAD_SNAPSHOT_USE_ZMQ"] def test_should_use_zmq_multinode_dp_attention(self): - self._publish(enable_dp_attention=True, nnodes=2, dp_size=2) + self._publish(attn_dp_size=2, nnodes=2) self.assertTrue(should_use_zmq()) @@ -304,10 +305,9 @@ class TestZmqReaderOwner(CustomTestCase): def _owners(self, **overrides): """Publish a config and return the callers that claim the socket.""" fields = dict( - enable_dp_attention=True, nnodes=2, node_rank=0, - dp_size=1, + attn_dp_size=1, load_balance_method="round_robin", tokenizer_worker_num=1, ) @@ -320,38 +320,43 @@ class TestZmqReaderOwner(CustomTestCase): override.restore() def test_zmq_disabled_no_owner(self): - self.assertEqual(self._owners(enable_dp_attention=False, nnodes=1), set()) + self.assertEqual(self._owners(nnodes=1), set()) def test_non_zero_node_rank_no_owner(self): self.assertEqual( - self._owners(node_rank=1, dp_size=4, tokenizer_worker_num=8), set() + self._owners(node_rank=1, attn_dp_size=4, tokenizer_worker_num=8), set() ) def test_tokenizer_manager_owns_when_dp1(self): - self.assertEqual(self._owners(dp_size=1), {"TokenizerManager"}) + # A single DP rank runs zmq mode only when it is forced. + with envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.override(True): + self.assertEqual(self._owners(attn_dp_size=1), {"TokenizerManager"}) def test_multi_tokenizer_router_owns_in_multi_tokenizer_dp1(self): - self.assertEqual( - self._owners(dp_size=1, tokenizer_worker_num=8), {"MultiTokenizerRouter"} - ) + with envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.override(True): + self.assertEqual( + self._owners(attn_dp_size=1, tokenizer_worker_num=8), + {"MultiTokenizerRouter"}, + ) def test_multi_tokenizer_router_owns_in_multi_tokenizer_round_robin(self): self.assertEqual( - self._owners(dp_size=4, tokenizer_worker_num=8), {"MultiTokenizerRouter"} + self._owners(attn_dp_size=4, tokenizer_worker_num=8), + {"MultiTokenizerRouter"}, ) def test_data_parallel_controller_owns_load_aware(self): for method in ("total_tokens", "total_requests"): self.assertEqual( self._owners( - dp_size=4, tokenizer_worker_num=8, load_balance_method=method + attn_dp_size=4, tokenizer_worker_num=8, load_balance_method=method ), {"DataParallelController"}, ) def test_tokenizer_manager_owns_dp4_round_robin(self): self.assertEqual( - self._owners(dp_size=4, tokenizer_worker_num=1), {"TokenizerManager"} + self._owners(attn_dp_size=4, tokenizer_worker_num=1), {"TokenizerManager"} ) def test_the_controller_answers_within_its_audited_namespaces(self): @@ -365,10 +370,9 @@ class TestZmqReaderOwner(CustomTestCase): import sglang.srt.runtime_context as rc fields = dict( - enable_dp_attention=True, nnodes=2, node_rank=0, - dp_size=4, + attn_dp_size=4, load_balance_method="total_tokens", tokenizer_worker_num=8, ) @@ -382,17 +386,19 @@ class TestZmqReaderOwner(CustomTestCase): self.assertTrue(zmq_reader_owner("DataParallelController")) def test_at_most_one_owner_across_configs(self): - for dp_size in (1, 4): - for tw in (1, 8): - for method in ("round_robin", "total_tokens", "total_requests"): - for node_rank in (0, 1): - owners = self._owners( - dp_size=dp_size, - tokenizer_worker_num=tw, - load_balance_method=method, - node_rank=node_rank, - ) - self.assertLessEqual(len(owners), 1, owners) + # Forced zmq mode puts a single DP rank in the sweep too. + with envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.override(True): + for attn_dp_size in (1, 4): + for tw in (1, 8): + for method in ("round_robin", "total_tokens", "total_requests"): + for node_rank in (0, 1): + owners = self._owners( + attn_dp_size=attn_dp_size, + tokenizer_worker_num=tw, + load_balance_method=method, + node_rank=node_rank, + ) + self.assertLessEqual(len(owners), 1, owners) class TestZmqAddr(CustomTestCase): diff --git a/test/registered/unit/managers/test_loadstat_wire.py b/test/registered/unit/managers/test_loadstat_wire.py index 02a55f6643..15bf6d3e0f 100644 --- a/test/registered/unit/managers/test_loadstat_wire.py +++ b/test/registered/unit/managers/test_loadstat_wire.py @@ -153,8 +153,7 @@ class TestLoadPublisherGating(CustomTestCase): def test_dp_attention_keys_the_load_port_by_attn_dp_rank(self): _, open_sock = self._build( tp_size=4, - dp_size=4, - enable_dp_attention=True, + attn_dp_size=4, ranks={"world_rank": 3, "dp_rank": 0}, ) open_sock.assert_called_once_with("tcp://*:5564") diff --git a/test/registered/unit/managers/test_mm_shm_error_consensus.py b/test/registered/unit/managers/test_mm_shm_error_consensus.py index c7fee26d62..26e647043c 100644 --- a/test/registered/unit/managers/test_mm_shm_error_consensus.py +++ b/test/registered/unit/managers/test_mm_shm_error_consensus.py @@ -121,7 +121,7 @@ def _run_consensus_rank(rank: int, world_size: int, init_file: str) -> None: ) try: req = _request(_failed_pointer() if rank == 1 else _successful_pointer()) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=world_size) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=world_size) receiver = _receiver() object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD) with ( @@ -169,7 +169,7 @@ def _run_image_receiver(rank, init_file, pipe): ), patch( "sglang.srt.managers.scheduler_components.request_receiver.get_parallel", - return_value=SimpleNamespace(enable_dp_attention=False, tp_size=2), + return_value=SimpleNamespace(attn_dp_enabled=False, tp_size=2), ), ): for base in [30, 90]: @@ -372,7 +372,7 @@ class TestShmRequestFailureConsensus(unittest.TestCase): def test_local_materialization_failure_becomes_request_error(self): req = _request(_failed_pointer()) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=1) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=1) with ( patch( @@ -396,7 +396,7 @@ class TestShmRequestFailureConsensus(unittest.TestCase): def test_peer_failure_rejects_the_local_request(self): req = _request(torch.zeros(1)) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=2) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=2) def inject_peer_failure(mask, **kwargs): mask.fill_(1) @@ -428,7 +428,7 @@ class TestShmRequestFailureConsensus(unittest.TestCase): failed_req = _request(torch.zeros(1), rid="failed") healthy_req = _request(torch.zeros(1), rid="healthy") batch = BatchTokenizedEmbeddingReqInput(batch=[failed_req, healthy_req]) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=1) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=1) def materialize(req): if req.rid == "failed": diff --git a/test/registered/unit/managers/test_pp_cp_rank_offsets.py b/test/registered/unit/managers/test_pp_cp_rank_offsets.py index fed3110607..b837ce0fac 100644 --- a/test/registered/unit/managers/test_pp_cp_rank_offsets.py +++ b/test/registered/unit/managers/test_pp_cp_rank_offsets.py @@ -36,9 +36,8 @@ def _published_topology(): ranks={"world_rank": 12, "dp_rank": 1}, tp_size=8, pp_size=2, - dp_size=2, + attn_dp_size=2, attn_cp_size=2, - enable_dp_attention=True, ) @@ -63,7 +62,6 @@ def _make_receiver() -> SchedulerRequestReceiver: attn_cp_cpu_group=attn_cp_group, world_group=world_group, server_args=SimpleNamespace( - enable_dp_attention=True, enable_dp_attention_local_control_broadcast=False, ), model_config=SimpleNamespace(is_multimodal=False), @@ -81,7 +79,7 @@ class TestRequestReceiverBroadcast(unittest.TestCase): receiver = _make_receiver() control_req = SimpleNamespace(kind="control") parallel = SimpleNamespace( - enable_dp_attention=True, + attn_dp_enabled=True, enable_dp_attention_local_control_broadcast=True, attn_tp_rank=0, attn_cp_rank=0, @@ -115,7 +113,7 @@ class TestRequestReceiverBroadcast(unittest.TestCase): receiver = _make_receiver() control_req = SimpleNamespace(kind="control") parallel = SimpleNamespace( - enable_dp_attention=True, + attn_dp_enabled=True, enable_dp_attention_local_control_broadcast=False, attn_tp_rank=0, attn_cp_rank=0, diff --git a/test/registered/unit/managers/test_prefill_delayer_gather.py b/test/registered/unit/managers/test_prefill_delayer_gather.py index 8cadc7334b..9c5177716d 100644 --- a/test/registered/unit/managers/test_prefill_delayer_gather.py +++ b/test/registered/unit/managers/test_prefill_delayer_gather.py @@ -13,10 +13,8 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") TOPOLOGIES = { "tp": dict(tp_size=4), "attn_cp": dict(tp_size=4, attn_cp_size=2), - "dp_attention": dict(tp_size=4, dp_size=2, enable_dp_attention=True), - "dp_attention_attn_cp": dict( - tp_size=8, dp_size=2, attn_cp_size=2, enable_dp_attention=True - ), + "dp_attention": dict(tp_size=4, attn_dp_size=2), + "dp_attention_attn_cp": dict(tp_size=8, attn_dp_size=2, attn_cp_size=2), } @@ -55,7 +53,7 @@ class TestPrefillDelayerGather(CustomTestCase): waiting_queue_len=5, ) parallel = get_parallel() - dp_groups = parallel.dp_size if parallel.enable_dp_attention else 1 + dp_groups = parallel.num_dp_ranks if parallel.attn_dp_enabled else 1 ranks_per_dp_group = parallel.tp_size // dp_groups local = torch.tensor([1, 0, 3, 4, 5]) expected = torch.stack( diff --git a/test/registered/unit/managers/test_prefill_delayer_high_watermark.py b/test/registered/unit/managers/test_prefill_delayer_high_watermark.py index 2631f4e5ba..3cb6bba4cb 100644 --- a/test/registered/unit/managers/test_prefill_delayer_high_watermark.py +++ b/test/registered/unit/managers/test_prefill_delayer_high_watermark.py @@ -33,8 +33,8 @@ class TestPrefillDelayerHighWatermark(CustomTestCase): self.assertEqual(tracker.observe_attempt(10), 10) delayer = MagicMock() - delayer.enable_dp_attention = True - delayer.dp_size = 1 + delayer.attn_dp_enabled = True + delayer.num_dp_ranks = 1 delayer._metrics_collector = None delayer._debug_log_enabled = False delayer._negotiate_should_allow_prefill.return_value = _NegotiateOutput( @@ -72,8 +72,8 @@ class TestPrefillDelayerHighWatermark(CustomTestCase): self.assertEqual(tracker.observe_attempt(100), 100) delayer = MagicMock() - delayer.enable_dp_attention = True - delayer.dp_size = 1 + delayer.attn_dp_enabled = True + delayer.num_dp_ranks = 1 delayer._metrics_collector = None delayer._debug_log_enabled = False delayer._negotiate_should_allow_prefill.return_value = _NegotiateOutput( diff --git a/test/registered/unit/managers/test_scheduler_internal_state_world_size.py b/test/registered/unit/managers/test_scheduler_internal_state_world_size.py index a36b6a27ca..741fb868d0 100644 --- a/test/registered/unit/managers/test_scheduler_internal_state_world_size.py +++ b/test/registered/unit/managers/test_scheduler_internal_state_world_size.py @@ -8,46 +8,45 @@ maybe_stub_sgl_kernel() from sglang.srt.managers.io_struct import GetInternalStateReq from sglang.srt.managers.scheduler import Scheduler -from sglang.srt.runtime_context import get_context +from sglang.srt.runtime_context import get_context, get_parallel from sglang.srt.server_args import compute_world_size register_cpu_ci(est_time=12, suite="base-a-test-cpu") -def _shape( - *, tp_size: int, pp_size: int, dp_size: int, enable_dp_attention: bool -) -> dict: - """The four `parallel` leaves the world size is computed from.""" - return { - "tp_size": tp_size, - "pp_size": pp_size, - "dp_size": dp_size, - "enable_dp_attention": enable_dp_attention, - } +def _shape(*, tp_size: int, pp_size: int, dp_size: int) -> dict: + """The three `parallel` leaves the world size is computed from.""" + return {"tp_size": tp_size, "pp_size": pp_size, "dp_size": dp_size} class TestComputeWorldSize(unittest.TestCase): def test_a_single_gpu_server_holds_one_gpu(self): """The default shape has to come out as one, or every consumer is off by a factor.""" - shape = _shape(tp_size=1, pp_size=1, dp_size=1, enable_dp_attention=False) + shape = _shape(tp_size=1, pp_size=1, dp_size=1) self.assertEqual(compute_world_size(**shape), 1) def test_tensor_and_pipeline_stages_multiply(self): """Each (pp_rank, tp_rank) pair is its own scheduler process on its own gpu.""" - shape = _shape(tp_size=2, pp_size=3, dp_size=1, enable_dp_attention=False) + shape = _shape(tp_size=2, pp_size=3, dp_size=1) self.assertEqual(compute_world_size(**shape), 6) def test_plain_data_parallel_replicas_each_hold_their_own_gpus(self): """Without dp attention every replica launches a full tensor-parallel group of its own.""" - shape = _shape(tp_size=2, pp_size=1, dp_size=2, enable_dp_attention=False) + shape = _shape(tp_size=2, pp_size=1, dp_size=2) self.assertEqual(compute_world_size(**shape), 4) def test_data_parallel_attention_shares_the_tensor_parallel_gpus(self): """With dp attention the dp ranks live inside the tensor-parallel world, not beside it.""" - shape = _shape(tp_size=4, pp_size=1, dp_size=2, enable_dp_attention=True) + with get_context().override_server_args(tp_size=4, attn_dp_size=2): + parallel = get_parallel() + shape = _shape( + tp_size=parallel.tp_size, + pp_size=parallel.pp_size, + dp_size=parallel.dp_size, + ) self.assertEqual(compute_world_size(**shape), 4) @@ -88,7 +87,7 @@ class TestSchedulerInternalStateWorldSize(unittest.TestCase): def test_the_internal_state_reports_the_whole_server(self): """A consumer sizing an external fleet reads the gpus the server occupies, not the declared sizes.""" - shape = _shape(tp_size=2, pp_size=1, dp_size=2, enable_dp_attention=False) + shape = _shape(tp_size=2, pp_size=1, dp_size=2) internal_state = self._get_internal_state(shape) diff --git a/test/registered/unit/managers/test_scheduler_recv_skipper.py b/test/registered/unit/managers/test_scheduler_recv_skipper.py index e912ffabef..72abd82d91 100644 --- a/test/registered/unit/managers/test_scheduler_recv_skipper.py +++ b/test/registered/unit/managers/test_scheduler_recv_skipper.py @@ -15,12 +15,13 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E4 register_cpu_ci(est_time=10, suite="base-a-test-cpu") -def _publish(case, interval, enable_dp_attention=False): +def _publish(case, interval, attn_dp_size=1): """Publish the config the skipper reads; the double is a published config, not an injected object.""" override = get_context().override_server_args( scheduler_recv_interval=interval, - enable_dp_attention=enable_dp_attention, + tp_size=attn_dp_size, + attn_dp_size=attn_dp_size, ) override.install() case.addCleanup(override.restore) @@ -40,8 +41,8 @@ class TestSchedulerRecvSkipper(CustomTestCase): self.assertIsNone(SchedulerRecvSkipper.maybe_create()) def test_enabled_under_dp_attention(self): - # Regression: the constructor used to assert `not enable_dp_attention`. - _publish(self, 50, enable_dp_attention=True) + # Regression: the constructor used to reject DP attention. + _publish(self, 50, attn_dp_size=2) skipper = SchedulerRecvSkipper.maybe_create() self.assertIsNotNone(skipper) @@ -69,12 +70,12 @@ class TestSchedulerRecvSkipper(CustomTestCase): def test_dp_uses_synced_mode_not_local(self): # Local EXTEND (weight 1000) must be ignored in favor of the synced # DECODE (weight 1); a recv here would mean the local mode leaked in. - _publish(self, 50, enable_dp_attention=True) + _publish(self, 50, attn_dp_size=2) skipper = SchedulerRecvSkipper.maybe_create() self.assertFalse(skipper.handle(_batch(ForwardMode.EXTEND, ForwardMode.DECODE))) def test_dp_synced_extend_triggers_recv(self): - _publish(self, 50, enable_dp_attention=True) + _publish(self, 50, attn_dp_size=2) skipper = SchedulerRecvSkipper.maybe_create() self.assertTrue(skipper.handle(_batch(ForwardMode.IDLE, ForwardMode.EXTEND))) diff --git a/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py b/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py index 2163f5a3d2..a4c7908c40 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py +++ b/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py @@ -65,7 +65,7 @@ def _make_options(**overrides): attn_cp_size=1, dcp_size=1, pp_size=1, - dp_size=1, + num_dp_ranks=1, ep_size=1, cpu_offload_gb=0, offload_group_size=-1, diff --git a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py index 68449056aa..395791b2b7 100644 --- a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py +++ b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py @@ -115,7 +115,7 @@ class TestMlpSyncPadUnpad(CustomTestCase): def test_draft_graph_gate_has_an_independent_dp_vote(self): sync_info = MLPSyncBatchInfo( - dp_size=1, + num_dp_ranks=1, tp_size=1, cp_size=1, num_tokens=1, @@ -306,8 +306,7 @@ class TestDraftScopeMlpSync(CustomTestCase): def setUp(self): override = get_context().override_server_args( tp_size=4, - dp_size=4, - enable_dp_attention=True, + attn_dp_size=4, cuda_graph_config=CudaGraphConfig(prefill=PhaseConfig(bs=[])), ) override.install() diff --git a/test/registered/unit/model_loader/test_weight_cache_protocol.py b/test/registered/unit/model_loader/test_weight_cache_protocol.py index f7e8d6b1c2..cbf8fec665 100644 --- a/test/registered/unit/model_loader/test_weight_cache_protocol.py +++ b/test/registered/unit/model_loader/test_weight_cache_protocol.py @@ -66,7 +66,7 @@ def _make_cache_config(**overrides) -> CacheConfig: moe_dp_size=1, moe_dp_rank=0, moe_ep_rank=0, - enable_dp_attention=False, + attn_dp_size=1, enable_dp_lm_head=False, attn_cp_size=1, moe_dense_tp_size=None, @@ -173,7 +173,7 @@ class TestCacheConfig(CustomTestCase): ("tp_rank", 1), ("moe_dp_rank", 1), ("moe_ep_rank", 1), - ("enable_dp_attention", True), + ("attn_dp_size", 2), ("moe_dense_tp_size", 1), ("moe_a2a_backend", "mooncake"), ("dtype", "torch.bfloat16"), @@ -294,10 +294,9 @@ class TestDaemonLaunchConfiguration(CustomTestCase): model_path="/models/demo", tp_size=8, pp_size=1, - dp_size=8, + attn_dp_size=8, ep_size=8, moe_dp_size=2, - enable_dp_attention=True, enable_dp_lm_head=True, attn_cp_size=2, moe_dense_tp_size=1, diff --git a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py index 42bb14029b..40ac508604 100644 --- a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py +++ b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py @@ -45,7 +45,7 @@ class TestRustServerExtension(CustomTestCase): return_value=SimpleNamespace( nnodes=1, pp_size=1, - dp_size=2, + num_dp_ranks=2, attn_dp_rank=1, tp_size=2, tp_rank=1, diff --git a/test/registered/unit/server_args/test_attn_dp_size.py b/test/registered/unit/server_args/test_attn_dp_size.py new file mode 100644 index 0000000000..6a70050f6e --- /dev/null +++ b/test/registered/unit/server_args/test_attn_dp_size.py @@ -0,0 +1,166 @@ +"""--attn-dp-size sets the attention data-parallel width; the deprecated +`--dp-size N --enable-dp-attention` spelling resolves to `--attn-dp-size N`.""" + +import argparse +import json +import logging +import os +import shutil +import tempfile +import unittest + +from sglang.srt.arg_groups.overrides import resolution_result +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, published_topology + +register_cpu_ci(est_time=20, suite="base-a-test-cpu") + +# A real config.json, so resolution runs past the dummy-model early return. +_MINI_CONFIG = { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "hidden_size": 16, + "intermediate_size": 32, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "num_hidden_layers": 2, + "vocab_size": 128, + "max_position_embeddings": 2048, +} + + +class TestAttnDpSize(CustomTestCase): + def setUp(self): + super().setUp() + environment = dict(os.environ) + + def restore(): + os.environ.clear() + os.environ.update(environment) + + self.addCleanup(restore) + self.path = tempfile.mkdtemp(prefix="attn_dp_size_") + self.addCleanup(shutil.rmtree, self.path, ignore_errors=True) + with open(os.path.join(self.path, "config.json"), "w") as handle: + json.dump(_MINI_CONFIG, handle) + + def resolve(self, **fields): + server_args = ServerArgs( + model_path=self.path, device="cuda", random_seed=42, **fields + ) + server_args.resolve_once() + return server_args + + def layout(self, server_args): + return tuple( + resolution_result(server_args, name) + for name in ("dp_size", "attn_dp_size", "enable_dp_attention") + ) + + def test_the_deprecated_spelling_moves_the_width_to_attn_dp_size(self): + for tp_size, width in ((2, 2), (4, 4), (4, 2), (8, 4)): + with self.subTest(tp_size=tp_size, width=width): + legacy = self.resolve( + tp_size=tp_size, dp_size=width, enable_dp_attention=True + ) + new = self.resolve(tp_size=tp_size, attn_dp_size=width) + self.assertEqual(self.layout(legacy), (1, width, False)) + self.assertEqual(legacy.resolved_dict(), new.resolved_dict()) + + def test_without_attention_dp_the_width_is_one(self): + for fields in ( + {"tp_size": 2}, + {"tp_size": 2, "dp_size": 2}, + # One DP group under the deprecated flag is no attention DP. + {"tp_size": 2, "dp_size": 1, "enable_dp_attention": True}, + ): + with self.subTest(**fields): + server_args = self.resolve(**fields) + self.assertEqual( + self.layout(server_args), (fields.get("dp_size", 1), 1, False) + ) + + def test_rejected_layouts(self): + for fields, message in ( + ( + {"tp_size": 4, "dp_size": 2, "attn_dp_size": 2}, + "replicas combined with attention data parallelism", + ), + ( + {"tp_size": 4, "attn_dp_size": 2, "enable_dp_attention": True}, + "pass --attn-dp-size alone", + ), + ({"tp_size": 2, "attn_dp_size": 0}, "must be positive"), + ): + with self.subTest(**fields): + with self.assertRaisesRegex(ValueError, message): + self.resolve(**fields) + + def test_both_cli_spellings_parse(self): + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + for flag in ("--attn-dp-size", "--attention-data-parallel-size"): + with self.subTest(flag=flag): + raw = parser.parse_args(["--model-path", self.path, flag, "2"]) + self.assertEqual(ServerArgs.from_cli_args(raw).attn_dp_size, 2) + + def test_the_deprecated_flag_warns_with_its_replacement(self): + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + argv = ["--model-path", self.path, "--dp-size", "2", "--enable-dp-attention"] + with self.assertLogs( + "sglang.srt.arg_groups.argparse_actions", "WARNING" + ) as logs: + raw = parser.parse_args(argv) + self.assertEqual(len(logs.output), 1) + self.assertIn("--attn-dp-size", logs.output[0]) + self.assertTrue(ServerArgs.from_cli_args(raw).enable_dp_attention) + + +class TestPublishedAttnDpSize(CustomTestCase): + """What the runtime reads: the attention-DP switch and the DP rank count.""" + + def test_the_published_layout(self): + for fields, (attn_dp_size, num_dp_ranks) in ( + ({"tp_size": 4, "attn_dp_size": 2}, (2, 2)), + ({"tp_size": 4, "dp_size": 2, "enable_dp_attention": True}, (2, 2)), + ({"tp_size": 2, "dp_size": 2}, (1, 2)), + ({"tp_size": 2}, (1, 1)), + ): + with self.subTest(**fields): + with published_topology(**fields): + parallel = get_parallel() + self.assertEqual(parallel.attn_dp_size, attn_dp_size) + self.assertEqual(parallel.num_dp_ranks, num_dp_ranks) + self.assertEqual(parallel.attn_dp_enabled, attn_dp_size > 1) + self.assertEqual( + parallel.attn_tp_size, fields["tp_size"] // attn_dp_size + ) + + def test_a_scale_joiner_runs_attention_dp_at_width_one(self): + with published_topology(tp_size=1, attn_dp_size=1, ep_join_mode="scale"): + parallel = get_parallel() + self.assertEqual(parallel.attn_dp_size, 1) + self.assertEqual(parallel.num_dp_ranks, 1) + self.assertTrue(parallel.attn_dp_enabled) + + def test_an_overridden_layout_publishes_the_same_leaves(self): + for fields, width in ( + ({"tp_size": 4, "attn_dp_size": 4}, 4), + ({"tp_size": 4, "attn_dp_size": 2}, 2), + ({"tp_size": 4, "dp_size": 2}, 1), + ): + with self.subTest(**fields): + with get_context().override_server_args(**fields): + self.assertEqual(get_parallel().attn_dp_size, width) + self.assertEqual(get_parallel().attn_dp_enabled, width > 1) + self.assertEqual( + get_parallel().num_dp_ranks, fields.get("dp_size", width) + ) + + +if __name__ == "__main__": + logging.basicConfig() + unittest.main() diff --git a/test/registered/unit/server_args/test_resolution_declarations.py b/test/registered/unit/server_args/test_resolution_declarations.py index 0fb39a9a21..7704a257e3 100644 --- a/test/registered/unit/server_args/test_resolution_declarations.py +++ b/test/registered/unit/server_args/test_resolution_declarations.py @@ -58,7 +58,7 @@ _SHAPES = ( "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 4, }, - {"dp_size": 2, "tp_size": 2, "enable_dp_attention": True}, + {"attn_dp_size": 2, "tp_size": 2}, {"enable_hierarchical_cache": True}, {"disaggregation_mode": "prefill"}, {"enable_lora": True, "max_lora_rank": 16}, @@ -77,6 +77,7 @@ _REACHED_BY_SHAPES = frozenset( "_speculative_draft_quantization_explicitly_set", "allowed_media_domains", "attention_backend", + "attn_dp_size", "chunked_prefill_size", "cuda_graph_config", "custom_weight_loader", @@ -84,7 +85,6 @@ _REACHED_BY_SHAPES = frozenset( "disable_cuda_graph", "disaggregation_ib_device", "dp_size", - "enable_dp_attention", "enable_dp_attention_local_control_broadcast", "enable_dp_lm_head", "enable_flashinfer_allreduce_fusion", @@ -761,7 +761,7 @@ class TestDeclaredValuesAreNotEditedLater(CustomTestCase): ("disaggregation", {"disaggregation_mode": "prefill"}), ("deterministic", {"enable_deterministic_inference": True}), ("speculative", {"speculative_algorithm": "EAGLE"}), - ("dp_attention", {"tp_size": 2, "dp_size": 2, "enable_dp_attention": True}), + ("dp_attention", {"tp_size": 2, "attn_dp_size": 2}), ): with self.subTest(shape=label): server_args, recorded = self._resolve_recording_each_entry(**supplied) diff --git a/test/registered/unit/server_args/test_resolution_is_reproducible.py b/test/registered/unit/server_args/test_resolution_is_reproducible.py index 4ed946b1ce..b55b6438c6 100644 --- a/test/registered/unit/server_args/test_resolution_is_reproducible.py +++ b/test/registered/unit/server_args/test_resolution_is_reproducible.py @@ -501,9 +501,8 @@ class TestALateDeclarationKeepsTheResolution(_RestoresProcessState, CustomTestCa server_args = ServerArgs( model_path=config_dir, device="cuda", - dp_size=2, + attn_dp_size=2, tp_size=2, - enable_dp_attention=True, random_seed=42, ) server_args.resolve_once() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 4949abe03a..be2872e2f8 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1523,8 +1523,7 @@ class TestFlashinferA2ADispatchType(CustomTestCase): moe_a2a_backend="flashinfer", moe_runner_backend=runner_backend, flashinfer_a2a_dispatch_type=dispatch_type, - enable_dp_attention=True, - dp_size=4, + attn_dp_size=4, tp_size=4, ) server_args._model_config = SimpleNamespace(nvfp4_moe_meta=None) @@ -1678,8 +1677,7 @@ class TestFlashinferMegaMoeConfig(CustomTestCase): quantization=quantization, moe_a2a_backend="flashinfer_megamoe", moe_runner_backend="flashinfer_megamoe", - enable_dp_attention=True, - dp_size=4, + attn_dp_size=4, tp_size=4, ) server_args._model_config = SimpleNamespace( @@ -1766,7 +1764,6 @@ class TestPortArgs(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = False port_args = PortArgs.init_new(server_args) @@ -1781,7 +1778,6 @@ class TestPortArgs(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.nccl_port = None - server_args.enable_dp_attention = False server_args.decoupled_spec_role = "verifier" server_args.decoupled_spec_bind_endpoint = "ipc:///tmp/v" server_args.decoupled_spec_connect_endpoints = ["ipc:///tmp/d"] @@ -1804,7 +1800,6 @@ class TestPortArgs(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.nccl_port = None - server_args.enable_dp_attention = False # decoupled_spec_role defaults to "null" port_args = PortArgs.init_new(server_args) @@ -1814,7 +1809,6 @@ class TestPortArgs(unittest.TestCase): def test_init_new_decoupled_role_requires_endpoints(self): server_args = ServerArgs(model_path="dummy") server_args.nccl_port = None - server_args.enable_dp_attention = False server_args.decoupled_spec_role = "drafter" # endpoints intentionally left as their None defaults @@ -1826,7 +1820,7 @@ class TestPortArgs(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 1 server_args.dist_init_addr = None @@ -1843,7 +1837,7 @@ class TestPortArgs(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 4 server_args.nnodes = 1 server_args.dist_init_addr = "192.168.1.1:25000" @@ -1861,7 +1855,7 @@ class TestPortArgs(unittest.TestCase): server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "192.168.1.1:25000" @@ -1879,7 +1873,7 @@ class TestPortArgs(unittest.TestCase): server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "192.168.1.1" @@ -1893,7 +1887,7 @@ class TestPortArgs(unittest.TestCase): server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "192.168.1.1:abc" @@ -2582,7 +2576,8 @@ class TestPipelineParallelCompat(CustomTestCase): speculative_algorithm="EAGLE", disaggregation_mode="null", speculative_adaptive=False, - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, ) with patch.object( validation_hook.envs.SGLANG_ENABLE_PP_SPEC, "get", return_value=True @@ -3249,8 +3244,7 @@ class TestDeepEPv2Args(CustomTestCase): disaggregation_mode="decode", max_running_requests=256, tp_size=8, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): validate_deepep_v2_dispatch_token_budget(args) @@ -3261,7 +3255,6 @@ class TestDeepEPv2Args(CustomTestCase): max_running_requests=256, tp_size=8, dp_size=1, - enable_dp_attention=False, ) with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): with self.assertRaisesRegex(ValueError, "decode CUDA graph"): @@ -3282,8 +3275,7 @@ class TestDeepEPv2Args(CustomTestCase): speculative_algorithm="EAGLE", speculative_num_draft_tokens=8, max_running_requests=256, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): with self.assertRaisesRegex(ValueError, "tokens/request=8"): @@ -3296,8 +3288,7 @@ class TestDeepEPv2Args(CustomTestCase): speculative_num_draft_tokens=4, speculative_adaptive=True, max_running_requests=128, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) with patch( "sglang.srt.arg_groups.moe_hook.max_speculative_num_draft_tokens", @@ -3663,7 +3654,7 @@ class TestTwoBatchOverlapBackend(CustomTestCase): With no EP a2a backend (moe_a2a_backend='none'), --enable-two-batch-overlap is only valid on the DeepSeek-V4 non-EP DP TP-MoE path (overlapping the DP all_gatherv / reduce_scatterv with the other ubatch's compute), which - requires --enable-dp-attention. This replaced the removed opt-in + requires attention DP (--attn-dp-size). This replaced the removed opt-in SGLANG_ENABLE_DP_TBO env: enabling DP TBO now needs no extra flag. dummy-model short-circuits __post_init__, so the guard handler is invoked @@ -3673,26 +3664,25 @@ class TestTwoBatchOverlapBackend(CustomTestCase): args = ServerArgs(model_path="dummy") args.enable_two_batch_overlap = True args.moe_a2a_backend = "none" - args.enable_dp_attention = False for key, value in overrides.items(): setattr(args, key, value) return args def test_no_a2a_without_dp_attention_raises(self): - args = self._args(enable_dp_attention=False) - with self.assertRaisesRegex(ValueError, "enable-dp-attention"): + args = self._args() + with self.assertRaisesRegex(ValueError, "attn-dp-size"): check_two_batch_overlap(args) def test_no_a2a_with_dp_attention_ok(self): - # DP TBO path is valid: --enable-dp-attention + --enable-two-batch-overlap + # DP TBO path is valid: --attn-dp-size + --enable-two-batch-overlap # with a2a backend 'none' must NOT raise (no SGLANG_ENABLE_DP_TBO needed). - args = self._args(enable_dp_attention=True) + args = self._args(attn_dp_size=2) check_two_batch_overlap(args) def test_ep_a2a_backend_ok_without_dp_attention(self): # EP a2a path (e.g. deepep) overlaps dispatch/combine; the guard does not # require dp-attention there. - args = self._args(moe_a2a_backend="deepep", enable_dp_attention=False) + args = self._args(moe_a2a_backend="deepep") check_two_batch_overlap(args) @@ -3951,9 +3941,8 @@ class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): # have settled by then; the dummy-model pipeline itself returns early. server_args = ServerArgs( model_path="dummy", - enable_dp_attention=True, + attn_dp_size=2, tp_size=2, - dp_size=2, chunked_prefill_size=8192, cuda_graph_config=CudaGraphConfig( prefill=PhaseConfig(backend=Backend.DISABLED) diff --git a/test/registered/unit/spec/test_draft_tp_context.py b/test/registered/unit/spec/test_draft_tp_context.py index d7ef2c3359..1e1b6795c6 100644 --- a/test/registered/unit/spec/test_draft_tp_context.py +++ b/test/registered/unit/spec/test_draft_tp_context.py @@ -21,9 +21,7 @@ class TestDraftTpContext(CustomTestCase): """Draft work enters the attention-TP placement only when it owns attention.""" def setUp(self): - topology = published_topology( - tp_size=4, dp_size=2, enable_dp_attention=True, ranks=dict(dp_rank=0) - ) + topology = published_topology(tp_size=4, attn_dp_size=2, ranks=dict(dp_rank=0)) topology.__enter__() self.addCleanup(topology.__exit__, None, None, None) self.whole_tp = _group(world_size=4, rank=0) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 9036047bc8..e77d007ddc 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -62,6 +62,7 @@ class TestBoundaryParallelismResolution(CustomTestCase): cp_strategy=None, tp_size=4, dp_size=1, + attn_dp_size=1, enable_aiter_allreduce_fusion=False, ), **options, @@ -171,6 +172,8 @@ class TestModelOverridableWhitelist(CustomTestCase): "page_size", "moe_runner_backend", "quantization", + "dp_size", + "attn_dp_size", "enable_dp_attention", "enable_attn_tp_input_scattered", "enable_dp_lm_head", @@ -589,7 +592,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): prefill_attention_backend=None, decode_attention_backend=None, disaggregation_mode="null", - enable_dp_attention=False, + attn_dp_size=1, enable_hierarchical_cache=False, ): args = SimpleNamespace( @@ -597,7 +600,8 @@ class TestGoldenModelOverrides(_IsolatedPublish): prefill_attention_backend=prefill_attention_backend, decode_attention_backend=decode_attention_backend, disaggregation_mode=disaggregation_mode, - enable_dp_attention=enable_dp_attention, + attn_dp_size=attn_dp_size, + ep_join_mode=None, enable_hierarchical_cache=enable_hierarchical_cache, ) mixer_types = [] @@ -651,7 +655,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): ): self._minicpm_overrides( architecture, - enable_dp_attention=True, + attn_dp_size=2, ) def test_minicpm_rejects_hierarchical_cache_for_hybrid_models(self): @@ -685,7 +689,8 @@ class TestGoldenModelOverrides(_IsolatedPublish): prefill_attention_backend=None, decode_attention_backend=None, disaggregation_mode="null", - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, enable_hierarchical_cache=False, ) config = SimpleNamespace( @@ -2387,7 +2392,8 @@ class TestGoldenModelOverrides(_IsolatedPublish): defaults = dict( flashinfer_allreduce_fusion_backend=None, tp_size=2, - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, nnodes=1, moe_a2a_backend="none", enforce_disable_flashinfer_allreduce_fusion=False, @@ -2423,9 +2429,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): _flashinfer_allreduce_fusion_auto_enable(_view(tp_size=1)), {} ) self.assertEqual( - _flashinfer_allreduce_fusion_auto_enable( - _view(enable_dp_attention=True) - ), + _flashinfer_allreduce_fusion_auto_enable(_view(attn_dp_size=2)), {}, ) self.assertEqual( @@ -3408,16 +3412,26 @@ class TestGoldenModelOverrides(_IsolatedPublish): self.assertEqual( _data_parallelism_defaults( - ResolvedView(SimpleNamespace(dp_size=1, ep_join_mode=None)) + ResolvedView( + SimpleNamespace(dp_size=1, attn_dp_size=1, ep_join_mode=None) + ) ), - {"enable_dp_attention": False, "enable_dp_lm_head": False}, - ) - self.assertEqual( - _data_parallelism_defaults( - ResolvedView(SimpleNamespace(dp_size=2, ep_join_mode=None)) - ), - {}, + {"enable_dp_lm_head": False}, ) + for dp_size, attn_dp_size in ((2, 1), (1, 2)): + with self.subTest(dp_size=dp_size, attn_dp_size=attn_dp_size): + self.assertEqual( + _data_parallelism_defaults( + ResolvedView( + SimpleNamespace( + dp_size=dp_size, + attn_dp_size=attn_dp_size, + ep_join_mode=None, + ) + ) + ), + {}, + ) self.assertEqual( _a2a_ep_size( @@ -3500,6 +3514,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): cp_strategy="zigzag", tp_size=8, dp_size=1, + attn_dp_size=1, ep_size=1, moe_a2a_backend="none", kv_cache_dtype="auto", @@ -3511,21 +3526,23 @@ class TestGoldenModelOverrides(_IsolatedPublish): { "attention_backend": "dsa", "page_size": 64, - "enable_dp_attention": True, + "attn_dp_size": 1, + "dp_size": 1, "moe_dense_tp_size": 1, "moe_a2a_backend": "deepep", "ep_size": 8, "attn_cp_size": 8, }, ) - # interleave CP with dp>1 must assert + # interleave CP with attention DP must assert with self.assertRaises(AssertionError): _deepseek_family_overrides( _args( enable_prefill_cp=True, cp_strategy="interleave", tp_size=8, - dp_size=2, + dp_size=1, + attn_dp_size=2, ), None, ) diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 5aedb9dc6f..462806429b 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -370,38 +370,38 @@ class TestAttentionRanksComeFromPublish(_IsolatedOverrides): from sglang.srt.layers.dp_attention import compute_dp_attention_world_info shapes = [ - (8, 1, 1, False), - (8, 2, 1, True), - (8, 4, 1, True), - (8, 2, 2, True), - (16, 4, 2, True), + (8, 1, 1), + (8, 2, 1), + (8, 4, 1), + (8, 2, 2), + (16, 4, 2), ] - for tp_size, dp_size, attn_cp_size, dp_attn in shapes: + for tp_size, attn_dp_size, attn_cp_size in shapes: for tp_rank in range(tp_size): reset_context() publish( ServerArgs( model_path="dummy", tp_size=tp_size, - dp_size=dp_size, + attn_dp_size=attn_dp_size, attn_cp_size=attn_cp_size, - enable_dp_attention=dp_attn, ), role="test", ranks=SpawnRanks(world_rank=tp_rank), ) want_tp, _, want_dp, _ = compute_dp_attention_world_info( - dp_attn, tp_rank, tp_size, dp_size, attn_cp_size + tp_rank, tp_size, attn_dp_size, attn_cp_size + ) + msg = ( + f"tp={tp_size} attn_dp={attn_dp_size} cp={attn_cp_size} " + f"rank={tp_rank}" ) - msg = f"tp={tp_size} dp={dp_size} cp={attn_cp_size} rank={tp_rank}" self.assertEqual(get_parallel().attn_tp_rank, want_tp, msg) self.assertEqual(get_parallel().attn_dp_rank, want_dp, msg) def test_the_rank_reads_without_a_process_group(self): publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ranks=SpawnRanks(world_rank=5), ) @@ -485,9 +485,7 @@ class TestStampedRanks(_IsolatedOverrides): self.addCleanup(restore) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=8, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=8), role="test", ranks=SpawnRanks(world_rank=3), ) @@ -512,9 +510,7 @@ class TestStampedRanks(_IsolatedOverrides): self.addCleanup(setattr, dp_flags, "use_world_group_for_gather", saved_gather) self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=8, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=8), role="test", ranks=SpawnRanks(world_rank=3), ) @@ -630,9 +626,7 @@ class TestAWidthReadStaysTraceable(_IsolatedOverrides): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ) @@ -1945,9 +1939,7 @@ class TestDerivedWidths(_IsolatedOverrides): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ) self.assertEqual(get_parallel().attn_tp_size, 4) @@ -2068,9 +2060,7 @@ class TestDerivedWidths(_IsolatedOverrides): side_effect=AssertionError("the group must not be consulted"), ): publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ) self.assertEqual(get_parallel().attn_tp_size, 4) @@ -2108,9 +2098,9 @@ class TestDerivedWidths(_IsolatedOverrides): widths from the same derivation `override_permanently`'s callers use.""" from sglang.srt.layers.dp_attention import compute_dp_attention_world_info - for tp_size, dp_size, attn_cp_size in ((8, 2, 1), (8, 2, 2), (16, 4, 2)): + for tp_size, dp_width, attn_cp_size in ((8, 2, 1), (8, 2, 2), (16, 4, 2)): _, attn_tp_size, _, attn_dp_size = compute_dp_attention_world_info( - True, 0, tp_size, dp_size, attn_cp_size + 0, tp_size, dp_width, attn_cp_size ) widths = derive_parallel_widths( tp_size=tp_size, @@ -2127,7 +2117,7 @@ class TestDerivedWidths(_IsolatedOverrides): def test_recomputing_from_published_leaves_matches_the_publish_bag(self): shapes = ( dict(tp_size=8), - dict(tp_size=8, dp_size=2, enable_dp_attention=True), + dict(tp_size=8, attn_dp_size=2), dict(tp_size=8, ep_size=4, moe_dp_size=2), dict(tp_size=8, dcp_size=8), ) @@ -2151,9 +2141,7 @@ class TestDerivedWidths(_IsolatedOverrides): recomputed = derive_parallel_widths( tp_size=parallel.tp_size, attn_cp_size=parallel.attn_cp_size, - attn_dp_size=( - parallel.dp_size if parallel.enable_dp_attention else 1 - ), + attn_dp_size=parallel.attn_dp_size, moe_ep_size=parallel.ep_size, moe_dp_size=parallel.moe_dp_size, dcp_size=parallel.dcp_size, @@ -2244,7 +2232,10 @@ class TestTheDerivedHalfIsDeclared(CustomTestCase): self.assertTrue(callable(getattr(importlib.import_module(module), attr))) def test_the_arithmetic_produces_nothing_that_is_not_declared(self): - from sglang.srt.runtime_context import _derived_widths + from sglang.srt.runtime_context import ( + _derived_widths, + _parallel_config_leaves, + ) produced = set( derive_parallel_widths( @@ -2257,7 +2248,10 @@ class TestTheDerivedHalfIsDeclared(CustomTestCase): dcp_enabled=False, ) ) - self.assertEqual(produced - set(_derived_widths()), set()) + # `attn_dp_size` is a configured leaf the widths carry along, so a + # process with nothing published can stamp the whole layout at once. + self.assertIn("attn_dp_size", _parallel_config_leaves()) + self.assertEqual(produced - set(_derived_widths()), {"attn_dp_size"}) def test_a_declared_quotient_is_not_a_record_field(self): """It has no operator input to preserve, and the record is what crosses @@ -2441,9 +2435,7 @@ class TestTheTopologyIdentities(CustomTestCase): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=2), role="scheduler", ranks=SpawnRanks(world_rank=3, dp_rank=1), ) @@ -2585,8 +2577,7 @@ class TestTheParallelPhase(CustomTestCase): ServerArgs( model_path="dummy", tp_size=4, - dp_size=2, - enable_dp_attention=True, + attn_dp_size=2, device="cpu", ), role="test", @@ -2630,17 +2621,17 @@ class TestTheParallelPhase(CustomTestCase): server_args = ServerArgs( model_path="dummy", device="cpu", - tp_size=4, + tp_size=8, + attn_dp_size=2, attn_cp_size=2, - enable_dp_attention=True, ) publish(server_args, role="test", ranks=SpawnRanks(world_rank=0)) # Identical shards contribute equally; reducing across CP replicas # would double the output even though the layer uses attention TP. tp_group = SimpleNamespace( - world_size=4, + world_size=8, rank_in_group=0, - all_reduce=Mock(side_effect=lambda x: x * 4), + all_reduce=Mock(side_effect=lambda x: x * 8), ) attn_tp_group = SimpleNamespace( world_size=2, @@ -2779,9 +2770,7 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=2), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) @@ -2798,14 +2787,14 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): self.assertEqual(parallel.attn_dp_rank, 0) self.assertEqual(parallel.attn_cp_size, 1) self.assertEqual(parallel.attn_cp_rank, 0) - # The scope leaves the deployment's replica count alone. - self.assertEqual(parallel.dp_size, 2) + # The scope leaves the deployment's DP rank count alone. + self.assertEqual(parallel.num_dp_ranks, 2) self.assertEqual( parallel.tp_size, parallel.attn_tp_size * parallel.attn_dp_size * parallel.attn_cp_size, ) self.assertEqual(get_parallel().attn_dp_size, 2) - self.assertEqual(get_parallel().dp_size, 2) + self.assertEqual(get_parallel().num_dp_ranks, 2) def test_a_full_width_swap_leaves_the_attention_layout_alone(self): from sglang.srt.distributed import parallel_state @@ -2813,9 +2802,7 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=2), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) @@ -2828,7 +2815,7 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): self.assertEqual(parallel.tp_size, 4) self.assertEqual(parallel.attn_dp_size, 2) self.assertEqual(parallel.attn_tp_size, 2) - self.assertEqual(parallel.dp_size, 2) + self.assertEqual(parallel.num_dp_ranks, 2) def test_a_narrowed_draft_still_indexes_the_targets_dp_sync(self): """A draft scope that owns its attention answers the DP gather width and @@ -2839,9 +2826,7 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=4, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=4), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) @@ -2866,9 +2851,7 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=4, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=4), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) diff --git a/test/registered/unit/test_server_args_migration.py b/test/registered/unit/test_server_args_migration.py index 0371f11a03..4a5270c081 100644 --- a/test/registered/unit/test_server_args_migration.py +++ b/test/registered/unit/test_server_args_migration.py @@ -178,9 +178,10 @@ class TestServerArgsAnnotatedCli(CustomTestCase): def test_each_deprecation_shape_does_its_job(self): """The four `Deprecated*Action` shapes, on a throwaway parser. - Only `DeprecatedStoreTrueAction` has a registration in `ServerArgs` - today (`--disable-cuda-graph`, covered above). The other three are kept - for the next flag that needs them, which makes this the only thing + Only `DeprecatedStoreTrueAction` has registrations in `ServerArgs` + today (`--disable-cuda-graph`, covered above, and + `--enable-dp-attention`). The other three are kept for the next flag + that needs them, which makes this the only thing standing between them and silent rot -- and it doubles as the worked example of which shape to reach for. """ diff --git a/test/registered/unit/utils/test_weight_checker.py b/test/registered/unit/utils/test_weight_checker.py index 2df370879d..874ea3b30c 100644 --- a/test/registered/unit/utils/test_weight_checker.py +++ b/test/registered/unit/utils/test_weight_checker.py @@ -761,8 +761,7 @@ class _ChecksumTestBase(CustomTestCase): self, published_topology( tp_size=4, - dp_size=2, - enable_dp_attention=True, + attn_dp_size=2, ranks={"world_rank": 2, "dp_rank": 1}, ), ) diff --git a/test/registered/utils/test_network_address.py b/test/registered/utils/test_network_address.py index d3ca555b0d..9600182aa0 100644 --- a/test/registered/utils/test_network_address.py +++ b/test/registered/utils/test_network_address.py @@ -246,7 +246,7 @@ class TestPortArgsIPv6(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]:25000" @@ -265,7 +265,7 @@ class TestPortArgsIPv6(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[invalid-ipv6]:25000" @@ -277,7 +277,7 @@ class TestPortArgsIPv6(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1:25000" @@ -289,7 +289,7 @@ class TestPortArgsIPv6(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]" @@ -301,7 +301,7 @@ class TestPortArgsIPv6(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]:abcde" @@ -313,7 +313,7 @@ class TestPortArgsIPv6(unittest.TestCase): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]#25000" diff --git a/test/registered/xpu/test_dsa_indexer_xpu.py b/test/registered/xpu/test_dsa_indexer_xpu.py index 292545e3b3..a05a326b54 100644 --- a/test/registered/xpu/test_dsa_indexer_xpu.py +++ b/test/registered/xpu/test_dsa_indexer_xpu.py @@ -236,7 +236,6 @@ class TestDSAIndexerXPU(CustomTestCase): @classmethod def setUpClass(cls): server_args = ServerArgs(model_path="dummy") - server_args.enable_dp_attention = False server_args.dsa_prefill_backend = "intel_xpu" server_args.dsa_decode_backend = "intel_xpu" server_args.dsa_topk_backend = "torch"