From 027208ae02fc6bb0c973714b490e38188dfd7df9 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:26:40 -0700 Subject: [PATCH] [Doc] layer-boundary/moe: Correct guides and comments that drifted from the code (#44896) --- docs/docs/advanced_features/expert_parallelism.mdx | 14 +++++++++----- docs/docs/developer_guide/layer_boundary.mdx | 11 ++++++++--- .../srt/layers/layer_boundary/residual/add_norm.py | 6 ++---- python/sglang/srt/layers/moe/utils.py | 7 ++++--- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/docs/advanced_features/expert_parallelism.mdx b/docs/docs/advanced_features/expert_parallelism.mdx index 077884fd9f..4bf8514221 100644 --- a/docs/docs/advanced_features/expert_parallelism.mdx +++ b/docs/docs/advanced_features/expert_parallelism.mdx @@ -148,14 +148,14 @@ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --moe-a2a-ba ## Extensible EP Framework -SGLang's EP framework provides modular abstractions for easy integration of custom kernels, backends, and optimizations. It decouples the MoE forward pass into stages (dispatch → pre-permute → core runner → post-permute → combine), enabling seamless extensions without refactoring core logic. +SGLang's EP framework provides modular abstractions for easy integration of custom kernels, backends, and optimizations. It decouples the MoE forward pass into stages (dispatch → pre-permute → core runner → post-permute → combine), enabling seamless extensions without refactoring core logic. A fused function registered for a layer's all-to-all and runner backends replaces the three middle stages. ### Framework Overview The framework centers on `FusedMoE` as the unified entry point for a single, extensible structure. Key components include: - **Dispatcher**: Manages dispatch/combine for backends like DeepEP (implements `BaseDispatcher` subclasses). - **MoeRunner**: Orchestrates grouped-GEMM execution via `MoeRunnerCore` implementations (e.g., `TritonRunnerCore`). -- **PermuteMethodPool**: Auto-registers layout conversions (e.g., pre/post-permute via `register_pre_permute` and `register_post_permute` for dynamic modes, or `register_fused_func` for static, torch.compile-compatible fused operations). +- **FusedOpPool** and **PermuteMethodPool**: Auto-register fused functions per all-to-all and runner backend pair (`register_fused_func`) and layout conversions (`register_pre_permute` and `register_post_permute`). `MoeRunner.run` calls the fused function when one is registered for the layer's pair and LoRA is disabled; otherwise it runs pre-permute, the runner core, and post-permute, or hands the dispatch output to a `DispatchMoeRunnerCore`. - **TopK Router**: Backend-agnostic expert selection. This design supports multiple backends via `--moe-a2a-backend` and `--moe-runner-backend`, with quantization integrated through a standardized `apply()` method. The computation flow ensures modularity: @@ -176,10 +176,12 @@ This design supports multiple backends via `--moe-a2a-backend` and `--moe-runner | [DispatchOutput] | | | v - | quant_method.apply -> MoeRunner.forward + | quant_method.apply -> MoeRunner.run | | | | | v - | | pre-permute + grouped_gemm + post-permute + | | fused func + | | or DispatchMoeRunnerCore.run_from_dispatch + | | or pre-permute + grouped_gemm + post-permute | | | | |-------------- | v @@ -205,11 +207,13 @@ To add a new backend: - **Fused Mode** (static, torch.compile-compatible): Use `register_fused_func` for end-to-end operations. - **Permute Mode** (dynamic): Register `register_pre_permute` and `register_post_permute` for flexible layouts. + When both are registered for a backend pair, the fused function takes precedence; the non-fused path still serves LoRA, which skips fused functions. + See the [MoE Refactor Implementation PR](https://github.com/sgl-project/sglang/pull/9269) for full changes, including type hints and config expansions. ### Examples -For an example implementation, see [moe_runner/triton.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/moe/moe_runner/triton.py), which demonstrates Triton-based grouped GEMMs with registered fused and permutation functions. +For an example implementation, see [moe_runner/triton.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/moe/moe_runner/triton.py), which demonstrates Triton-based grouped GEMMs with a fused function registered for the `none` all-to-all backend and permutation functions as the fallback path, used for example with LoRA. ## Computation and Communication Overlap diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 3d5811b226..bafe138624 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -308,18 +308,24 @@ Paths in this table are relative to `python/sglang/srt/layers/layer_boundary/`. | `contracts.py` | Resolved input/output contracts, edge declarations, batch variants, and bound step records | | `layout.py` | Token axes, layouts, named sum groups, and topology/batch facts | | `construction.py` | Build a `StagePlan` and select its batch variant | -| `boundary.py` | Select and bind producer/consumer boundary operations from contracts | +| `boundary.py` | Select and bind producer/consumer boundary operations from contracts; for prefill CP, one `CpMoves` (`contracts.py`) pairs the FFN-input gather, the take-back of a complete output, and, where there is one, the reduce-scatter that completes a sum | | `prepare.py` | Execute completion, update, read, and input movement in the selected order | | `ops.py` | Collective and token-row movement primitives | | `stage.py` | Model-facing `StageBoundary` methods | | `exit.py` | Select output completion and publish scoped compute flags | | `output.py` | Unreduced outputs, deferred finalizes and output transforms | +| `facts.py` | What binding keeps of a stage it does not run (`facts_of()`): a stack's neighbours on other pipeline ranks and a bound stage's producer | +| `row_transport.py` | Row transports that run a stage's compute on token rows other than its residual's (`RowTransport`, `ComputeTransport`) | | `residual/__init__.py` | `ResidualUpdate`/`ResidualReadout` protocols and `LayerResidualOps` | | `residual/stream.py` | Per-forward state and ownership of outstanding work | | `residual/batch.py` | Model-stack entry, final norm, pipeline-parallel (PP) transport, snapshots, and extra contributions | | `residual/access.py` | Tensor access and materialization helpers | | `residual/add_norm.py`, `residual/mhc.py` | Plain add/norm and MHC read/update implementations | -| `adapters/` | Attention input, CP, TBO, branches, and LoRA integration | +| `residual/ihc.py`, `residual/gated.py` | Gated and gated-injection hyper-connection read/update implementations | +| `residual/post_norm.py` | Post-norm residual: each stage's output is normalized before it is added | +| `residual/attn_bank.py` | A residual read that aggregates a bank of residual snapshots | +| `residual/short_conv.py` | Deferred channelwise convolution followed by residual add and normalization | +| `adapters/` | Attention input, TBO, branches, and LoRA integration | | `fusions/` | Consumer all-reduce fusion and CuteDSL integration | Models normally use factories, `StageBoundary`, and `residual.batch`. Keep row arithmetic, process-group selection, and calls such as `token_axis_sizes()` inside the boundary implementation. @@ -363,7 +369,6 @@ Producer GEMM plus reduce-scatter fusion is not a generic implemented API here. ### Special adapters - **Attention/DeepSeek Sparse Attention (DSA) adapters:** retain `AttnTpContext` and its input-scattered/latent-input hooks. They are not a general asynchronous dependency scheduler. -- **CP adapters:** pair gather, return-to-local-rows, and eligible reduce-scatter operations using the same token mapping. - **LoRA adapter:** publishes the selected compute input's rows for LoRA execution. - **TBO and branch adapters:** manage layer exits and entries that cannot be represented as a normal sequential prepare call. diff --git a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py index 59e8651a24..133147674b 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py @@ -136,15 +136,13 @@ def _fused_rmsnorm_fp8_per_token_quant( return (out_fp8, scale.unsqueeze(1)) -# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465 -# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True). +# Largest batch the non-CuTe-DSL FlashInfer all-reduce fusion runs on; +# it also sizes that fusion's workspace, pre-allocated before graph capture. FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048 def flashinfer_ar_fusion_applies(batch_size: int): return ( - # NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion - # Ref: https://github.com/sgl-project/sglang/issues/17237 (_is_sm90_supported or _is_sm100_supported) and _is_flashinfer_available and not is_dp_attention_enabled() diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index abea470255..9086da5da9 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -716,9 +716,10 @@ def is_moe_input_scattered_across_dp_ranks() -> bool: def should_use_dp_reduce_scatterv(): """ - Use reduce_scatterv in the standard dispatcher's combine() for DP attention - with EP, replacing the default all-reduce + dp_scatter path. - Only changes the combine (post-kernel) communication; dispatch is unchanged. + Whether the parallel configuration allows reduce_scatterv for returning an + FFN output over attention DP (DP attention with EP), replacing the default + all-reduce + dp_scatter path; a stage boundary also needs boundary_reduction + rsv or rs+rsv. Only that post-expert return changes; dispatch is unchanged. The reduce_scatterv group is the global TP group, while its variable split sizes are one entry per attention-DP rank. Therefore this optimization is