[Refactor] Group communicator fusion and CP adapters (#41547)
This commit is contained in:
@@ -103,7 +103,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
# Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP.
|
||||
# The DSA / MLA CP gather and reduce-scatter
|
||||
# (communicator_dsa_cp) assume it.
|
||||
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
|
||||
attn_cp_size = cfg.tp_size // cfg.dp_size
|
||||
overrides["attn_cp_size"] = attn_cp_size
|
||||
logger.warning(
|
||||
@@ -168,7 +168,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
# FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP.
|
||||
# The DSA / MLA CP gather and reduce-scatter
|
||||
# (communicator_dsa_cp) assume it.
|
||||
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
|
||||
attn_cp_size = cfg.tp_size // cfg.dp_size
|
||||
overrides["attn_cp_size"] = attn_cp_size
|
||||
logger.warning(
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Backend implementations of stage-boundary fusion.
|
||||
|
||||
The boundary selects these candidates; compute kernels stay with their backend.
|
||||
"""
|
||||
@@ -33,6 +33,10 @@ from sglang.srt.layers.communicator.adapters.attention import (
|
||||
_redistribute_to_attn_tp_shards,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.communicator.layout import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
@@ -48,10 +52,6 @@ from sglang.srt.layers.communicator.output import (
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import StageRead, StageUpdate
|
||||
from sglang.srt.layers.communicator.residual.add_norm import ADD, NORM_READ
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
|
||||
@@ -303,7 +303,7 @@ class BaseRunner(ABC):
|
||||
resolve_flashinfer_allreduce_fusion_backend()
|
||||
if not mr.is_draft_worker:
|
||||
# A draft installs no fusion communicator.
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
prepare_cutedsl_fusion,
|
||||
)
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ from sglang.srt.layers.communicator import (
|
||||
get_attn_tp_context,
|
||||
layer_input_buffer,
|
||||
)
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
)
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
@@ -1107,7 +1107,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
if deferred_finalize and get_forward().defer_moe_finalize:
|
||||
# deferred_finalize excludes _shared_expert_tp1, so the shared add folds in.
|
||||
assert shared_output is not None
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
return MoeFinalizeHandoff.from_flashinfer(
|
||||
final_hidden_states,
|
||||
@@ -2639,7 +2641,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
and _use_mnnvl_cutedsl_fusion()
|
||||
):
|
||||
# Dense layers too: selecting cutedsl turns the legacy fusion off.
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
fusions = CuteDSLFusion()
|
||||
return LayerCommunicator(
|
||||
@@ -2971,7 +2973,9 @@ class DeepseekV2Model(nn.Module):
|
||||
# llama_4_scaling: for supporting Mistral-Large-3 model
|
||||
self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None)
|
||||
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import install_cutedsl_fusion
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
install_cutedsl_fusion,
|
||||
)
|
||||
|
||||
install_cutedsl_fusion(
|
||||
# PP pads self.layers with PPMissingLayer, which has no communicator.
|
||||
|
||||
@@ -71,7 +71,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
dsa_cp_gather_hidden_states,
|
||||
dsa_cp_reduce_scatter_hidden_states,
|
||||
)
|
||||
|
||||
@@ -859,7 +859,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
if defer_finalize:
|
||||
if shared_output is None:
|
||||
raise RuntimeError("Qwen deferred finalize requires shared output")
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
return MoeFinalizeHandoff.from_flashinfer(
|
||||
final_hidden_states,
|
||||
|
||||
@@ -217,7 +217,7 @@ def _use_mnnvl_cutedsl_fusion(config: Qwen3_5TextConfig, is_nextn: bool) -> bool
|
||||
def _layer_fusions(config: Qwen3_5TextConfig, is_nextn: bool):
|
||||
"""The CuTe DSL kernels a layer tries before its own, when they are on."""
|
||||
if _use_mnnvl_cutedsl_fusion(config, is_nextn):
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusion
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import CuteDSLFusion
|
||||
|
||||
return CuteDSLFusion()
|
||||
return None
|
||||
@@ -1730,7 +1730,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
"layers: "
|
||||
f"{unsupported_layers}"
|
||||
)
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import (
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
install_cutedsl_fusion,
|
||||
)
|
||||
|
||||
@@ -1877,7 +1877,9 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
# The final layer has no successor to consume its deferred MoE tail.
|
||||
is_deferred_finalize = False
|
||||
if self.flashinfer_mnnvl_cutedsl_fusion is not None:
|
||||
from sglang.srt.layers.moe.cutedsl_ar_fusion import MoeFinalizeHandoff
|
||||
from sglang.srt.layers.communicator.fusions.cutedsl import (
|
||||
MoeFinalizeHandoff,
|
||||
)
|
||||
|
||||
is_deferred_finalize = isinstance(hidden_states, MoeFinalizeHandoff)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user