diff --git a/python/sglang/srt/batch_overlap/operations_strategy.py b/python/sglang/srt/batch_overlap/operations_strategy.py index af2171e3d8..9fa3b54dc3 100644 --- a/python/sglang/srt/batch_overlap/operations_strategy.py +++ b/python/sglang/srt/batch_overlap/operations_strategy.py @@ -63,7 +63,10 @@ class OperationsStrategy: for layer in layers ] ) - elif layer_name == "DeepseekV4DecoderLayer": + # Other model families do not need to import the V4 implementation. + from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayerBase + + if isinstance(layers[0], DeepseekV4DecoderLayerBase): return OperationsStrategy.concat( [ _compute_moe_deepseek_v4_layer_operations_strategy_tbo( @@ -72,8 +75,7 @@ class OperationsStrategy: for layer in layers ] ) - else: - raise NotImplementedError + raise NotImplementedError def _assert_all_same(items: List): diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 75e7894223..fa20f75d5c 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.contracts import ( CpMoves, EdgeContract, EntryPath, + ExitFacts, ProducerReduction, StageKind, StagePath, @@ -126,6 +127,7 @@ class StagePlan: is_branch=False, terminal=False, writes_at_handoff=False, + row_transport=None, finishes_directly=False, qkv_latent_func=None, fusions=None, @@ -201,9 +203,15 @@ class StagePlan: returns_over_dp=out.returns_over_dp, writes_at_handoff=writes_at_handoff, ) - self.paths[variant] = msgspec.structs.replace( - path, exit=exit_facts(kind, self, variant, path) + fixed_exit = ( + ExitFacts( + writes_residual=path.output.update.applied_at_exit + or writes_at_handoff + ) + if row_transport is not None + else exit_facts(kind, self, variant, path) ) + self.paths[variant] = msgspec.structs.replace(path, exit=fixed_exit) @property def incoming_residual_rows(self): @@ -278,10 +286,16 @@ def _bind_stage( if incoming.entries.keys() != outgoing.exits.keys(): raise ValueError("incoming and outgoing batch variants disagree") update = declaration.update - if pipeline_handoff and not (update.is_plain_add or update.applied_at_exit): + if pipeline_handoff and not ( + update.is_plain_add + or update.applied_at_exit + or (declaration.writes_at_handoff and declaration.row_transport is not None) + ): raise NotImplementedError( "pipeline boundaries require a producer-written residual" ) + if declaration.row_transport is not None and declaration.row_transport.bind is None: + raise ValueError("a local stage requires its compute transport") if declaration.sum_impl is not None and declaration.sum_impl.run is None: raise ValueError("a local stage requires its custom sum implementation") if declaration.output_complete and declaration.sum_impl is not None: @@ -311,7 +325,10 @@ def _bind_stage( "an update applied at the stage's exit with a gather over attention CP" ) attn_input_adapter = None - if declaration.kind is StageKind.ATTENTION: + if ( + declaration.kind is StageKind.ATTENTION + and declaration.row_transport is None + ): # On an input-scattered batch the step gathers the rows itself for # an attention whose QKV hook does not gather them after the # projection. @@ -332,6 +349,7 @@ def _bind_stage( is_branch=declaration.prepared_from is not None, terminal=declaration.terminal, writes_at_handoff=declaration.writes_at_handoff, + row_transport=declaration.row_transport, attn_tp_gather=declaration.attn_tp_gather, # The exit runs its consumer's gather: the next stage's, or the # final read's. diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 7b741d3de5..07e6c22a60 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -314,6 +314,7 @@ class ExitFacts(msgspec.Struct, frozen=True): is not one all-reduce, may still be left to a fused consumer. """ + writes_residual: bool = False may_defer_sum: bool = False defers_mixer_sum: bool = False sum_in_reduce_scatter: bool = False diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 93f4fca370..e75721206d 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -55,6 +55,7 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_READOUT, PLAIN_ADD, ) +from sglang.srt.layers.layer_boundary.row_transport import RowTransport from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks from sglang.srt.runtime_context import get_exec, get_parallel @@ -305,6 +306,9 @@ class StageDeclaration: sum_impl: Optional implementation of the declared output sum using the model's custom collective. Its batch predicate, when supplied, preserves eligibility for carrying that sum to the consumer. + row_transport: Explicit source, compute and return rows for a stage + whose operand moves independently of its residual. The bound + invocation owns output completion and inverse transport. attn_tp_gather: Optional implementation of the gather over attention TP that brings rows into this stage: at its own entry, or at its producer's exit, which runs its consumer's gather. Given this rank's @@ -340,6 +344,7 @@ class StageDeclaration: output_complete: bool = False attn_tp_gather: Optional[Callable] = None sum_impl: Optional[SumImplementation] = None + row_transport: Optional[RowTransport] = None # Only declarations participate in construction, never executable stages. previous: Optional[StageDeclaration] = None prepared_from: Optional[StageDeclaration] = None @@ -436,6 +441,7 @@ def declare_attn( attn_tp_gather=None, in_token_order=False, sum_impl=None, + row_transport=None, ): """Declare attention or a mixer; construct its executable boundary later. @@ -479,6 +485,7 @@ def declare_attn( attn_tp_gather=attn_tp_gather, in_token_order=in_token_order, sum_impl=sum_impl, + row_transport=row_transport, ) @@ -493,6 +500,7 @@ def declare_ffn( exit_rows=None, output_complete=False, sum_impl=None, + row_transport=None, ): """Declare a dense or MoE FFN independently of its compute module. @@ -526,6 +534,7 @@ def declare_ffn( exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse), output_complete=output_complete, sum_impl=sum_impl, + row_transport=row_transport, ) @@ -620,6 +629,39 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None): and the stage after it (None when it ends the stack or a branch).""" axes, attention, local, full = _row_layouts(variant) _reject_unsupported(stage) + if stage.row_transport is not None: + if ( + stage.sum_impl is not None + or stage.output_transform is not None + or stage.output_complete + or stage.in_token_order + ): + raise NotImplementedError( + "operand transport cannot bypass another output contract" + ) + if stage.exit_rows not in (None, ExitRows.ATTENTION): + raise NotImplementedError( + "operand transport declares its return rows explicitly" + ) + try: + rows = stage.row_transport.rows[variant] + except KeyError: + raise NotImplementedError( + "operand transport does not support this batch variant" + ) from None + if rows.group is not None and not isinstance(rows.group, SumGroup): + raise TypeError("operand transport requires a declared SumGroup") + if rows.decode_group is not None and not isinstance( + rows.decode_group, SumGroup + ): + raise TypeError("operand transport requires a declared decode SumGroup") + if rows.source != arrival.produced.layout or rows.returned != rows.source: + raise NotImplementedError("compute transport must return its source rows") + declaration = StageContract( + InputContract(rows.source, read=stage.read), + OutputContract(rows.returned, group=rows.group, update=stage.update), + ) + return _Flow(declaration, arrival, arrival.residual, rows.returned) if stage.kind is StageKind.FFN: declaration, residual, returned = _resolve_ffn( variant, @@ -811,8 +853,13 @@ def _connect_line( # their row choices still determine what crosses the pipeline boundary. needs_unpadded = any( stage is not None - and stage.kind is StageKind.FFN - and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size) + and ( + stage.row_transport is not None + or ( + stage.kind is StageKind.FFN + and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size) + ) + ) for stage in (before, *line, after) ) variants = tuple( @@ -1074,8 +1121,9 @@ def _handed_off(declaration): return replace( declaration, exit_rows=ExitRows.ATTENTION, - writes_at_handoff=_ffn_on_rank_rows( - declaration.sparse, declaration.dense_tp_size + writes_at_handoff=( + declaration.row_transport is not None + or _ffn_on_rank_rows(declaration.sparse, declaration.dense_tp_size) ), ) diff --git a/python/sglang/srt/layers/layer_boundary/facts.py b/python/sglang/srt/layers/layer_boundary/facts.py index 155bdcb452..1fc8054566 100644 --- a/python/sglang/srt/layers/layer_boundary/facts.py +++ b/python/sglang/srt/layers/layer_boundary/facts.py @@ -114,6 +114,9 @@ def facts_of(declaration): sum_impl=declaration.sum_impl.facts() if declaration.sum_impl is not None else None, + row_transport=declaration.row_transport.facts() + if declaration.row_transport is not None + else None, previous=None, prepared_from=None, ) diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index 417e62b2ed..7169ce824e 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -48,9 +48,11 @@ def clear(forward_batch): """Release invocation state on completion or an exception.""" stream = forward_batch.residual_stream if stream is not None: + if stream.transport is not None: + stream.transport[1].release() if stream.pending is not None: stream.pending.release() - stream.pending = stream.residual = None + stream.pending = stream.residual = stream.transport = None forward_batch.residual_stream = None diff --git a/python/sglang/srt/layers/layer_boundary/residual/stream.py b/python/sglang/srt/layers/layer_boundary/residual/stream.py index 4b70a96c4a..b784e5506e 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/stream.py +++ b/python/sglang/srt/layers/layer_boundary/residual/stream.py @@ -101,11 +101,12 @@ class ResidualStream: selection live on the bound stage, not on tensor values. """ - __slots__ = ("residual", "pending") + __slots__ = ("residual", "pending", "transport") def __init__(self, residual=None): self.residual = residual self.pending = None + self.transport = None @classmethod def from_handoff(cls, hidden, residual, update): diff --git a/python/sglang/srt/layers/layer_boundary/row_transport.py b/python/sglang/srt/layers/layer_boundary/row_transport.py new file mode 100644 index 0000000000..4f53e8661e --- /dev/null +++ b/python/sglang/srt/layers/layer_boundary/row_transport.py @@ -0,0 +1,161 @@ +"""Transport a compute operand without moving its source residual.""" + +from contextlib import nullcontext +from typing import Callable, Mapping, Optional + +import msgspec + +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, _sum_group +from sglang.srt.layers.layer_boundary.output import ( + DeferredFinalize, + UnreducedOutput, + complete_owed, +) +from sglang.srt.layers.layer_boundary.residual import bind_update + + +class TransportRows(msgspec.Struct, frozen=True): + """The source/read, compute, and completed output rows of one adapter. + + The residual stays on source rows. ``group`` names the compute output's + outstanding sum; None means the computation (e.g. EP combine) completes it. + The boundary completes that sum and returns source rows, or publishes an + explicit completion handle on source rows. It never applies the update. + """ + + source: Layout + compute: Layout + returned: Layout + group: Optional[SumGroup] = None + decode_group: Optional[SumGroup] = None + + +class RowTransport(msgspec.Struct, frozen=True): + """Declare separate compute/residual rows and bind one invocation's moves. + + ``bind(input, batch, context, rows, overlapped_rows)`` returns an invocation + with ``input`` and ``scope()``. The boundary checks its selected rows, + prepares its output, completes owed work and runs its return move. + Facts retain row contracts only. + """ + + rows: Mapping[BatchVariant, TransportRows] + bind: Optional[Callable] = None + overlapped_rows: Optional[Mapping[BatchVariant, TransportRows]] = None + + def facts(self): + return RowTransport( + dict(self.rows), + overlapped_rows=( + dict(self.overlapped_rows) if self.overlapped_rows is not None else None + ), + ) + + +class ComputeTransport: + """One prepared operand with declared rows and output completion policy. + + ``prepare_output`` exposes the compute result's outstanding work. + The boundary completes it before ``return_output`` moves the result back; + a move that performs the sum declares ``sum_in_return`` before compute. + Residual update and consumer normalization stay with the boundary. + """ + + @staticmethod + def scope(): + return nullcontext() + + is_overlapped = False + defers_sum = False + sum_in_return = False + + def release(self): + pass + + def prepare_output(self, output): + return output + + def return_output(self, output): + return output + + +def validate_invocation(invocation, ordinary, overlapped): + expected = overlapped if invocation.is_overlapped else ordinary + if expected is None or invocation.rows != expected: + raise ValueError("compute transport does not match its declared rows and sum") + if invocation.defers_sum and ( + invocation.sum_in_return or expected.compute != expected.returned + ): + raise ValueError("a deferred transport sum must already be on its return rows") + if invocation.sum_in_return and expected.group is None: + raise ValueError("a transport move cannot complete an undeclared sum") + + +class TransportExit: + """Consume a prepared transport once, then publish the residual update.""" + + def __init__(self, stage, batch, *, update_context=None): + self.stage = stage + self.batch = batch + self.stream = batch.residual_stream + owner, self.invocation = self.stream.transport + if owner is not stage: + raise RuntimeError("compute transport belongs to another stage") + self.path = stage.plan.path_for(batch) + self.update = bind_update(self.path.output.update, batch, update_context) + self.publish = ( + TransportExit._write + if self.path.exit.writes_residual + else TransportExit._record + ) + self.scope = self.invocation.scope() + self.used = False + + def __enter__(self): + try: + self.scope.__enter__() + except BaseException: + self.stream.transport = None + raise + return self + + def __exit__(self, *exc): + try: + result = self.scope.__exit__(*exc) + finally: + self.invocation.release() + self.stream.transport = None + if exc[0] is None and not self.used: + raise RuntimeError("compute transport exited without finish") + return result + + def finish(self, output): + if self.used: + raise RuntimeError("compute transport was already consumed") + self.used = True + output = self.invocation.prepare_output(output) + rows = self.invocation.rows + groups = tuple( + group for group in (rows.group, rows.decode_group) if group is not None + ) + if isinstance(output, UnreducedOutput): + if output.complete_sum is not None or output.reduce_to_dp_local is not None: + raise ValueError("operand transport requires its declared sum group") + if not any(_sum_group(group) is output.group for group in groups): + raise ValueError("transport output owes an undeclared sum group") + elif ( + isinstance(output, DeferredFinalize) + and output.completion_group not in groups + ): + raise ValueError("transport finalize owes an undeclared sum group") + if not self.invocation.defers_sum or self.path.exit.writes_residual: + output = complete_owed(output) + output = self.invocation.return_output(output) + return self.publish(self, output) + + def _write(self, output): + return self.stream.write(self.update.update(output, self.stream.residual)) + + def _record(self, output): + return self.stream.record(output, self.update) diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 86f64ad668..94c5eff8dc 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -40,6 +40,11 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( ) from sglang.srt.layers.layer_boundary.residual.batch import stream_of from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum, ResidualStream +from sglang.srt.layers.layer_boundary.row_transport import ( + ComputeTransport, + TransportExit, + validate_invocation, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_forward @@ -96,6 +101,7 @@ class StageBoundary: (``call``: what the stage's read takes, e.g. the attention's ``quant_format``).""" entry = self.entry(forward_batch, steps) + context = call.get("read_context") if ( stream.pending is None and stream.residual is None @@ -123,6 +129,25 @@ class StageBoundary: hidden_states, forward_batch, self.plan.qkv_latent_func ) stream.write(residual) + transport = self.declaration.row_transport + if transport is not None: + if stream.transport is not None: + raise RuntimeError("cannot replace an unconsumed compute transport") + variant = self.plan.variant_for(forward_batch) + ordinary = transport.rows[variant] + overlapped = ( + transport.overlapped_rows.get(variant) + if transport.overlapped_rows is not None + else None + ) + invocation = transport.bind( + hidden_states, forward_batch, context, ordinary, overlapped + ) + if not isinstance(invocation, ComputeTransport): + raise TypeError("row transport must bind a ComputeTransport invocation") + validate_invocation(invocation, ordinary, overlapped) + stream.transport = (self, invocation) + hidden_states = invocation.input return hidden_states, stream @property @@ -175,6 +200,9 @@ class StageBoundary: actual contribution, or an FFN's or a mixer's output with its sum completed or carried on. An FFN whose boundary has fusions (which may take a MoE finalize handoff) finishes through exit() instead.""" + if self.declaration.row_transport is not None: + with self.exit(forward_batch, update_context=update_context) as output: + return output.finish(hidden_states) if self.kind is StageKind.ATTENTION and self.plan.finishes_directly: produced = self.plan.path_for(forward_batch).output completion = ( @@ -201,6 +229,13 @@ class StageBoundary: hidden_states ) + def transport_context(self, forward_batch): + """Borrow this prepared compute's operand transport until finish().""" + stream = stream_of(forward_batch) + if stream.transport is None or stream.transport[0] is not self: + raise RuntimeError("no prepared compute transport for this stage") + return stream.transport[1] + def exit(self, forward_batch, *, update_context=None): """Select and scope one FFN or single-stage mixer's output decision, for an FFN whose MoE may hand off its finalize (see finish()). @@ -215,6 +250,8 @@ class StageBoundary: when compute is skipped for an empty input. It completes or carries work using the same decision rather than selecting a second path. """ + if self.declaration.row_transport is not None: + return TransportExit(self, forward_batch, update_context=update_context) result = ( self.plan.output.ffn_exit( forward_batch, diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 30fe44c4bc..0f10d5f2ab 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -2155,7 +2155,7 @@ def enable_num_token_non_padded(): return True # DP attention + TP MoE gathers each rank's whole padded chunk into the MoE, # so the model needs the real count to zero the pad rows (see - # deepseek_v4 _run_moe_ffn_dp_sync). + # deepseek_common.moe_transport). return _dp_tp_moe_masks_pad_rows() diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py index 4b84840dbd..8ac5d7a9c5 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py @@ -11,8 +11,6 @@ from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled from sglang.srt.environ import envs from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.layers.moe.utils import should_skip_post_experts_all_reduce -from sglang.srt.models.deepseek_v2 import MoEOutput -from sglang.srt.models.deepseek_v4_mhc import AttnOutput from sglang.srt.runtime_context import ( get_exec, get_forward, @@ -572,83 +570,6 @@ def apply_moe_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor: ) -def forward_hc_pre_from_prev_fused_boundary( - layer, - positions: torch.Tensor, - hidden_states: Optional[torch.Tensor], - input_ids: torch.Tensor, - forward_batch, - input_ids_global: torch.Tensor, - prev_pre: Optional[torch.Tensor], - pending_post: Optional[Tuple[torch.Tensor, ...]], - defer_post: bool, -) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]: - """ROCm form of DeepseekV4DecoderLayer.forward_hc_pre_from_prev. - pending_post is the previous layer's unapplied FFN hc_post (x, residual, post, - comb); with defer_post this layer's is returned the same way and hidden_states is None.""" - if pending_post is None: - pending_post = (None, hidden_states, None, None) - residual, x, attn_coefficients = hc_boundary( - layer, - *pending_post, - prev_pre, - layer.hc_attn_fn, - layer.hc_attn_scale, - layer.hc_attn_base, - ) - # the boundary's reduce + sinkhorn rides in the norm launch - x, x_quant = layer._input_norm( - x, allow_aiter_quant=False, coefficients=attn_coefficients - ) - with layer.self_attn.maybe_use_decode_attn_tp(forward_batch): - mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch) - x = layer.self_attn( - x=x, - positions=positions, - forward_batch=forward_batch, - x_quant=x_quant, - defer_all_reduce=mhc is not None, - ) - if mhc is not None: - # wo_b honors the deferred reduce under attention_mhc_fusion's gates; - # fail loud rather than fall back to an unfused reduce. - assert isinstance(x, AttnOutput) - residual = apply_attention_mhc(x.partial, mhc) - x = None - residual, x, ffn_coefficients = hc_boundary( - layer, - x, - residual, - attn_coefficients.post if x is not None else None, - attn_coefficients.comb if x is not None else None, - attn_coefficients.pre, - layer.hc_ffn_fn, - layer.hc_ffn_scale, - layer.hc_ffn_base, - ) - x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients) - mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch) - x = layer._run_moe_ffn_dp_sync( - x, - forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, - return_moe_output=mhc is not None, - ) - ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors() - if mhc is not None: - # The MoE TP path honors the deferred output under moe_mhc_fusion's gates; - # fail loud rather than fall back to an unfused reduce. The fused post - # carries the reduce, so take the merged rows without one. - assert isinstance(x, MoEOutput) - # Reduction already applied post. The next boundary consumes this - # materialized residual, including when it would normally defer post. - return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None - if defer_post: - return None, ffn_pre, (x, residual, ffn_post, ffn_comb) - return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None - - def _gfx95_dense_post_attention_norm(layer, x: torch.Tensor, coefficients): """layer.post_attention_layernorm(x) hosting the pending reduce + sinkhorn where the gfx950 norm kernel is available; the module norm plus a standalone reduce + diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py index 51f47d3075..f0a871fe5b 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py @@ -21,7 +21,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context from sglang.srt.models.deepseek_common.amd import deepseek_v4_gfx95_dense as gfx95_dense from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( # noqa: F401 deepseek_v4 reaches apply_attention_mhc through this module apply_attention_mhc, - forward_hc_pre_from_prev_fused_boundary, ) from sglang.srt.runtime_context import get_exec from sglang.srt.utils import is_gfx95_supported @@ -150,7 +149,7 @@ def attention_inv_rope( def init_decoder_layer(layer, quant_config) -> None: - """The ROCm state of DeepseekV4DecoderLayer.__init__. input_layernorm also emits the + """The ROCm state of DeepseekV4DecoderLayerBase.__init__. input_layernorm also emits the pre-quantized operand of the dense projections (whether wqkv_a consumes native MXFP8 resolves on first use), and on gfx950 one launch runs hc_post, the pre-collapse and the mixing stats; the kernel only supports hc_mult 4.""" @@ -203,48 +202,3 @@ def input_norm( if coefficients is not None: coefficients.materialize() return layer.input_layernorm(hidden_states), None - - -# ---- DeepseekV4Model layer loop ---- - - -def forward_layer_fused_boundary( - model, - i: int, - state, - *, - positions: torch.Tensor, - input_ids: torch.Tensor, - forward_batch, - input_ids_global: torch.Tensor, - capture_dspark: bool, -): - """Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next - layer's boundary launch unless that layer reads the residual stream first (Engram gate, - DSpark capture) or there is none; hidden_states is None while a post is pending.""" - nxt = i + 1 - defer_post = ( - nxt < model.end_layer - and model.layers[nxt].engram is None - and not (capture_dspark and nxt in model.dspark_layers_to_capture) - ) - # The boundary speaks its own (hidden_states, prev_pre, pending_post) triple; - # the loop speaks HcState. This is the only place the two meet. - from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState - - pending = isinstance(state.streams, HcPending) - hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary( - model.layers[i], - positions=positions, - hidden_states=None if pending else state.streams, - input_ids=input_ids, - forward_batch=forward_batch, - input_ids_global=input_ids_global, - prev_pre=state.pre, - pending_post=tuple(state.streams) if pending else None, - defer_post=defer_post, - ) - return HcState( - hidden_states if pending_post is None else HcPending(*pending_post), - prev_pre, - ) diff --git a/python/sglang/srt/models/deepseek_common/moe_transport.py b/python/sglang/srt/models/deepseek_common/moe_transport.py new file mode 100644 index 0000000000..92c16d0afe --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/moe_transport.py @@ -0,0 +1,433 @@ +"""MoE operand transport and completion, preserving source residual rows.""" + +from contextlib import contextmanager, nullcontext + +import torch + +from sglang.srt.environ import envs +from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather +from sglang.srt.layers.cp.utils import is_cp_active +from sglang.srt.layers.dp_attention import ( + _tbo_event, + attn_tp_all_gather, + attn_tp_all_gather_into_tensor, + dp_gather_partial, + dp_gather_replicate, + dp_reduce_scatter_tensor, + dp_reduce_scatterv_async, + dp_scatter, + get_dp_global_num_tokens, + get_dp_tbo_comm_stream, + get_global_dp_buffer, + get_global_dp_buffer_len, + get_local_dp_buffer, + get_local_dp_buffer_len, + get_tbo_persistent_buffer, + is_dp_gatherv_active, +) +from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter +from sglang.srt.layers.layer_boundary.output import DeferredFinalize +from sglang.srt.layers.layer_boundary.row_transport import ComputeTransport +from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv +from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.srt.utils import get_bool_env_var, is_hip + +_is_hip = is_hip() +SHARED_EXPERT_LOCAL = get_bool_env_var("SGLANG_DP_SHARED_EXPERT_LOCAL") + + +def shared_expert_runs_locally(moe): + """Whether the replicated shared expert can run before the DP gather.""" + return ( + SHARED_EXPERT_LOCAL + and getattr(moe, "shared_experts", None) is not None + and getattr(moe, "_shared_expert_tp1", False) + ) + + +@contextmanager +def _every_row_routed(forward_batch, num_rows: int): + # Under CP the real rows are not a prefix, so every row must be routed. + saved = ( + forward_batch.num_token_non_padded, + forward_batch.global_num_token_non_padded_cpu, + ) + if saved[0] is not None: + forward_batch.num_token_non_padded = torch.full_like(saved[0], num_rows) + forward_batch.global_num_token_non_padded_cpu = num_rows + try: + yield + finally: + ( + forward_batch.num_token_non_padded, + forward_batch.global_num_token_non_padded_cpu, + ) = saved + + +def _mask_dp_pad_rows(local_hidden_states, forward_batch) -> None: + n_real = forward_batch.num_token_non_padded + num_rows = local_hidden_states.shape[0] + if n_real is None or num_rows == 0: + return + pad = torch.arange(num_rows, device=local_hidden_states.device) >= n_real + local_hidden_states.masked_fill_( + pad.view(-1, *([1] * (local_hidden_states.dim() - 1))), 0 + ) + + +def _attn_tp_all_gather_chunks( + local: torch.Tensor, chunk_rows: list[int] +) -> torch.Tensor: + # Tensor all_gather stays on aiter AG / pynccl under graph capture; the list + # all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture. + max_rows = max(chunk_rows) + padded = local + if local.shape[0] != max_rows: + padded = local.new_zeros((max_rows, *local.shape[1:])) + padded[: local.shape[0]].copy_(local) + gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:])) + attn_tp_all_gather_into_tensor(gathered, padded.contiguous()) + if all(rows == max_rows for rows in chunk_rows): + return gathered + return torch.cat( + [ + gathered[i * max_rows : i * max_rows + rows] + for i, rows in enumerate(chunk_rows) + ] + ) + + +class MoeRowTransport(ComputeTransport): + def __init__( + self, + moe, + hidden_states, + forward_batch, + input_ids, + input_ids_global, + *, + cp_return_contiguous=False, + use_cp=None, + moe_gather=None, + attn_scatter=None, + ): + _use_cp = is_cp_active(forward_batch) if use_cp is None else use_cp + _use_tp_moe_gather = ( + moe_gather + if moe_gather is not None + else ( + not _use_cp + and get_parallel().attn_dp_size > 1 + and get_moe_a2a_backend().is_none() + ) + ) + _use_tp_attn_a2a_scatter = ( + attn_scatter + if attn_scatter is not None + else ( + not _use_cp + and get_parallel().attn_tp_size > 1 + and not get_moe_a2a_backend().is_none() + ) + ) + # Variable-length combine sums TP-sharded expert output while returning + # each DP rank's rows. The producer leaves that sum to the transport. + _use_reduce_scatterv = ( + _use_tp_moe_gather + and is_dp_gatherv_active() + and forward_batch.dp_padding_mode is not None + and not forward_batch.dp_padding_mode.is_max_len() + ) + # Equal-chunk combine requires MAX_LEN padding and one attention rank + # per DP shard; the variable-length path takes precedence. + _use_reduce_scatter = ( + envs.SGLANG_DP_USE_REDUCE_SCATTER.get() + and _use_tp_moe_gather + and not _use_reduce_scatterv + and not should_use_dp_reduce_scatterv() + and forward_batch.dp_padding_mode is not None + and forward_batch.dp_padding_mode.is_max_len() + and get_parallel().tp_size == get_parallel().attn_dp_size + ) + mlp_reduce_scatter = ( + _use_cp + or _use_reduce_scatterv + or _use_reduce_scatter + or (_use_tp_moe_gather and should_use_dp_reduce_scatterv()) + ) + # A replicated shared expert can run on local rows before the gather; + # its output is added after returning the routed experts' local slice. + _shared_local = None + do_shared_local = _use_tp_moe_gather and shared_expert_runs_locally(moe) + if _use_cp: + moe_a2a_backend = get_moe_a2a_backend() + if moe_a2a_backend.is_none(): + hidden_states = attn_cp_interleave_gather(hidden_states) + else: + assert ( + moe_a2a_backend.is_deepep() + or moe_a2a_backend.is_megamoe() + or moe_a2a_backend.is_mori() + ), ( + "CP requires moe_a2a_backend in ('deepep', 'megamoe', 'mori'), " + f"got {moe_a2a_backend.value!r}." + ) + elif _use_tp_moe_gather: + hidden_states, local_hidden_states = ( + get_global_dp_buffer(get_parallel().tp_group), + hidden_states, + ) + if do_shared_local and local_hidden_states.shape[0] > 0: + _shared_local = moe._forward_shared_experts(local_hidden_states) + # Pad rows are not refreshed during graph replay. Stale bf16-max + # values can overflow the MoE and contaminate the combine output. + if _is_hip: + _mask_dp_pad_rows(local_hidden_states, forward_batch) + # self_attn has already reduced across attention TP, so these hidden + # states are replicated and must not be summed by a partial gather. + dp_gather_replicate(hidden_states, local_hidden_states, forward_batch) + _a2a_scatter_chunks = None + if _use_tp_attn_a2a_scatter: + s, r = get_parallel().attn_tp_size, get_parallel().attn_tp_rank + _a2a_scatter_chunks = list(hidden_states.tensor_split(s)) + hidden_states = _a2a_scatter_chunks[r].contiguous() + # DSpark next-token layers are not hash-routed and intentionally do not + # carry token IDs. Only split IDs for callers that actually provide them. + if input_ids is not None: + input_ids = input_ids.tensor_split(s)[r].contiguous() + if input_ids_global is not None: + input_ids_global = input_ids_global.tensor_split(s)[r].contiguous() + gathered_rows = ( + _every_row_routed(forward_batch, hidden_states.shape[0]) + if _use_cp and get_moe_a2a_backend().is_none() + else nullcontext() + ) + self.input = hidden_states + self.input_ids, self.input_ids_global = input_ids, input_ids_global + self.batch = forward_batch + self.cp_return_contiguous = cp_return_contiguous + self._use_cp = _use_cp + self._use_tp_moe_gather = _use_tp_moe_gather + self._use_tp_attn_a2a_scatter = _use_tp_attn_a2a_scatter + self._use_reduce_scatterv = _use_reduce_scatterv + self._use_reduce_scatter = _use_reduce_scatter + self.mlp_reduce_scatter = mlp_reduce_scatter + self._shared_local = _shared_local + self.do_shared_local = do_shared_local + self._a2a_scatter_chunks = _a2a_scatter_chunks + self.gathered_rows = gathered_rows + + def return_output(self, hidden_states): + forward_batch = self.batch + _use_cp = self._use_cp + _use_tp_moe_gather = self._use_tp_moe_gather + _use_tp_attn_a2a_scatter = self._use_tp_attn_a2a_scatter + _use_reduce_scatterv = self._use_reduce_scatterv + _use_reduce_scatter = self._use_reduce_scatter + _shared_local = self._shared_local + _a2a_scatter_chunks = self._a2a_scatter_chunks + if _use_cp and get_moe_a2a_backend().is_none(): + if self.cp_return_contiguous: + parallel = get_parallel() + hidden_states = parallel.tp_group.all_reduce(hidden_states) + hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[ + parallel.attn_cp_rank + ].contiguous() + else: + hidden_states = attn_cp_interleave_reduce_scatter(hidden_states) + elif _use_tp_moe_gather: + hidden_states, global_hidden_states = ( + get_local_dp_buffer(get_parallel().tp_group), + hidden_states, + ) + if should_use_dp_reduce_scatterv() or _use_reduce_scatterv: + get_parallel().tp_group.reduce_scatterv( + global_hidden_states, + output=hidden_states, + sizes=get_dp_global_num_tokens(), + ) + elif _use_reduce_scatter: + dp_reduce_scatter_tensor(hidden_states, global_hidden_states) + else: + dp_scatter(hidden_states, global_hidden_states, forward_batch) + if _shared_local is not None: + n = hidden_states.shape[0] + hidden_states = hidden_states + _shared_local[:n] + if _use_tp_attn_a2a_scatter: + assert _a2a_scatter_chunks is not None + if _is_hip: + hidden_states = _attn_tp_all_gather_chunks( + hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks] + ) + else: + gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks] + attn_tp_all_gather(gathered, hidden_states.contiguous()) + hidden_states = torch.cat(gathered) + return hidden_states + + @contextmanager + def scope(self): + with ( + get_forward().scoped( + mlp_reduce_scatter=self.mlp_reduce_scatter, defer_moe_finalize=False + ), + self.gathered_rows, + ): + yield + + def release(self): + self.input = self._shared_local = self._a2a_scatter_chunks = None + + +def _tbo_gather_moe_input(moe, local, batch, sub, do_shared_local): + shared_local = ( + moe._forward_shared_experts(local) + if do_shared_local and local.shape[0] > 0 + else None + ) + global_hidden = get_tbo_persistent_buffer( + ("gh", sub), + get_global_dp_buffer_len(), + local.shape[1], + local.dtype, + local.device, + ) + comm = get_dp_tbo_comm_stream() + compute = torch.cuda.current_stream() + with torch.cuda.stream(comm): + comm.wait_stream(compute) + dp_gather_partial(global_hidden, local, batch) + event = _tbo_event(("gather", sub)) + event.record(comm) + return global_hidden, event, shared_local + + +def _tbo_combine_moe_output(output, sub): + local = get_tbo_persistent_buffer( + ("lo", sub), + get_local_dp_buffer_len(), + output.shape[1], + output.dtype, + output.device, + ) + event = dp_reduce_scatterv_async( + local, output, get_dp_global_num_tokens(), event_key=("combine", sub) + ) + return local, event + + +def _tbo_wait_moe_output(local, event, shared_local): + torch.cuda.current_stream().wait_event(event) + return local + shared_local[: local.shape[0]] if shared_local is not None else local + + +class MoeFinalize(DeferredFinalize, frozen=True): + pieces: object = None + partial: object = None + + def complete(self, *, reduce=True): + from sglang.srt.layers.moe import post_experts_all_reduce + from sglang.srt.layers.moe.utils import should_add_replicated_moe_output + + with get_forward().scoped(mlp_reduce_scatter=not reduce): + value = ( + self.pieces.get_merged() if self.pieces is not None else self.partial + ) + if reduce: + value = post_experts_all_reduce(value) + if ( + self.pieces is not None + and self.pieces.shared is not None + and self.pieces.shared_is_replicated + and should_add_replicated_moe_output() + ): + value += self.pieces.shared + return value + + +class OverlappedMoeTransport(ComputeTransport): + """The TBO issue/wait protocol, owned by one microbatch. + + EP ops already return source rows. The TP path gathers normalized 2-D + input and sums/scatters expert output asynchronously; residual coefficients + stay on the source rows throughout. Neither finish path sums again. + """ + + is_overlapped = True + + def __init__(self, moe, hidden_states, batch, subbatch_index, rows): + self.rows = rows + self.moe, self.input, self.batch = moe, hidden_states, batch + self.ep = not get_moe_a2a_backend().is_none() + self.sub = subbatch_index + self.do_shared_local = not self.ep and shared_expert_runs_locally(moe) + self.shared_local = None + self.gather_event = self.combine_event = None + self.gather_keepalive = self.combine_keepalive = None + self.global_hidden = self.local_out = None + self.combined = False + + def release(self): + events = (self.gather_event, self.combine_event) + if any(event is not None for event in events): + current = torch.cuda.current_stream() + for event in events: + if event is not None: + current.wait_event(event) + self.gather_event = self.combine_event = None + self.gather_keepalive = self.combine_keepalive = None + self.global_hidden = self.local_out = self.shared_local = self.input = None + + def issue_gather(self): + if self.ep or self.gather_event is not None or self.global_hidden is not None: + raise RuntimeError("TBO gather cannot be issued twice") + local = self.input + self.global_hidden, self.gather_event, self.shared_local = ( + _tbo_gather_moe_input( + self.moe, local, self.batch, self.sub, self.do_shared_local + ) + ) + self.gather_keepalive = local + self.input = None + + def wait_gather(self): + if self.gather_event is None: + raise RuntimeError("TBO gather was not issued") + torch.cuda.current_stream().wait_event(self.gather_event) + self.gather_event = self.gather_keepalive = None + return self.global_hidden + + def issue_combine(self, output): + if self.ep or self.combine_event is not None or self.combined: + raise RuntimeError("TBO combine cannot be issued twice") + self.local_out, self.combine_event = _tbo_combine_moe_output(output, self.sub) + self.combine_keepalive = output + self.global_hidden = None + + def wait_combine(self): + if self.combine_event is None: + raise RuntimeError("TBO combine was not issued") + value = _tbo_wait_moe_output( + self.local_out, self.combine_event, self.shared_local + ) + self.combine_event = self.combine_keepalive = None + self.local_out = self.shared_local = None + self.combined = True + return value + + def prepare_output(self, output): + if not self.ep and not self.combined: + raise RuntimeError("TBO finish requires the combine wait") + return output + + +def _tbo_run_moe(layer, hidden, batch, do_shared_local): + global_ids = batch._tbo_global_input_ids + with get_forward().scoped(mlp_reduce_scatter=True): + return layer.mlp( + hidden, + batch, + input_ids=global_ids, + input_ids_global=global_ids, + skip_shared_experts=do_shared_local, + ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 4255da8467..9b1b597ec8 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -618,6 +618,13 @@ class MoEOutput(NamedTuple): ) +class MergedMoEOutput(MoEOutput): + """Merged routed rows with a replicated shared output still separate.""" + + def get_merged(self) -> torch.Tensor: + return self.routed + + class DeepseekV2MoE(nn.Module): def __init__( self, @@ -982,10 +989,13 @@ class DeepseekV2MoE(nn.Module): input_ids_global: Optional[torch.Tensor] = None, skip_shared_experts: bool = False, return_moe_output: bool = False, + leave_output_sum: bool = False, ) -> Union[torch.Tensor, MoEOutput]: """``return_moe_output`` asks for the pieces unmerged, as `MoEOutput`; only the TP paths honor it. Distinct from `get_forward().defer_moe_finalize`, - the V2/V3 layer-boundary handoff.""" + the V2/V3 layer-boundary handoff. ``leave_output_sum`` returns those + pieces with the selected finalize kernel, for + a boundary that owns reduction and the replicated shared add.""" from sglang.srt.layers.moe.mega_moe import forward_mega_moe, should_use_mega_moe if should_use_mega_moe(self, hidden_states): @@ -1007,6 +1017,20 @@ class DeepseekV2MoE(nn.Module): if not self._enable_a2a_moe: if self._can_dual_stream_graph(hidden_states): fwd = get_forward() + if leave_output_sum: + routed, shared = dsv4_flashinfer_moe_dual_stream_outputs( + hidden_states, + self.layer_id, + fwd.mlp_reduce_scatter, + self._shared_expert_tp1, + ) + return MergedMoEOutput( + routed, + shared if self._shared_expert_tp1 else None, + self.experts, + 1.0, + self._shared_expert_tp1, + ) return dsv2_flashinfer_moe_dual_stream_graph( hidden_states, self.layer_id, @@ -1027,6 +1051,7 @@ class DeepseekV2MoE(nn.Module): num_token_non_padded=num_token_non_padded, use_vision_topk=use_vision_topk, return_moe_output=return_moe_output, + leave_output_sum=leave_output_sum, ) else: return self.forward_normal( @@ -1038,6 +1063,7 @@ class DeepseekV2MoE(nn.Module): num_token_non_padded=num_token_non_padded, use_vision_topk=use_vision_topk, return_moe_output=return_moe_output, + leave_output_sum=leave_output_sum, ) else: return self.forward_deepep( @@ -1053,6 +1079,7 @@ class DeepseekV2MoE(nn.Module): num_token_non_padded: Optional[torch.Tensor] = None, use_vision_topk: bool = False, return_moe_output: bool = False, + leave_output_sum: bool = False, ) -> Union[torch.Tensor, MoEOutput]: # Note(kpham-sgl): issue order satisfies 3 constraints: # - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream; @@ -1195,7 +1222,7 @@ class DeepseekV2MoE(nn.Module): reduce=sum_post_experts_output, ) - if return_moe_output: + if return_moe_output or leave_output_sum: return MoEOutput( routed=final_hidden_states, shared=shared_output, @@ -1238,6 +1265,7 @@ class DeepseekV2MoE(nn.Module): num_token_non_padded: Optional[torch.Tensor] = None, use_vision_topk: bool = False, return_moe_output: bool = False, + leave_output_sum: bool = False, ) -> Union[torch.Tensor, MoEOutput]: if hasattr(self, "shared_experts") and use_intel_amx_backend( self.shared_experts.gate_up_proj @@ -1373,7 +1401,7 @@ class DeepseekV2MoE(nn.Module): pre_quant_input=pre_quant_input, ) - if return_moe_output: + if return_moe_output or leave_output_sum: return MoEOutput( routed=final_hidden_states, shared=shared_output, @@ -3494,4 +3522,48 @@ def dsv2_flashinfer_moe_dual_stream_graph( return moe_fusion.forward_normal_dual_stream(hidden_states) +def _fake_dsv4_dual_stream_outputs( + hidden_states, layer_id, mlp_reduce_scatter, shared_replicated +): + return torch.empty_like(hidden_states), ( + torch.empty_like(hidden_states) + if shared_replicated + else hidden_states.new_empty((0,)) + ) + + +@register_custom_op(fake_impl=_fake_dsv4_dual_stream_outputs) +def dsv4_flashinfer_moe_dual_stream_outputs( + hidden_states: torch.Tensor, + layer_id: int, + mlp_reduce_scatter: bool, + shared_replicated: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _run_dsv4_dual_stream_outputs( + hidden_states, layer_id, mlp_reduce_scatter, shared_replicated + ) + + +def _run_dsv4_dual_stream_outputs( + hidden_states: torch.Tensor, + layer_id: int, + mlp_reduce_scatter: bool, + shared_replicated: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + forward_context = get_tc_piecewise_forward_context() + assert forward_context is not None and forward_context.moe_fusions is not None + moe = forward_context.moe_fusions[layer_id] + assert moe is not None + with get_forward().scoped( + mlp_reduce_scatter=mlp_reduce_scatter, + flashinfer_trtllm_bypass=True, + lora_batch_layout=LoRABatchLayout.TP_GLOBAL, + defer_moe_finalize=False, + ): + pieces = moe.forward_normal_dual_stream(hidden_states, leave_output_sum=True) + assert pieces.shared_is_replicated == shared_replicated + merged = pieces.get_merged() + return merged, pieces.shared if shared_replicated else merged.new_empty((0,)) + + EntryClass = [DeepseekV2ForCausalLM, DeepseekV3ForCausalLM, DeepseekV32ForCausalLM] diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index c23474abe1..ef6df00ba2 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -68,7 +68,6 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( ) from sglang.srt.layers.attention.dsv4.indexer import C4Indexer from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx -from sglang.srt.layers.cp.interleave import attn_cp_interleave_gather from sglang.srt.layers.cp.utils import ( cp_gather_full_sequence_states, cp_materialize_global_token_order, @@ -76,33 +75,19 @@ from sglang.srt.layers.cp.utils import ( ) from sglang.srt.layers.deep_gemm_wrapper.configurer import DEEPGEMM_SCALE_UE8M0 from sglang.srt.layers.dp_attention import ( - _tbo_event, - attn_tp_all_gather, - attn_tp_all_gather_into_tensor, attn_tp_all_reduce, - dp_gather_partial, dp_gather_replicate, - dp_reduce_scatter_tensor, - dp_reduce_scatterv_async, - dp_scatter, - get_dp_global_num_tokens, - get_dp_tbo_comm_stream, - get_global_dp_buffer, get_global_dp_buffer_len, - get_local_dp_buffer, - get_local_dp_buffer_len, - get_tbo_persistent_buffer, is_allocation_symmetric, is_dp_attention_enabled, - is_dp_gatherv_active, ) from sglang.srt.layers.engram import Engram, EngramHasher, EngramLayout -from sglang.srt.layers.layer_boundary import get_attn_tp_context -from sglang.srt.layers.layer_boundary.ops import attn_cp_interleave_reduce_scatter +from sglang.srt.layers.layer_boundary import append_stages, get_attn_tp_context +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor -from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scatterv +from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.utils import ( is_shared_experts_fusion_disabled, @@ -114,9 +99,6 @@ from sglang.srt.layers.quantization.fp8_utils import ( Mxfp8DenseGemmBackend, view_aiter_fused_rms_transposed_fp8_scale, ) -from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( - can_fuse_all_reduce, -) from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput from sglang.srt.layers.rotary_embedding import get_rope_wrapper from sglang.srt.layers.utils import PPMissingLayer, get_layer_id @@ -158,10 +140,10 @@ from sglang.srt.model_loader.weight_utils import ( RUNAI_STREAMER_TENSOR_ATTR, default_weight_loader, ) +from sglang.srt.models import deepseek_v4_boundary as hc_boundary from sglang.srt.models import deepseek_v4_mhc as mhc from sglang.srt.models.dbrx import ReplicatedLinear from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( - apply_mhc_post_pre_boundary, is_cross_layer_mhc_fusion_enabled, ) from sglang.srt.models.deepseek_common.decoder_swa_bounded_replay import ( @@ -169,6 +151,10 @@ from sglang.srt.models.deepseek_common.decoder_swa_bounded_replay import ( exit_late_layer_dp, late_layer_dp_counts, ) +from sglang.srt.models.deepseek_common.moe_transport import ( + OverlappedMoeTransport, + _tbo_run_moe, +) from sglang.srt.models.deepseek_common.utils import ( _use_aiter_bpreshuffle_gfx95, is_wint4afp8_or_wint4a16_config, @@ -181,6 +167,11 @@ from sglang.srt.models.deepseek_v2 import ( _is_npu, _is_xpu, ) +from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE +from sglang.srt.models.deepseek_v4_platform_stage import ( + NonCudaDecoderMixin, + forward_layer_fused_boundary, +) from sglang.srt.models.deepseek_v41_vit import Aligner, ViT from sglang.srt.models.utils import WeightsMapper from sglang.srt.multimodal.deepseek_v41_image_processing import ( @@ -284,7 +275,6 @@ def wo_a_fp8_gemm_enabled(quant_config: Optional[QuantizationConfig]) -> bool: _NPU_BF16_WO_A_GEMM = _is_npu and envs.SGLANG_OPT_NPU_BF16_WO_A_GEMM.get() -_MHC_POST_MULT_VALUE = 2.0 _HC_PRENORM_DEEPGEMM_MIN_TOKENS = 1024 DEEPSEEK_V4_STACKED_PARAMS_MAPPING: List[Tuple[str, str, int]] = [ @@ -361,7 +351,7 @@ def _flashinfer_hc_pre( rms_eps=rms_eps, mhc_pre_eps=hc_eps, mhc_sinkhorn_eps=hc_eps, - mhc_post_mult_value=_MHC_POST_MULT_VALUE, + mhc_post_mult_value=MHC_POST_MULT_VALUE, sinkhorn_repeat=sinkhorn_iters, num_splits=n_splits, ) @@ -369,14 +359,11 @@ def _flashinfer_hc_pre( _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip -# PoC: compute the (replicated TP1) shared expert on LOCAL hidden before the dp -# gather instead of on the gathered global buffer. Requires -# SGLANG_SHARED_EXPERT_TP1=1 (replicated shared expert). Default OFF. -_SHARED_EXPERT_LOCAL = get_bool_env_var("SGLANG_DP_SHARED_EXPERT_LOCAL") _is_gfx95_supported = is_gfx95_supported() _is_gfx942_supported = is_gfx942_supported() _is_gfx1250_supported = is_gfx1250_supported() + if _use_aiter: if _is_gfx95_supported or _is_gfx1250_supported: from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant @@ -2655,9 +2642,14 @@ class MQALayer(MqaAttentionBase): defer_all_reduce = defer_all_reduce and ( self.attn_tp_size > 1 - and self.attn_tp_size == get_parallel().tp_size - and self.wo_b.reduce_results - and not self.wo_b.use_decode_attn_tp + and ( + _is_cuda + or ( + self.attn_tp_size == get_parallel().tp_size + and self.wo_b.reduce_results + and not self.wo_b.use_decode_attn_tp + ) + ) and not should_skip_mlp_all_reduce() ) o, _ = self.wo_b( @@ -2683,51 +2675,11 @@ class MQALayer(MqaAttentionBase): positions=state.positions, forward_batch=state.forward_batch, x_quant=state.pop("attn_x_quant"), + defer_all_reduce=_is_cuda, ) -@contextmanager -def _every_row_routed(forward_batch: ForwardBatch, num_rows: int): - # Under CP the real rows are not a prefix, so every row must be routed. - saved = ( - forward_batch.num_token_non_padded, - forward_batch.global_num_token_non_padded_cpu, - ) - if saved[0] is not None: - forward_batch.num_token_non_padded = torch.full_like(saved[0], num_rows) - forward_batch.global_num_token_non_padded_cpu = num_rows - try: - yield - finally: - ( - forward_batch.num_token_non_padded, - forward_batch.global_num_token_non_padded_cpu, - ) = saved - - -def _attn_tp_all_gather_chunks( - local: torch.Tensor, chunk_rows: List[int] -) -> torch.Tensor: - # Tensor all_gather stays on aiter AG / pynccl under graph capture; the list - # all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture. - max_rows = max(chunk_rows) - padded = local - if local.shape[0] != max_rows: - padded = local.new_zeros((max_rows, *local.shape[1:])) - padded[: local.shape[0]].copy_(local) - gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:])) - attn_tp_all_gather_into_tensor(gathered, padded.contiguous()) - if all(rows == max_rows for rows in chunk_rows): - return gathered - return torch.cat( - [ - gathered[i * max_rows : i * max_rows + rows] - for i, rows in enumerate(chunk_rows) - ] - ) - - -class DeepseekV4DecoderLayer(nn.Module): +class DeepseekV4DecoderLayerBase(nn.Module): def __init__( self, config: DeepSeekV4Config, @@ -2741,12 +2693,21 @@ class DeepseekV4DecoderLayer(nn.Module): engram_layout: Optional[EngramLayout] = None, hc_stats_stream: Optional[torch.cuda.Stream] = None, moe_routed_quant_stream: Optional[torch.cuda.Stream] = None, + *, + boundary_pre_from_prev: Optional[bool] = None, + boundary_mhc_fusions: bool = True, ) -> None: super().__init__() self.hc_stats_stream = hc_stats_stream self.config = config self.hidden_size = config.hidden_size self.layer_id = layer_id + self.boundary_pre_from_prev = ( + config.hc_pre_from_prev_sublayer and not is_nextn + if boundary_pre_from_prev is None + else boundary_pre_from_prev + ) + self.boundary_mhc_fusions = boundary_mhc_fusions self.self_attn = self._build_self_attn( config=config, layer_id=layer_id, @@ -2822,9 +2783,6 @@ class DeepseekV4DecoderLayer(nn.Module): self._hc_attn_tf32_parts = self._hc_ffn_tf32_parts = None self._hc_attn_bf16_parts = self._hc_ffn_bf16_parts = None self._init_hyper_connections() - self._next_layer: Optional[DeepseekV4DecoderLayer] = None - self.local_boundary: Optional[mhc.HcNextBoundary] = None - self.next_boundary: Optional[mhc.HcNextBoundary] = None self.engram = None if engram_layout is not None and layer_id in engram_layout.layer_ids: self.engram = Engram( @@ -2856,14 +2814,6 @@ class DeepseekV4DecoderLayer(nn.Module): compress_ratio_override=compress_ratio_override, ) - def bind_next(self, next_layer: Optional[nn.Module]) -> None: - """Save the next layer for mhc. Must be called before `_init_boundaries`""" - if not self.hc_pre_from_prev_sublayer: - return - self._next_layer = None - if isinstance(next_layer, DeepseekV4DecoderLayer) and next_layer.engram is None: - self._next_layer = next_layer - def _init_hyper_connections(self) -> None: """(Re)bind the two hyper-connections to the current mixing weights.""" if not self.hc_pre_from_prev_sublayer: @@ -2887,25 +2837,6 @@ class DeepseekV4DecoderLayer(nn.Module): self._hc_ffn_bf16_parts, ) - def _init_boundaries(self) -> None: - """Resolved on first forward: `accepts_mxfp8_swizzled_input` is not settled - until `process_weights_after_loading`.""" - if self.local_boundary is not None: - return - # ATTN[i] -> MOE[i] - self.local_boundary = mhc.make_boundary( - self.post_attention_layernorm, - accepts_mxfp8=False, - hc=self.ffn_hc, - ) - # MOE[i] -> ATTN[i+1] - if self._next_layer is not None: - self.next_boundary = mhc.make_boundary( - self._next_layer.input_layernorm, - accepts_mxfp8=self._next_layer.self_attn.accepts_mxfp8_swizzled_input(), - hc=self._next_layer.attn_hc, - ) - def refresh_mhc_norm_weight_cache(self): # Cache bf16 norm weights so the fused path does not allocate/cast per forward. self._input_layernorm_weight_bf16 = ( @@ -2962,8 +2893,6 @@ class DeepseekV4DecoderLayer(nn.Module): # re-run this, so drop them and re-resolve on the next forward. self.__dict__.pop("_can_fuse_attn_mhc", None) self.__dict__.pop("_can_fuse_ffn_mhc", None) - self.local_boundary = None - self.next_boundary = None def _input_norm( self, @@ -3042,7 +2971,7 @@ class DeepseekV4DecoderLayer(nn.Module): rms_eps=self.rms_norm_eps, hc_pre_eps=self.hc_eps, hc_sinkhorn_eps=self.hc_eps, - hc_post_mult_value=_MHC_POST_MULT_VALUE, + hc_post_mult_value=MHC_POST_MULT_VALUE, sinkhorn_repeat=self.hc_sinkhorn_iters, **norm_kwargs, ) @@ -3076,7 +3005,7 @@ class DeepseekV4DecoderLayer(nn.Module): rms_eps=self.rms_norm_eps, hc_pre_eps=self.hc_eps, hc_sinkhorn_eps=self.hc_eps, - hc_post_mult_value=_MHC_POST_MULT_VALUE, + hc_post_mult_value=MHC_POST_MULT_VALUE, sinkhorn_repeat=self.hc_sinkhorn_iters, **norm_kwargs, ) @@ -3093,7 +3022,7 @@ class DeepseekV4DecoderLayer(nn.Module): rms_eps=self.rms_norm_eps, hc_pre_eps=self.hc_eps, hc_sinkhorn_eps=self.hc_eps, - hc_post_mult_value=_MHC_POST_MULT_VALUE, + hc_post_mult_value=MHC_POST_MULT_VALUE, sinkhorn_repeat=self.hc_sinkhorn_iters, ) return y, post.squeeze(-1), comb, False @@ -3153,187 +3082,6 @@ class DeepseekV4DecoderLayer(nn.Module): ) -> torch.Tensor: return mhc.post(self.hc_cfg, x, residual, post, comb) - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - input_ids: torch.Tensor, - forward_batch: ForwardBatch, - input_ids_global: torch.Tensor, - prev_residual: Optional[torch.Tensor] = None, - prev_post: Optional[torch.Tensor] = None, - prev_comb: Optional[torch.Tensor] = None, - ) -> Tuple[ - torch.Tensor, - Optional[torch.Tensor], - Optional[torch.Tensor], - Optional[torch.Tensor], - ]: - use_fused = self.use_fused_mhc_post_pre - - if prev_residual is not None and use_fused: - # Dispatch cascade: aiter HIP (gfx95) -> Triton (gfx95 small-batch - # <=64 tokens, or gfx1250 all sizes) -> TileLang -> None. - input_norm_weight = ( - self._input_layernorm_weight_bf16 - if self._input_layernorm_weight_bf16 is not None - else self.input_layernorm.weight.data - ) - fused = apply_mhc_post_pre_boundary( - hidden_states, - prev_residual, - prev_post, - prev_comb, - self.hc_attn_fn, - self.hc_attn_scale, - self.hc_attn_base, - self.hc_mult, - self.rms_norm_eps, - self.hc_eps, - _MHC_POST_MULT_VALUE, - self.hc_sinkhorn_iters, - input_norm_weight, - self.input_layernorm.variance_epsilon, - fn_transpose=False, - ) - if fused is not None: - residual, hidden_states, post, comb, norm_fused = fused - if not norm_fused: - # Triton fused post+pre (gfx95 small-batch or gfx1250) returns - # norm_fused=False — the input layernorm is NOT folded. - # gfx95 takes the fp8-quant path; gfx1250 takes plain layernorm. - hidden_states, x_quant = self._input_norm( - hidden_states, allow_aiter_quant=_is_gfx95_supported - ) - else: - x_quant = None - else: - hidden_states = self.hc_post( - hidden_states, prev_residual, prev_post, prev_comb - ) - residual = hidden_states - hidden_states, post, comb, norm_fused = self.hc_pre( - hidden_states, - self.hc_attn_fn, - self.hc_attn_scale, - self.hc_attn_base, - norm=self.input_layernorm, - forward_batch=forward_batch, - ) - if not norm_fused: - # gfx1250 takes plain layernorm here - hidden_states, x_quant = self._input_norm( - hidden_states, allow_aiter_quant=_is_gfx95_supported - ) - else: - x_quant = None - else: - residual = hidden_states - hidden_states, post, comb, norm_fused = self.hc_pre( - hidden_states, - self.hc_attn_fn, - self.hc_attn_scale, - self.hc_attn_base, - norm=self.input_layernorm, - forward_batch=forward_batch, - ) - if not norm_fused: - # gfx1250 takes plain layernorm here - hidden_states, x_quant = self._input_norm( - hidden_states, allow_aiter_quant=_is_gfx95_supported - ) - else: - x_quant = None - - with self.self_attn.maybe_use_decode_attn_tp(forward_batch): - hidden_states = self.self_attn( - x=hidden_states, - positions=positions, - forward_batch=forward_batch, - x_quant=x_quant, - ) - - if use_fused: - post_attn_norm_weight = ( - self._post_attention_layernorm_weight_bf16 - if self._post_attention_layernorm_weight_bf16 is not None - else self.post_attention_layernorm.weight.data - ) - fused = apply_mhc_post_pre_boundary( - hidden_states, - residual, - post, - comb, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - self.hc_mult, - self.rms_norm_eps, - self.hc_eps, - _MHC_POST_MULT_VALUE, - self.hc_sinkhorn_iters, - post_attn_norm_weight, - self.post_attention_layernorm.variance_epsilon, - fn_transpose=True, - ) - if fused is not None: - residual, hidden_states, post, comb, norm_fused = fused - if not norm_fused: - hidden_states = self.post_attention_layernorm(hidden_states) - else: - hidden_states = self.hc_post(hidden_states, residual, post, comb) - residual = hidden_states - hidden_states, post, comb, norm_fused = self.hc_pre( - hidden_states, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - norm=self.post_attention_layernorm, - forward_batch=forward_batch, - ) - if not norm_fused: - hidden_states = self.post_attention_layernorm(hidden_states) - else: - hidden_states = self.hc_post(hidden_states, residual, post, comb) - residual = hidden_states - hidden_states, post, comb, norm_fused = self.hc_pre( - hidden_states, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - norm=self.post_attention_layernorm, - forward_batch=forward_batch, - ) - if not norm_fused: - hidden_states = self.post_attention_layernorm(hidden_states) - - hidden_states = self._run_moe_ffn_dp_sync( - hidden_states, - forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, - ) - - if not use_fused: - hidden_states = self.hc_post(hidden_states, residual, post, comb) - return hidden_states, None, None, None - - # Return the deferred FFN hc_post state; the next layer consumes it with - # cross-layer fusion, and the final layer is completed in DeepseekV4Model. - return hidden_states, residual, post, comb - - @functools.cached_property - def _can_fuse_attn_mhc(self) -> bool: - """The static half of the attention fuse gate (dsv4.1 only); load-time - facts, dropped by `refresh_mhc_norm_weight_cache`. Row capacity is the - dynamic half, checked per forward via `can_fuse_all_reduce`.""" - return ( - get_parallel().tp_size == self.self_attn.attn_tp_size == 4 - and self.self_attn.wo_b.reduce_results - and self.local_boundary.norm_fusable - and mhc.can_fuse_post(self.hc_cfg) - ) - @functools.cached_property def _can_fuse_ffn_mhc(self) -> bool: """The MoE counterpart of `_can_fuse_attn_mhc`.""" @@ -3345,541 +3093,6 @@ class DeepseekV4DecoderLayer(nn.Module): and mhc.can_fuse_post(self.hc_cfg) ) - def forward_hc_pre_from_prev( - self, - positions: torch.Tensor, - state: mhc.HcState, - input_ids: torch.Tensor, - forward_batch: ForwardBatch, - input_ids_global: torch.Tensor, - ) -> mhc.HcState: - """The layer's two hyper-connections, each collapsing with the previous one's - pre-mix.""" - self._init_boundaries() - stats_stream = None - if mhc.use_stats_stream(self.hc_cfg, forward_batch, state.residual): - stats_stream = self.hc_stats_stream - - def run_attn_hc(state: mhc.HcState) -> mhc.HcState: - assert self.attn_hc is not None - residual = state.residual - quantized = [] if self.self_attn.accepts_mxfp8_swizzled_input() else None - mhc.fork_stats_stream(stats_stream) - x = mhc.combine(self.attn_hc, state, quantized) - precomputed = state.stats - state.release() - del state - world_size = self.self_attn.attn_tp_size - fuse_all_reduce_mhc = self._can_fuse_attn_mhc and can_fuse_all_reduce( - x.shape[0], self.hc_cfg.hidden - ) - with self.self_attn.maybe_use_decode_attn_tp(forward_batch): - y = self.self_attn( - x=x, - positions=positions, - forward_batch=forward_batch, - x_quant=quantized[0] if quantized else None, - defer_all_reduce=fuse_all_reduce_mhc, - ) - del x - return mhc.run_attn_post( - self.attn_hc, - y, - residual, - stats_stream=stats_stream, - next=self.local_boundary, - world_size=world_size, - precomputed=precomputed, - ) - - def run_ffn_hc(state: mhc.HcState) -> mhc.HcState: - assert self.ffn_hc is not None - residual = state.residual - mhc.fork_stats_stream(stats_stream) - x = mhc.combine(self.ffn_hc, state) - precomputed = state.stats - state.release() - del state - fuse_all_reduce_mhc = self._can_fuse_ffn_mhc and can_fuse_all_reduce( - x.shape[0], self.hc_cfg.hidden - ) - y = self._run_moe_ffn_dp_sync( - x, - forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, - return_moe_output=fuse_all_reduce_mhc, - ) - del x - return mhc.run_moe_post( - self.ffn_hc, - y, - residual, - stats_stream=stats_stream, - next=self.next_boundary, - world_size=self.mlp.tp_size, - precomputed=precomputed, - ) - - return run_ffn_hc(run_attn_hc(state)) - - def _run_moe_ffn_dp_sync( - self, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - *, - input_ids: Optional[torch.Tensor], - input_ids_global: Optional[torch.Tensor], - return_moe_output: bool = False, - ) -> Union[torch.Tensor, deepseek_v2.MoEOutput]: - _use_cp = is_cp_active(forward_batch) - _use_tp_moe_gather = ( - not _use_cp - and get_parallel().attn_dp_size > 1 - and get_moe_a2a_backend().is_none() - ) - _use_tp_attn_a2a_scatter = ( - not _use_cp - and get_parallel().attn_tp_size > 1 - and not get_moe_a2a_backend().is_none() - ) - # symmetric gather+scatter for the no-EP TP-MoE dp-attn path: - # all_gatherv gather (in self.mlp's dp_gather) + reduce_scatterv combine. - # The experts ARE TP-sharded by intermediate (moe_tp_size==tp_size), so - # the post-experts reduce is a SUM. reduce_scatterv does that sum+scatter - # in ONE op, REPLACING the MoE-internal post-experts all_reduce — so we - # MUST tell the MoE to skip it (mlp_reduce_scatter=True) or it - # double-reduces. Env-gated via SGLANG_DP_USE_GATHERV, default OFF. - _use_reduce_scatterv = ( - _use_tp_moe_gather - and is_dp_gatherv_active() - and forward_batch.dp_padding_mode is not None - and not forward_batch.dp_padding_mode.is_max_len() - ) - # SGLANG_DP_USE_REDUCE_SCATTER: in the MAX_LEN decode path (equal per-rank - # padding, gatherv inactive, no EP), replace the MoE-internal post-experts - # all_reduce + dp_scatter with an equal-chunk reduce_scatter. On ROCm this - # uses the aiter custom kernel (so BOTH gather and combine are aiter custom), - # elsewhere RCCL reduce_scatter; either way it cuts combine traffic ~2x vs - # all_reduce. tp_size==attn_dp_size required so the global buffer splits - # evenly into per-rank chunks. - _use_reduce_scatter = ( - envs.SGLANG_DP_USE_REDUCE_SCATTER.get() - and _use_tp_moe_gather - and not _use_reduce_scatterv - and not should_use_dp_reduce_scatterv() - and forward_batch.dp_padding_mode is not None - and forward_batch.dp_padding_mode.is_max_len() - and get_parallel().tp_size == get_parallel().attn_dp_size - ) - mlp_reduce_scatter = ( - _use_cp - or _use_reduce_scatterv - or _use_reduce_scatter - or (_use_tp_moe_gather and should_use_dp_reduce_scatterv()) - ) - # PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): compute the replicated shared expert - # on LOCAL hidden before the gather and add it back after the combine - # (reduce_scatterv OR dp_scatter), instead of on the gathered global buffer. - # Applies to BOTH prefill and decode: the shared expert is a per-token MLP, - # so computing it on this rank's local tokens (M_local rows) is identical to - # computing it on the gathered global buffer (M_global rows) and keeping the - # local slice -- but costs 1/attn_dp_size the rows. With a replicated (TP1) shared - # expert this cancels the TP1 "full-dim" cost in decode (M_local * dim == - # M_global * dim/tp), so decode no longer pays the ~attn_dp_size x penalty. - _shared_local = None - _do_shared_local = ( - _SHARED_EXPERT_LOCAL - and _use_tp_moe_gather - and getattr(self.mlp, "shared_experts", None) is not None - and getattr(self.mlp, "_shared_expert_tp1", False) - ) - if _use_cp: - moe_a2a_backend = get_moe_a2a_backend() - if moe_a2a_backend.is_none(): - hidden_states = attn_cp_interleave_gather(hidden_states) - else: - assert ( - moe_a2a_backend.is_deepep() - or moe_a2a_backend.is_megamoe() - or moe_a2a_backend.is_mori() - ), ( - "CP requires moe_a2a_backend in ('deepep', 'megamoe', 'mori'), " - f"got {moe_a2a_backend.value!r}." - ) - elif _use_tp_moe_gather: - hidden_states, local_hidden_states = ( - get_global_dp_buffer(get_parallel().tp_group), - hidden_states, - ) - if _do_shared_local and local_hidden_states.shape[0] > 0: - _shared_local = self.mlp._forward_shared_experts(local_hidden_states) - # The gather ships the whole padded chunk and the MoE runs on it, but - # graph replay never refreshes pad rows, so they hold stale pool - # memory (bf16-max scale) that overflows the MoE and the combine. - if _is_hip: - _mask_dp_pad_rows(local_hidden_states, forward_batch) - # self_attn has already reduced across attention TP, so these hidden - # states are replicated and must not be summed by a partial gather. - dp_gather_replicate(hidden_states, local_hidden_states, forward_batch) - _a2a_scatter_chunks: Optional[List[torch.Tensor]] = None - if _use_tp_attn_a2a_scatter: - s, r = get_parallel().attn_tp_size, get_parallel().attn_tp_rank - _a2a_scatter_chunks = list(hidden_states.tensor_split(s)) - hidden_states = _a2a_scatter_chunks[r].contiguous() - # DSpark next-token layers are not hash-routed and intentionally do not - # carry token IDs. Only split IDs for callers that actually provide them. - if input_ids is not None: - input_ids = input_ids.tensor_split(s)[r].contiguous() - if input_ids_global is not None: - input_ids_global = input_ids_global.tensor_split(s)[r].contiguous() - # Skip the MoE-internal post-experts all_reduce when we will do the - # reduce via reduce_scatterv/reduce_scatter at the combine below - # (else double-reduce). - gathered_rows = ( - _every_row_routed(forward_batch, hidden_states.shape[0]) - if _use_cp and get_moe_a2a_backend().is_none() - else nullcontext() - ) - with ( - get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter), - gathered_rows, - ): - return_moe_output = ( - return_moe_output - and not _use_cp - and not _use_tp_moe_gather - and not _use_tp_attn_a2a_scatter - ) - hidden_states = self.mlp( - hidden_states, - forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, - skip_shared_experts=_do_shared_local, - return_moe_output=return_moe_output, - ) - - if _use_cp and get_moe_a2a_backend().is_none(): - if self.config.model_type == "deepseek_v41": - parallel = get_parallel() - hidden_states = parallel.tp_group.all_reduce(hidden_states) - hidden_states = hidden_states.tensor_split(parallel.attn_cp_size)[ - parallel.attn_cp_rank - ].contiguous() - else: - hidden_states = attn_cp_interleave_reduce_scatter(hidden_states) - elif _use_tp_moe_gather: - hidden_states, global_hidden_states = ( - get_local_dp_buffer(get_parallel().tp_group), - hidden_states, - ) - if should_use_dp_reduce_scatterv() or _use_reduce_scatterv: - # SUM the TP-sharded per-rank partial expert outputs AND scatter - # each rank its own token slice, in one op. Correct because the - # MoE-internal all_reduce was skipped (mlp_reduce_scatter above). - # This is the symmetric inverse of the all_gatherv gather. - get_parallel().tp_group.reduce_scatterv( - global_hidden_states, - output=hidden_states, - sizes=get_dp_global_num_tokens(), - ) - elif _use_reduce_scatter: - # Equal-chunk reduce_scatter: SUM the TP-sharded per-rank partial - # expert outputs AND scatter each rank its own (MAX_LEN-padded) - # token chunk in one op (symmetric inverse of the MAX_LEN - # all_gather). Correct because the MoE-internal all_reduce was - # skipped (mlp_reduce_scatter above). dp_reduce_scatter_tensor - # routes to the equal-chunk reduce_scatter_tensor here (its - # variable-length reduce_scatterv branch is gated by - # is_dp_gatherv_active(), which is False under MAX_LEN), which in - # turn uses the aiter custom kernel when it fits (else RCCL). - dp_reduce_scatter_tensor(hidden_states, global_hidden_states) - else: - dp_scatter(hidden_states, global_hidden_states, forward_batch) - # PoC: add the locally-computed shared-expert output to this rank's - # reduce-scattered / dp-scattered local slice (skipped inside self.mlp - # above). Covers both prefill (gatherv) and decode (dp_scatter). - if _shared_local is not None: - n = hidden_states.shape[0] - hidden_states = hidden_states + _shared_local[:n] - if _use_tp_attn_a2a_scatter: - assert _a2a_scatter_chunks is not None - if _is_hip: - hidden_states = _attn_tp_all_gather_chunks( - hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks] - ) - else: - gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks] - attn_tp_all_gather(gathered, hidden_states.contiguous()) - hidden_states = torch.cat(gathered) - return hidden_states - - # ------------------------------------------------------------------ - # TBO op decomposition (prefill two-batch-overlap, EP / mori path) - # - # These mirror the NON-fused branch of ``forward`` (cross-layer mHC - # fusion is disabled under TBO, so every layer is self-contained), split - # into ops so the operations engine can overlap one ubatch's MoE a2a - # dispatch/combine with the other ubatch's attention + expert GEMM. - # The MoE ops themselves (op_gate / op_select_experts / op_dispatch_a/b / - # op_experts / op_combine_a/b / op_shared_experts / op_output) are reused - # as-is from ``self.mlp`` (DeepseekV2MoE) — they decompose ``forward_deepep``. - # ------------------------------------------------------------------ - def op_mhc_prepare_attn( - self, - state, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - residual: Optional[torch.Tensor] = None, - tbo_subbatch_index: Optional[int] = None, - **kwargs, - ): - # Non-fused attention-side mHC pre + input layernorm. - attn_residual = hidden_states - hidden_states, post, comb, norm_fused = self.hc_pre( - hidden_states, - self.hc_attn_fn, - self.hc_attn_scale, - self.hc_attn_base, - norm=self.input_layernorm, - forward_batch=forward_batch, - ) - if not norm_fused: - hidden_states, x_quant = self._input_norm(hidden_states) - else: - x_quant = None - - state.attn_residual = attn_residual - state.attn_post = post - state.attn_comb = comb - state.hidden_states_after_input_norm = hidden_states - state.attn_x_quant = x_quant - # mori's op_output slices final_hidden_states[:num_tokens]. - if get_moe_a2a_backend().is_mori(): - state.num_tokens = attn_residual.shape[0] - state.update( - dict( - forward_batch=forward_batch, - positions=positions, - tbo_subbatch_index=tbo_subbatch_index, - ) - ) - - def op_mhc_post_attn_pre_mlp(self, state): - # Close the attention mHC (hc_post), then open the FFN-side mHC pre + - # post-attention layernorm. Produces the 2D MoE input. - # - # Pop each boundary tensor from the state EXACTLY ONCE, up front, and - # reuse the locals for both the fused attempt and the non-fused - # fallback. apply_mhc_post_pre_boundary() returns None when it declines - # to fuse -- most importantly for the 0-token DP two-batch-overlap idle - # ubatch -- in which case control must fall through to the unfused - # hc_post. Popping in the fused call's arguments and again in the - # fallback would double-pop -> KeyError: 'hidden_states_after_attn' on - # every idle DP rank. use_fused_mhc_post_pre is on whenever the aiter - # gfx95 mHC path is available (is_cross_layer_mhc_fusion_enabled), so - # this fallback is reached under DP regardless of the TileLang env. - hidden_states_after_attn = state.pop("hidden_states_after_attn") - attn_residual = state.pop("attn_residual") - attn_post = state.pop("attn_post") - attn_comb = state.pop("attn_comb") - - if self.use_fused_mhc_post_pre: - post_attn_norm_weight = ( - self._post_attention_layernorm_weight_bf16 - if self._post_attention_layernorm_weight_bf16 is not None - else self.post_attention_layernorm.weight.data - ) - fused = apply_mhc_post_pre_boundary( - hidden_states_after_attn, - attn_residual, - attn_post, - attn_comb, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - self.hc_mult, - self.rms_norm_eps, - self.hc_eps, - _MHC_POST_MULT_VALUE, - self.hc_sinkhorn_iters, - post_attn_norm_weight, - self.post_attention_layernorm.variance_epsilon, - fn_transpose=True, - ) - if fused is not None: - ffn_residual, hidden_states, post, comb, norm_fused = fused - if not norm_fused: - # The Triton fused post+pre skips the post-attention - # layernorm (norm_fused=False); apply it before the MoE, - # matching the unfused hc_pre path below. - hidden_states = self.post_attention_layernorm(hidden_states) - state.ffn_residual = ffn_residual - state.ffn_post = post - state.ffn_comb = comb - state.hidden_states_mlp_input = hidden_states - return - - hidden_states = self.hc_post( - hidden_states_after_attn, - attn_residual, - attn_post, - attn_comb, - ) - ffn_residual = hidden_states - hidden_states, post, comb, norm_fused = self.hc_pre( - hidden_states, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - norm=self.post_attention_layernorm, - forward_batch=state.forward_batch, - ) - if not norm_fused: - hidden_states = self.post_attention_layernorm(hidden_states) - state.ffn_residual = ffn_residual - state.ffn_post = post - state.ffn_comb = comb - state.hidden_states_mlp_input = hidden_states - - def op_mhc_postprocess(self, state): - # Close the FFN mHC (hc_post) and emit the next layer's input dict. - hidden_states = self.hc_post( - state.pop("hidden_states_mlp_output"), - state.pop("ffn_residual"), - state.pop("ffn_post"), - state.pop("ffn_comb"), - ) - output = dict( - positions=state.positions, - hidden_states=hidden_states, - # DSV4 non-fused layers carry no residual across layers; the key is - # required by the next layer's op_mhc_prepare_attn (ignored). - residual=None, - forward_batch=state.forward_batch, - tbo_subbatch_index=state.tbo_subbatch_index, - ) - state.clear( - expect_keys={ - "positions", - "forward_batch", - "tbo_subbatch_index", - } - ) - return output - - # ------------------------------------------------------------------ - # Non-EP (DP TP-MoE) TBO ops. Overlap the DP all_gatherv (pre-MoE gather) - # + reduce_scatterv (post-MoE combine) with the OTHER ubatch's attn+MoE - # compute. Used when moe_a2a_backend is "none" (DP-attention, TP-MoE) — - # the path ATOM uses for DSV4 (+~7.7% prefill). Replaces the EP mori - # op_dispatch/op_combine. op_mhc_* and op_attn are reused (local hidden). - # ------------------------------------------------------------------ - def op_gather_a(self, state): - # Launch the all_gatherv (local hidden -> global buffer) + the input_ids - # replicate-gather on the shared comm stream; record an event. - fb = state.forward_batch - local = state.pop("hidden_states_mlp_input") # LOCAL [M_local, hidden] - # Shared-expert-local: compute on LOCAL hidden before the gather; added - # back after the combine (same as the non-fused forward). Skipped in the - # global MoE via skip_shared_experts. - do_shared_local = ( - _SHARED_EXPERT_LOCAL - and getattr(self.mlp, "shared_experts", None) is not None - and getattr(self.mlp, "_shared_expert_tp1", False) - ) - state.do_shared_local = do_shared_local - state.shared_local = ( - self.mlp._forward_shared_experts(local) - if (do_shared_local and local.shape[0] > 0) - else None - ) - # Persistent grow-only scratch (keyed per ubatch) instead of a fresh - # torch.empty each layer -> stops the allocator's `reserved` from - # ballooning at large prefill chunks. input_ids_global is gathered ONCE - # per ubatch in _forward_layers_tbo (cached on fb), not here. - sub = state.tbo_subbatch_index - global_rows = get_global_dp_buffer_len() - global_hidden = get_tbo_persistent_buffer( - ("gh", sub), global_rows, local.shape[1], local.dtype, local.device - ) - comm = get_dp_tbo_comm_stream() - compute = torch.cuda.current_stream() - with torch.cuda.stream(comm): - comm.wait_stream(compute) - dp_gather_partial(global_hidden, local, fb) - state.gather_event = _tbo_event(("gather", sub)) - state.gather_event.record(comm) - state.gather_keepalive = local - state.global_hidden = global_hidden - - def op_gather_b(self, state): - torch.cuda.current_stream().wait_event(state.pop("gather_event")) - # Compute now ordered after the gather -> the gather input is safe to - # release (freed on the compute stream, no record_stream deferral). - state.pop("gather_keepalive") - - def op_moe(self, state): - # MoE (gate/topk/experts) on the GLOBAL gathered buffer. mlp_reduce_scatter - # skips the MoE-internal all_reduce (we reduce_scatterv in op_combine). - fb = state.forward_batch - global_hidden = state.pop("global_hidden") - global_ids = fb._tbo_global_input_ids - with get_forward().scoped(mlp_reduce_scatter=True): - state.global_expert_out = self.mlp( - global_hidden, - fb, - input_ids=global_ids, - input_ids_global=global_ids, - skip_shared_experts=state.do_shared_local, - ) - - def op_combine_a(self, state): - # Launch reduce_scatterv (global partial expert sums -> per-rank local) on - # the comm stream; record an event. Symmetric inverse of the all_gatherv. - global_out = state.pop("global_expert_out") - local_out = get_tbo_persistent_buffer( - ("lo", state.tbo_subbatch_index), - get_local_dp_buffer_len(), - global_out.shape[1], - global_out.dtype, - global_out.device, - ) - state.combine_event = dp_reduce_scatterv_async( - local_out, - global_out, - get_dp_global_num_tokens(), - event_key=("combine", state.tbo_subbatch_index), - ) - state.local_out = local_out - # Keep the (variable-size) MoE output alive until op_combine_b waits on - # the combine event (replaces record_stream; avoids reserved churn). - state.combine_keepalive = global_out - - def op_combine_b(self, state): - torch.cuda.current_stream().wait_event(state.pop("combine_event")) - state.pop("combine_keepalive") - hidden = state.pop("local_out") - shared_local = state.pop("shared_local") - state.pop("do_shared_local") - if shared_local is not None: - n = hidden.shape[0] - hidden = hidden + shared_local[:n] - state.hidden_states_mlp_output = hidden - - -def _mask_dp_pad_rows(local_hidden_states, forward_batch) -> None: - n_real = forward_batch.num_token_non_padded - num_rows = local_hidden_states.shape[0] - if n_real is None or num_rows == 0: - return - pad = torch.arange(num_rows, device=local_hidden_states.device) >= n_real - local_hidden_states.masked_fill_( - pad.view(-1, *([1] * (local_hidden_states.dim() - 1))), 0 - ) - def _gather_late_layer_ids( input_ids: torch.Tensor, counts: List[int], forward_batch: ForwardBatch @@ -3908,6 +3121,201 @@ def _scatter_tail_rows( return full +class DeepseekV4CudaDecoderLayer(DeepseekV4DecoderLayerBase): + """CUDA execution with declared residual reads and writes.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.self_attn.wo_b.reduce_results = False + self.mlp.reduce_results = False + declarations = hc_boundary.stages(self.config, self.layer_id, layer=self) + self.attn_boundary, self.ffn_boundary = append_stages( + (declarations[0], self.input_layernorm), + (declarations[1], self.post_attention_layernorm), + ) + + def _run_boundary_attention( + self, hidden_states, positions, forward_batch, quantized + ): + return self.self_attn( + x=hidden_states, + positions=positions, + forward_batch=forward_batch, + x_quant=quantized, + defer_all_reduce=True, + ) + + def _bind_attention_transport( + self, hidden_states, batch, call, rows, overlapped_rows + ): + return hc_boundary.AttentionTransport(self, hidden_states, batch, call, rows) + + def _bind_moe_transport(self, hidden_states, batch, call, rows, overlapped_rows): + if call.tbo: + if overlapped_rows is None: + raise NotImplementedError( + "this operand transport has no overlapped path" + ) + return OverlappedMoeTransport( + self.mlp, hidden_states, batch, call.tbo_subbatch_index, overlapped_rows + ) + return hc_boundary.MoeTransport(self, hidden_states, batch, call, rows) + + def forward( + self, + positions, + hidden_states, + input_ids, + forward_batch, + input_ids_global, + *, + allow_cross_layer_fusion=True, + ): + call = hc_boundary.LayerCall( + input_ids, input_ids_global, allow_cross_layer_fusion + ) + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, read_context=call + ) + quantized = residual_batch.written_residual(forward_batch).quantized + with self.self_attn.maybe_use_decode_attn_tp(forward_batch): + hidden_states = self._run_boundary_attention( + hidden_states, positions, forward_batch, quantized + ) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare( + hidden_states, forward_batch, read_context=call + ) + transport = self.ffn_boundary.transport_context(forward_batch) + with self.ffn_boundary.exit(forward_batch) as output: + hidden_states = self.mlp( + hidden_states, + forward_batch, + input_ids=transport.input_ids, + input_ids_global=transport.input_ids_global, + skip_shared_experts=transport.do_shared_local, + return_moe_output=transport.return_moe_output, + leave_output_sum=True, + ) + return output.finish(hidden_states) + + @functools.cached_property + def _can_fuse_attn_mhc(self) -> bool: + """The static half of the attention fuse gate (dsv4.1 only); load-time + facts, dropped by `refresh_mhc_norm_weight_cache`. Row capacity is the + dynamic half, checked per forward via `can_fuse_all_reduce`.""" + boundary = mhc.make_boundary( + self.post_attention_layernorm, accepts_mxfp8=False, hc=self.ffn_hc + ) + return ( + get_parallel().tp_size == self.self_attn.attn_tp_size == 4 + and boundary.norm_fusable + and mhc.can_fuse_post(self.hc_cfg) + ) + + def op_mhc_prepare_attn( + self, + state, + positions, + hidden_states, + forward_batch, + residual=None, + tbo_subbatch_index=None, + **kwargs, + ): + residual_batch.start(forward_batch) + call = hc_boundary.LayerCall( + forward_batch.input_ids, + getattr(forward_batch, "_tbo_global_input_ids", forward_batch.input_ids), + allow_cross_layer_fusion=False, + tbo=True, + tbo_subbatch_index=tbo_subbatch_index, + ) + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, read_context=call + ) + state.hidden_states_after_input_norm = hidden_states + state.attn_x_quant = residual_batch.written_residual(forward_batch).quantized + if get_moe_a2a_backend().is_mori(): + state.num_tokens = hidden_states.shape[0] + state.update( + dict( + forward_batch=forward_batch, + positions=positions, + tbo_subbatch_index=tbo_subbatch_index, + ) + ) + + def op_mhc_post_attn_pre_mlp(self, state): + batch = state.forward_batch + hidden_states = self.attn_boundary.finish( + state.pop("hidden_states_after_attn"), batch + ) + call = hc_boundary.LayerCall( + batch.input_ids, + getattr(batch, "_tbo_global_input_ids", batch.input_ids), + allow_cross_layer_fusion=False, + tbo=True, + tbo_subbatch_index=state.tbo_subbatch_index, + ) + state.hidden_states_mlp_input = self.ffn_boundary.prepare( + hidden_states, batch, read_context=call + ) + + def op_mhc_postprocess(self, state): + batch = state.forward_batch + hidden_states = self.ffn_boundary.finish( + state.pop("hidden_states_mlp_output"), batch + ) + hidden_states = residual_batch.read_output( + hidden_states, batch, hc_boundary.FinalRead(), None + ) + output = dict( + positions=state.positions, + hidden_states=hidden_states, + residual=None, + forward_batch=batch, + tbo_subbatch_index=state.tbo_subbatch_index, + ) + state.clear(expect_keys={"positions", "forward_batch", "tbo_subbatch_index"}) + return output + + def op_gather_a(self, state): + state.pop("hidden_states_mlp_input") + self.ffn_boundary.transport_context(state.forward_batch).issue_gather() + + def op_gather_b(self, state): + self.ffn_boundary.transport_context(state.forward_batch).wait_gather() + + def op_moe(self, state): + transport = self.ffn_boundary.transport_context(state.forward_batch) + state.global_expert_out = _tbo_run_moe( + self, + transport.global_hidden, + state.forward_batch, + transport.do_shared_local, + ) + + def op_combine_a(self, state): + self.ffn_boundary.transport_context(state.forward_batch).issue_combine( + state.pop("global_expert_out") + ) + + def op_combine_b(self, state): + state.hidden_states_mlp_output = self.ffn_boundary.transport_context( + state.forward_batch + ).wait_combine() + + +class DeepseekV4PlatformDecoderLayer(NonCudaDecoderMixin, DeepseekV4DecoderLayerBase): + """Non-CUDA execution with platform mHC state and fusion.""" + + +DeepseekV4DecoderLayer = ( + DeepseekV4CudaDecoderLayer if _is_cuda else DeepseekV4PlatformDecoderLayer +) + + class DeepseekV4Model(nn.Module): fall_back_to_pt_during_load = False @@ -3988,6 +3396,10 @@ class DeepseekV4Model(nn.Module): moe_routed_quant_stream=self.moe_routed_quant_stream, ), prefix=add_prefix("layers", prefix), + final_read=hc_boundary.FinalRead() if _is_cuda else None, + stage_facts=functools.partial(hc_boundary.stages, config) + if _is_cuda + else None, ) if self.pp_group.is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -4018,9 +3430,11 @@ class DeepseekV4Model(nn.Module): ), ) - self.use_fused_mhc_post_pre = ( - is_cross_layer_mhc_fusion_enabled() or _is_fused_mhc_post_pre_enabled_xpu() - ) + if not _is_cuda: + self.use_fused_mhc_post_pre = ( + is_cross_layer_mhc_fusion_enabled() + or _is_fused_mhc_post_pre_enabled_xpu() + ) self.dspark_layers_to_capture: Optional[List[int]] = None @@ -4178,12 +3592,34 @@ class DeepseekV4Model(nn.Module): ) tail_sentinel = positions.new_zeros((1,)) if tail_in_graph else None saved_full = None - # A pending post never meets a residual reader: the HIP boundary's defer_post - # gate excludes Engram/DSpark-capture layers and the model end. + # The HIP defer_post gate excludes residual readers. CUDA materializes + # its producer update explicitly before these readers below. # mHC assumes full token rows per rank; LayerNorm SP needs its own path. assert not get_forward().sp_active state = mhc.HcState(hidden_states) + if _is_cuda: + residual_batch.start(forward_batch) + hidden_states = residual_batch.set_written(state, forward_batch) for i in range(self.start_layer, self.end_layer): + if _is_cuda: + layer = self.layers[i] + if ( + (tail is not None and i == self.late_layer_start) + or layer.engram is not None + or (capture_dspark and i in self.dspark_layers_to_capture) + ): + call = hc_boundary.LayerCall( + input_ids, + input_ids_global, + ) + next_boundary = ( + hc_boundary.HcRead(layer, True) + .bind(forward_batch, call) + .next_boundary + ) + hidden_states = state = hc_boundary.materialize( + hidden_states, forward_batch, next=next_boundary + ) if tail is not None and i == self.late_layer_start: # Decode reaches back at most SWA_WINDOW positions. if tail_in_graph: @@ -4195,6 +3631,8 @@ class DeepseekV4Model(nn.Module): else: saved_full = attn_backend.enter_late_layer_tail(forward_batch) state = state.take_rows(tail.rows) + if _is_cuda: + hidden_states = residual_batch.set_written(state, forward_batch) input_ids = tail.rows(input_ids) if not ( get_parallel().attn_dp_size > 1 and get_moe_a2a_backend().is_none() @@ -4234,6 +3672,8 @@ class DeepseekV4Model(nn.Module): hidden_states, ) state = state.with_residual(hidden_states) + if _is_cuda: + hidden_states = residual_batch.set_written(state, forward_batch) if capture_dspark and i in self.dspark_layers_to_capture: # The draft head reads the attention input of its target layers. aux = state.residual @@ -4246,25 +3686,39 @@ class DeepseekV4Model(nn.Module): else get_global_expert_distribution_recorder().with_current_layer(i) ) with ctx: - if _is_hip and self.layers[i].hc_boundary_fused: - state = _hip.forward_layer_fused_boundary( - self, - i, - state, + if _is_cuda: + hidden_states = self.layers[i]( positions=positions, + hidden_states=hidden_states, input_ids=input_ids, forward_batch=forward_batch, input_ids_global=input_ids_global, - capture_dspark=capture_dspark, ) - continue - state = self.layers[i].forward_hc_pre_from_prev( - positions=positions, - state=state, - input_ids=input_ids, - forward_batch=forward_batch, - input_ids_global=input_ids_global, - ) + else: + if _is_hip and self.layers[i].hc_boundary_fused: + state = forward_layer_fused_boundary( + self, + i, + state, + positions=positions, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + capture_dspark=capture_dspark, + ) + continue + state = self.layers[i].forward_hc_pre_from_prev( + positions=positions, + state=state, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + ) + if _is_cuda: + state = residual_batch.read_output( + hidden_states, forward_batch, hc_boundary.FinalRead(), None + ) + residual_batch.clear(forward_batch) state = state.materialized(self.layers[self.end_layer - 1].hc_cfg) if saved_dp is not None: exit_late_layer_dp(forward_batch, saved_dp) @@ -4382,15 +3836,20 @@ class DeepseekV4Model(nn.Module): tp_group.all_gatherv(padded_ids, sizes=sizes, output=gids) child._tbo_global_input_ids = gids - outputs_arr = execute_overlapped_operations( - inputs_arr=inputs_arr, - operations_arr=[operations_strategy.operations] * 2, - delta_stages=[0, operations_strategy.tbo_delta_stages], - ) + try: + outputs_arr = execute_overlapped_operations( + inputs_arr=inputs_arr, + operations_arr=[operations_strategy.operations] * 2, + delta_stages=[0, operations_strategy.tbo_delta_stages], + ) - return _model_forward_tbo_merge_key( - outputs_arr[0], outputs_arr[1], "hidden_states", hidden_states.shape[0] - ) + return _model_forward_tbo_merge_key( + outputs_arr[0], outputs_arr[1], "hidden_states", hidden_states.shape[0] + ) + finally: + if _is_cuda: + for child in forward_batch.tbo_children: + residual_batch.clear(child) @torch.no_grad() def forward( @@ -4400,6 +3859,26 @@ class DeepseekV4Model(nn.Module): forward_batch: ForwardBatch, input_embeds: Optional[torch.Tensor], pp_proxy_tensors: Optional[PPProxyTensors] = None, + ) -> Union[torch.Tensor, PPProxyTensors]: + if not _is_cuda: + return self._forward_impl( + input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors + ) + try: + return self._forward_impl( + input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors + ) + finally: + residual_batch.clear(forward_batch) + + @torch.no_grad() + def _forward_impl( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor], + pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> Union[torch.Tensor, PPProxyTensors]: if self.pp_group.is_first_rank: if input_embeds is None: @@ -4489,7 +3968,9 @@ class DeepseekV4Model(nn.Module): forward_batch=forward_batch, ) else: - use_fused = self.use_fused_mhc_post_pre + if _is_cuda: + residual_batch.start(forward_batch) + use_fused = not _is_cuda and self.use_fused_mhc_post_pre prev_residual, prev_post, prev_comb = None, None, None last_layer = None for i in range(self.start_layer, self.end_layer): @@ -4501,25 +3982,42 @@ class DeepseekV4Model(nn.Module): else get_global_expert_distribution_recorder().with_current_layer(i) ) with ctx: - hidden_states, prev_residual, prev_post, prev_comb = layer( - positions=positions, - hidden_states=hidden_states, - forward_batch=forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, - prev_residual=prev_residual, - prev_post=prev_post, - prev_comb=prev_comb, - ) + if _is_cuda: + hidden_states = layer( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + ) + else: + hidden_states, prev_residual, prev_post, prev_comb = layer( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + prev_residual=prev_residual, + prev_post=prev_post, + prev_comb=prev_comb, + ) if capture_dspark and i in self.dspark_layers_to_capture: - if use_fused: + if _is_cuda: + completed = hc_boundary.materialize( + hidden_states, forward_batch, consume=False + ) + elif use_fused: completed = layer.hc_post( hidden_states, prev_residual, prev_post, prev_comb ) else: completed = hidden_states dspark_aux_hidden_states.append(completed.mean(dim=1)) - if use_fused and last_layer is not None: + if _is_cuda: + hidden_states = residual_batch.read_output( + hidden_states, forward_batch, hc_boundary.FinalRead(), None + ) + elif use_fused and last_layer is not None: hidden_states = last_layer.hc_post( hidden_states, prev_residual, prev_post, prev_comb ) @@ -4997,7 +4495,7 @@ class DeepseekV4ForCausalLM(nn.Module): layer.refresh_mhc_norm_weight_cache() layers = self.model.layers for i, layer in enumerate(layers): - if isinstance(layer, DeepseekV4DecoderLayer): + if isinstance(layer, DeepseekV4PlatformDecoderLayer): layer.bind_next(layers[i + 1] if i + 1 < len(layers) else None) def precompile_kernels_after_loading(self) -> None: @@ -5130,7 +4628,7 @@ class DeepseekV4ForCausalLM(nn.Module): rms_eps=layer.rms_norm_eps, hc_pre_eps=layer.hc_eps, hc_sinkhorn_eps=layer.hc_eps, - hc_post_mult_value=_MHC_POST_MULT_VALUE, + hc_post_mult_value=MHC_POST_MULT_VALUE, sinkhorn_repeat=layer.hc_sinkhorn_iters, n_splits=1, n_splits_pre=32, diff --git a/python/sglang/srt/models/deepseek_v4_boundary.py b/python/sglang/srt/models/deepseek_v4_boundary.py new file mode 100644 index 0000000000..452443a1e8 --- /dev/null +++ b/python/sglang/srt/models/deepseek_v4_boundary.py @@ -0,0 +1,540 @@ +"""DeepSeek hyper-connection reads, updates and operand row transport.""" + +from functools import partial + +import msgspec +import torch + +from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx +from sglang.srt.layers.layer_boundary.contracts import BatchVariant, ReadoutFusion +from sglang.srt.layers.layer_boundary.factories import declare_attn, declare_ffn +from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts +from sglang.srt.layers.layer_boundary.layout import ( + Layout, + SumGroup, + TokenAxis, + _sum_group, +) +from sglang.srt.layers.layer_boundary.output import UnreducedOutput +from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.row_transport import ( + ComputeTransport, + RowTransport, + TransportRows, +) +from sglang.srt.layers.moe import get_moe_a2a_backend +from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + can_fuse_all_reduce, +) +from sglang.srt.models import deepseek_v4_mhc as mhc +from sglang.srt.models.deepseek_common.moe_transport import ( + MoeFinalize, + MoeRowTransport, +) +from sglang.srt.models.deepseek_v2 import MoEOutput +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import is_gfx95_supported + +_is_gfx95_supported = is_gfx95_supported() + + +class LayerCall(msgspec.Struct, frozen=True): + input_ids: object = None + input_ids_global: object = None + allow_cross_layer_fusion: bool = True + tbo: bool = False + tbo_subbatch_index: object = None + + +class ReadState(msgspec.Struct): + """The source residual and coefficients of one sublayer invocation.""" + + streams: torch.Tensor + post: object = None + comb: object = None + quantized: object = None + stats: object = None + stats_stream: object = None + call: object = None + + +class HcUpdate: + bind = None + replay_update = None + is_plain_add = False + outlives_layer = True + quantized_sum = False + + def __init__(self, layer, attention, *, applied_at_exit=False): + self.layer = layer + self.attention = attention + self.applied_at_exit = applied_at_exit + self.writes_stream = False + + def slice_residual_attn_tp(self, residual): + raise NotImplementedError("mHC coefficients remain on their source rows") + + def gather_residual_attn_tp(self, residual): + raise NotImplementedError("mHC coefficients remain on their source rows") + + def post(self, output, state, *, next=None): + if not self.layer.boundary_pre_from_prev: + return self.layer.hc_post(output, state.streams, state.post, state.comb) + hc = self.layer.attn_hc if self.attention else self.layer.ffn_hc + if self.attention: + return mhc.run_attn_post( + hc, + output, + state.streams, + stats_stream=state.stats_stream, + next=next, + world_size=self.layer.self_attn.attn_tp_size, + precomputed=state.stats, + ) + return mhc.run_moe_post( + hc, + output, + state.streams, + stats_stream=state.stats_stream, + next=next if state.call.allow_cross_layer_fusion else None, + world_size=self.layer.mlp.tp_size, + precomputed=state.stats, + ) + + def update(self, output, state): + return self.post(output, state) + + +class HcRead: + is_plain_norm = False + reads_before_dp_gather = True + reads_after_attn_tp_gather = True + gathering_reads = () + + def __init__(self, layer, attention): + self.layer = layer + self.attention = attention + self.completing_fusions = tuple( + ReadoutFusion( + group, + self._complete_read, + reads=True, + carried=True, + accepts_update=lambda update: isinstance(update, HcUpdate), + ) + for group in (SumGroup.ATTN_TP, SumGroup.MOE_OUTPUT) + ) + + def bind(self, batch, call): + return HcReadCall(self, batch, call) + + @staticmethod + def init_residual(hidden_states): + return hidden_states + + def _complete_read(self, output, state, batch, norm, *, update, read_context=None): + if not self.layer.boundary_pre_from_prev: + return None + if isinstance(output, MoeFinalize): + if output.pieces is None: + return None + output = output.pieces + elif isinstance(output, UnreducedOutput): + if not update.attention: + return None + output = mhc.AttnOutput(output.partial) + else: + return None + call = read_context if read_context is not None else state.call + read = self.bind(batch, call) + return read._after_post( + update.post(output, state, next=read.next_boundary), norm + ) + + +class HcReadCall(HcRead): + def __init__(self, spec, batch, call): + self.spec = spec + self.layer = spec.layer + self.attention = spec.attention + self.batch = batch + self.call = call + self.completing_fusions = tuple( + msgspec.structs.replace( + f, run=partial(spec._complete_read, read_context=call) + ) + for f in spec.completing_fusions + ) + + def bind(self, batch, call=None): + if batch is not self.batch or (call is not None and call is not self.call): + raise ValueError("an invocation read cannot bind another call") + return self + + @property + def next_boundary(self): + if not self.layer.boundary_mhc_fusions or ( + self.attention and self.layer.engram is not None + ): + return None + hc = self.layer.attn_hc if self.attention else self.layer.ffn_hc + return mhc.make_boundary( + self.layer.input_layernorm + if self.attention + else self.layer.post_attention_layernorm, + accepts_mxfp8=self.attention + and self.layer.self_attn.accepts_mxfp8_swizzled_input(), + hc=hc, + ) + + def _after_post(self, residual, norm): + return self.read(residual, norm) + + def read( + self, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + if quant_format or post_residual_addition is not None: + raise NotImplementedError("mHC read supplies its own projection format") + layer = self.layer + if layer.boundary_pre_from_prev: + if not isinstance(residual, mhc.HcState): + residual = mhc.HcState(residual) + streams = residual.residual + stream = ( + layer.hc_stats_stream + if mhc.use_stats_stream(layer.hc_cfg, self.batch, streams) + else None + ) + mhc.fork_stats_stream(stream) + quantized = ( + [] + if self.attention and layer.self_attn.accepts_mxfp8_swizzled_input() + else None + ) + hc = layer.attn_hc if self.attention else layer.ffn_hc + x = mhc.combine(hc, residual, quantized) + stats = residual.stats + residual.release() + return x, ReadState( + streams, + quantized=quantized[0] if quantized else None, + stats=stats, + stats_stream=stream, + call=self.call, + ) + fn, scale, base = self._parameters() + x, post, comb, norm_fused = layer.hc_pre( + residual, + fn, + scale, + base, + norm=norm if layer.boundary_mhc_fusions else None, + forward_batch=self.batch, + ) + quantized = None + if not norm_fused: + if self.attention: + x, quantized = layer._input_norm( + x, allow_aiter_quant=_is_gfx95_supported + ) + else: + x = norm(x) + return x, ReadState(residual, post, comb, quantized, call=self.call) + + def _parameters(self): + layer = self.layer + if self.attention: + return layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base + return layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base + + def update_and_read(self, update, value, residual, norm, **kwargs): + if not isinstance(update, HcUpdate): + raise TypeError("mHC read requires its producer's update") + if self.layer.boundary_pre_from_prev: + return self.read( + update.post(value, residual, next=self.next_boundary), norm + ) + layer = self.layer + if layer.boundary_mhc_fusions and layer.use_fused_mhc_post_pre: + from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + apply_mhc_post_pre_boundary, + ) + + fn, scale, base = self._parameters() + cached = ( + layer._input_layernorm_weight_bf16 + if self.attention + else layer._post_attention_layernorm_weight_bf16 + ) + fused = apply_mhc_post_pre_boundary( + value, + residual.streams, + residual.post, + residual.comb, + fn, + scale, + base, + layer.hc_mult, + layer.rms_norm_eps, + layer.hc_eps, + mhc.MHC_POST_MULT_VALUE, + layer.hc_sinkhorn_iters, + cached if cached is not None else norm.weight.data, + norm.variance_epsilon, + fn_transpose=not self.attention, + ) + if fused is not None: + streams, x, post, comb, norm_fused = fused + quantized = None + if not norm_fused: + if self.attention: + x, quantized = layer._input_norm( + x, allow_aiter_quant=_is_gfx95_supported + ) + else: + x = norm(x) + return x, ReadState(streams, post, comb, quantized, call=self.call) + return self.read(update.update(value, residual), norm) + + +class FinalRead(PendingUpdateRead): + bind = None + + @staticmethod + def _complete(output, residual, batch, norm, *, update): + if ( + not isinstance(output, MoeFinalize) + or output.pieces is None + or not update.layer.boundary_pre_from_prev + ): + return None + return update.post(output.pieces, residual) + + completing_fusions = ( + ReadoutFusion( + SumGroup.MOE_OUTPUT, + _complete.__func__, + reads=True, + carried=True, + accepts_update=lambda update: isinstance(update, HcUpdate), + ), + ) + + @staticmethod + def accepts_pending_update(update): + return isinstance(update, HcUpdate) + + @staticmethod + def init_residual(value): + return value + + @staticmethod + def read(value, norm): + return value + + @staticmethod + def update_and_read(update, value, residual, norm): + return update.update(value, residual) + + +def materialize(hidden_states, batch, *, consume=True, next=None): + """Read mHC streams at a PP, head, Engram or capture seam.""" + stream = residual_batch.stream_of(batch) + stream.check(hidden_states) + pending = stream.pending + if pending is None: + return hidden_states + value = pending.for_boundary() + if isinstance(value, MoeFinalize) and value.pieces is not None: + result = pending.update.post(value.pieces, stream.residual, next=next) + elif isinstance(value, UnreducedOutput): + result = pending.update.post( + mhc.AttnOutput(value.partial), stream.residual, next=next + ) + else: + result = pending.update.post(pending.complete(), stream.residual, next=next) + if consume: + stream.write(result) + return result + + +def stages(config, layer_id, *, layer=None): + from sglang.srt.layers.layer_boundary.layout import token_axis_sizes + + ep = not get_moe_a2a_backend().is_none() + table_attn, table_ffn, overlapped_ffn = {}, {}, {} + for variant in (BatchVariant.ORDINARY, BatchVariant.CONTEXT_PARALLEL): + cp = variant is BatchVariant.CONTEXT_PARALLEL + source = Layout.sharded_over( + TokenAxis.ATTN_DP, + *((TokenAxis.ATTN_CP,) if cp else ()), + axis_sizes=token_axis_sizes(cp_active=cp), + ) + compute = Layout.sharded_over( + *( + (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP) + if cp and ep + else ((TokenAxis.ATTN_DP, TokenAxis.ATTN_TP) if ep else ()) + ), + axis_sizes=token_axis_sizes(cp_active=cp), + ) + table_attn[variant] = TransportRows( + source, + source, + source, + SumGroup.ATTN_TP, + SumGroup.ATTN_CP if get_parallel().attn_cp_size > 1 else None, + ) + table_ffn[variant] = TransportRows( + source, compute, source, None if ep else SumGroup.MOE_OUTPUT + ) + if not cp: + overlapped_ffn[variant] = TransportRows( + source, + source if ep else compute, + source, + None if ep else SumGroup.MOE_OUTPUT, + ) + if layer is None: + from sglang.srt.models.deepseek_v4 import ( + _is_fused_mhc_post_pre_enabled_xpu, + is_cross_layer_mhc_fusion_enabled, + ) + + lagged = config.hc_pre_from_prev_sublayer + fused = ( + is_cross_layer_mhc_fusion_enabled() or _is_fused_mhc_post_pre_enabled_xpu() + ) + else: + lagged = layer.boundary_pre_from_prev + fused = layer.boundary_mhc_fusions and layer.use_fused_mhc_post_pre + applied = False if lagged else not fused + if layer is None: + read = ReadFacts.of(HcRead(None, True)) + au = UpdateFacts.of(HcUpdate(None, True)) + fu = UpdateFacts.of(HcUpdate(None, False, applied_at_exit=applied)) + else: + read = HcRead(layer, True) + au = HcUpdate(layer, True) + fu = HcUpdate(layer, False, applied_at_exit=applied) + for variant in (BatchVariant.UNPADDED, BatchVariant.INPUT_SCATTERED): + table_attn[variant] = table_attn[BatchVariant.ORDINARY] + table_ffn[variant] = table_ffn[BatchVariant.ORDINARY] + # TBO keeps source rows until expert dispatch, including an uneven final + # microbatch. It does not shard these rows across attention TP. + overlapped_ffn[BatchVariant.UNPADDED] = overlapped_ffn[BatchVariant.ORDINARY] + return ( + declare_attn( + read=read, + update=au, + gathers_attn_tp_input=False, + row_transport=RowTransport( + table_attn, + layer._bind_attention_transport if layer is not None else None, + ), + ), + declare_ffn( + sparse=True, + read=HcRead(layer, False) if layer is not None else read, + update=fu, + row_transport=RowTransport( + table_ffn, + layer._bind_moe_transport if layer is not None else None, + overlapped_rows=overlapped_ffn, + ), + ), + ) + + +class AttentionTransport(ComputeTransport): + def __init__(self, layer, hidden_states, batch, call, rows): + if rows.source != rows.compute or rows.source != rows.returned: + raise ValueError("attention transport must preserve its source rows") + if rows.group is not SumGroup.ATTN_TP or rows.decode_group not in ( + None, + SumGroup.ATTN_CP, + ): + raise ValueError("attention transport does not match its projection groups") + self.num_rows = hidden_states.shape[0] + self.rows = rows + self.input = hidden_states + self.defers_sum = ( + layer.boundary_mhc_fusions + and layer.boundary_pre_from_prev + and layer._can_fuse_attn_mhc + and can_fuse_all_reduce(hidden_states.shape[0], layer.hc_cfg.hidden) + ) + + def release(self): + self.input = None + + def prepare_output(self, output): + self.input = None + value = output.partial if isinstance(output, mhc.AttnOutput) else output + if value.shape[0] != self.num_rows: + raise ValueError("attention output changed its source rows") + if not isinstance(output, mhc.AttnOutput): + return output + ctx = get_cp_decode_attn_tp_ctx() + group = self.rows.decode_group if ctx.use_decode_attn_tp else self.rows.group + if group is None: + raise ValueError("attention output owes no declared transport sum") + return UnreducedOutput(output.partial, group=_sum_group(group)) + + +class MoeTransport(MoeRowTransport): + def __init__(self, layer, hidden_states, forward_batch, call, rows): + self.source_rows = hidden_states.shape[0] + self.rows = rows + if (rows.group is None) != (not get_moe_a2a_backend().is_none()): + raise ValueError("MoE backend does not match its transport output contract") + cp = TokenAxis.ATTN_CP in rows.source.sharded + super().__init__( + layer.mlp, + hidden_states, + forward_batch, + call.input_ids, + call.input_ids_global, + cp_return_contiguous=layer.config.model_type == "deepseek_v41", + use_cp=cp, + moe_gather=not cp + and TokenAxis.ATTN_DP in rows.source.sharded - rows.compute.sharded, + attn_scatter=not cp + and TokenAxis.ATTN_TP in rows.compute.sharded - rows.source.sharded, + ) + self.sum_in_return = self.mlp_reduce_scatter and rows.group is not None + self.return_moe_output = self.defers_sum = ( + layer.boundary_mhc_fusions + and layer.boundary_pre_from_prev + and layer._can_fuse_ffn_mhc + and can_fuse_all_reduce(self.input.shape[0], layer.hc_cfg.hidden) + and not self._use_cp + and not self._use_tp_moe_gather + and not self._use_tp_attn_a2a_scatter + ) + + def prepare_output(self, hidden_states): + self.input = None + if isinstance(hidden_states, MoEOutput): + completion = MoeFinalize(hidden_states) + return ( + completion.complete(reduce=False) + if self.mlp_reduce_scatter + else completion + ) + if self.rows.group is not None and not self.mlp_reduce_scatter: + return MoeFinalize(partial=hidden_states) + return hidden_states + + def return_output(self, hidden_states): + if self.defers_sum: + return hidden_states + value = super().return_output(hidden_states) + if value.shape[0] != self.source_rows: + raise ValueError("MoE transport did not return its source rows") + return value diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 62b5776658..b1c0248806 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -2,6 +2,7 @@ from __future__ import annotations import copy import logging +from contextlib import nullcontext from typing import Iterable, List, Optional, Tuple import msgspec @@ -19,6 +20,8 @@ from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layer_boundary import layer_stack +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled @@ -33,18 +36,22 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models import deepseek_v4_boundary as hc_boundary from sglang.srt.models import deepseek_v4_mhc as mhc from sglang.srt.models.dbrx import ReplicatedLinear from sglang.srt.models.deepseek_v4 import ( DEEPSEEK_V4_STACKED_PARAMS_MAPPING, - DeepseekV4DecoderLayer, + DeepseekV4CudaDecoderLayer, DeepseekV4ForCausalLM, + DeepseekV4PlatformDecoderLayer, MqaAttentionBase, _apply_wo_a_bf16_matmul, _dequant_fp8_wo_a_streaming, + _is_cuda, hc_head_torch, make_hc_head_params, ) +from sglang.srt.models.deepseek_v4_platform_stage import NonCudaDSparkMixin from sglang.srt.models.dspark import ( BaseDSparkMarkovHead, DSparkConfidenceHead, @@ -249,6 +256,8 @@ class DSparkAttention(MqaAttentionBase): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, + *, + defer_all_reduce: bool = False, ) -> torch.Tensor: # Idle ranks have no draft KV rows or attention metadata, but still # participate in the shared MoE forward later in the draft stage. @@ -370,8 +379,17 @@ class DSparkAttention(MqaAttentionBase): ) else: o = torch.einsum("bgd,grd->bgr", o.float(), wo_a.float()).to(q.dtype) - out, _ = self.wo_b(o.reshape(o.shape[0], o.shape[1] * o.shape[2])) - return out + value = o.reshape(o.shape[0], o.shape[1] * o.shape[2]) + out, _ = self.wo_b(value, skip_all_reduce=defer_all_reduce) + return ( + mhc.AttnOutput(out) + if _is_cuda and defer_all_reduce and self.attn_tp_size > 1 + else out + ) + + @staticmethod + def accepts_mxfp8_swizzled_input(): + return False def _resolve_dspark_pool() -> DeepSeekV4TokenToKVPool: @@ -555,7 +573,7 @@ def _dspark_stage_config(config: DeepSeekV4Config) -> DeepSeekV4Config: return stage_config -class DSparkV4Stage(DeepseekV4DecoderLayer): +class DSparkV4StageMixin: def __init__( self, config: DeepSeekV4Config, @@ -575,6 +593,8 @@ class DSparkV4Stage(DeepseekV4DecoderLayer): quant_config=quant_config, prefix=prefix, is_nextn=True, + boundary_pre_from_prev=config.hc_pre_from_prev_sublayer, + boundary_mhc_fusions=False, alt_streams=alt_streams, hc_stats_stream=hc_stats_stream, moe_routed_quant_stream=moe_routed_quant_stream, @@ -628,99 +648,33 @@ class DSparkV4Stage(DeepseekV4DecoderLayer): alt_streams=alt_streams, ) - def _hc_pre_block( - self, - x: torch.Tensor, - hc_fn: torch.Tensor, - hc_scale: torch.Tensor, - hc_base: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - y, post, comb, _ = self.hc_pre(x, hc_fn, hc_scale, hc_base) - return y, post, comb - def _hc_post_block( - self, - x: torch.Tensor, - residual: torch.Tensor, - post: torch.Tensor, - comb: torch.Tensor, - ) -> torch.Tensor: - return self.hc_post(x, residual, post, comb) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - prev_pre: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - if self.hc_pre_from_prev_sublayer: - return self._forward_hc_pre_from_prev( - positions, hidden_states, forward_batch, prev_pre - ) - residual = hidden_states - x, post, comb = self._hc_pre_block( - hidden_states, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base +class DSparkV4CudaStage(DSparkV4StageMixin, DeepseekV4CudaDecoderLayer): + def _run_boundary_attention( + self, hidden_states, positions, forward_batch, quantized + ): + return self.self_attn( + positions, hidden_states, forward_batch, defer_all_reduce=True ) - x = self.input_layernorm(x) - with self.self_attn.maybe_use_decode_attn_tp(forward_batch): - x = self.self_attn(positions, x, forward_batch) - x = self._hc_post_block(x, residual, post, comb) - residual = x - x, post, comb = self._hc_pre_block( - x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base + def forward(self, positions, hidden_states, forward_batch): + return super().forward( + positions, + hidden_states, + None, + forward_batch, + None, + allow_cross_layer_fusion=False, ) - x = self.post_attention_layernorm(x) - x = self._run_ffn(x, forward_batch) - x = self._hc_post_block(x, residual, post, comb) - return x, None - def _forward_hc_pre_from_prev( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - prev_pre: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - # The draft stages run the plain four steps: no fused post, no seams, so - # they need nothing from `bind_next` and never hand anything forward. - assert self.attn_hc is not None and self.ffn_hc is not None - stats_stream = None - if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states): - stats_stream = self.hc_stats_stream - residual = hidden_states - mhc.fork_stats_stream(stats_stream) - x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre)) - with self.self_attn.maybe_use_decode_attn_tp(forward_batch): - x = self.self_attn(positions, x, forward_batch) - attn_pre, attn_post, attn_comb = mhc.mix_stats( - self.attn_hc, hidden_states, stats_stream - ) - if stats_stream is not None: - torch.cuda.current_stream().wait_stream(stats_stream) - hidden_states = self.hc_post(x, residual, attn_post, attn_comb) +class DSparkV4PlatformStage( + NonCudaDSparkMixin, DSparkV4StageMixin, DeepseekV4PlatformDecoderLayer +): + """Non-CUDA draft with platform mHC operations.""" - residual = hidden_states - mhc.fork_stats_stream(stats_stream) - x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre)) - x = self._run_ffn(x, forward_batch) - ffn_pre, ffn_post, ffn_comb = mhc.mix_stats( - self.ffn_hc, hidden_states, stats_stream - ) - if stats_stream is not None: - torch.cuda.current_stream().wait_stream(stats_stream) - hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb) - return hidden_states, ffn_pre - def _run_ffn(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: - shape = x.shape - x = x.reshape(-1, self.dim) - y = self._run_moe_ffn_dp_sync( - x, forward_batch, input_ids=None, input_ids_global=None - ) - return y.view(shape) +DSparkV4Stage = DSparkV4CudaStage if _is_cuda else DSparkV4PlatformStage class DeepseekV4ForCausalLMDSpark(nn.Module): @@ -812,23 +766,28 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): and getattr(config, "hc_pre_from_prev_sublayer", False) else None ) - self.stages = nn.ModuleList( - [ - DSparkV4Stage( - config=config, - layer_id=stage_id, - stage_id=stage_id, - num_stages=self.num_stages, - num_target_layers=self.num_target_features, - quant_config=quant_config, - prefix=add_prefix(f"stages.{stage_id}", prefix), - alt_streams=self.alt_streams, - hc_stats_stream=self.hc_stats_stream, - moe_routed_quant_stream=self.moe_routed_quant_stream, - ) - for stage_id in range(self.num_stages) - ] - ) + with ( + layer_stack(final_read=hc_boundary.FinalRead()) + if _is_cuda + else nullcontext() + ): + self.stages = nn.ModuleList( + [ + DSparkV4Stage( + config=config, + layer_id=stage_id, + stage_id=stage_id, + num_stages=self.num_stages, + num_target_layers=self.num_target_features, + quant_config=quant_config, + prefix=add_prefix(f"stages.{stage_id}", prefix), + alt_streams=self.alt_streams, + hc_stats_stream=self.hc_stats_stream, + moe_routed_quant_stream=self.moe_routed_quant_stream, + ) + for stage_id in range(self.num_stages) + ] + ) self.markov_head = DSparkV4MarkovHead( vocab_size=int(config.vocab_size), markov_rank=int(dspark_config.markov_rank), @@ -942,20 +901,37 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): get_embedding: bool = False, pp_proxy_tensors=None, ) -> LogitsProcessorOutput: - del get_embedding, pp_proxy_tensors + if not _is_cuda: + return self._forward_impl(input_ids, positions, forward_batch, input_embeds) + try: + return self._forward_impl(input_ids, positions, forward_batch, input_embeds) + finally: + residual_batch.clear(forward_batch) + + def _forward_impl(self, input_ids, positions, forward_batch, input_embeds): if input_embeds is None: input_embeds = self.forward_embed(input_ids) x = input_embeds pre = None - # Draft MoE layers have no recorder layer index, so keep them out of the stats. + if _is_cuda: + residual_batch.start(forward_batch) + # Draft stages do not have recorder layer indices. with get_global_expert_distribution_recorder().disable_this_region(): for stage in self.stages: - x, pre = stage(positions, x, forward_batch, pre) + if _is_cuda: + x = stage(positions, x, forward_batch) + else: + x, pre = stage(positions, x, forward_batch, pre) + if _is_cuda: + x = residual_batch.read_output( + x, forward_batch, hc_boundary.FinalRead(), None + ) + if self.hc_pre_from_prev_sublayer: + pre, x = x.pre, x.residual if self.hc_pre_from_prev_sublayer: from sglang.kernels.ops.layernorm.mhc import hc_combine x = hc_combine(x.flatten(1).float(), pre, self.hc_mult, x.dtype) - return LogitsProcessorOutput(next_token_logits=None, hidden_states=x) def collapse_hc_head(self, x: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/models/deepseek_v4_mhc.py b/python/sglang/srt/models/deepseek_v4_mhc.py index b483f71754..8cb607926b 100644 --- a/python/sglang/srt/models/deepseek_v4_mhc.py +++ b/python/sglang/srt/models/deepseek_v4_mhc.py @@ -52,6 +52,9 @@ _is_xpu = is_xpu() HcTriplet = Tuple[torch.Tensor, torch.Tensor, torch.Tensor] +MHC_POST_MULT_VALUE = 2.0 + + class HcConfig(NamedTuple): """Everything about a layer's mHC that is fixed for the model's lifetime.""" diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index d278403808..3734c500ce 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -1,4 +1,5 @@ import logging +from contextlib import nullcontext from typing import Iterable, Optional, Tuple import torch @@ -18,6 +19,8 @@ from sglang.srt.layers.dp_attention import ( get_global_dp_buffer_len, is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary import layer_stack +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -30,9 +33,11 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.models import deepseek_v4_boundary as hc_boundary from sglang.srt.models.deepseek_v4 import ( DeepseekV4DecoderLayer, DeepseekV4ForCausalLM, + _is_cuda, _is_npu, wo_a_fp8_gemm_enabled, ) @@ -95,15 +100,20 @@ class DeepseekV4ModelNextN(nn.Module): else: prefix = add_prefix("decoder", prefix) - self.decoder = DeepseekV4DecoderLayer( - config, - layer_id=0, - quant_config=quant_config, - is_nextn=True, - prefix=prefix, - alt_streams=None, - compress_ratio_override=COMPRESS_RATIO_NEXTN_LAYER, - ) + with ( + layer_stack(final_read=hc_boundary.FinalRead()) + if _is_cuda + else nullcontext() + ): + self.decoder = DeepseekV4DecoderLayer( + config, + layer_id=0, + quant_config=quant_config, + is_nextn=True, + prefix=prefix, + alt_streams=None, + compress_ratio_override=COMPRESS_RATIO_NEXTN_LAYER, + ) self.shared_head = nn.Module() self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -187,17 +197,35 @@ class DeepseekV4ModelNextN(nn.Module): # decoder layer reads the memoized gather instead of re-gathering. prime_rope_cos_sin([self.decoder.self_attn], forward_batch, positions) - hidden_states, residual, post, comb = self.decoder( - positions=positions, - hidden_states=hidden_states, - forward_batch=forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, - ) - if residual is not None: - # NextN has a single decoder layer, so no later layer can consume a - # deferred fused hc_post state. - hidden_states = self.decoder.hc_post(hidden_states, residual, post, comb) + if _is_cuda: + residual_batch.start(forward_batch) + try: + hidden_states = self.decoder( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + ) + hidden_states = residual_batch.read_output( + hidden_states, forward_batch, hc_boundary.FinalRead(), None + ) + finally: + residual_batch.clear(forward_batch) + else: + hidden_states, residual, post, comb = self.decoder( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + ) + if residual is not None: + # NextN has a single decoder layer, so no later layer can consume a + # deferred fused hc_post state. + hidden_states = self.decoder.hc_post( + hidden_states, residual, post, comb + ) pre_hc_head = hidden_states.flatten(1) diff --git a/python/sglang/srt/models/deepseek_v4_platform_stage.py b/python/sglang/srt/models/deepseek_v4_platform_stage.py new file mode 100644 index 0000000000..2734acf78f --- /dev/null +++ b/python/sglang/srt/models/deepseek_v4_platform_stage.py @@ -0,0 +1,716 @@ +"""Non-CUDA DeepSeek V4 decoder and draft execution. + +The mixins own platform mHC state, fusion, and TBO operations. +""" + +from __future__ import annotations + +from functools import cached_property +from typing import TYPE_CHECKING, Optional, Tuple, Union + +import torch + +from sglang.srt.layers.moe import get_moe_a2a_backend +from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + can_fuse_all_reduce, +) +from sglang.srt.models import deepseek_v4_mhc as mhc +from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + _gfx95_dense_post_attention_norm, + apply_attention_mhc, + apply_mhc_post_pre_boundary, + apply_moe_mhc, + attention_mhc_fusion, +) +from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + hc_boundary as make_amd_hc_boundary, +) +from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + moe_mhc_fusion, +) +from sglang.srt.models.deepseek_common.moe_transport import ( + MoeRowTransport, + _tbo_combine_moe_output, + _tbo_gather_moe_input, + _tbo_run_moe, + _tbo_wait_moe_output, + shared_expert_runs_locally, +) +from sglang.srt.models.deepseek_v2 import MoEOutput +from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE, AttnOutput +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import is_gfx95_supported + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + from sglang.srt.models import deepseek_v2 + +_is_gfx95_supported = is_gfx95_supported() + + +def _prepare_ffn_input(layer, hidden_states, residual, post, comb, forward_batch): + """Close attention and prepare the FFN with the platform's mHC policy.""" + if layer.use_fused_mhc_post_pre: + post_attn_norm_weight = ( + layer._post_attention_layernorm_weight_bf16 + if layer._post_attention_layernorm_weight_bf16 is not None + else layer.post_attention_layernorm.weight.data + ) + fused = apply_mhc_post_pre_boundary( + hidden_states, + residual, + post, + comb, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + layer.hc_mult, + layer.rms_norm_eps, + layer.hc_eps, + MHC_POST_MULT_VALUE, + layer.hc_sinkhorn_iters, + post_attn_norm_weight, + layer.post_attention_layernorm.variance_epsilon, + fn_transpose=True, + ) + if fused is not None: + residual, hidden_states, post, comb, norm_fused = fused + if not norm_fused: + hidden_states = layer.post_attention_layernorm(hidden_states) + return residual, hidden_states, post, comb + hidden_states = layer.hc_post(hidden_states, residual, post, comb) + residual = hidden_states + hidden_states, post, comb, norm_fused = layer.hc_pre( + hidden_states, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + norm=layer.post_attention_layernorm, + forward_batch=forward_batch, + ) + if not norm_fused: + hidden_states = layer.post_attention_layernorm(hidden_states) + + return residual, hidden_states, post, comb + + +class NonCudaDecoderMixin: + """Platform tuple and HcState execution for target and NextN decoders. + + CUDA mHC completion lives in deepseek_v4_boundary; operand movement and + finalization live in deepseek_common.moe_transport. Changes to completion + ordering must be checked in both implementations. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._next_layer = None + self.local_boundary = None + self.next_boundary = None + + def bind_next(self, next_layer): + if not self.hc_pre_from_prev_sublayer: + return + self._next_layer = None + if isinstance(next_layer, NonCudaDecoderMixin) and next_layer.engram is None: + self._next_layer = next_layer + + def refresh_mhc_norm_weight_cache(self): + super().refresh_mhc_norm_weight_cache() + self.local_boundary = None + self.next_boundary = None + + @cached_property + def _can_fuse_attn_mhc(self): + return ( + get_parallel().tp_size == self.self_attn.attn_tp_size == 4 + and self.self_attn.wo_b.reduce_results + and self.local_boundary.norm_fusable + and mhc.can_fuse_post(self.hc_cfg) + ) + + def forward_hc_pre_from_prev( + self, + positions: torch.Tensor, + state: mhc.HcState, + input_ids: torch.Tensor, + forward_batch: ForwardBatch, + input_ids_global: torch.Tensor, + ) -> mhc.HcState: + """The layer's two hyper-connections, each collapsing with the previous one's + pre-mix.""" + self._init_boundaries() + stats_stream = None + if mhc.use_stats_stream(self.hc_cfg, forward_batch, state.residual): + stats_stream = self.hc_stats_stream + + def run_attn_hc(state: mhc.HcState) -> mhc.HcState: + assert self.attn_hc is not None + residual = state.residual + quantized = [] if self.self_attn.accepts_mxfp8_swizzled_input() else None + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.attn_hc, state, quantized) + precomputed = state.stats + state.release() + del state + world_size = self.self_attn.attn_tp_size + fuse_all_reduce_mhc = self._can_fuse_attn_mhc and can_fuse_all_reduce( + x.shape[0], self.hc_cfg.hidden + ) + with self.self_attn.maybe_use_decode_attn_tp(forward_batch): + y = self.self_attn( + x=x, + positions=positions, + forward_batch=forward_batch, + x_quant=quantized[0] if quantized else None, + defer_all_reduce=fuse_all_reduce_mhc, + ) + del x + return mhc.run_attn_post( + self.attn_hc, + y, + residual, + stats_stream=stats_stream, + next=self.local_boundary, + world_size=world_size, + precomputed=precomputed, + ) + + def run_ffn_hc(state: mhc.HcState) -> mhc.HcState: + assert self.ffn_hc is not None + residual = state.residual + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.ffn_hc, state) + precomputed = state.stats + state.release() + del state + fuse_all_reduce_mhc = self._can_fuse_ffn_mhc and can_fuse_all_reduce( + x.shape[0], self.hc_cfg.hidden + ) + y = self._run_moe_ffn_dp_sync( + x, + forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + return_moe_output=fuse_all_reduce_mhc, + ) + del x + return mhc.run_moe_post( + self.ffn_hc, + y, + residual, + stats_stream=stats_stream, + next=self.next_boundary, + world_size=self.mlp.tp_size, + precomputed=precomputed, + ) + + return run_ffn_hc(run_attn_hc(state)) + + def _init_boundaries(self) -> None: + """Resolved on first forward: `accepts_mxfp8_swizzled_input` is not settled + until `process_weights_after_loading`.""" + if self.local_boundary is not None: + return + # ATTN[i] -> MOE[i] + self.local_boundary = mhc.make_boundary( + self.post_attention_layernorm, + accepts_mxfp8=False, + hc=self.ffn_hc, + ) + # MOE[i] -> ATTN[i+1] + if self._next_layer is not None: + self.next_boundary = mhc.make_boundary( + self._next_layer.input_layernorm, + accepts_mxfp8=self._next_layer.self_attn.accepts_mxfp8_swizzled_input(), + hc=self._next_layer.attn_hc, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + forward_batch: ForwardBatch, + input_ids_global: torch.Tensor, + prev_residual: Optional[torch.Tensor] = None, + prev_post: Optional[torch.Tensor] = None, + prev_comb: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + use_fused = self.use_fused_mhc_post_pre + + fused = None + if prev_residual is not None and use_fused: + # Dispatch cascade: aiter HIP (gfx95) -> Triton (gfx95 small-batch + # <=64 tokens, or gfx1250 all sizes) -> TileLang -> None. + input_norm_weight = ( + self._input_layernorm_weight_bf16 + if self._input_layernorm_weight_bf16 is not None + else self.input_layernorm.weight.data + ) + fused = apply_mhc_post_pre_boundary( + hidden_states, + prev_residual, + prev_post, + prev_comb, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + self.hc_mult, + self.rms_norm_eps, + self.hc_eps, + MHC_POST_MULT_VALUE, + self.hc_sinkhorn_iters, + input_norm_weight, + self.input_layernorm.variance_epsilon, + fn_transpose=False, + ) + if fused is not None: + residual, hidden_states, post, comb, norm_fused = fused + else: + if prev_residual is not None and use_fused: + hidden_states = self.hc_post( + hidden_states, prev_residual, prev_post, prev_comb + ) + residual = hidden_states + hidden_states, post, comb, norm_fused = self.hc_pre( + hidden_states, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + norm=self.input_layernorm, + forward_batch=forward_batch, + ) + if not norm_fused: + # Triton post+pre can leave norm separate: gfx95 may quantize it, + # while gfx1250 must keep the plain layernorm path. + hidden_states, x_quant = self._input_norm( + hidden_states, allow_aiter_quant=_is_gfx95_supported + ) + else: + x_quant = None + + with self.self_attn.maybe_use_decode_attn_tp(forward_batch): + hidden_states = self.self_attn( + x=hidden_states, + positions=positions, + forward_batch=forward_batch, + x_quant=x_quant, + ) + + del fused + residual, hidden_states, post, comb = _prepare_ffn_input( + self, hidden_states, residual, post, comb, forward_batch + ) + + hidden_states = self._run_moe_ffn_dp_sync( + hidden_states, + forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + ) + + if not use_fused: + hidden_states = self.hc_post(hidden_states, residual, post, comb) + return hidden_states, None, None, None + + # Return the deferred FFN hc_post state; the next layer consumes it with + # cross-layer fusion, and the final layer is completed in DeepseekV4Model. + return hidden_states, residual, post, comb + + def _run_moe_ffn_dp_sync( + self, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + *, + input_ids: Optional[torch.Tensor], + input_ids_global: Optional[torch.Tensor], + return_moe_output: bool = False, + ) -> Union[torch.Tensor, deepseek_v2.MoEOutput]: + prepared = MoeRowTransport( + self.mlp, + hidden_states, + forward_batch, + input_ids, + input_ids_global, + cp_return_contiguous=self.config.model_type == "deepseek_v41", + ) + with prepared.scope(): + return_moe_output = ( + return_moe_output + and not prepared._use_cp + and not prepared._use_tp_moe_gather + and not prepared._use_tp_attn_a2a_scatter + ) + hidden_states = self.mlp( + prepared.input, + forward_batch, + input_ids=prepared.input_ids, + input_ids_global=prepared.input_ids_global, + skip_shared_experts=prepared.do_shared_local, + return_moe_output=return_moe_output, + ) + prepared.input = None + return prepared.return_output(hidden_states) + + def op_mhc_prepare_attn( + self, + state, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + residual: Optional[torch.Tensor] = None, + tbo_subbatch_index: Optional[int] = None, + **kwargs, + ): + # Non-fused attention-side mHC pre + input layernorm. + attn_residual = hidden_states + hidden_states, post, comb, norm_fused = self.hc_pre( + hidden_states, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + norm=self.input_layernorm, + forward_batch=forward_batch, + ) + if not norm_fused: + hidden_states, x_quant = self._input_norm(hidden_states) + else: + x_quant = None + + state.attn_residual = attn_residual + state.attn_post = post + state.attn_comb = comb + state.hidden_states_after_input_norm = hidden_states + state.attn_x_quant = x_quant + # mori's op_output slices final_hidden_states[:num_tokens]. + if get_moe_a2a_backend().is_mori(): + state.num_tokens = attn_residual.shape[0] + state.update( + dict( + forward_batch=forward_batch, + positions=positions, + tbo_subbatch_index=tbo_subbatch_index, + ) + ) + + def op_mhc_post_attn_pre_mlp(self, state): + # Close the attention mHC (hc_post), then open the FFN-side mHC pre + + # post-attention layernorm. Produces the 2D MoE input. + # + # Pop each boundary tensor from the state EXACTLY ONCE, up front, and + # reuse the locals for both the fused attempt and the non-fused + # fallback. apply_mhc_post_pre_boundary() returns None when it declines + # to fuse -- most importantly for the 0-token DP two-batch-overlap idle + # ubatch -- in which case control must fall through to the unfused + # hc_post. Popping in the fused call's arguments and again in the + # fallback would double-pop -> KeyError: 'hidden_states_after_attn' on + # every idle DP rank. use_fused_mhc_post_pre is on whenever the aiter + # gfx95 mHC path is available (is_cross_layer_mhc_fusion_enabled), so + # this fallback is reached under DP regardless of the TileLang env. + hidden_states_after_attn = state.pop("hidden_states_after_attn") + attn_residual = state.pop("attn_residual") + attn_post = state.pop("attn_post") + attn_comb = state.pop("attn_comb") + + residual, hidden_states, post, comb = _prepare_ffn_input( + self, + hidden_states_after_attn, + attn_residual, + attn_post, + attn_comb, + state.forward_batch, + ) + state.ffn_residual = residual + state.ffn_post = post + state.ffn_comb = comb + state.hidden_states_mlp_input = hidden_states + + def op_mhc_postprocess(self, state): + # Close the FFN mHC (hc_post) and emit the next layer's input dict. + hidden_states = self.hc_post( + state.pop("hidden_states_mlp_output"), + state.pop("ffn_residual"), + state.pop("ffn_post"), + state.pop("ffn_comb"), + ) + output = dict( + positions=state.positions, + hidden_states=hidden_states, + # DSV4 non-fused layers carry no residual across layers; the key is + # required by the next layer's op_mhc_prepare_attn (ignored). + residual=None, + forward_batch=state.forward_batch, + tbo_subbatch_index=state.tbo_subbatch_index, + ) + state.clear( + expect_keys={ + "positions", + "forward_batch", + "tbo_subbatch_index", + } + ) + return output + + def op_gather_a(self, state): + local = state.pop("hidden_states_mlp_input") + state.do_shared_local = shared_expert_runs_locally(self.mlp) + state.global_hidden, state.gather_event, state.shared_local = ( + _tbo_gather_moe_input( + self.mlp, + local, + state.forward_batch, + state.tbo_subbatch_index, + state.do_shared_local, + ) + ) + state.gather_keepalive = local + + def op_gather_b(self, state): + torch.cuda.current_stream().wait_event(state.pop("gather_event")) + # Compute now ordered after the gather -> the gather input is safe to + # release (freed on the compute stream, no record_stream deferral). + state.pop("gather_keepalive") + + def op_moe(self, state): + state.global_expert_out = _tbo_run_moe( + self, state.pop("global_hidden"), state.forward_batch, state.do_shared_local + ) + + def op_combine_a(self, state): + output = state.pop("global_expert_out") + state.local_out, state.combine_event = _tbo_combine_moe_output( + output, state.tbo_subbatch_index + ) + state.combine_keepalive = output + + def op_combine_b(self, state): + state.hidden_states_mlp_output = _tbo_wait_moe_output( + state.pop("local_out"), + state.pop("combine_event"), + state.pop("shared_local"), + ) + state.pop("combine_keepalive") + state.pop("do_shared_local") + + +class NonCudaDSparkMixin(NonCudaDecoderMixin): + """Draft stage execution with plain norms and its attention call.""" + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + prev_pre: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + if self.hc_pre_from_prev_sublayer: + return self._forward_hc_pre_from_prev( + positions, hidden_states, forward_batch, prev_pre + ) + residual = hidden_states + x, post, comb = self._hc_pre_block( + hidden_states, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base + ) + x = self.input_layernorm(x) + with self.self_attn.maybe_use_decode_attn_tp(forward_batch): + x = self.self_attn(positions, x, forward_batch) + x = self._hc_post_block(x, residual, post, comb) + + residual = x + x, post, comb = self._hc_pre_block( + x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base + ) + x = self.post_attention_layernorm(x) + x = self._run_ffn(x, forward_batch) + x = self._hc_post_block(x, residual, post, comb) + return x, None + + def _hc_pre_block( + self, + x: torch.Tensor, + hc_fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + y, post, comb, _ = self.hc_pre(x, hc_fn, hc_scale, hc_base) + return y, post, comb + + def _hc_post_block( + self, + x: torch.Tensor, + residual: torch.Tensor, + post: torch.Tensor, + comb: torch.Tensor, + ) -> torch.Tensor: + return self.hc_post(x, residual, post, comb) + + def _forward_hc_pre_from_prev( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + prev_pre: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + # The draft stages run the plain four steps: no fused post, no seams, so + # they need nothing from `bind_next` and never hand anything forward. + assert self.attn_hc is not None and self.ffn_hc is not None + stats_stream = None + if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states): + stats_stream = self.hc_stats_stream + + residual = hidden_states + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre)) + with self.self_attn.maybe_use_decode_attn_tp(forward_batch): + x = self.self_attn(positions, x, forward_batch) + attn_pre, attn_post, attn_comb = mhc.mix_stats( + self.attn_hc, hidden_states, stats_stream + ) + if stats_stream is not None: + torch.cuda.current_stream().wait_stream(stats_stream) + hidden_states = self.hc_post(x, residual, attn_post, attn_comb) + + residual = hidden_states + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre)) + x = self._run_ffn(x, forward_batch) + ffn_pre, ffn_post, ffn_comb = mhc.mix_stats( + self.ffn_hc, hidden_states, stats_stream + ) + if stats_stream is not None: + torch.cuda.current_stream().wait_stream(stats_stream) + hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb) + return hidden_states, ffn_pre + + def _run_ffn(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: + shape = x.shape + x = x.reshape(-1, self.dim) + y = self._run_moe_ffn_dp_sync( + x, forward_batch, input_ids=None, input_ids_global=None + ) + return y.view(shape) + + +def forward_layer_fused_boundary( + model, + i: int, + state, + *, + positions: torch.Tensor, + input_ids: torch.Tensor, + forward_batch, + input_ids_global: torch.Tensor, + capture_dspark: bool, +): + """Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next + layer's boundary launch unless that layer reads the residual stream first (Engram gate, + DSpark capture) or there is none; hidden_states is None while a post is pending.""" + nxt = i + 1 + defer_post = ( + nxt < model.end_layer + and model.layers[nxt].engram is None + and not (capture_dspark and nxt in model.dspark_layers_to_capture) + ) + # The boundary speaks its own (hidden_states, prev_pre, pending_post) triple; + # the loop speaks HcState. This is the only place the two meet. + from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState + + pending = isinstance(state.streams, HcPending) + hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary( + model.layers[i], + positions=positions, + hidden_states=None if pending else state.streams, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + prev_pre=state.pre, + pending_post=tuple(state.streams) if pending else None, + defer_post=defer_post, + ) + return HcState( + hidden_states if pending_post is None else HcPending(*pending_post), + prev_pre, + ) + + +def forward_hc_pre_from_prev_fused_boundary( + layer, + positions: torch.Tensor, + hidden_states: Optional[torch.Tensor], + input_ids: torch.Tensor, + forward_batch, + input_ids_global: torch.Tensor, + prev_pre: Optional[torch.Tensor], + pending_post: Optional[Tuple[torch.Tensor, ...]], + defer_post: bool, +) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]: + """ROCm fused attention and FFN hyper-connections. + pending_post is the previous layer's unapplied FFN hc_post (x, residual, post, + comb); with defer_post this layer's is returned the same way and hidden_states is None.""" + if pending_post is None: + pending_post = (None, hidden_states, None, None) + residual, x, attn_coefficients = make_amd_hc_boundary( + layer, + *pending_post, + prev_pre, + layer.hc_attn_fn, + layer.hc_attn_scale, + layer.hc_attn_base, + ) + # the boundary's reduce + sinkhorn rides in the norm launch + x, x_quant = layer._input_norm( + x, allow_aiter_quant=False, coefficients=attn_coefficients + ) + with layer.self_attn.maybe_use_decode_attn_tp(forward_batch): + mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch) + x = layer.self_attn( + x=x, + positions=positions, + forward_batch=forward_batch, + x_quant=x_quant, + defer_all_reduce=mhc is not None, + ) + if mhc is not None: + # wo_b honors the deferred reduce under attention_mhc_fusion's gates; + # fail loud rather than fall back to an unfused reduce. + assert isinstance(x, AttnOutput) + residual = apply_attention_mhc(x.partial, mhc) + x = None + residual, x, ffn_coefficients = make_amd_hc_boundary( + layer, + x, + residual, + attn_coefficients.post if x is not None else None, + attn_coefficients.comb if x is not None else None, + attn_coefficients.pre, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + ) + x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients) + mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch) + x = layer._run_moe_ffn_dp_sync( + x, + forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + return_moe_output=mhc is not None, + ) + ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors() + if mhc is not None: + # The MoE TP path honors the deferred output under moe_mhc_fusion's gates; + # fail loud rather than fall back to an unfused reduce. The fused post + # carries the reduce, so take the merged rows without one. + assert isinstance(x, MoEOutput) + # Reduction already applied post. The next boundary consumes this + # materialized residual, including when it would normally defer post. + return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None + if defer_post: + return None, ffn_pre, (x, residual, ffn_post, ffn_comb) + return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 40f4ea3501..862203e6f0 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -137,7 +137,7 @@ if _use_aiter_gfx95: logger = logging.getLogger(__name__) -# Matches DeepSeek-V4's _MHC_POST_MULT_VALUE; the fused and unfused boundaries +# Matches deepseek_v4_mhc.MHC_POST_MULT_VALUE; the fused and unfused boundaries # must agree on it. _MHC_POST_MULT_VALUE = 2.0 diff --git a/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py b/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py new file mode 100644 index 0000000000..3acf49b51c --- /dev/null +++ b/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py @@ -0,0 +1,153 @@ +"""CUDA dispatch, fake shapes and replay for the uncompleted dual-stream outputs.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.models import deepseek_v2 +from sglang.srt.runtime_context import get_forward +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b", runner_config="1-gpu-small") + + +class DualStreamOperand: + """Small CUDA computations isolate the custom-op boundary from expert kernels.""" + + _can_dual_stream_graph = deepseek_v2.DeepseekV2MoE._can_dual_stream_graph + + def __init__(self, replicated): + self.layer_id = 0 + self._shared_expert_tp1 = replicated + self.alt_stream = torch.cuda.Stream() + self.num_fused_shared_experts = 0 + self._enable_a2a_moe = self._fuse_shared_experts_inside_sbo = False + self.shared_experts = object() + self.is_hash = False + self.gate = SimpleNamespace(e_score_correction_bias_vl=None) + self.experts = SimpleNamespace( + use_flashinfer_trtllm_moe=True, + quant_method=None, + should_fuse_routed_scaling_factor_in_topk=False, + ) + self.calls = 0 + + def forward_normal_dual_stream(self, value, *, leave_output_sum): + assert leave_output_sum + assert get_forward().flashinfer_trtllm_bypass + assert not get_forward().defer_moe_finalize + self.calls += 1 + current = torch.cuda.current_stream() + self.alt_stream.wait_stream(current) + routed = torch.sin(value) * 0.25 + with torch.cuda.stream(self.alt_stream): + shared = torch.cos(value) * 0.75 + current.wait_stream(self.alt_stream) + return deepseek_v2.MoEOutput( + routed, shared, self.experts, 1.0, self._shared_expert_tp1 + ) + + +class TestDualStreamOutputs(CustomTestCase): + @classmethod + def setUpClass(cls): + if torch.version.cuda is None or not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA required") + + def test_dispatch_compile_and_replay_preserve_separate_replicated_shared(self): + for replicated in (False, True): + with self.subTest(replicated=replicated): + operand = DualStreamOperand(replicated) + value = torch.linspace(-0.5, 0.5, 128, device="cuda").reshape(2, 64) + context = SimpleNamespace(moe_fusions={0: operand}) + with ( + patch.object(deepseek_v2, "_enable_pcg_dsv2_dual_stream", True), + patch.object( + deepseek_v2, "is_in_breakable_cuda_graph", return_value=True + ), + patch.object( + deepseek_v2, + "get_moe_runner_backend", + return_value=SimpleNamespace(is_flashinfer_trtllm=lambda: True), + ), + patch.object( + deepseek_v2, + "get_exec", + return_value=SimpleNamespace( + moe=SimpleNamespace(enable_eplb=False) + ), + ), + patch.object( + deepseek_v2, + "get_tc_piecewise_forward_context", + return_value=context, + ), + patch( + "sglang.srt.layers.moe.mega_moe.should_use_mega_moe", + return_value=False, + ), + ): + selected = deepseek_v2.DeepseekV2MoE.forward( + operand, value, leave_output_sum=True + ) + self.assertIsInstance(selected, deepseek_v2.MergedMoEOutput) + self.assertEqual(operand.calls, 1) + if replicated: + torch.testing.assert_close( + selected.routed, torch.sin(value) * 0.25 + ) + torch.testing.assert_close( + selected.shared, torch.cos(value) * 0.75 + ) + else: + self.assertIsNone(selected.shared) + torch.testing.assert_close( + selected.routed, + torch.sin(value) * 0.25 + torch.cos(value) * 0.75, + ) + + def invoke(x): + return deepseek_v2.dsv4_flashinfer_moe_dual_stream_outputs( + x, 0, False, replicated + ) + + torch.library.opcheck( + invoke_op + := deepseek_v2.dsv4_flashinfer_moe_dual_stream_outputs, + (value, 0, False, replicated), + test_utils=("test_schema", "test_faketensor"), + ) + compiled = torch.compile(invoke, fullgraph=True) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + compiled(value) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + routed, shared = compiled(value) + count_after_capture = operand.calls + for offset in (0.2, -0.1, 0.7): + value.add_(offset) + graph.replay() + expected_routed, expected_shared = ( + torch.sin(value) * 0.25, + torch.cos(value) * 0.75, + ) + self.assertEqual(operand.calls, count_after_capture) + if replicated: + torch.testing.assert_close(routed, expected_routed) + torch.testing.assert_close(shared, expected_shared) + else: + self.assertEqual(shared.numel(), 0) + torch.testing.assert_close( + routed, expected_routed + expected_shared + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py index 6084fdd4c6..d54456a4c5 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py @@ -367,7 +367,6 @@ def test_mhc_refresh_without_deepgemm_prenorm(): base = torch.zeros(24, device="cuda") layer = _hopper_mhc_layer(norm, w, scale, base) layer._can_fuse_attn_mhc = layer._can_fuse_ffn_mhc = True - layer.local_boundary = layer.next_boundary = object() with ( override_platform(is_sm100=True, is_sm90=False), envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.override(True), @@ -381,7 +380,6 @@ def test_mhc_refresh_without_deepgemm_prenorm(): assert layer.attn_hc.bf16_parts is None assert "_can_fuse_attn_mhc" not in layer.__dict__ assert "_can_fuse_ffn_mhc" not in layer.__dict__ - assert layer.local_boundary is None and layer.next_boundary is None @pytest.mark.parametrize( diff --git a/test/registered/unit/layer_boundary/test_moe_output_reduction.py b/test/registered/unit/layer_boundary/test_moe_output_reduction.py index b515105b53..d71cf20a45 100644 --- a/test/registered/unit/layer_boundary/test_moe_output_reduction.py +++ b/test/registered/unit/layer_boundary/test_moe_output_reduction.py @@ -1,15 +1,12 @@ """When a MoE block all-reduces its own output, and why it does not.""" -import ast import contextlib import types import unittest -from pathlib import Path from unittest.mock import patch import torch -import sglang from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.utils import ( post_experts_output_is_complete, @@ -22,8 +19,6 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") -MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" - def a2a(name=None): names = ("flashinfer", "pplx", "flashinfer_megamoe") @@ -166,84 +161,180 @@ class TestReplicatedMoeOutput(CustomTestCase): class TestModelsWithExplicitDpCompletion(CustomTestCase): def test_direct_dp_exits_publish_their_selected_sum(self): - # Execute the real orchestration methods without constructing weights. - # DeepSeek V4 owns its DP exit instead of using a stage boundary. - import __future__ - - filename, method = "deepseek_v4.py", "_run_moe_ffn_dp_sync" - path = MODELS_DIR / filename - node = next( - n - for n in ast.walk(ast.parse(path.read_text())) - if isinstance(n, ast.FunctionDef) and n.name == method + from sglang.srt.layers import dp_attention + from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis + from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream + from sglang.srt.layers.layer_boundary.row_transport import ( + TransportExit, + TransportRows, ) - for use_rsv in (False, True): - with self.subTest(model=filename, reduce_scatterv=use_rsv): - trace = [] + from sglang.srt.models import deepseek_v4 + from sglang.srt.models import deepseek_v4_boundary as boundary + from sglang.srt.models.deepseek_common import moe_transport - def rsv(value, *, output, sizes): - trace.append("RSv") - output.copy_(value[:2] * 2) + for non_cuda in (False, True): + for use_rsv in (False, True): + with self.subTest(non_cuda=non_cuda, reduce_scatterv=use_rsv): + trace = [] - def scatter(output, value, batch): - trace.append("slice") - output.copy_(value[:2]) + def rsv(value, *, output, sizes): + trace.append("RSv") + output.copy_(value[:2] * 2) - group = types.SimpleNamespace(reduce_scatterv=rsv) - parallel = types.SimpleNamespace( - attn_dp_size=2, attn_tp_size=1, tp_size=2, tp_group=group - ) - namespace = dict( - torch=torch, - get_forward=get_forward, - get_parallel=lambda: parallel, - get_moe_a2a_backend=lambda: a2a(), - should_use_dp_reduce_scatterv=lambda: use_rsv, - is_dp_gatherv_active=lambda: False, - is_cp_active=lambda batch: False, - envs=types.SimpleNamespace( - SGLANG_DP_USE_REDUCE_SCATTER=types.SimpleNamespace( - get=lambda: False + def scatter(output, value, batch): + trace.append("slice") + output.copy_(value[:2]) + + def reduce(value): + self.assertFalse(get_forward().mlp_reduce_scatter) + trace.append("AR") + return value * 2 + + def compute(value, batch, **kwargs): + self.assertEqual(get_forward().mlp_reduce_scatter, use_rsv) + return reduce(value) if non_cuda and not use_rsv else value + + group = types.SimpleNamespace(reduce_scatterv=rsv) + parallel = types.SimpleNamespace( + attn_dp_size=2, attn_tp_size=1, tp_size=2, tp_group=group + ) + model = types.SimpleNamespace( + dsa_enable_prefill_cp=False, + boundary_mhc_fusions=False, + config=types.SimpleNamespace(model_type="deepseek_v4"), + mlp=compute, + ) + batch = types.SimpleNamespace( + dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: True) + ) + rows = TransportRows( + Layout(frozenset({TokenAxis.ATTN_DP})), + Layout(frozenset()), + Layout(frozenset({TokenAxis.ATTN_DP})), + SumGroup.MOE_OUTPUT, + ) + with contextlib.ExitStack() as patches: + patches.enter_context( + get_forward().scoped(mlp_reduce_scatter=False) ) - ), - _SHARED_EXPERT_LOCAL=False, - _is_hip=False, - nullcontext=contextlib.nullcontext, - get_global_dp_buffer=lambda g: torch.empty(4, 3), - get_local_dp_buffer=lambda g: torch.empty(2, 3), - get_dp_global_num_tokens=lambda: [2, 2], - dp_gather_replicate=lambda output, value, batch: output.fill_(1), - dp_scatter=scatter, - ) - exec( - compile( - ast.Module(body=[node], type_ignores=[]), - str(path), - "exec", - flags=__future__.annotations.compiler_flag, - ), - namespace, - ) - model = types.SimpleNamespace( - dsa_enable_prefill_cp=False, - mlp=lambda value, batch, **kwargs: reduce_moe_output(value), - ) - batch = types.SimpleNamespace( - dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: True) - ) - kwargs = dict(input_ids=None, input_ids_global=None) - with ( - moe_config(), - get_forward().scoped(mlp_reduce_scatter=False), - patch( - "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", - side_effect=lambda value: trace.append("AR") or value * 2, - ), - ): - output = namespace[method](model, torch.ones(2, 3), batch, **kwargs) - self.assertFalse(get_forward().mlp_reduce_scatter) - self.assertEqual(trace, ["RSv"] if use_rsv else ["AR", "slice"]) - torch.testing.assert_close(output, torch.full((2, 3), 2.0)) + for module in (moe_transport, boundary): + patches.enter_context( + patch.object( + module, "get_parallel", return_value=parallel + ) + ) + for context in ( + patch.object( + moe_transport, "get_moe_a2a_backend", return_value=a2a() + ), + patch.object(moe_transport, "SHARED_EXPERT_LOCAL", False), + patch.object( + moe_transport, "is_cp_active", return_value=False + ), + patch.object( + moe_transport, + "is_dp_gatherv_active", + return_value=False, + ), + patch.object( + moe_transport, + "should_use_dp_reduce_scatterv", + return_value=use_rsv, + ), + patch.object( + moe_transport, + "get_global_dp_buffer", + side_effect=lambda g: torch.empty(4, 3), + ), + patch.object( + moe_transport, + "get_local_dp_buffer", + side_effect=lambda g: torch.empty(2, 3), + ), + patch.object( + moe_transport, + "get_dp_global_num_tokens", + return_value=[2, 2], + ), + patch.object( + moe_transport, + "dp_gather_replicate", + side_effect=lambda out, value, batch: out.fill_(1), + ), + patch.object( + moe_transport, "dp_scatter", side_effect=scatter + ), + patch( + "sglang.srt.layers.moe.should_use_dp_reduce_scatterv", + return_value=use_rsv, + ), + deepseek_v4.envs.SGLANG_DP_USE_REDUCE_SCATTER.override( + False + ), + patch.object( + dp_attention, "is_dp_gatherv_active", return_value=False + ), + patch.object( + dp_attention, + "get_global_dp_buffer", + side_effect=lambda g: torch.empty(4, 3), + ), + patch.object( + dp_attention, + "get_local_dp_buffer", + side_effect=lambda g: torch.empty(2, 3), + ), + patch.object( + dp_attention, + "get_dp_global_num_tokens", + return_value=[2, 2], + ), + patch.object( + dp_attention, "dp_scatter", side_effect=scatter + ), + patch( + "sglang.srt.layers.moe.post_experts_all_reduce", + side_effect=reduce, + ), + ): + patches.enter_context(context) + if non_cuda: + output = ( + deepseek_v4.NonCudaDecoderMixin._run_moe_ffn_dp_sync( + model, + torch.ones(2, 3), + batch, + input_ids=None, + input_ids_global=None, + ) + ) + else: + transport = boundary.MoeTransport( + model, + torch.ones(2, 3), + batch, + boundary.LayerCall(None), + rows, + ) + stage = types.SimpleNamespace( + plan=types.SimpleNamespace( + path_for=lambda b: types.SimpleNamespace( + output=types.SimpleNamespace(update=PLAIN_ADD), + exit=types.SimpleNamespace( + writes_residual=False + ), + ) + ) + ) + batch.residual_stream = ResidualStream(torch.ones(2, 3)) + batch.residual_stream.transport = (stage, transport) + with TransportExit(stage, batch) as completion: + value = model.mlp(transport.input, batch) + output = completion.finish(value) + self.assertFalse(get_forward().mlp_reduce_scatter) + self.assertEqual(trace, ["RSv"] if use_rsv else ["AR", "slice"]) + torch.testing.assert_close(output, torch.full((2, 3), 2.0)) if __name__ == "__main__": diff --git a/test/registered/unit/layer_boundary/test_producer_completion.py b/test/registered/unit/layer_boundary/test_producer_completion.py index 03c670d079..a78059ca10 100644 --- a/test/registered/unit/layer_boundary/test_producer_completion.py +++ b/test/registered/unit/layer_boundary/test_producer_completion.py @@ -371,6 +371,7 @@ class TestProducerCompletion(CustomTestCase): def test_neighbour_facts_do_not_retain_layer_state(self): import test_kimi_k3_stage_kernels as k3_fixture + from sglang.srt.layers.layer_boundary.row_transport import RowTransport from sglang.srt.models import kimi_k3 seen = set() @@ -416,6 +417,7 @@ class TestProducerCompletion(CustomTestCase): update=update, output_transform=OutputTransform(update.layer.forward), sum_impl=SumImplementation(update.layer.forward), + row_transport=RowTransport({}, bind=update.layer.forward), ) facts = facts_of(declaration) self.assertIsInstance(facts.read, ReadFacts) diff --git a/test/registered/unit/layer_boundary/test_residual_stream.py b/test/registered/unit/layer_boundary/test_residual_stream.py index 89f987f3c1..5fa8861c30 100644 --- a/test/registered/unit/layer_boundary/test_residual_stream.py +++ b/test/registered/unit/layer_boundary/test_residual_stream.py @@ -68,7 +68,7 @@ class TestResidualStream(CustomTestCase): steps = StagePath(EntryPath(boundary.prepare, rows), OutputContract(rows), None) stage = StageBoundary( SimpleNamespace(norm=None), - declaration=SimpleNamespace(kind=StageKind.ATTENTION), + declaration=SimpleNamespace(kind=StageKind.ATTENTION, row_transport=None), ) value = torch.ones(2, 4) hidden, stream = stage._prepare(value, ResidualStream(), None, steps) @@ -115,7 +115,7 @@ class TestResidualStream(CustomTestCase): steps = StagePath(EntryPath(boundary.prepare, rows), OutputContract(rows), None) stage = StageBoundary( SimpleNamespace(norm=None), - declaration=SimpleNamespace(kind=StageKind.ATTENTION), + declaration=SimpleNamespace(kind=StageKind.ATTENTION, row_transport=None), ) for increment in (2, 7): with self.subTest(increment=increment): @@ -419,7 +419,7 @@ class TestResidualStream(CustomTestCase): steps = StagePath(EntryPath(boundary.prepare, rows), OutputContract(rows), None) stage = StageBoundary( SimpleNamespace(norm=None), - declaration=SimpleNamespace(kind=StageKind.ATTENTION), + declaration=SimpleNamespace(kind=StageKind.ATTENTION, row_transport=None), ) partial = torch.ones(4, 4) partial_ref = weakref.ref(partial) diff --git a/test/registered/unit/layer_boundary/test_row_transport.py b/test/registered/unit/layer_boundary/test_row_transport.py new file mode 100644 index 0000000000..1acff3c847 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_row_transport.py @@ -0,0 +1,185 @@ +"""Compute rows move independently of the source residual and finish once.""" + +import unittest +from contextlib import contextmanager +from types import SimpleNamespace + +import test_declared_decoder_boundary as fixture +import torch + +from sglang.srt.layers.layer_boundary import ( + append_stages, + declare_attn, + declare_ffn, + layer_stack, +) +from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.layout import Layout, TokenAxis +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD, PlainAdd +from sglang.srt.layers.layer_boundary.row_transport import ( + ComputeTransport, + RowTransport, + TransportRows, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class GatheredOperand(ComputeTransport): + def __init__(self, source, events, rows): + self.rows = rows + self.input = torch.cat((source, source + 7)) + self.events = events + + @contextmanager + def scope(self): + self.events.append("enter") + try: + yield + finally: + self.events.append("exit") + + def return_output(self, output): + self.events.append("sum-return") + left, right = output.chunk(2) + return left + right + + +class TestRowTransport(unittest.TestCase): + def build(self, events, *, in_token_order=False, update=PLAIN_ADD): + parallel = fixture.parallel_of(attn_dp=2, attn_tp=1, pp_size=1) + source = Layout.sharded_over( + TokenAxis.ATTN_DP, axis_sizes={TokenAxis.ATTN_DP: 2} + ) + rows = TransportRows(source, Layout(frozenset()), source) + transport = RowTransport( + { + variant: rows + for variant in BatchVariant + if variant is not BatchVariant.SEQUENCE_PARALLEL + }, + lambda x, batch, context, rows, overlapped: GatheredOperand( + x, events, rows + ), + ) + with fixture.planning(parallel), layer_stack(): + (stage,) = append_stages( + ( + declare_attn(row_transport=transport, in_token_order=in_token_order) + if in_token_order + else declare_ffn(row_transport=transport, update=update), + fixture.Norm(), + ) + ) + return parallel, stage + + def test_transport_rejects_unhandled_token_order_contract(self): + with self.assertRaisesRegex(NotImplementedError, "another output contract"): + self.build([], in_token_order=True) + + def batch(self): + return SimpleNamespace( + input_ids=torch.tensor([1, 2]), + forward_mode=ForwardMode.DECODE, + residual_stream=None, + dp_padding_mode=None, + ) + + def test_operand_moves_after_read_and_residual_stays_on_source_rows(self): + events = [] + parallel, stage = self.build(events) + batch = self.batch() + source = torch.tensor([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]]) + residual_batch.start(batch) + with fixture.planning(parallel): + operand = stage.prepare(source, batch) + self.assertIs(batch.residual_stream.residual, source) + torch.testing.assert_close(operand, torch.cat((source * 2, source * 2 + 7))) + with stage.exit(batch) as output: + hidden = output.finish(operand * 3) + with self.assertRaisesRegex(RuntimeError, "already consumed"): + output.finish(operand) + torch.testing.assert_close(hidden, source * 12 + 21) + self.assertIs(batch.residual_stream.pending.update, PLAIN_ADD) + self.assertIsNone(batch.residual_stream.transport) + self.assertEqual(events, ["enter", "sum-return", "exit"]) + + def test_transport_binds_update_context_once_for_finish_and_exit(self): + for route in ("finish", "exit"): + for at_exit in (False, True): + bindings = [] + applied = [] + + class BoundUpdate(PlainAdd): + applied_at_exit = at_exit + + def __init__(self, context): + self.context = context + + def update(self, value, residual): + applied.append(self.context) + return super().update(value, residual) + + class Update(PlainAdd): + applied_at_exit = at_exit + + def bind(self, batch, context): + assert context is not None, "missing update context" + bindings.append(context) + return BoundUpdate(context) + + parallel, stage = self.build([], update=Update()) + for context in (17, 29): + with self.subTest(route=route, at_exit=at_exit, context=context): + batch = self.batch() + residual_batch.start(batch) + source = torch.ones(2, 4) + with fixture.planning(parallel): + operand = stage.prepare(source, batch) + if route == "finish": + result = stage.finish( + operand * 3, batch, update_context=context + ) + else: + with stage.exit( + batch, update_context=context + ) as output: + result = output.finish(operand * 3) + if at_exit: + self.assertEqual(applied[-1], context) + else: + self.assertEqual( + batch.residual_stream.pending.update.context, + context, + ) + result = residual_batch.final_norm( + result, batch, fixture.Norm() + ) + torch.testing.assert_close(result, source * 26 + 42) + self.assertIsNone(batch.residual_stream) + self.assertEqual(bindings, [17, 29]) + + def test_successful_scope_requires_finish_and_error_releases_invocation(self): + for fails in (False, True): + with self.subTest(compute_error=fails): + events = [] + parallel, stage = self.build(events) + batch = self.batch() + residual_batch.start(batch) + with fixture.planning(parallel): + stage.prepare(torch.ones(2, 4), batch) + with self.assertRaisesRegex( + RuntimeError, "compute failed" if fails else "without finish" + ): + with stage.exit(batch): + if fails: + raise RuntimeError("compute failed") + self.assertIsNone(batch.residual_stream.transport) + self.assertEqual(events, ["enter", "exit"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_deepseek_moe_transport.py b/test/registered/unit/models/test_deepseek_moe_transport.py new file mode 100644 index 0000000000..2e25c46029 --- /dev/null +++ b/test/registered/unit/models/test_deepseek_moe_transport.py @@ -0,0 +1,117 @@ +"""DeepSeek MoE transport isolates overlapping microbatches.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup +from sglang.srt.layers.layer_boundary.row_transport import TransportRows +from sglang.srt.models.deepseek_common import moe_transport +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestOverlappedTransport(unittest.TestCase): + def test_interleaved_microbatches_keep_events_and_buffers_separate(self): + trace = [] + compute = SimpleNamespace( + wait_event=lambda event: trace.append(("wait", event.key)) + ) + comm = SimpleNamespace(wait_stream=lambda stream: trace.append(("fork",))) + buffers = {} + + def buffer(key, n, h, dtype, device): + value = torch.empty(n, h, dtype=dtype, device=device) + buffers[key] = value + return value + + def gather(out, value, batch): + trace.append(("gather", batch.sub)) + out.copy_(torch.cat((value, value + 10))) + + def event(key): + return SimpleNamespace( + key=key, record=lambda stream: trace.append(("record", key)) + ) + + def combine(out, value, sizes, *, event_key): + trace.append(("combine", event_key)) + out.copy_(value[: out.shape[0]] * 2) + return event(event_key) + + layer = SimpleNamespace(mlp=SimpleNamespace()) + inputs = (torch.arange(6.0).view(2, 3), torch.tensor([[7.0, 11.0, 13.0]])) + batches = [SimpleNamespace(sub=i) for i in range(2)] + lengths = [4, 2] + with ( + patch( + "sglang.srt.models.deepseek_common.moe_transport.get_moe_a2a_backend", + return_value=SimpleNamespace(is_none=lambda: True), + ), + patch( + "sglang.srt.models.deepseek_common.moe_transport.SHARED_EXPERT_LOCAL", + False, + ), + patch.object(torch.cuda, "current_stream", return_value=compute), + patch.object( + torch.cuda, "stream", side_effect=lambda stream: nullcontext() + ), + patch.object(moe_transport, "get_dp_tbo_comm_stream", return_value=comm), + patch.object( + moe_transport, "get_tbo_persistent_buffer", side_effect=buffer + ), + patch.object( + moe_transport, "get_global_dp_buffer_len", side_effect=lengths + ), + patch.object(moe_transport, "get_local_dp_buffer_len", side_effect=[2, 1]), + patch.object( + moe_transport, "get_dp_global_num_tokens", side_effect=[[2, 2], [1, 1]] + ), + patch.object(moe_transport, "dp_gather_partial", side_effect=gather), + patch.object(moe_transport, "_tbo_event", side_effect=event), + patch.object( + moe_transport, "dp_reduce_scatterv_async", side_effect=combine + ), + ): + calls = [ + moe_transport.OverlappedMoeTransport( + layer.mlp, + x, + batch, + i, + TransportRows( + Layout(frozenset()), + Layout(frozenset()), + Layout(frozenset()), + SumGroup.MOE_OUTPUT, + ), + ) + for i, (x, batch) in enumerate(zip(inputs, batches)) + ] + for call in calls: + with self.assertRaisesRegex(RuntimeError, "combine wait"): + call.prepare_output(call.input) + call.issue_gather() + for call in calls: + x = call.wait_gather() + call.issue_combine(x * 3) + for call, expected in zip(calls, inputs): + output = call.wait_combine() + torch.testing.assert_close(call.prepare_output(output), expected * 6) + self.assertIsNone(call.gather_keepalive) + self.assertIsNone(call.combine_keepalive) + self.assertIsNone(call.global_hidden) + with self.assertRaisesRegex(RuntimeError, "not issued"): + call.wait_combine() + self.assertEqual( + [x for x in trace if x[0] == "gather"], [("gather", 0), ("gather", 1)] + ) + self.assertIsNot(buffers[("gh", 0)], buffers[("gh", 1)]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py b/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py index 6897db6454..83d84d8f78 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py @@ -196,14 +196,14 @@ class TestAmdFusedMhcAttnBoundaryFallback(unittest.TestCase): layer's deferred post was never applied and every subsequent layer computed on corrupted activations. - Drives the real ``DeepseekV4DecoderLayer.forward`` on a mocked layer with the + Drives the real ``NonCudaDecoderMixin.forward`` on a mocked layer with the fused dispatcher forced to ``None`` and halts at ``self_attn`` via a sentinel, so only the boundary fallback executes. """ def test_fallback_closes_previous_post_before_pre(self): try: - from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + from sglang.srt.models.deepseek_v4_platform_stage import NonCudaDecoderMixin except Exception as e: # pragma: no cover - env without full model deps self.skipTest(f"deepseek_v4 import unavailable: {e}") @@ -224,12 +224,12 @@ class TestAmdFusedMhcAttnBoundaryFallback(unittest.TestCase): with ( mock.patch( - "sglang.srt.models.deepseek_v4.apply_mhc_post_pre_boundary", + "sglang.srt.models.deepseek_v4_platform_stage.apply_mhc_post_pre_boundary", return_value=None, ), self.assertRaises(_StopForward), ): - DeepseekV4DecoderLayer.forward( + NonCudaDecoderMixin.forward( layer, positions=object(), hidden_states=hs_in, @@ -263,14 +263,17 @@ class TestAmdFusedMhcNormFusedHandling(unittest.TestCase): silently corrupting every subsequent layer. The fix mirrors the unfused ``hc_pre`` branch: apply the input layernorm when ``norm_fused`` is False. - Drives the real ``DeepseekV4DecoderLayer.forward`` with the boundary forced to + Drives the real ``NonCudaDecoderMixin.forward`` with the boundary forced to return ``norm_fused=False`` and halts at ``self_attn`` via a sentinel. """ def test_fused_success_applies_input_layernorm_when_not_norm_fused(self): try: import sglang.srt.models.deepseek_v4 as deepseek_v4 - from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + from sglang.srt.models.deepseek_v4 import ( + DeepseekV4DecoderLayer, + ) + from sglang.srt.models.deepseek_v4_platform_stage import NonCudaDecoderMixin except Exception as e: # pragma: no cover - env without full model deps self.skipTest(f"deepseek_v4 import unavailable: {e}") @@ -297,14 +300,13 @@ class TestAmdFusedMhcNormFusedHandling(unittest.TestCase): with ( mock.patch.object(deepseek_v4, "_is_hip", False), mock.patch.object(deepseek_v4, "_use_aiter", False), - mock.patch.object(deepseek_v4, "_is_gfx95_supported", False), mock.patch( - "sglang.srt.models.deepseek_v4.apply_mhc_post_pre_boundary", + "sglang.srt.models.deepseek_v4_platform_stage.apply_mhc_post_pre_boundary", return_value=(residual, fused_hs, post, comb, False), ), self.assertRaises(_StopForward), ): - DeepseekV4DecoderLayer.forward( + NonCudaDecoderMixin.forward( layer, positions=object(), hidden_states=object(), diff --git a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py index f035cf4791..2cf1f45573 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py @@ -176,7 +176,7 @@ def _replay_fused_and_unfused(group, run, check, *, residual, pre): ) def test_model_handoff_and_graph_replay(group, rows, verify): from sglang.srt.model_executor.forward_batch_info import ForwardMode - from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + from sglang.srt.models.deepseek_v4_platform_stage import ( forward_hc_pre_from_prev_fused_boundary, ) @@ -262,15 +262,18 @@ def _moe_stub(rank, *, dual, shared_tp1): def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): from sglang.srt.layers.moe import MoeA2ABackend from sglang.srt.model_executor.forward_batch_info import ForwardMode - from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + from sglang.srt.models.deepseek_v4 import ( + DeepseekV4DecoderLayer, + DeepseekV4PlatformDecoderLayer, + ) + from sglang.srt.models.deepseek_v4_platform_stage import ( forward_hc_pre_from_prev_fused_boundary, ) - from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer layer = _layer(group) layer.mlp = _moe_stub(group.rank_in_group, dual=dual, shared_tp1=shared_tp1) layer._run_moe_ffn_dp_sync = lambda *a, **kw: ( - DeepseekV4DecoderLayer._run_moe_ffn_dp_sync(layer, *a, **kw) + DeepseekV4PlatformDecoderLayer._run_moe_ffn_dp_sync(layer, *a, **kw) ) layer.hc_post = lambda *a: DeepseekV4DecoderLayer.hc_post(layer, *a) residual = torch.randn(rows, 4, 5120, device="cuda", dtype=torch.bfloat16) @@ -298,7 +301,7 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): "sglang.srt.layers.moe.get_moe_a2a_backend", return_value=MoeA2ABackend.NONE ), patch( - "sglang.srt.models.deepseek_v4.get_moe_a2a_backend", + "sglang.srt.models.deepseek_common.moe_transport.get_moe_a2a_backend", return_value=MoeA2ABackend.NONE, ), _no_deterministic_inference(), diff --git a/test/registered/unit/models/test_deepseek_v4_boundary.py b/test/registered/unit/models/test_deepseek_v4_boundary.py new file mode 100644 index 0000000000..94fe68b27e --- /dev/null +++ b/test/registered/unit/models/test_deepseek_v4_boundary.py @@ -0,0 +1,254 @@ +"""mHC consumer ownership, replicated shared add, and source row contracts.""" + +import unittest +import weakref +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup +from sglang.srt.layers.layer_boundary.row_transport import TransportRows +from sglang.srt.models import deepseek_v4_boundary as boundary +from sglang.srt.models import deepseek_v4_mhc as mhc +from sglang.srt.models.deepseek_common import moe_transport +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestHcConsumer(unittest.TestCase): + def test_dspark_tap_hands_materialized_stream_to_next_layer(self): + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models import deepseek_v4 + + class Layer(torch.nn.Module): + engram = None + boundary_mhc_fusions = False + boundary_pre_from_prev = True + hc_cfg = mhc.HcConfig(2, 4, 1e-5, 1e-6, 4, True, False) + + def __init__(self, value): + super().__init__() + self.value = value + + def forward(self, hidden_states, forward_batch, **kwargs): + stream = residual_batch.stream_of(forward_batch) + stream.check(hidden_states) + update = boundary.HcUpdate(self, False) + update.post = lambda value, state, **kw: mhc.HcState( + state.residual + value, torch.ones(3, 2) + ) + return stream.record(torch.full((3, 2, 4), self.value), update) + + model = deepseek_v4.DeepseekV4Model.__new__(deepseek_v4.DeepseekV4Model) + torch.nn.Module.__init__(model) + model.layers = torch.nn.ModuleList([Layer(1.0), Layer(2.0)]) + model.start_layer, model.end_layer = 0, 2 + model.pp_group = SimpleNamespace(world_size=1) + model.engram_hasher = model.late_layer_start = None + model.dspark_layers_to_capture = [1] + positions = torch.arange(3) + batch = SimpleNamespace( + positions=positions, + input_ids=positions, + forward_mode=ForwardMode.EXTEND, + residual_stream=None, + ) + source = torch.arange(24.0).reshape(3, 2, 4) + aux = [] + recorder = SimpleNamespace(with_current_layer=lambda _: nullcontext()) + with ( + patch.object(deepseek_v4, "_is_cuda", True), + patch.object(deepseek_v4, "_is_hip", False), + patch.object(deepseek_v4, "is_cp_active", return_value=False), + patch.object(deepseek_v4, "check_cuda_graph_backend", return_value=False), + patch.object( + deepseek_v4, + "get_global_expert_distribution_recorder", + return_value=recorder, + ), + ): + output, pre, tail = model._forward_layers_hc_pre_from_prev( + positions, source, batch, positions, positions, True, aux + ) + torch.testing.assert_close(output, source + 3) + torch.testing.assert_close(aux[0], (source + 1).mean(1)) + torch.testing.assert_close(pre, torch.ones(3, 2)) + self.assertIsNone(tail) + self.assertIsNone(batch.residual_stream) + + def test_replicated_shared_is_added_after_the_sum(self): + routed = torch.tensor([[1.0, 1.0]], dtype=torch.bfloat16) + other = torch.tensor([[-1.0, -0.5]], dtype=torch.bfloat16) + shared = torch.tensor([[0.01, 0.003]], dtype=torch.bfloat16) + pieces = SimpleNamespace( + get_merged=lambda: routed, shared=shared, shared_is_replicated=True + ) + with ( + patch( + "sglang.srt.layers.moe.post_experts_all_reduce", + side_effect=lambda x: x + other, + ) as reduce, + patch( + "sglang.srt.layers.moe.utils.should_add_replicated_moe_output", + return_value=True, + ), + ): + output = boundary.MoeFinalize(pieces).complete() + reduce.assert_called_once_with(routed) + torch.testing.assert_close(output, (routed + other) + shared, atol=0, rtol=0) + self.assertFalse(torch.equal(output, (routed + shared) + other)) + + +class TestAttentionCompletion(unittest.TestCase): + def test_projection_and_deferred_sum_preserve_full_precision_policy(self): + from sglang.srt.layers import linear + from sglang.srt.layers.moe import utils as moe_utils + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models import deepseek_v4 + from sglang.srt.runtime_context import get_forward + + group = SimpleNamespace(all_reduce=Mock(side_effect=lambda value: value * 2)) + parallel = SimpleNamespace(tp_size=4, tp_group=group, attn_tp_group=group) + + class Projection: + tp_group = group + input_is_parallel = True + bias = None + skip_bias_add = False + reduce_results = False + use_decode_attn_tp = False + use_dp_attention_reduce = False + parallel_group = "attn_tp" + quant_method = SimpleNamespace(apply=lambda layer, value, bias: value + 1) + + def __call__(self, value, **kwargs): + return linear.RowParallelLinear.forward(self, value, **kwargs) + + attention = deepseek_v4.MQALayer.__new__(deepseek_v4.MQALayer) + torch.nn.Module.__init__(attention) + attention.attn_tp_size = 2 + attention.wo_b = Projection() + layer = SimpleNamespace(boundary_mhc_fusions=False) + value = torch.arange(12.0).view(3, 4) + rows = TransportRows( + Layout(frozenset()), + Layout(frozenset()), + Layout(frozenset()), + SumGroup.ATTN_TP, + ) + for quant in (False, True): + for mode in (ForwardMode.EXTEND, ForwardMode.DECODE): + batch = SimpleNamespace(forward_mode=mode) + with ( + self.subTest(quant=quant, mode=mode), + get_forward().scoped(sp_active=False), + patch.object(linear, "get_group_rank_size", return_value=(0, 2)), + patch.object(linear, "get_parallel", return_value=parallel), + patch.object( + linear, + "get_exec", + return_value=SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=quant) + ), + ), + patch.object( + linear, + "use_symmetric_memory", + side_effect=lambda *a, **k: nullcontext(), + ), + patch.object(linear, "is_allocation_symmetric", return_value=False), + patch.object( + linear, "should_skip_mlp_all_reduce", return_value=False + ), + patch.object( + linear, + "tensor_model_parallel_all_reduce", + side_effect=group.all_reduce, + ), + patch.object( + linear, + "tensor_model_parallel_quant_all_reduce", + side_effect=AssertionError("unexpected quantized sum"), + ), + patch.object( + moe_utils, "should_skip_mlp_all_reduce", return_value=False + ), + patch.object(deepseek_v4, "get_parallel", return_value=parallel), + patch.object(boundary, "_sum_group", return_value=group), + patch.object( + deepseek_v4, "attn_tp_all_reduce", side_effect=group.all_reduce + ), + patch.object( + boundary, + "get_cp_decode_attn_tp_ctx", + return_value=SimpleNamespace(use_decode_attn_tp=False), + ), + ): + # The production projection never passes forward_batch to + # RowParallelLinear, so this flag does not quantize its sum. + with patch.object(deepseek_v4, "_is_cuda", False): + immediate = attention._project_wo_b( + value, defer_all_reduce=True + ) + with patch.object(deepseek_v4, "_is_cuda", True): + deferred = attention._project_wo_b(value, defer_all_reduce=True) + transport = boundary.AttentionTransport( + layer, value, batch, None, rows + ) + completed = transport.prepare_output(deferred).complete() + torch.testing.assert_close(completed, immediate, atol=0, rtol=0) + torch.testing.assert_close(completed, (value + 1) * 2, atol=0, rtol=0) + + +class TestMoeSourceRows(unittest.TestCase): + def test_cp_operand_is_released_before_return_and_v41_uses_contiguous_rows(self): + from sglang.srt.layers.layer_boundary.layout import TokenAxis + + layer = SimpleNamespace(boundary_mhc_fusions=False, mlp=SimpleNamespace()) + rows = TransportRows( + Layout.sharded_over(TokenAxis.ATTN_CP, axis_sizes={TokenAxis.ATTN_CP: 2}), + Layout.sharded_over(axis_sizes={TokenAxis.ATTN_CP: 2}), + Layout.sharded_over(TokenAxis.ATTN_CP, axis_sizes={TokenAxis.ATTN_CP: 2}), + SumGroup.MOE_OUTPUT, + ) + backend = SimpleNamespace(is_none=lambda: True) + group = SimpleNamespace(all_reduce=lambda value: value * 2) + parallel = SimpleNamespace(attn_cp_size=2, attn_cp_rank=1, tp_group=group) + batch = SimpleNamespace(dp_padding_mode=None) + with ( + patch.object(moe_transport, "get_moe_a2a_backend", return_value=backend), + patch.object(boundary, "get_moe_a2a_backend", return_value=backend), + patch.object(moe_transport, "get_parallel", return_value=parallel), + patch.object( + moe_transport, + "attn_cp_interleave_gather", + side_effect=lambda value: torch.cat((value, value)), + ), + patch.object( + moe_transport, + "attn_cp_interleave_reduce_scatter", + side_effect=lambda value: value[1::2] * 2, + ) as interleave, + ): + for kind in ("deepseek_v4", "deepseek_v41"): + layer.config = SimpleNamespace(model_type=kind) + call = boundary.MoeTransport( + layer, torch.ones(2, 3), batch, boundary.LayerCall(), rows + ) + reference = weakref.ref(call.input) + value = torch.arange(12.0).reshape(4, 3) + output = call.prepare_output(value) + self.assertIsNone(reference()) + result = call.return_output(output) + expected = value[1::2] if kind == "deepseek_v4" else value[2:] + torch.testing.assert_close(result, expected * 2) + call.release() + interleave.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_deepseek_v4_pipeline.py b/test/registered/unit/models/test_deepseek_v4_pipeline.py new file mode 100644 index 0000000000..9ffe2bc7ad --- /dev/null +++ b/test/registered/unit/models/test_deepseek_v4_pipeline.py @@ -0,0 +1,190 @@ +"""Hyper-connection stage construction and completion across pipeline ranks.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.models import deepseek_v4_boundary as boundary +from sglang.srt.models import deepseek_v4_mhc as mhc +from sglang.srt.models.deepseek_common import moe_transport +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestPipelineCompletion(unittest.TestCase): + def test_real_stages_write_mhc_once_before_pipeline_handoff(self): + from sglang.srt.layers.layer_boundary import append_stages, layer_stack + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models import deepseek_v4 + from sglang.srt.models.deepseek_v2 import MoEOutput + from sglang.test import boundary_fixtures as fixture + + for lagged in (False, True): + for fused in (False, True): + with self.subTest(lagged=lagged, fused=fused): + config = SimpleNamespace( + hc_pre_from_prev_sublayer=lagged, + model_type="deepseek_v41" if lagged else "deepseek_v4", + ) + layer = SimpleNamespace( + config=config, + boundary_pre_from_prev=lagged, + boundary_mhc_fusions=fused, + use_fused_mhc_post_pre=False, + _can_fuse_ffn_mhc=fused, + _can_fuse_attn_mhc=False, + hc_cfg=SimpleNamespace(hidden=4), + mlp=SimpleNamespace(tp_size=2), + ffn_hc=object(), + attn_hc=object(), + self_attn=SimpleNamespace( + attn_tp_size=2, accepts_mxfp8_swizzled_input=lambda: False + ), + engram=None, + input_layernorm=fixture.Norm(), + post_attention_layernorm=fixture.Norm(), + hc_post=Mock( + side_effect=lambda value, streams, post, comb: ( + streams + value[:, None] + ) + ), + ) + layer._bind_attention_transport = lambda *args: ( + deepseek_v4.DeepseekV4CudaDecoderLayer._bind_attention_transport( + layer, *args + ) + ) + layer._bind_moe_transport = lambda *args: ( + deepseek_v4.DeepseekV4CudaDecoderLayer._bind_moe_transport( + layer, *args + ) + ) + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=2) + source = torch.arange(24.0).reshape(3, 2, 4) + batch = SimpleNamespace( + input_ids=torch.arange(3), + forward_mode=ForwardMode.EXTEND, + attn_cp_metadata=None, + residual_stream=ResidualStream(source), + ) + call = boundary.LayerCall() + sums = Mock(side_effect=lambda value: value * 2) + attn_sums = Mock(side_effect=lambda value: value * 2) + attn_group = SimpleNamespace(all_reduce=attn_sums) + post = Mock( + side_effect=lambda hc, value, streams, **kw: ( + streams + value[:, None] + ) + ) + with ( + fixture.planning(parallel), + patch( + "sglang.srt.models.deepseek_v4_boundary.get_moe_a2a_backend", + return_value=SimpleNamespace(is_none=lambda: True), + ), + patch.object( + moe_transport, + "get_moe_a2a_backend", + return_value=SimpleNamespace(is_none=lambda: True), + ), + patch.object(boundary, "get_parallel", return_value=parallel), + patch.object( + moe_transport, "get_parallel", return_value=parallel + ), + patch("sglang.srt.layers.moe.post_experts_all_reduce", sums), + patch.object(mhc, "run_moe_post", post), + patch.object(mhc, "run_attn_post", post), + patch.object(mhc, "make_boundary", return_value=None), + patch.object( + boundary, + "_sum_group", + return_value=attn_group, + ), + patch( + "sglang.srt.layers.layer_boundary.row_transport._sum_group", + return_value=attn_group, + ), + patch( + "sglang.srt.models.deepseek_v4_boundary.get_cp_decode_attn_tp_ctx", + return_value=SimpleNamespace(use_decode_attn_tp=False), + ), + patch.object( + deepseek_v4, + "is_cross_layer_mhc_fusion_enabled", + return_value=fused, + ), + patch.object( + deepseek_v4, + "_is_fused_mhc_post_pre_enabled_xpu", + return_value=False, + ), + patch( + "sglang.srt.models.deepseek_v4_boundary.can_fuse_all_reduce", + return_value=True, + ), + patch.object( + boundary.HcReadCall, + "read", + side_effect=lambda value, norm, quant_format="", **kw: ( + value.mean(1), + boundary.ReadState(value, call=call), + ), + ), + ): + with layer_stack( + next_layers=[lambda: boundary.stages(config, 1)] + ): + declarations = boundary.stages(config, 0, layer=layer) + attn, ffn = append_stages( + *((d, fixture.Norm()) for d in declarations) + ) + operand = attn.prepare(source, batch, read_context=call) + attended = attn.finish( + mhc.AttnOutput(torch.ones_like(operand)), batch + ) + operand = ffn.prepare(attended, batch, read_context=call) + with ffn.exit(batch) as exit: + pieces = MoEOutput( + torch.ones_like(operand), + None, + SimpleNamespace(quant_method=None), + 1.0, + False, + ) + output = exit.finish(pieces) + self.assertIsNone(batch.residual_stream.pending) + self.assertIsNone(batch.residual_stream.transport) + wire = residual_batch.to_pp(output, batch) + with layer_stack( + previous_layers=[lambda: boundary.stages(config, 0)], + final_read=boundary.FinalRead(), + ): + receiver, _ = append_stages( + *( + (d, fixture.Norm()) + for d in boundary.stages(config, 1, layer=layer) + ) + ) + received_batch = SimpleNamespace(**vars(batch)) + received = receiver.from_pp(wire, received_batch) + read = receiver.prepare( + received, received_batch, read_context=call + ) + torch.testing.assert_close(read, (source + 4).mean(1)) + # Receiving an already-completed output must not reduce it again. + self.assertEqual(attn_sums.call_count, 1) + residual_batch.clear(received_batch) + torch.testing.assert_close(output, source + 4) + torch.testing.assert_close(wire["hidden_states"], source + 4) + sums.assert_called_once() + self.assertEqual((post if lagged else layer.hc_post).call_count, 2) + self.assertIsNone(batch.residual_stream) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_deepseek_v4_platform_dispatch.py b/test/registered/unit/models/test_deepseek_v4_platform_dispatch.py new file mode 100644 index 0000000000..9b10fe88ad --- /dev/null +++ b/test/registered/unit/models/test_deepseek_v4_platform_dispatch.py @@ -0,0 +1,250 @@ +"""Non-CUDA platforms keep decoder, draft and operation contracts.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.models import deepseek_v4 +from sglang.srt.models import deepseek_v4_dspark as dspark +from sglang.srt.models import deepseek_v4_nextn as nextn +from sglang.srt.models import deepseek_v4_platform_stage as platform_stage +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestPlatformDispatch(unittest.TestCase): + def test_cuda_main_loop_materializes_and_clears_output(self): + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + + model = deepseek_v4.DeepseekV4Model.__new__(deepseek_v4.DeepseekV4Model) + torch.nn.Module.__init__(model) + model.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True) + model.hc_mult = 2 + model.hc_pre_from_prev_sublayer = False + model.dspark_layers_to_capture = None + model.start_layer, model.end_layer = 0, 1 + model._can_run_tbo = lambda batch: False + model.hc_head = lambda value, *args: value.mean(1) + model.hc_head_fn = model.hc_head_scale = model.hc_head_base = None + model.norm = torch.nn.Identity() + + def layer(**kwargs): + return residual_batch.set_written( + kwargs["hidden_states"] + 3, kwargs["forward_batch"] + ) + + model.layers = [layer] + ids = torch.tensor([1]) + batch = SimpleNamespace(input_ids=ids, attn_cp_metadata=None) + with ( + patch.object(deepseek_v4, "_is_cuda", True), + patch.object(deepseek_v4, "_is_npu", False), + patch.object( + deepseek_v4, + "get_parallel", + return_value=SimpleNamespace(attn_dp_size=1), + ), + patch.object(deepseek_v4, "get_attn_backend", return_value=object()), + patch.object(deepseek_v4, "check_cuda_graph_backend", return_value=False), + patch.object( + deepseek_v4, + "get_global_expert_distribution_recorder", + return_value=SimpleNamespace( + with_current_layer=lambda i: nullcontext() + ), + ), + patch( + "sglang.kernels.ops.layernorm.mhc.hc_broadcast", + side_effect=lambda value, n: value[:, None].repeat(1, n, 1), + ), + ): + result = model(ids, ids, batch, torch.ones(1, 2)) + torch.testing.assert_close(result[0], torch.full((1, 2), 4.0)) + self.assertIsNone(batch.residual_stream) + + def test_nextn_exception_releases_invocation_transport(self): + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + + model = nextn.DeepseekV4ModelNextN.__new__(nextn.DeepseekV4ModelNextN) + torch.nn.Module.__init__(model) + model.config = SimpleNamespace(hidden_size=2) + model.hc_mult = 2 + model.enorm = model.hnorm = torch.nn.Identity() + model.e_proj = model.h_proj = lambda x: (x, None) + transport = SimpleNamespace(release=Mock()) + + def fail(**kwargs): + batch = kwargs["forward_batch"] + residual_batch.stream_of(batch).transport = (object(), transport) + raise RuntimeError("decoder failed") + + model.decoder = fail + model.hc_head = lambda value, *args: value.mean(1) + model.hc_head_fn = model.hc_head_scale = model.hc_head_base = None + model.shared_head = SimpleNamespace(norm=torch.nn.Identity()) + ids = torch.tensor([1]) + batch = SimpleNamespace( + spec_info=SimpleNamespace(hidden_states=torch.ones(1, 2, 2)) + ) + with ( + patch.object(nextn, "_is_cuda", True), + patch.object(nextn, "_is_npu", False), + patch.object( + nextn, "get_parallel", return_value=SimpleNamespace(attn_dp_size=1) + ), + patch.object(nextn, "dsa_use_prefill_cp", return_value=False), + ): + with self.assertRaisesRegex(RuntimeError, "decoder failed"): + model(ids, ids, batch, input_embeds=torch.ones(1, 2)) + self.assertIsNone(batch.residual_stream) + transport.release.assert_called_once() + model.decoder = lambda **kwargs: residual_batch.set_written( + kwargs["hidden_states"], kwargs["forward_batch"] + ) + result, _ = model(ids, ids, batch, input_embeds=torch.ones(1, 2)) + torch.testing.assert_close(result, torch.full((1, 2), 2.0)) + self.assertIsNone(batch.residual_stream) + + def test_non_cuda_nextn_completes_tensor_tail(self): + model = nextn.DeepseekV4ModelNextN.__new__(nextn.DeepseekV4ModelNextN) + torch.nn.Module.__init__(model) + model.config = SimpleNamespace(hidden_size=2) + model.hc_mult = 2 + model.enorm = model.hnorm = torch.nn.Identity() + model.e_proj = model.h_proj = lambda x: (x, None) + streams = torch.arange(4.0).reshape(1, 2, 2) + model.decoder = Mock(return_value=(streams, streams + 1, object(), object())) + completed = streams + 5 + model.decoder.hc_post.return_value = completed + model.hc_head = lambda value, *args: value.mean(1) + model.hc_head_fn = model.hc_head_scale = model.hc_head_base = None + model.shared_head = SimpleNamespace(norm=torch.nn.Identity()) + ids = torch.tensor([1]) + batch = SimpleNamespace(spec_info=SimpleNamespace(hidden_states=streams)) + with ( + patch.object(nextn, "_is_cuda", False), + patch.object(nextn, "_is_npu", False), + patch.object( + nextn, "get_parallel", return_value=SimpleNamespace(attn_dp_size=1) + ), + patch.object(nextn, "dsa_use_prefill_cp", return_value=False), + ): + result, pre_head = model(ids, ids, batch, input_embeds=torch.ones(1, 2)) + model.decoder.hc_post.assert_called_once() + torch.testing.assert_close(pre_head, completed.flatten(1)) + torch.testing.assert_close(result, completed.mean(1)) + + def test_non_cuda_dspark_uses_plain_input_norm(self): + stage = dspark.DSparkV4PlatformStage.__new__(dspark.DSparkV4PlatformStage) + torch.nn.Module.__init__(stage) + stage.hc_pre_from_prev_sublayer = False + for name in ( + "hc_attn_fn", + "hc_attn_scale", + "hc_attn_base", + "hc_ffn_fn", + "hc_ffn_scale", + "hc_ffn_base", + ): + setattr(stage, name, object()) + stage.input_layernorm = stage.post_attention_layernorm = torch.nn.Identity() + stage.hc_pre = lambda value, *args: (value + 0.5, object(), object(), False) + stage.hc_post = lambda value, residual, *args: value + residual + stage.self_attn = Mock(side_effect=lambda positions, value, batch: value * 2) + stage.self_attn.maybe_use_decode_attn_tp.return_value = nullcontext() + stage._run_ffn = lambda value, batch: value + 7 + stage._input_norm = Mock(side_effect=AssertionError("changed norm route")) + value = torch.ones(1, 2, 2) + result, pre = stage(torch.tensor([0]), value, object()) + first = (value + 0.5) * 2 + value + torch.testing.assert_close(result, (first + 0.5 + 7) + first) + self.assertIsNone(pre) + + +class TestStageLifecycle(unittest.TestCase): + def make_layer(self, cuda, *, lagged=True): + cls = ( + deepseek_v4.DeepseekV4CudaDecoderLayer + if cuda + else deepseek_v4.DeepseekV4PlatformDecoderLayer + ) + config = SimpleNamespace( + hidden_size=8, + hc_pre_from_prev_sublayer=lagged, + model_type="deepseek_v41", + vision_n_layers=0, + hc_mult=4, + hc_sinkhorn_iters=20, + hc_eps=1e-6, + rms_norm_eps=1e-6, + ) + attention = SimpleNamespace( + wo_b=SimpleNamespace(reduce_results=True), + attn_tp_size=4, + ) + moe = SimpleNamespace(reduce_results=True) + with ( + patch.object(deepseek_v4, "_is_cuda", cuda), + patch.object(deepseek_v4, "_is_hip", False), + patch.object(deepseek_v4, "is_dsa_enable_prefill_cp", return_value=False), + patch.object( + cls, + "_build_self_attn", + return_value=attention, + ), + patch.object(deepseek_v4.deepseek_v2, "DeepseekV2MoE", return_value=moe), + patch.object( + deepseek_v4.hc_boundary, "stages", return_value=(object(), object()) + ), + patch.object( + deepseek_v4, + "append_stages", + return_value=(object(), object()), + ), + ): + return cls(config, 0) + + def test_weight_refresh_invalidates_only_owned_boundary_caches(self): + for cuda in (False, True): + with self.subTest(cuda=cuda): + layer = self.make_layer(cuda) + old_hc = layer.attn_hc + layer.__dict__["_can_fuse_attn_mhc"] = True + layer.__dict__["_can_fuse_ffn_mhc"] = True + if not cuda: + layer.local_boundary = layer.next_boundary = object() + with patch.object( + deepseek_v4, + "get_platform", + return_value=SimpleNamespace(is_sm100=False, is_sm90=False), + ): + layer.refresh_mhc_norm_weight_cache() + self.assertNotIn("_can_fuse_attn_mhc", layer.__dict__) + self.assertNotIn("_can_fuse_ffn_mhc", layer.__dict__) + self.assertIsNot(layer.attn_hc, old_hc) + self.assertEqual( + layer._input_layernorm_weight_bf16.dtype, torch.bfloat16 + ) + if not cuda: + self.assertIsNone(layer.local_boundary) + self.assertIsNone(layer.next_boundary) + + def test_non_cuda_partial_attention_cannot_fuse_mhc(self): + layer = self.make_layer(False) + layer.self_attn.wo_b.reduce_results = False + layer.local_boundary = SimpleNamespace(norm_fusable=True) + with ( + patch.object( + platform_stage, "get_parallel", return_value=SimpleNamespace(tp_size=4) + ), + patch.object(platform_stage.mhc, "can_fuse_post", return_value=True), + ): + self.assertFalse(layer._can_fuse_attn_mhc) + + +if __name__ == "__main__": + unittest.main()