[Refactor] layer-boundary: Bind residual completion to producer updates (#43605)

This commit is contained in:
Cheng Wan
2026-10-15 02:04:02 -07:00
committed by GitHub
parent d59c3448b0
commit e1f9fce7b3
27 changed files with 1445 additions and 213 deletions
@@ -173,6 +173,13 @@ class _TransformedRead:
completing_fusions = ()
gathering_reads = ()
@property
def bind(self):
return self._bind if self.inner.bind is not None else None
def _bind(self, forward_batch, context):
return _TransformedRead(self.inner.bind(forward_batch, context), self.transform)
@property
def reads_before_dp_gather(self):
return self.inner.reads_before_dp_gather
@@ -341,6 +348,14 @@ def _bind_entry_path(
step=entry.step,
is_plain_add=is_plain_add,
carried_fusions=carried_fusions if plain else (),
carried_read_fusions=tuple(
f
for f in edge.need.read.completing_fusions
if f.carried
and not f.scatters
and (is_plain_add or f.accepts_update is not None)
),
read=edge.need.read,
expected_sum=declared_sum,
completed_step=completed_step,
written_step=written_step,
@@ -367,8 +382,8 @@ def bind_exit(
Args:
edge: Output contract and destination/residual rows. Deferred updates
must outlive the producer; pipeline handoffs require a plain add
or a residual already written by the producer.
must outlive the producer. Pipeline handoff eligibility is checked
when the stage is constructed.
cp_moves: Context-parallel return operations, when the edge needs them.
attn_tp_gather: The stage's implementation of a gather over attention
TP, tried before the default one.
@@ -383,10 +398,6 @@ def bind_exit(
raise NotImplementedError(
"a deferred update must guarantee its lifetime across layers"
)
if not update.is_plain_add and get_parallel().pp_size > 1:
raise NotImplementedError(
"pipeline boundaries require a plain add or a producer-written residual"
)
if edge.need.layout != edge.residual_to:
raise NotImplementedError(f"{edge=}")
return _select_exit_move(
@@ -398,17 +409,20 @@ def bind_exit(
)
def _read_fusions(read, owes, is_plain_add, *, scatters):
"""The kernels a read supplies that complete ``owes`` on the rows this
entry completes it on. They take the residual add themselves, so only a
plain one."""
if not is_plain_add:
return ()
return tuple(
f
for f in read.completing_fusions
if f.completes is owes and f.scatters == scatters
def _read_fusion_args(read, owes, is_plain_add, *, scatters):
"""Select callback slots for this group and row placement once. Stateful
readers resolve those slots on the invocation, stateless readers directly
retain their kernels. Non-plain updates require explicit acceptance."""
indices = tuple(
i
for i, f in enumerate(read.completing_fusions)
if (is_plain_add or f.accepts_update is not None)
and f.completes is owes
and f.scatters == scatters
)
if read.bind is not None:
return {"read_fusion_indices": indices}
return {"read_fusions": tuple(read.completing_fusions[i] for i in indices)}
def _select_entry_step(
@@ -484,7 +498,7 @@ def _select_entry_step(
enters_stack=enters_stack,
)
return inner._replace(
step=partial(_move_before_read, move=move, read=inner.step)
step=partial(_move_before_read, move=move, read_step=inner.step)
)
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
@@ -501,7 +515,7 @@ def _select_entry_step(
if owes is SumGroup.ATTN_TP:
step = partial(
_attn_tp_reduce_scatter_update_read,
read_fusions=_read_fusions(read, owes, is_plain_add, scatters=True),
**_read_fusion_args(read, owes, is_plain_add, scatters=True),
)
else:
step = _attn_tp_slice_update_read
@@ -581,7 +595,14 @@ def _select_entry_step(
enters_stack=enters_stack,
read=read,
read_gathers=(
read.gathering_reads if is_plain_add or arrives_written else ()
read.gathering_reads
if read.bind is None and (is_plain_add or arrives_written)
else ()
),
read_gather_indices=(
tuple(range(len(read.gathering_reads)))
if read.bind is not None and (is_plain_add or arrives_written)
else ()
),
),
input_move=(
@@ -639,13 +660,13 @@ def _select_entry_step(
):
raise NotImplementedError(f"{produced=} {need=}")
fused = tuple(f for f in fusions if f.completes is owes)
read_fused = _read_fusions(read, owes, is_plain_add, scatters=False)
read_fused = _read_fusion_args(read, owes, is_plain_add, scatters=False)
return EntryStep(
partial(
_reduce_update_read,
gathers_residual=gathers_residual,
fusions=tuple(f.run for f in fused),
read_fusions=read_fused,
**read_fused,
group=owes,
read=read,
),
@@ -50,6 +50,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
_attn_input_default,
_attn_input_scattered,
)
from sglang.srt.layers.layer_boundary.residual import takes_pending_update
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.runtime_context import (
get_forward,
@@ -269,6 +270,7 @@ def _bind_stage(
*,
final_read=None,
capture_preserves_residual=None,
pipeline_handoff=False,
**options,
):
if incoming.consumer != declaration or outgoing.producer != declaration:
@@ -276,7 +278,15 @@ def _bind_stage(
if incoming.entries.keys() != outgoing.exits.keys():
raise ValueError("incoming and outgoing batch variants disagree")
update = declaration.update
if declaration.terminal and not (update.is_plain_add or update.applied_at_exit):
if pipeline_handoff and not (update.is_plain_add or update.applied_at_exit):
raise NotImplementedError(
"pipeline boundaries require a producer-written residual"
)
if declaration.terminal and not (
update.is_plain_add
or update.applied_at_exit
or takes_pending_update(final_read, update)
):
# The final read adds the last output into the residual itself, as a
# plain add, before its norm.
raise NotImplementedError(
@@ -186,13 +186,14 @@ class FfnInputFusion(msgspec.Struct, frozen=True):
class ReadoutFusion(msgspec.Struct, frozen=True):
"""A kernel a stage's read supplies (its ``completing_fusions``) that
completes the sum its input owes together with the residual add, for a
read that is not the residual's plain norm and so takes no FfnInputFusion.
completes the input's sum and producer update before the consumer read.
With no ``accepts_update`` predicate the update must be plain addition.
Otherwise ``run`` also receives the actual producer's ``update`` keyword.
``run(hidden_states, residual, forward_batch)`` returns the written stream,
the completed sum plus the residual, which the read then reads; or None
when it does not take the batch, before touching its inputs or starting a
collective."""
or, with ``reads=True``, takes ``norm`` and returns (input, residual).
A carried candidate may receive UnreducedOutput or DeferredFinalize.
Returning None must precede any collective or mutation of input/state."""
# The group whose sum it completes.
completes: SumGroup
@@ -204,6 +205,12 @@ class ReadoutFusion(msgspec.Struct, frozen=True):
# Whether it also does the read: run(hidden_states, residual,
# forward_batch, norm) then returns the read's (input, residual).
reads: bool = False
# None restricts the existing kernel to a plain addition. An explicit
# predicate validates the actual update before run receives it as a keyword.
accepts_update: Optional[Callable] = None
# Also accepts dynamically carried output; run must decline unsupported
# completion handles without starting communication or changing state.
carried: bool = False
class CpMoves(msgspec.Struct, frozen=True):
+34 -11
View File
@@ -49,6 +49,7 @@ from sglang.srt.layers.layer_boundary.ops import (
from sglang.srt.layers.layer_boundary.output import (
UnreducedOutput,
)
from sglang.srt.layers.layer_boundary.residual import bind_update
from sglang.srt.layers.layer_boundary.residual.add_norm import (
aiter_ar_fusion_applies,
flashinfer_ar_fusion_applies,
@@ -116,7 +117,7 @@ def exit_facts(kind, plan, variant, path) -> ExitFacts:
)
may_defer_sum = (
tp
and not plan.terminal
and (not plan.terminal or output.may_defer_to_next)
and _sum_deferral_allowed(plan, output, scattered=scattered)
# Under attention DP the next layer must also run postprocess's
# scatter back to this rank's tokens, and nothing more.
@@ -199,10 +200,12 @@ class ExitPolicy:
forward_batch: ForwardBatch,
*,
already_reduced: bool = False,
update_context=None,
):
"""Complete the output of the operation-scheduled producer path, which
never defers: the sum it owes, then its move and write-back."""
steps = self.plan.path_for(forward_batch)
update = bind_update(steps.output.update, forward_batch, update_context)
if already_reduced and steps.output_move_completes_sum:
raise ValueError("already-reduced output cannot use a reduce-scatter move")
hidden_states, residual = self._complete_now(
@@ -212,6 +215,7 @@ class ExitPolicy:
dp_step=None,
steps=steps,
output_move=steps.output_move,
update=update,
owes_sum=not already_reduced
and steps.output.group is not None
and not steps.output_move_completes_sum,
@@ -220,7 +224,7 @@ class ExitPolicy:
hidden_states,
residual,
stream,
steps.output.update,
update,
)
@staticmethod
@@ -260,7 +264,9 @@ class ExitPolicy:
group on a sparse layer, the TP group a dense MLP reduces over."""
return _sum_group(steps.output.group)
def _decide(self, forward_batch: ForwardBatch, steps) -> ExitDecision:
def _decide(
self, forward_batch: ForwardBatch, steps, *, update=None
) -> ExitDecision:
"""Decide once, before the FFN runs, what completes its output's sum:
the next layer's input, or this layer's postprocess step."""
dp_step = self._dp_reduce_scatter_step(forward_batch, steps)
@@ -271,6 +277,7 @@ class ExitPolicy:
dp_step=dp_step,
steps=steps,
output_move=steps.output_move,
update=update,
# A reduce-scatter completes the sum on its way back.
owes_sum=steps.output.group is not None and not sum_in_reduce_scatter,
)
@@ -340,6 +347,7 @@ class ExitPolicy:
steps,
output_move: Optional[Callable],
owes_sum: bool,
update=None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This layer's postprocess, run with the attention-DP step already
chosen: the sum the output owes, the move back to where the next layer
@@ -364,27 +372,40 @@ class ExitPolicy:
residual=residual,
forward_batch=forward_batch,
)
update = steps.output.update
if update is None:
update = bind_update(steps.output.update, forward_batch)
if residual is not None and (update.applied_at_exit or steps.writes_at_handoff):
hidden_states = update.update(hidden_states, residual)
residual = None
return hidden_states, residual
def mixer_exit(
self, forward_batch: ForwardBatch, *, stream: ResidualStream
self,
forward_batch: ForwardBatch,
*,
stream: ResidualStream,
update_context=None,
) -> MixerExit:
"""Decide once whether this stage's mixer (an attention-like stage)
output's sum is completed here or carried on. Call ``finish`` on the
result with the mixer's output."""
return MixerExit(self, forward_batch, stream=stream)
return MixerExit(
self, forward_batch, stream=stream, update_context=update_context
)
def ffn_exit(
self, forward_batch: ForwardBatch, *, stream: ResidualStream
self,
forward_batch: ForwardBatch,
*,
stream: ResidualStream,
update_context=None,
) -> FfnExit:
"""Decide once how this layer's FFN output reduction completes. Use the
result as a context manager around the FFN call when the FFN may hand
off its MoE finalize, then call ``finish``."""
return FfnExit(self, forward_batch, stream=stream)
return FfnExit(
self, forward_batch, stream=stream, update_context=update_context
)
def _defers_sum(
self,
@@ -485,12 +506,13 @@ class MixerExit:
forward_batch: ForwardBatch,
*,
stream: ResidualStream,
update_context=None,
):
steps = boundary.plan.path_for(forward_batch)
produced = steps.output
self._stream = stream
self._forward_batch = forward_batch
self._update = produced.update
self._update = bind_update(produced.update, forward_batch, update_context)
self._group = produced.group
self._declared_sum = produced.group if produced.always_partial else None
self._defers = steps.exit.defers_mixer_sum
@@ -552,16 +574,17 @@ class FfnExit:
forward_batch: ForwardBatch,
*,
stream: ResidualStream,
update_context=None,
):
self._stream = stream
self.boundary = boundary
# The same path the entry took: the flags that select it hold for the
# whole call, and a stage keeps no state between its entry and exit.
steps = boundary.plan.path_for(forward_batch)
completion = boundary._decide(forward_batch, steps)
self._update = bind_update(steps.output.update, forward_batch, update_context)
completion = boundary._decide(forward_batch, steps, update=self._update)
self.defer_moe_finalize = completion.defer_moe_finalize
self._complete = completion.complete
self._update = steps.output.update
# A reduce-scatter that completes the sum on the way back is the
# next input's only on an input-scattered batch.
self._declared_sum = steps.exit.sum_left_to_next_input
@@ -46,6 +46,7 @@ from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.residual import (
ResidualReadout,
ResidualUpdate,
takes_pending_update,
)
from sglang.srt.layers.layer_boundary.residual.add_norm import (
NORM_QUANT_READOUT,
@@ -137,6 +138,7 @@ def _resolve_ffn(
update=PLAIN_ADD,
dense_tp_size=None,
output_complete=False,
terminal_accepts_update=False,
):
parallel = get_parallel()
if dense_tp_size not in (None, 1, parallel.attn_tp_size, parallel.tp_size):
@@ -244,7 +246,7 @@ def _resolve_ffn(
rows,
group=group,
may_defer_to_next=may_leave
and not terminal
and (not terminal or terminal_accepts_update)
and not update.applied_at_exit
and update.outlives_layer
and output_transform is None,
@@ -372,7 +374,9 @@ def _require(protocol, implementation, facts):
def _check_update(update):
_require("ResidualUpdate", update, _UPDATE_FACTS)
_require("ResidualUpdate", update, _UPDATE_FACTS + ("bind",))
if update.bind is not None and not callable(update.bind):
raise TypeError("ResidualUpdate.bind must be callable or None")
name = type(update).__name__
if update.writes_stream and (update.is_plain_add or not update.applied_at_exit):
raise ValueError(
@@ -386,7 +390,9 @@ def _check_update(update):
def _check_read(read):
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS)
_require("ResidualReadout", read, _READ_FACTS + _READ_KERNELS + ("bind",))
if read.bind is not None and not callable(read.bind):
raise TypeError("ResidualReadout.bind must be callable or None")
@dataclass(frozen=True)
@@ -576,7 +582,7 @@ def _stack_arrival(variant):
)
def _resolve_stage(stage, variant, arrival, following=None):
def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
"""``stage``'s data flow for ``variant``, given what arrives at its entry
and the stage after it (None when it ends the stack or a branch)."""
axes, attention, local, full = _row_layouts(variant)
@@ -591,6 +597,8 @@ def _resolve_stage(stage, variant, arrival, following=None):
update=stage.update,
dense_tp_size=stage.dense_tp_size,
output_complete=stage.output_complete,
terminal_accepts_update=stage.terminal
and takes_pending_update(final_read, stage.update),
)
exit_rows = resolve_exit_rows(stage.exit_rows)
if exit_rows is ExitRows.ATTENTION or (
@@ -745,7 +753,9 @@ def _exit_edge(stage, flow, entry):
)
def _connect_line(line, origins, *, before=None, after=None, arrivals=None):
def _connect_line(
line, origins, *, before=None, after=None, arrivals=None, final_read=None
):
"""The connections along a chain of stages, each resolved once per batch
variant: into ``line[0]``, between each two, and out of ``line[-1]`` to
``after`` (a stage on the next rank, or None at the end of the stack or a
@@ -754,12 +764,33 @@ def _connect_line(line, origins, *, before=None, after=None, arrivals=None):
rank's own entry is taken as the stack input's. An error resolving a stage
names the append at ``origins[i]`` that declared ``line[i]``, or the one
next to the neighbouring rank's stage."""
variants = tuple(_active_variants())
# The global dense-TP default may be overridden by every FFN declaration.
# Odd row counts need a separate path only when some connected FFN would
# otherwise slice them over attention TP. Keep neighbours in this decision:
# their row choices still determine what crosses the pipeline boundary.
needs_unpadded = any(
stage is not None
and stage.kind is StageKind.FFN
and _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size)
for stage in (before, *line, after)
)
variants = tuple(
variant
for variant in _active_variants()
if variant is not BatchVariant.UNPADDED or needs_unpadded
)
def resolve(position, stage, arrivals, following=None):
try:
return {
v: _resolve_stage(stage, v, arrivals[v], following) for v in variants
v: _resolve_stage(
stage,
v,
arrivals[v],
following,
final_read=final_read if stage.terminal else None,
)
for v in variants
}
except Exception as error:
_note_origin(error, origins[position])
@@ -1072,6 +1103,7 @@ def _bind_stack(appends, *, previous, following, final_read=None):
[append.origin for append, _ in stages],
before=chained[0].previous,
after=following,
final_read=final_read if following is None else None,
)
_bind_line(
stages,
@@ -1145,6 +1177,8 @@ def _bind_line(stages, chained, connections, bound, sources, *, final_read=None)
connections[position],
connections[position + 1],
final_read=final_read if position == last else None,
pipeline_handoff=position == last
and connections[position + 1].consumer is not None,
capture_preserves_residual=(
_preserves_residual(boundaries[position + 1])
if consumer is not None
@@ -28,6 +28,8 @@ import msgspec
class UpdateFacts(msgspec.Struct, frozen=True):
"""The capabilities a ResidualUpdate declares (see ResidualUpdate)."""
bind = None
is_plain_add: bool
applied_at_exit: bool
outlives_layer: bool
@@ -52,6 +54,8 @@ class ReadFacts(msgspec.Struct, frozen=True):
Binding takes kernels only from the reads of the stages it binds, never
from a stage recorded by its facts, so these supply none."""
bind = None
is_plain_norm: bool
reads_before_dp_gather: bool
reads_after_attn_tp_gather: bool
@@ -72,6 +72,12 @@ class DeferredFinalize(msgspec.Struct, frozen=True):
there may do that work together with its own; anything else passes it
through complete_owed(), which calls ``complete()``."""
@property
def completion_group(self):
from sglang.srt.layers.layer_boundary.layout import SumGroup
return SumGroup.MOE_OUTPUT
def complete(self) -> torch.Tensor:
"""Do the owed work, unfused, and return the complete output."""
raise NotImplementedError
@@ -51,7 +51,11 @@ from sglang.srt.layers.layer_boundary.output import (
UnreducedOutput,
complete_owed,
)
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
from sglang.srt.layers.layer_boundary.residual import (
ResidualReadout,
ResidualUpdate,
bind_read,
)
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -77,19 +81,38 @@ from sglang.srt.layers.layer_boundary.ops import (
def _run_read_fusions(
read_fusions, hidden_states, residual, forward_batch, norm, *, read, **call
read_fusions,
hidden_states,
residual,
forward_batch,
norm,
*,
read,
read_fusion_indices: Tuple[int, ...] = (),
update=PLAIN_ADD,
**call,
):
"""The first of the read's own kernels (ReadoutFusion) that takes the
batch: one that completes the sum with the residual add hands the written
stream to ``read``; one that also reads returns the read. None when every
kernel declines."""
if read_fusion_indices:
read_fusions = (read.completing_fusions[i] for i in read_fusion_indices)
for fused in read_fusions:
args = {}
if fused.accepts_update is None:
if not update.is_plain_add:
continue
else:
if not fused.accepts_update(update):
continue
args["update"] = update
if fused.reads:
result = fused.run(hidden_states, residual, forward_batch, norm)
result = fused.run(hidden_states, residual, forward_batch, norm, **args)
if result is not None:
return result
continue
stream = fused.run(hidden_states, residual, forward_batch)
stream = fused.run(hidden_states, residual, forward_batch, **args)
if stream is not None:
return read.read(stream, norm, **call)
return None
@@ -105,6 +128,7 @@ def _reduce_update_read(
gathers_residual: bool,
fusions: Tuple[Callable, ...],
read_fusions: Tuple[ReadoutFusion, ...] = (),
read_fusion_indices: Tuple[int, ...] = (),
group: SumGroup = SumGroup.ATTN_TP,
read: ResidualReadout = NORM_READOUT,
update: ResidualUpdate = PLAIN_ADD,
@@ -125,6 +149,8 @@ def _reduce_update_read(
forward_batch,
norm,
read=read,
read_fusion_indices=read_fusion_indices,
update=update,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
@@ -271,12 +297,15 @@ def _run_entry(
step: Callable,
is_plain_add: bool,
carried_fusions: Tuple[Callable, ...],
carried_read_fusions: Tuple[ReadoutFusion, ...] = (),
read: ResidualReadout = NORM_READOUT,
expected_sum: Optional[SumGroup] = None,
completed_step: Optional[Callable] = None,
pending=None,
written_step: Optional[Callable] = None,
written: bool = False,
update: ResidualUpdate = PLAIN_ADD,
read_context=None,
**call,
):
"""A boundary's half into a stage: complete what the value carries, a sum
@@ -287,6 +316,17 @@ def _run_entry(
whose declared sum is already completed uses ``completed_step`` instead;
a pending declared sum must match ``expected_sum``. ``call`` is what the stage's read takes (the attention's
``quant_format`` and ``post_residual_addition``)."""
bind = read.bind
if bind is not None:
bound = bind_read(read, forward_batch, read_context)
declared = read.completing_fusions
actual = bound.completing_fusions
carried_read_fusions = tuple(
actual[i]
for i, spec in enumerate(declared)
if any(spec is selected for selected in carried_read_fusions)
)
read = bound
if residual is not None and update.is_plain_add != is_plain_add:
raise RuntimeError("producer update does not match the boundary's capability")
if pending is not None:
@@ -299,7 +339,7 @@ def _run_entry(
step = completed_step
if written and written_step is not None:
step = written_step
if not isinstance(hidden_states, torch.Tensor):
if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)):
owed = hidden_states
if residual is None:
raise RuntimeError(f"{type(owed).__name__} requires residual input")
@@ -311,6 +351,30 @@ def _run_entry(
if isinstance(owed, UnreducedOutput) and owed.reduce_to_dp_local is not None:
hidden_states = complete_owed(owed)
else:
read_candidates = tuple(
fused
for fused in carried_read_fusions
if (
isinstance(owed, DeferredFinalize)
and fused.completes is owed.completion_group
)
or (
isinstance(owed, UnreducedOutput)
and _sum_group(fused.completes) is owed.group
)
)
result = _run_read_fusions(
read_candidates,
owed,
residual,
forward_batch,
norm,
read=read,
update=update,
**call,
)
if result is not None:
return result
for fused in carried_fusions:
result = fused(
owed, residual, forward_batch, call.get("post_residual_addition")
@@ -321,6 +385,8 @@ def _run_entry(
hidden_states = complete_owed(owed)
else:
hidden_states = owed.partial
if bind is not None:
call["read"] = read
return step(hidden_states, residual, forward_batch, norm, update=update, **call)
@@ -335,6 +401,7 @@ def _update_read(
enters_stack: bool,
read: ResidualReadout,
read_gathers: Tuple[Callable, ...] = (),
read_gather_indices: Tuple[int, ...] = (),
update: ResidualUpdate = PLAIN_ADD,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
@@ -350,6 +417,8 @@ def _update_read(
hidden_states, residual = pre_move(hidden_states, residual)
if enters:
hidden_states, residual = read.init_residual(hidden_states), None
if read_gather_indices:
read_gathers = (read.gathering_reads[i] for i in read_gather_indices)
for fused in read_gathers:
result = fused(hidden_states, residual, norm)
if result is not None:
@@ -377,6 +446,7 @@ def _attn_tp_reduce_scatter_update_read(
cache=None,
scatters_residual: bool,
read_fusions: Tuple[ReadoutFusion, ...] = (),
read_fusion_indices: Tuple[int, ...] = (),
read: ResidualReadout = NORM_READOUT,
update: ResidualUpdate = PLAIN_ADD,
):
@@ -385,7 +455,14 @@ def _attn_tp_reduce_scatter_update_read(
of the read's ``read_fusions`` that takes the batch does all of this
instead."""
result = _run_read_fusions(
read_fusions, hidden_states, residual, forward_batch, norm, read=read
read_fusions,
hidden_states,
residual,
forward_batch,
norm,
read=read,
read_fusion_indices=read_fusion_indices,
update=update,
)
if result is not None:
return result
@@ -474,11 +551,11 @@ def _then_attn_cp_gather(
def _move_before_read(
hidden_states, residual, forward_batch, norm, *, move, read, **call
hidden_states, residual, forward_batch, norm, *, move, read_step, **call
):
"""Complete and place a producer contribution before its residual update."""
hidden_states, residual = move(hidden_states, residual, forward_batch)
return read(hidden_states, residual, forward_batch, norm, **call)
return read_step(hidden_states, residual, forward_batch, norm, **call)
def _then_moe_cp_gather(
@@ -490,6 +567,7 @@ def _then_moe_cp_gather(
cache=None,
gather: Callable,
update: ResidualUpdate = PLAIN_ADD,
**call,
):
"""Read each shard, then gather the MoE group's tokens on its CP path.
@@ -498,7 +576,7 @@ def _then_moe_cp_gather(
ordinary path. The residual stays on this rank's attention rows.
"""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, norm, update=update, cache=cache
hidden_states, residual, forward_batch, norm, update=update, cache=cache, **call
)
hidden_states = moe_cp_gather(
hidden_states,
@@ -16,7 +16,7 @@ stage reads its input back from it (its read). The producer of a boundary
declares the update, the consumer the read; the boundary steps complete sums
and move tokens around the two."""
from typing import NamedTuple, Optional, Protocol, Tuple
from typing import Callable, NamedTuple, Optional, Protocol, Tuple
import torch
@@ -39,10 +39,18 @@ class ResidualUpdate(Protocol):
This is a numerical policy, separate from being a plain add; full
precision is always correct.
An optional ``bind(forward_batch, context)`` returns an update owning this call's
dynamic metadata. The stage binds it once before recording a contribution;
every capability must match the declaration. Consumption releases the
invocation update with its contribution.
The shard conversion methods must move any update-associated state together
with the residual. Nonlinear updates must not claim is_plain_add.
"""
# None explicitly declares a stateless update. Bindings take (batch, context).
bind: Optional[Callable] = None
# A plain add, which one rank may run before the sum it adds into
# completes (the DP partial order, a residual that joins the attention
# output's sum), and which a fused add + norm kernel may run.
@@ -108,6 +116,11 @@ class ResidualReadout(Protocol):
snapshot.
"""
# bind(batch, context) may replace callback implementations, preserving the
# order and capabilities of completing_fusions and the gathering_reads slots.
# None explicitly declares a stateless consumer.
bind: Optional[Callable] = None
# The input is the residual's norm, in the quantization the call asks for,
# and the residual is left as it is: what a fused add + norm kernel computes,
# and what may run on the rows a sum completes onto.
@@ -175,6 +188,73 @@ class ResidualReadout(Protocol):
"""
def bind_read(read, forward_batch, context=None):
"""Bind a consumer invocation without changing its declared read capabilities."""
if read.bind is None:
return read
bound = read.bind(forward_batch, context)
for fact in (
"is_plain_norm",
"reads_before_dp_gather",
"reads_after_attn_tp_gather",
):
if getattr(bound, fact) != getattr(read, fact):
raise ValueError("invocation read changed its declared capability")
# Callback slots are selected when the edge is constructed. Binding
# replaces their implementations, not their group or row capabilities.
declared = read.completing_fusions
actual = bound.completing_fusions
if len(actual) != len(declared) or len(bound.gathering_reads) != len(
read.gathering_reads
):
raise ValueError("invocation read changed its declared callbacks")
for spec, fused in zip(declared, actual):
if (spec.completes, spec.scatters, spec.reads, spec.carried) != (
fused.completes,
fused.scatters,
fused.reads,
fused.carried,
) or (spec.accepts_update is None) != (fused.accepts_update is None):
raise ValueError("invocation read changed its declared callbacks")
return bound
def bind_update(update, forward_batch, context=None):
"""Bind optional per-invocation producer state before recording output.
``bind(batch, context)`` must preserve every declared update capability and return
an update owning its invocation context with ``bind = None``. Stateless updates
are unchanged.
"""
if update.bind is None:
return update
actual = update.bind(forward_batch, context)
for fact in (
"is_plain_add",
"applied_at_exit",
"outlives_layer",
"writes_stream",
"quantized_sum",
):
if getattr(actual, fact) != getattr(update, fact):
raise ValueError("invocation update changed a declared capability")
if actual.bind is not None:
raise ValueError("invocation update must have bind = None")
return actual
def takes_pending_update(read, update):
"""Whether a final read explicitly accepts this producer's update."""
return isinstance(read, PendingUpdateRead) and read.accepts_pending_update(update)
class PendingUpdateRead:
"""Explicit consumer interface for a terminal pending producer update."""
def accepts_pending_update(self, update: ResidualUpdate) -> bool:
raise NotImplementedError
class FinalRead(Protocol):
"""The model's read of the layer stack's output, its final norm.
@@ -182,6 +262,12 @@ class FinalRead(Protocol):
with the output and the residual, returning ``(normed, residual)``: the
residual add followed by the norm, so a pending update must be a plain add.
A ResidualReadout can instead explicitly supply
``accepts_pending_update(update)`` and consume the pending producer update
through residual_batch.read_output(output, batch, read, norm). Construction
checks the declared update; execution checks the actual invocation. This
does not change the ordinary two-input final norm contract.
Optional, each read where it is used with the default given:
attn_tp_gather: Its own gather over attention TP of rows the stack's
last stage leaves on this rank's attention-TP slice, returning
@@ -273,6 +273,8 @@ def _norm_quant_kernel(quant_format: str):
class PlainAdd:
"""A plain residual: the output is added into it."""
bind = None
is_plain_add = True
applied_at_exit = False
outlives_layer = True
@@ -296,6 +298,8 @@ class ReplaceAtExit:
previous residual is dropped. The stream is complete: the producer has its
boundary complete each part's sum first (StageBoundary.sum_part)."""
bind = None
is_plain_add = False
applied_at_exit = True
outlives_layer = True
@@ -330,6 +334,8 @@ class NormQuantReadout:
It is not applied by the aiter mxfp4 and fp8-group kernels, after an
update other than a plain add, or by ``read``, which has no add."""
bind = None
is_plain_norm = True
completing_fusions = ()
gathering_reads = ()
@@ -370,6 +376,8 @@ class NormReadout:
``quant_format`` and a ``post_residual_addition``, except that the latter
is not applied after an update other than a plain add."""
bind = None
is_plain_norm = True
completing_fusions = ()
gathering_reads = ()
@@ -417,6 +425,8 @@ class UnfusedNormReadout(NormReadout):
two steps: the update rounds to the activation dtype before the norm, as
in models that add their residual themselves. No fused kernel takes it."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -210,6 +210,8 @@ class _BankReadout:
is not the residual's plain norm and no add+norm kernel may claim it. It
reads this rank's own rows, which the bank holds, before any DP gather."""
bind = None
is_plain_norm = False
reads_before_dp_gather = True
completing_fusions = ()
@@ -35,6 +35,16 @@ def stream_of(forward_batch):
return stream
def clear(forward_batch):
"""Release invocation state on completion or an exception."""
stream = forward_batch.residual_stream
if stream is not None:
if stream.pending is not None:
stream.pending.release()
stream.pending = stream.residual = None
forward_batch.residual_stream = None
def written_residual(forward_batch):
"""The residual the last prepare wrote, for a stage whose compute reads it
too (e.g. to write the next stream at its exit). Borrowed: do not modify."""
@@ -56,6 +66,17 @@ def take_output(hidden_states, forward_batch):
return hidden_states
def _plain_output(hidden_states, forward_batch):
stream = stream_of(forward_batch)
stream.check(hidden_states)
if stream.pending is not None and not stream.pending.update.is_plain_add:
raise NotImplementedError(
"output/residual pair requires a plain residual update; "
"apply the producer update before export"
)
return stream
def final_norm(
hidden_states,
forward_batch,
@@ -86,12 +107,12 @@ def final_norm(
Normalized tensor. Add and norm remain together to preserve the kernel's
accumulation/rounding order; a snapshot is not used as the norm input.
"""
hidden_states, residual = stream_of(forward_batch).export(
hidden_states, residual = _plain_output(hidden_states, forward_batch).export(
hidden_states, takes_handoff=finalize_norm is not None
)
# The terminal consumer now owns the pair. Do not keep layer buffers alive
# through logits processing or the next forward on this batch.
forward_batch.residual_stream = None
clear(forward_batch)
from sglang.srt.layers.layer_boundary.output import DeferredFinalize
if isinstance(hidden_states, DeferredFinalize):
@@ -114,6 +135,83 @@ def final_norm(
)
def read_output(hidden_states, forward_batch, read, norm, *, read_context=None):
"""Consume the stack's output with an explicitly declared final read.
This entry accepts producer updates through ``accepts_pending_update``.
Completion candidates see the actual update before any collective; their
fallback completes the owed output once, then applies update_and_read.
A bindable read receives read_context once before consuming the output.
Ordinary model final norms continue to use final_norm().
"""
from sglang.srt.layers.layer_boundary.layout import _sum_group
from sglang.srt.layers.layer_boundary.output import (
DeferredFinalize,
UnreducedOutput,
)
from sglang.srt.layers.layer_boundary.prepare import _run_read_fusions
from sglang.srt.layers.layer_boundary.residual import (
bind_read,
takes_pending_update,
)
from sglang.srt.layers.layer_boundary.residual.stream import DeclaredSum
stream = stream_of(forward_batch)
stream.check(hidden_states)
pending = stream.pending
try:
if pending is not None and not takes_pending_update(read, pending.update):
raise NotImplementedError(
"final read does not accept the pending producer update"
)
read = bind_read(read, forward_batch, read_context)
if pending is None:
value = stream.residual
if value is None:
value = read.init_residual(hidden_states)
return read.read(value, norm)
if not takes_pending_update(read, pending.update):
raise NotImplementedError(
"final read does not accept the pending producer update"
)
owed = pending.owed
candidates = tuple(
f
for f in read.completing_fusions
if not f.scatters
and (
(isinstance(owed, DeclaredSum) and f.completes is owed.group)
or (
isinstance(owed, UnreducedOutput)
and f.carried
and owed.reduce_to_dp_local is None
and _sum_group(f.completes) is owed.group
)
or (
isinstance(owed, DeferredFinalize)
and f.carried
and f.completes is owed.completion_group
)
)
)
result = _run_read_fusions(
candidates,
pending.for_boundary(),
stream.residual,
forward_batch,
norm,
read=read,
update=pending.update,
)
if result is None:
value = pending.complete()
result = read.update_and_read(pending.update, value, stream.residual, norm)
return result
finally:
# The terminal consumer owns the result, including on an exception.
clear(forward_batch)
def to_pp(hidden_states, forward_batch):
"""Export hidden_states and residual as PPProxyTensors.
@@ -129,8 +227,10 @@ def to_pp(hidden_states, forward_batch):
and is sent as hidden_states alone; the receiver's from_pp reconstructs it
as written.
"""
hidden_states, residual = stream_of(forward_batch).export(hidden_states)
forward_batch.residual_stream = None
hidden_states, residual = _plain_output(hidden_states, forward_batch).export(
hidden_states
)
clear(forward_batch)
tensors = {"hidden_states": hidden_states}
if residual is not None:
tensors["residual"] = residual
@@ -134,6 +134,8 @@ class _AttnReadout:
the previous layer's output: such a layer takes its input written back. A
``post_residual_addition`` is not applied."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -167,6 +169,8 @@ class _AttnUpdate:
residual its read produced. The injection is not a plain add, so it runs
only once the sum it writes in is complete."""
bind = None
is_plain_add = False
applied_at_exit = False
outlives_layer = False
@@ -189,6 +193,8 @@ class _AttnUpdate:
class _FfnReadout:
"""The attention output's injection and the FFN input's mix."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -220,6 +226,8 @@ class _FfnUpdate:
"""The gated injection of the FFN output, which this layer runs itself. The
streams stay widened: the layer stack's terminal read mixes them down."""
bind = None
is_plain_add = False
applied_at_exit = True
outlives_layer = False
@@ -103,6 +103,8 @@ class _AttnReadout:
already hold the previous layer's output: an iHC layer takes its input
written back. A ``post_residual_addition`` is not applied."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -129,6 +131,8 @@ class _AttnUpdate:
It is not a plain add, so it runs only once the sum it writes in is
complete."""
bind = None
is_plain_add = False
applied_at_exit = False
outlives_layer = False
@@ -152,6 +156,8 @@ class _FfnReadout:
"""The attention output's write-back and the FFN input's gated mix and
norm, fused in post_pre when the layer provides it."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -181,6 +187,8 @@ class _FfnUpdate:
layer stack's terminal read is a learned head rather than a contraction,
so it belongs to the final norm."""
bind = None
is_plain_add = False
applied_at_exit = True
outlives_layer = False
@@ -139,6 +139,8 @@ class _AttnReadout:
layer's output: an MHC layer takes its input written back. A
``post_residual_addition`` is not applied."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -164,6 +166,8 @@ class _AttnUpdate:
"""hc_post with the coefficients the attention's read produced. It is not a
plain add, so it runs only once the sum it writes in is complete."""
bind = None
is_plain_add = False
applied_at_exit = False
outlives_layer = False
@@ -187,6 +191,8 @@ class _FfnReadout:
"""The attention output's hc_post and the FFN input's hc_pre and norm, fused
in hc_ffn_post_pre when it takes the batch."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -215,6 +221,8 @@ class _FfnUpdate:
layer runs itself; the last layer also contracts the streams into the
hidden states the layer stack hands on."""
bind = None
is_plain_add = False
applied_at_exit = True
outlives_layer = False
@@ -24,6 +24,8 @@ class PostNormAdd:
"""``residual + norm(output)``. It is not a plain add, so it runs only on a
complete output. ``applied_at_exit`` makes the stage write it itself."""
bind = None
is_plain_add = False
outlives_layer = True
writes_stream = False
@@ -46,6 +48,8 @@ class PostNormAdd:
class PlainReadout:
"""The input is the residual itself; the stage's norm is not applied."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
@@ -46,7 +46,7 @@ class Contribution(msgspec.Struct):
"""
value: Optional[torch.Tensor]
update: ResidualUpdate
update: Optional[ResidualUpdate]
owed: Union[UnreducedOutput, DeclaredSum, DeferredFinalize, None] = None
def for_boundary(self):
@@ -66,9 +66,10 @@ class Contribution(msgspec.Struct):
return value
def release(self):
"""Drop the tensors of a contribution its consumer has taken."""
"""Drop a consumed contribution and its per-invocation update state."""
self.value = None
self.owed = None
self.update = None
class OwedOutput(msgspec.Struct, frozen=True):
@@ -134,7 +135,8 @@ class ResidualStream:
Args:
output: Complete tensor, UnreducedOutput, or producer-specific DeferredFinalize.
update: Producer operation that the next boundary must apply.
declared_sum: SumGroup statically owed by a raw tensor, or None. Cannot
declared_sum: The producer's outstanding sum contract,
statically owed by a raw tensor; None when complete. Cannot
be combined with another output wrapper's completion contract.
Returns:
@@ -146,13 +148,23 @@ class ResidualStream:
if declared_sum is not None:
if not isinstance(output, torch.Tensor):
raise RuntimeError("a declared sum cannot carry another completion")
contribution = Contribution(output, update, DeclaredSum(declared_sum))
owed = (
declared_sum
if isinstance(declared_sum, DeclaredSum)
else DeclaredSum(declared_sum)
)
contribution = Contribution(output, update, owed)
elif isinstance(output, UnreducedOutput):
contribution = Contribution(output.partial, update, output)
elif isinstance(output, DeferredFinalize):
contribution = Contribution(None, update, output)
else:
contribution = Contribution(output, update)
bind = update.bind
if bind is not None:
raise RuntimeError(
"bind an invocation update at the stage before recording it"
)
self.pending = contribution
return OwedOutput(contribution) if contribution.owed is not None else output
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
)
from sglang.srt.layers.layer_boundary.layout import SumGroup
from sglang.srt.layers.layer_boundary.ops import sum_output
from sglang.srt.layers.layer_boundary.residual import bind_update
from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp
from sglang.srt.layers.layer_boundary.residual.add_norm import (
PLAIN_ADD,
@@ -168,7 +169,7 @@ class StageBoundary:
)
return hidden_states
def finish(self, hidden_states, forward_batch):
def finish(self, hidden_states, forward_batch, *, update_context=None):
"""Hand this stage's output to the following stage: an attention's
actual contribution, or an FFN's or a mixer's output with its sum
completed or carried on. An FFN whose boundary has fusions (which may
@@ -177,16 +178,18 @@ class StageBoundary:
produced = self.plan.path_for(forward_batch).output
return stream_of(forward_batch).record(
hidden_states,
produced.update,
bind_update(produced.update, forward_batch, update_context),
declared_sum=produced.group if produced.always_partial else None,
)
if self.kind is StageKind.FFN and self.plan.fusions is not None:
raise RuntimeError(
"an FFN whose boundary has fusions finishes through exit()"
)
return self.exit(forward_batch).finish(hidden_states)
return self.exit(forward_batch, update_context=update_context).finish(
hidden_states
)
def exit(self, forward_batch):
def exit(self, forward_batch, *, update_context=None):
"""Select and scope one FFN or single-stage mixer's output decision,
for an FFN whose MoE may hand off its finalize (see finish()).
@@ -201,10 +204,16 @@ class StageBoundary:
using the same decision rather than selecting a second path.
"""
result = (
self.plan.output.ffn_exit(forward_batch, stream=stream_of(forward_batch))
self.plan.output.ffn_exit(
forward_batch,
stream=stream_of(forward_batch),
update_context=update_context,
)
if self.kind is StageKind.FFN
else self.plan.output.mixer_exit(
forward_batch, stream=stream_of(forward_batch)
forward_batch,
stream=stream_of(forward_batch),
update_context=update_context,
)
)
return result
@@ -260,7 +269,14 @@ class StageBoundary:
hidden_states = stream.complete(hidden_states)
return hidden_states, stream.snapshot(hidden_states)
def complete_now(self, hidden_states, forward_batch, *, already_reduced=False):
def complete_now(
self,
hidden_states,
forward_batch,
*,
already_reduced=False,
update_context=None,
):
"""Complete an FFN output outside exit(), including its residual update.
already_reduced is for a producer kernel that fused the output sum.
@@ -271,6 +287,7 @@ class StageBoundary:
stream_of(forward_batch),
forward_batch,
already_reduced=already_reduced,
update_context=update_context,
)
def sum_part(self, hidden_states, forward_batch, group: SumGroup):
@@ -347,6 +364,7 @@ class StageBoundary:
post_residual_addition: Optional[torch.Tensor] = None,
quant_format: str = "",
capture: Optional[Callable] = None,
read_context=None,
):
# Aux consumers need a materialized output before the input norm.
# Complete communication here, preserving the existing add+norm kernel
@@ -373,6 +391,7 @@ class StageBoundary:
steps,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
read_context=read_context,
)
entry = steps.entry
if capture is not None and not capture_before_read:
+2
View File
@@ -133,6 +133,8 @@ class _ResidualMergeRead:
sum as the new residual, and the input norm of that sum in the model
dtype. The producer's update must be a plain add."""
bind = None
is_plain_norm = False
completing_fusions = ()
gathering_reads = ()
+132
View File
@@ -1,8 +1,11 @@
"""Small fixtures around production boundaries; no parallel layout recipes."""
from contextlib import contextmanager
from types import SimpleNamespace
from unittest import mock
from unittest.mock import patch
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.layer_boundary import factories
from sglang.srt.layers.layer_boundary.construction import StagePlan
from sglang.srt.layers.layer_boundary.factories import (
@@ -14,6 +17,8 @@ from sglang.srt.layers.layer_boundary.factories import (
from sglang.srt.layers.layer_boundary.ops import keep_output
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_RESIDUAL_OPS
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import utils as moe_utils
from sglang.test.communicator_patch import patch_communicator
def build_stages(*stages, previous=None, terminal=False):
@@ -189,3 +194,130 @@ def postprocess_output(boundary, hidden, residual, forward_batch):
def identity_input(hidden_states, forward_batch):
return hidden_states
def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
fields = dict(
tp_size=attn_dp * attn_cp * attn_tp,
tp_rank=0,
attn_dp_size=attn_dp,
attn_tp_size=attn_tp,
attn_tp_rank=0,
attn_cp_size=attn_cp,
attn_cp_rank=0,
moe_dense_tp_size=None,
enable_prefill_cp=False,
enable_cp_tp_group_sharing=False,
moe_ep_size=1,
moe_tp_size=attn_dp * attn_cp * attn_tp,
moe_dp_size=1,
dwdp_size=1,
attn_dp_enabled=attn_dp > 1,
enable_attn_tp_input_scattered=False,
disable_attn_tp_gather=False,
tp_group=SimpleNamespace(
name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp))
),
attn_tp_group=SimpleNamespace(name="attn_tp", ranks=list(range(attn_tp))),
attn_cp_group=SimpleNamespace(name="attn_cp", ranks=list(range(attn_cp))),
)
fields.update(overrides)
return SimpleNamespace(**fields)
@contextmanager
def planning(
parallel,
*,
sp=False,
a2a=False,
dsa_cp=False,
boundary_reduction="rs+rsv",
):
"""What layer planning and communicator construction read, without the
process-wide parallel state. ``parallel`` may be a callable, for a
per-thread parallel state. ``dsa_cp``: the prefill CP is DSA's (MLA's is
the same to the communicator)."""
get_parallel = parallel if callable(parallel) else (lambda: parallel)
def moe_cp_gathers():
return get_parallel().attn_cp_size > get_parallel().moe_dp_size
with (
patch_communicator("get_parallel", get_parallel),
patch_communicator("is_dsa_enable_prefill_cp", lambda: dsa_cp),
patch_communicator("is_mla_cp_enabled", lambda: False),
patch_communicator(
"get_moe_cp_size",
lambda: get_parallel().attn_cp_size if moe_cp_gathers() else 1,
),
patch.object(layernorm_sp, "layernorm_sp_enabled", lambda: sp),
patch_communicator(
"get_spec", lambda: SimpleNamespace(speculative_algorithm=None)
),
patch_communicator(
"get_moe_a2a_backend",
lambda: SimpleNamespace(is_none=lambda: not a2a),
),
patch_communicator("is_moe_input_scattered_across_dp_ranks", lambda: a2a),
patch_communicator("is_enable_moe_cp_allgather", moe_cp_gathers),
patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=False)),
# A MoE whose EP and TP sums merge: its output's group is the TP group.
patch_communicator(
"post_experts_reduction_group", lambda: get_parallel().tp_group
),
# Planning asks whether a dense layer gathers for two-batch overlap.
patch_communicator(
"get_exec",
lambda: SimpleNamespace(
comm=SimpleNamespace(
boundary_reduction=boundary_reduction,
enable_quant_communications=False,
),
overlap=SimpleNamespace(enable_two_batch_overlap=False),
),
),
# What the MoE predicates an exit's decisions read, in their module.
patch.object(moe_utils, "get_parallel", get_parallel),
patch.object(
moe_utils,
"get_moe_a2a_backend",
lambda: SimpleNamespace(
is_none=lambda: not a2a,
is_flashinfer=lambda: False,
is_pplx=lambda: False,
is_flashinfer_megamoe=lambda: False,
),
),
patch.object(moe_utils, "get_lora", lambda: SimpleNamespace(enable_lora=False)),
patch.object(
moe_utils,
"get_exec",
lambda: SimpleNamespace(
comm=SimpleNamespace(enable_quant_communications=False)
),
),
patch.object(
moe_utils, "post_experts_reduction_group", lambda: get_parallel().tp_group
),
patch.object(
moe_utils, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False
),
patch.object(
moe_utils,
"is_dp_attention_enabled",
lambda: get_parallel().attn_dp_size > 1,
),
):
yield
class Norm:
"""add + norm stand-in: (2 * (h + r), h + r); one input: 2 * h."""
def __call__(self, x, residual=None, post_residual_addition=None):
assert post_residual_addition is None
if residual is None:
return x * 2
s = x + residual
return s * 2, s