[Refactor] Carry residual state across stage boundaries (#41549)
This commit is contained in:
@@ -17,9 +17,10 @@ from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import (
|
||||
CommunicateContext,
|
||||
Layout,
|
||||
reduce_output,
|
||||
tbo_split_moves,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.access import finish_layer_stack
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
get_moe_a2a_backend,
|
||||
@@ -964,12 +965,25 @@ def _model_forward_tbo(
|
||||
operations_strategy: OperationsStrategy,
|
||||
layer_input_rows: Layout,
|
||||
):
|
||||
inputs["hidden_states"] = reduce_output(inputs["hidden_states"])
|
||||
parent_batch = inputs["forward_batch"]
|
||||
stream = inputs["residual"]
|
||||
pending = stream.pending if isinstance(stream, ResidualStream) else None
|
||||
inputs["hidden_states"], inputs["residual"] = finish_layer_stack(
|
||||
inputs["hidden_states"], inputs["residual"], inputs["forward_batch"]
|
||||
)
|
||||
inputs_arr = _model_forward_tbo_split_inputs(
|
||||
**inputs, layer_input_rows=layer_input_rows
|
||||
)
|
||||
original_hidden_states_len = inputs["hidden_states"].shape[0]
|
||||
del inputs
|
||||
if isinstance(stream, ResidualStream):
|
||||
for part in inputs_arr:
|
||||
part["hidden_states"], part["residual"] = ResidualStream.arrive(
|
||||
part["hidden_states"],
|
||||
part["residual"],
|
||||
pending.update if pending is not None else None,
|
||||
)
|
||||
part["forward_batch"].residual_stream = part["residual"]
|
||||
del inputs, stream, pending
|
||||
|
||||
context = (
|
||||
empty_context()
|
||||
@@ -986,7 +1000,13 @@ def _model_forward_tbo(
|
||||
delta_stages=[0, operations_strategy.tbo_delta_stages],
|
||||
)
|
||||
|
||||
return _model_forward_tbo_merge_outputs(*outputs_arr, original_hidden_states_len)
|
||||
hidden_states, residual = _model_forward_tbo_merge_outputs(
|
||||
*outputs_arr, original_hidden_states_len
|
||||
)
|
||||
parent_batch.residual_stream = (
|
||||
residual if isinstance(residual, ResidualStream) else None
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy):
|
||||
@@ -1100,6 +1120,21 @@ def _model_forward_filter_inputs(
|
||||
|
||||
|
||||
def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
|
||||
stream_a, stream_b = output_a["residual"], output_b["residual"]
|
||||
has_stream = isinstance(stream_a, ResidualStream)
|
||||
assert has_stream == isinstance(stream_b, ResidualStream)
|
||||
update = None
|
||||
if has_stream:
|
||||
pending_a, pending_b = stream_a.pending, stream_b.pending
|
||||
assert (pending_a is None) == (pending_b is None)
|
||||
if pending_a is not None:
|
||||
assert pending_a.update is pending_b.update
|
||||
update = pending_a.update
|
||||
for output in (output_a, output_b):
|
||||
output["hidden_states"], output["residual"] = finish_layer_stack(
|
||||
output["hidden_states"], output["residual"], output["forward_batch"]
|
||||
)
|
||||
|
||||
def _handle_key(name):
|
||||
value_a = output_a[name]
|
||||
value_b = output_b[name]
|
||||
@@ -1117,7 +1152,12 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
|
||||
res[slice(s1, t1)] = value_b[: t1 - s1]
|
||||
return res
|
||||
|
||||
return _handle_key("hidden_states"), _handle_key("residual")
|
||||
hidden, residual = _handle_key("hidden_states"), _handle_key("residual")
|
||||
return (
|
||||
ResidualStream.arrive(hidden, residual, update)
|
||||
if has_stream
|
||||
else (hidden, residual)
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------- Utilities and wrappers ---------------------------------------
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.layers.communicator.ops import (
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
_consumer_step,
|
||||
_dispatch_consumer,
|
||||
_hand_qkv_hook_its_input,
|
||||
_mlp_input_dp_partial,
|
||||
_mlp_input_dp_replicate,
|
||||
@@ -96,8 +97,9 @@ class StageOutput(msgspec.Struct, frozen=True):
|
||||
# Whether it leaves the sum to the attention-DP reduce_scatterv whenever that
|
||||
# combine applies, as a MoE block does without any flag.
|
||||
leaves_for_reduce_scatterv: bool = False
|
||||
# How the producer's output is written into the residual.
|
||||
update: StageUpdate = ADD
|
||||
# How the producer's output is written into the residual. An arrival
|
||||
# description has no producer object; its edge declares capabilities only.
|
||||
update: Optional[StageUpdate] = ADD
|
||||
|
||||
|
||||
class StageDecl(msgspec.Struct, frozen=True):
|
||||
@@ -140,6 +142,8 @@ class EdgeDecl(msgspec.Struct, frozen=True):
|
||||
# Whether the residual is added into one rank's share of the produced sum
|
||||
# before that sum completes, instead of after it.
|
||||
residual_joins_sum: bool = False
|
||||
# Capabilities allowed to arrive from another layer, not its update object.
|
||||
update_capabilities: Tuple[bool, ...] = ()
|
||||
|
||||
|
||||
class DecoderLayerEdges(msgspec.Struct, frozen=True):
|
||||
@@ -159,13 +163,13 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
construction (``input_owes``); a sum the previous layer leaves for a batch
|
||||
comes with the value."""
|
||||
owes = sides.input_owes is not None
|
||||
# The previous layer's FFN output, written in as this layer's FFN writes its
|
||||
# own.
|
||||
# Cross-layer arrival describes rows and owed work. The actual update
|
||||
# arrives in the stream; this model family supplies only its capability.
|
||||
arrived = StageOutput(
|
||||
sides.input_rows,
|
||||
group=sides.input_owes,
|
||||
always_leaves=owes,
|
||||
update=sides.ffn.output.update,
|
||||
update=None,
|
||||
)
|
||||
# Completing what the input owes leaves it and the residual on each
|
||||
# attention-TP rank's slice.
|
||||
@@ -180,6 +184,7 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges:
|
||||
need=sides.attention.input,
|
||||
residual=sides.input_rows,
|
||||
residual_to=attention_rows,
|
||||
update_capabilities=(sides.ffn.output.update.adds_plainly,),
|
||||
),
|
||||
into_ffn=EdgeDecl(
|
||||
produced=sides.attention.output,
|
||||
@@ -227,22 +232,28 @@ def stage_edges(
|
||||
first, and what it may leave of its sum comes with the value; otherwise the
|
||||
value is complete. The residual follows the input onto a finer slice and
|
||||
stays where it is when the input is gathered."""
|
||||
update = previous.update if previous is not None else ADD
|
||||
adds_plainly = previous.update.adds_plainly if previous is not None else True
|
||||
arrived = (
|
||||
StageOutput(
|
||||
rows,
|
||||
group=previous.group,
|
||||
always_leaves=previous.always_leaves,
|
||||
leaves_for_next_layer=previous.leaves_for_next_layer,
|
||||
update=update,
|
||||
update=None,
|
||||
)
|
||||
if previous is not None
|
||||
and (previous.always_leaves or previous.leaves_for_next_layer)
|
||||
else StageOutput(rows, update=update)
|
||||
else StageOutput(rows, update=None)
|
||||
)
|
||||
during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows
|
||||
return (
|
||||
EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during),
|
||||
EdgeDecl(
|
||||
produced=arrived,
|
||||
need=stage.input,
|
||||
residual=rows,
|
||||
residual_to=during,
|
||||
update_capabilities=(adds_plainly,),
|
||||
),
|
||||
EdgeDecl(
|
||||
produced=stage.output,
|
||||
need=StageInput(rows),
|
||||
@@ -622,6 +633,56 @@ def make_boundary(
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
"""Bind only the update capabilities that can reach this edge.
|
||||
|
||||
Execution receives the producer's actual update from the residual stream.
|
||||
A single capability needs no dispatcher; multiple capabilities select one
|
||||
of the already constructed paths at the consumer.
|
||||
"""
|
||||
capabilities = edge.update_capabilities
|
||||
if not capabilities:
|
||||
if edge.produced.update is None:
|
||||
raise ValueError("an arrival must declare its update capabilities")
|
||||
capabilities = (edge.produced.update.adds_plainly,)
|
||||
paths = {
|
||||
capability: _bind_consumer(
|
||||
edge,
|
||||
adds_plainly=capability,
|
||||
fusions=fusions,
|
||||
carried_fusions=carried_fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
for capability in capabilities
|
||||
}
|
||||
first = next(iter(paths.values()))
|
||||
if len(paths) == 1:
|
||||
return first
|
||||
if any(path.input_move != first.input_move for path in paths.values()):
|
||||
raise NotImplementedError("update capabilities require different input moves")
|
||||
return msgspec.structs.replace(
|
||||
first,
|
||||
prepare=partial(
|
||||
_dispatch_consumer,
|
||||
paths={capability: path.prepare for capability, path in paths.items()},
|
||||
),
|
||||
fused=tuple(
|
||||
f for f in fusions if any(f in path.fused for path in paths.values())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _bind_consumer(
|
||||
edge: EdgeDecl,
|
||||
*,
|
||||
adds_plainly: bool,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
carried_fusions: Tuple[Callable, ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> Boundary:
|
||||
"""The consumer's half of ``edge``, chosen from the edge's declarations and
|
||||
the producer's update: what a value carries for a batch (a sum or a handoff
|
||||
@@ -633,28 +694,45 @@ def make_boundary(
|
||||
over attention CP; ``enters_stack`` for the edge into the layer stack's
|
||||
first stage. The steps read only this edge's declarations, never what the
|
||||
producer chose for a batch."""
|
||||
update = edge.produced.update
|
||||
# A fused kernel runs the add and the norm itself.
|
||||
plain = update.adds_plainly and edge.need.read.norms_plainly
|
||||
plain = adds_plainly and edge.need.read.norms_plainly
|
||||
step, fused, input_move = _select_input_steps(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
update=update,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=fusions if plain else (),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
# A written stream entering the first physical layer must not run enter
|
||||
# again (e.g. MHC expansion). Both alternatives are bound at construction.
|
||||
written_step = None
|
||||
if enters_stack:
|
||||
written_step, _, _ = _select_input_steps(
|
||||
edge.produced,
|
||||
residual=edge.residual,
|
||||
residual_to=edge.residual_to,
|
||||
need=edge.need,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=fusions if plain else (),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=edge.residual_joins_sum,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=False,
|
||||
)
|
||||
return Boundary(
|
||||
edge,
|
||||
prepare=partial(
|
||||
_consumer_step,
|
||||
step=step,
|
||||
adds_plainly=adds_plainly,
|
||||
carried_fusions=carried_fusions if plain else (),
|
||||
owes_by_construction=edge.produced.always_leaves,
|
||||
written_step=written_step,
|
||||
),
|
||||
input_move=input_move,
|
||||
fused=fused,
|
||||
@@ -668,6 +746,16 @@ def make_output_boundary(
|
||||
rows the layer hands on (``edge.need``), whose consumer runs in the next
|
||||
layer: the postprocess that moves the output there. ``cp_moves`` for an
|
||||
edge that returns across attention CP."""
|
||||
update = edge.produced.update
|
||||
if not getattr(update, "at_producer", False):
|
||||
if not getattr(update, "can_defer_across_layers", False):
|
||||
raise NotImplementedError(
|
||||
"a deferred update must guarantee its lifetime across layers"
|
||||
)
|
||||
if not update.adds_plainly and get_parallel().pp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"pipeline boundaries require a plain add or a producer-written residual"
|
||||
)
|
||||
if edge.need.layout != edge.residual_to:
|
||||
raise NotImplementedError(f"{edge=}")
|
||||
returns_over_dp, output_move, completes_sum = _select_ffn_output_move(
|
||||
@@ -740,7 +828,7 @@ def _select_input_steps(
|
||||
residual: Layout,
|
||||
residual_to: Layout,
|
||||
need: StageInput,
|
||||
update: StageUpdate,
|
||||
adds_plainly: bool,
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
force_layernorm_before_gather: bool,
|
||||
residual_joins_sum: bool,
|
||||
@@ -781,7 +869,6 @@ def _select_input_steps(
|
||||
_mlp_input_scatter if owes is SumGroup.ATTN_TP else _mlp_input_slice,
|
||||
scatters_residual=residual != residual_to,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
@@ -793,7 +880,7 @@ def _select_input_steps(
|
||||
if gathered or owes is not SumGroup.ATTN_TP:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(_mlp_input_on_residual_shard, read=read, update=update),
|
||||
partial(_mlp_input_on_residual_shard, read=read),
|
||||
(),
|
||||
None,
|
||||
)
|
||||
@@ -812,7 +899,6 @@ def _select_input_steps(
|
||||
layer_input=tp_reduce_scatter,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
@@ -826,7 +912,7 @@ def _select_input_steps(
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=StageInput(produced.layout, read=read),
|
||||
update=update,
|
||||
adds_plainly=adds_plainly,
|
||||
fusions=fusions,
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
@@ -849,7 +935,6 @@ def _select_input_steps(
|
||||
layer_input=None,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
CommunicateSimpleFn._scattered_to_tp_attn_full,
|
||||
@@ -878,7 +963,6 @@ def _select_input_steps(
|
||||
layer_input=None,
|
||||
enters_stack=enters_stack,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
@@ -886,7 +970,7 @@ def _select_input_steps(
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if owes is not SumGroup.ATTN_TP or not update.adds_plainly:
|
||||
if owes is not SumGroup.ATTN_TP or not adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_residual_into_sum, read=read), (), None
|
||||
# A sum over TP completes only on rows every TP rank holds: the TP group
|
||||
@@ -903,7 +987,6 @@ def _select_input_steps(
|
||||
fusions=tuple(f.run for f in fused),
|
||||
group=owes,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
fused,
|
||||
None,
|
||||
@@ -920,7 +1003,7 @@ def _select_input_steps(
|
||||
if (
|
||||
owes_attention_tp
|
||||
and not force_layernorm_before_gather
|
||||
and update.adds_plainly
|
||||
and adds_plainly
|
||||
and read.norms_plainly
|
||||
):
|
||||
return (
|
||||
@@ -940,7 +1023,6 @@ def _select_input_steps(
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
update=update,
|
||||
),
|
||||
(),
|
||||
None,
|
||||
|
||||
@@ -72,6 +72,7 @@ from sglang.srt.layers.communicator.output import (
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import LayerResidual
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.communicator.residual.access import (
|
||||
add_to_output,
|
||||
buffer,
|
||||
@@ -79,6 +80,7 @@ from sglang.srt.layers.communicator.residual.access import (
|
||||
fold,
|
||||
from_pp,
|
||||
snapshot,
|
||||
written,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
PLAIN_RESIDUAL,
|
||||
@@ -87,6 +89,7 @@ from sglang.srt.layers.communicator.residual.add_norm import (
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.mhc import MHCState
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
@@ -195,6 +198,15 @@ class StageCommunicator:
|
||||
``quant_format``)."""
|
||||
entry = self.entry(forward_batch, steps)
|
||||
context = self._layer._context
|
||||
stream = residual_batch.current(forward_batch)
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
if stream is not None:
|
||||
if stream.pending is not None:
|
||||
call["update"] = stream.pending.update
|
||||
if stream.pending is None and stream.residual is not None:
|
||||
call["written"] = True
|
||||
hidden_states, residual = stream.input(hidden_states)
|
||||
hidden_states, residual = entry.prepare(
|
||||
hidden_states, residual, forward_batch, self.norm, context, **call
|
||||
)
|
||||
@@ -208,7 +220,8 @@ class StageCommunicator:
|
||||
hidden_states = entry.handoff(
|
||||
hidden_states, forward_batch, self._layer.qkv_latent_func
|
||||
)
|
||||
return hidden_states, residual
|
||||
stream.write(residual)
|
||||
return hidden_states, stream
|
||||
|
||||
|
||||
class LayerFusions(Protocol):
|
||||
@@ -231,6 +244,7 @@ class LayerCommunicator:
|
||||
add_to_output = staticmethod(add_to_output)
|
||||
buffer = staticmethod(buffer)
|
||||
fold = staticmethod(fold)
|
||||
written = staticmethod(written)
|
||||
finish_layer_stack = staticmethod(finish_layer_stack)
|
||||
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA
|
||||
@@ -602,15 +616,24 @@ class LayerCommunicator:
|
||||
storage = buffer(hidden_states)
|
||||
if storage is not None and storage.shape[0] == 0:
|
||||
return hidden_states, None
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states = (
|
||||
residual.complete(hidden_states)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else reduce_output(hidden_states)
|
||||
)
|
||||
return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input)
|
||||
|
||||
def from_pp(self, tensors, *, allow_missing_residual: bool = False):
|
||||
return from_pp(
|
||||
def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False):
|
||||
hidden_states, residual = from_pp(
|
||||
tensors,
|
||||
residual_in_hidden=self._residual.ffn_update.at_producer,
|
||||
allow_missing_residual=allow_missing_residual,
|
||||
)
|
||||
hidden_states, stream = ResidualStream.arrive(
|
||||
hidden_states, residual, self._residual.ffn_update
|
||||
)
|
||||
forward_batch.residual_stream = stream
|
||||
return hidden_states, stream
|
||||
|
||||
def prepare_attn_and_capture_last_layer_outputs(
|
||||
self,
|
||||
@@ -629,23 +652,26 @@ class LayerCommunicator:
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
residual_value = residual.residual
|
||||
move = self.attn.entry(forward_batch).input_move
|
||||
gathered_last_layer_output = (
|
||||
residual
|
||||
residual_value
|
||||
if move is None
|
||||
else move(
|
||||
hidden_states=residual,
|
||||
hidden_states=residual_value,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
if (
|
||||
gathered_last_layer_output is residual
|
||||
gathered_last_layer_output is residual_value
|
||||
# An accumulator that copies on append already holds a snapshot.
|
||||
and not getattr(captured_last_layer_outputs, "copies_on_append", False)
|
||||
and not self._post_attn_residual_is_read_only(residual, forward_batch)
|
||||
and not self._post_attn_residual_is_read_only(
|
||||
residual_value, forward_batch
|
||||
)
|
||||
):
|
||||
gathered_last_layer_output = residual.clone()
|
||||
gathered_last_layer_output = residual_value.clone()
|
||||
captured_last_layer_outputs.append(gathered_last_layer_output)
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -836,7 +862,18 @@ class LayerCommunicator:
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
return self.ffn.prepare(hidden_states, residual, forward_batch, steps)
|
||||
if isinstance(residual, ResidualStream) and self.stage_edges is None:
|
||||
# The paired boundary records the attention contribution once.
|
||||
hidden_states = residual.leave(
|
||||
hidden_states, self._residual.attention_update
|
||||
)
|
||||
return self.ffn.prepare(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
steps,
|
||||
update=self._residual.attention_update,
|
||||
)
|
||||
|
||||
def maybe_prefetch_next_full_attention_kv(
|
||||
self,
|
||||
@@ -851,12 +888,28 @@ class LayerCommunicator:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
return self._complete_ffn_output_now(
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
hidden_states, residual = self._complete_ffn_output_now(
|
||||
hidden_states,
|
||||
residual,
|
||||
stream.residual if stream is not None else residual,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=self._postprocess_dp_step(forward_batch),
|
||||
)
|
||||
return self._leave_ffn_output(
|
||||
hidden_states,
|
||||
residual,
|
||||
stream,
|
||||
self._batch_steps(forward_batch).ffn_output.update,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _leave_ffn_output(hidden_states, residual, stream, update):
|
||||
if stream is None:
|
||||
return hidden_states, residual
|
||||
stream.residual = residual
|
||||
if update.at_producer:
|
||||
return stream.write(hidden_states), stream
|
||||
return stream.leave(hidden_states, update), stream
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1037,10 +1090,15 @@ class LayerCommunicator:
|
||||
to the rows this FFN needs and its residual's rows."""
|
||||
rows, residual_rows, _ = source._branch_rows(forward_batch)
|
||||
to, residual_to, _ = self._branch_rows(forward_batch)
|
||||
return (
|
||||
move_rows(hidden_states, rows, to, forward_batch),
|
||||
move_rows(residual, residual_rows, residual_to, forward_batch),
|
||||
)
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
if stream.pending is not None:
|
||||
raise RuntimeError("a branch must start from a prepared stage input")
|
||||
residual = stream.residual
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, residual_rows, residual_to, forward_batch)
|
||||
forward_batch.residual_stream = ResidualStream(residual)
|
||||
return hidden_states, forward_batch.residual_stream
|
||||
|
||||
def branch_output(
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
@@ -1064,9 +1122,19 @@ class LayerCommunicator:
|
||||
this layer hands on."""
|
||||
_, _, rows = source._branch_rows(forward_batch)
|
||||
_, _, to = self._branch_rows(forward_batch)
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
update = None
|
||||
if stream is not None:
|
||||
stream.check(hidden_states)
|
||||
update = stream.pending.update
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
hidden_states = move_rows(hidden_states, rows, to, forward_batch)
|
||||
residual = move_rows(residual, rows, to, forward_batch)
|
||||
return contribution + hidden_states, residual
|
||||
output = contribution + hidden_states
|
||||
if stream is not None:
|
||||
stream.write(residual)
|
||||
return stream.leave(output, update), stream
|
||||
return output, residual
|
||||
|
||||
def _select_ffn_exit_fusions(
|
||||
self,
|
||||
@@ -1203,10 +1271,11 @@ class MixerExit:
|
||||
its input), and when it may leave it and the fused kernel takes it into the
|
||||
next input norm."""
|
||||
|
||||
__slots__ = ("skips_reduction", "_hands_on", "_scope")
|
||||
__slots__ = ("skips_reduction", "_hands_on", "_scope", "_update")
|
||||
|
||||
def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch):
|
||||
produced = communicator._batch_steps(forward_batch).ffn_output
|
||||
self._update = produced.update
|
||||
self._hands_on = (
|
||||
produced.leaves_for_next_layer
|
||||
and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch)
|
||||
@@ -1221,13 +1290,15 @@ class MixerExit:
|
||||
def __exit__(self, *exc_info):
|
||||
return self._scope.__exit__(*exc_info)
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> Union[torch.Tensor, UnreducedOutput]:
|
||||
def finish(self, hidden_states: torch.Tensor, residual=None):
|
||||
"""The mixer's output: its partial sum to the FFN stage after it, as a
|
||||
value that owes the sum to a mixer after it, or complete."""
|
||||
if self._hands_on:
|
||||
return UnreducedOutput(hidden_states, group=get_parallel().tp_group)
|
||||
hidden_states = UnreducedOutput(
|
||||
hidden_states, group=get_parallel().tp_group
|
||||
)
|
||||
if isinstance(residual, ResidualStream):
|
||||
return residual.leave(hidden_states, self._update), residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1243,6 +1314,7 @@ class FfnExit:
|
||||
"fuse_mlp_allreduce",
|
||||
"mlp_reduce_scatter",
|
||||
"_complete",
|
||||
"_update",
|
||||
"_scope",
|
||||
)
|
||||
|
||||
@@ -1254,6 +1326,7 @@ class FfnExit:
|
||||
self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce
|
||||
self.mlp_reduce_scatter = completion.mlp_reduce_scatter
|
||||
self._complete = completion.complete
|
||||
self._update = communicator._batch_steps(forward_batch).ffn_output.update
|
||||
self._scope = get_forward().scoped(
|
||||
fuse_mlp_allreduce=self.fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=self.mlp_reduce_scatter,
|
||||
@@ -1271,11 +1344,17 @@ class FfnExit:
|
||||
self, hidden_states: torch.Tensor, residual: torch.Tensor
|
||||
) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]:
|
||||
"""Leave the reduction to the next layer's input, or postprocess."""
|
||||
stream = residual_batch.current(self.forward_batch)
|
||||
if stream is not None:
|
||||
residual = stream.residual
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
# A deferred MoE finalize handoff, consumed by the next prepare_attn.
|
||||
assert self.defer_moe_finalize, "unrequested deferred MoE handoff"
|
||||
return hidden_states, residual
|
||||
return self._complete(hidden_states, residual)
|
||||
else:
|
||||
hidden_states, residual = self._complete(hidden_states, residual)
|
||||
return self.communicator._leave_ffn_output(
|
||||
hidden_states, residual, stream, self._update
|
||||
)
|
||||
|
||||
|
||||
class MHCLayerCommunicator(LayerCommunicator):
|
||||
|
||||
@@ -320,6 +320,7 @@ def _mlp_input_dp_partial(
|
||||
gathers_residual: bool,
|
||||
places_cp_shards: bool = False,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Attention DP: one rank adds the residual, the gather sums it, then read
|
||||
the input from the sum. With ``places_cp_shards`` each CP rank puts its
|
||||
@@ -384,6 +385,22 @@ def _tp_all_gather_scattered_rows(
|
||||
return output
|
||||
|
||||
|
||||
def _dispatch_consumer(
|
||||
hidden_states, residual, forward_batch, norm, context, *, paths, update=ADD, **call
|
||||
):
|
||||
if residual is None:
|
||||
# Enter/read-only paths do not execute an update.
|
||||
prepare = paths[False]
|
||||
else:
|
||||
try:
|
||||
prepare = paths[update.adds_plainly]
|
||||
except KeyError:
|
||||
raise RuntimeError("producer update has no bound input path") from None
|
||||
return prepare(
|
||||
hidden_states, residual, forward_batch, norm, context, update=update, **call
|
||||
)
|
||||
|
||||
|
||||
def _consumer_step(
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput],
|
||||
residual: Optional[torch.Tensor],
|
||||
@@ -392,8 +409,12 @@ def _consumer_step(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
step: Callable,
|
||||
adds_plainly: bool,
|
||||
carried_fusions: Tuple[Callable, ...],
|
||||
owes_by_construction: bool = False,
|
||||
written_step: Optional[Callable] = None,
|
||||
written: bool = False,
|
||||
update: StageUpdate = ADD,
|
||||
**call,
|
||||
):
|
||||
"""A boundary's half into a stage: complete what the value carries, a sum
|
||||
@@ -404,6 +425,10 @@ def _consumer_step(
|
||||
that owes its sum by construction (``owes_by_construction``) is that sum's
|
||||
only carrier. ``call`` is what the stage's read takes (the attention's
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if residual is not None and update.adds_plainly != adds_plainly:
|
||||
raise RuntimeError("producer update does not match the boundary's capability")
|
||||
if written and written_step is not None:
|
||||
step = written_step
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
owed = hidden_states
|
||||
if residual is None:
|
||||
@@ -429,7 +454,9 @@ def _consumer_step(
|
||||
hidden_states = reduce_output(owed)
|
||||
else:
|
||||
hidden_states = owed.partial
|
||||
return step(hidden_states, residual, forward_batch, norm, context, **call)
|
||||
return step(
|
||||
hidden_states, residual, forward_batch, norm, context, update=update, **call
|
||||
)
|
||||
|
||||
|
||||
def _read_input(
|
||||
@@ -442,7 +469,7 @@ def _read_input(
|
||||
layer_input: Optional[Callable],
|
||||
enters_stack: bool,
|
||||
read: StageRead,
|
||||
update: StageUpdate,
|
||||
update: StageUpdate = ADD,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
@@ -535,6 +562,7 @@ def _mlp_input_residual_into_sum(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
read: StageRead = NORM_READ,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
return _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states, residual, layernorm, context, read
|
||||
@@ -565,12 +593,13 @@ def _mlp_input_gather_attention_cp(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
|
||||
equal length, over the attention-CP group. The residual stays on the
|
||||
shard."""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, context
|
||||
hidden_states, residual, forward_batch, layernorm, context, update=update
|
||||
)
|
||||
return dsa_cp_gather_hidden_states(hidden_states), residual
|
||||
|
||||
@@ -583,6 +612,7 @@ def _mlp_input_gather_moe_cp(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
gather: Callable,
|
||||
update: StageUpdate = ADD,
|
||||
):
|
||||
"""Gather for the FFN, then over the MoE-CP group so each rank holds all
|
||||
tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at
|
||||
@@ -596,7 +626,7 @@ def _mlp_input_gather_moe_cp(
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, layernorm, context
|
||||
hidden_states, residual, forward_batch, layernorm, context, update=update
|
||||
)
|
||||
|
||||
rows = moe_cp_gathered_rows(forward_batch)
|
||||
|
||||
@@ -31,6 +31,9 @@ class StageUpdate(Protocol):
|
||||
# The stage writes its output into the residual itself at its end, instead
|
||||
# of leaving that to the next stage's read.
|
||||
at_producer: bool
|
||||
# Its parameters and state remain valid after leaving the producer layer.
|
||||
# Stateful or offloaded implementations must not opt in without that guarantee.
|
||||
can_defer_across_layers: bool
|
||||
|
||||
def update(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the output written into it."""
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.output import (
|
||||
UnreducedOutput,
|
||||
reduce_output,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
|
||||
|
||||
@@ -32,6 +33,8 @@ def buffer(
|
||||
"""Storage that can be reused after prepare consumes the input. A finalize
|
||||
handoff has no reusable layer-output tensor. This does not complete or read
|
||||
the value held in the storage."""
|
||||
if isinstance(hidden_states, OwedOutput):
|
||||
return hidden_states.contribution.value
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
return hidden_states.partial
|
||||
if isinstance(hidden_states, HandoffOutput):
|
||||
@@ -42,18 +45,35 @@ def buffer(
|
||||
def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple:
|
||||
"""Complete the output before adding an extra contribution exactly once.
|
||||
Leave its residual update for the next prepare call."""
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
hidden_states = (
|
||||
residual.complete(hidden_states)
|
||||
if isinstance(residual, ResidualStream)
|
||||
else reduce_output(hidden_states)
|
||||
)
|
||||
hidden_states.add_(extra)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def fold(hidden_states, residual) -> Tuple[torch.Tensor, None]:
|
||||
def fold(hidden_states, residual):
|
||||
"""Finish a plain layer output and fold its residual into a complete value.
|
||||
The following stage receives it with no outstanding residual addition."""
|
||||
stream = residual if isinstance(residual, ResidualStream) else None
|
||||
if stream is not None:
|
||||
hidden_states, residual = stream.finish(hidden_states)
|
||||
hidden_states = reduce_output(hidden_states)
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
return hidden_states, None
|
||||
return (
|
||||
(stream.write(hidden_states), stream)
|
||||
if stream is not None
|
||||
else (hidden_states, None)
|
||||
)
|
||||
|
||||
|
||||
def written(hidden_states, forward_batch):
|
||||
"""Re-enter after a computation that already updated the full residual."""
|
||||
forward_batch.residual_stream = ResidualStream(hidden_states)
|
||||
return hidden_states, forward_batch.residual_stream
|
||||
|
||||
|
||||
def finish_layer_stack(
|
||||
@@ -68,6 +88,13 @@ def finish_layer_stack(
|
||||
pipeline rank, or any other consumer outside the layers. A final norm
|
||||
that does a producer's handoff together with its own work
|
||||
(``final_norm_takes_handoff``) receives it as it is."""
|
||||
if isinstance(residual, ResidualStream):
|
||||
stream = forward_batch.residual_stream
|
||||
if residual is not stream:
|
||||
raise RuntimeError("residual alias belongs to a different invocation")
|
||||
output = stream.finish(hidden_states, takes_handoff=final_norm_takes_handoff)
|
||||
forward_batch.residual_stream = None
|
||||
return output
|
||||
if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput):
|
||||
return hidden_states, residual
|
||||
return reduce_output(hidden_states), residual
|
||||
@@ -99,6 +126,8 @@ def snapshot(
|
||||
) -> torch.Tensor:
|
||||
"""Copy a complete output with its plain residual. A statically declared
|
||||
sum is reduced on a copy; the main output and residual remain unchanged."""
|
||||
if isinstance(residual, ResidualStream):
|
||||
return residual.snapshot(hidden_states, group=group)
|
||||
if group is not None:
|
||||
hidden_states = group.all_reduce(hidden_states.clone())
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
|
||||
@@ -267,6 +267,7 @@ class Add:
|
||||
|
||||
adds_plainly = True
|
||||
at_producer = False
|
||||
can_defer_across_layers = True
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Own one invocation's residual stream, including independent TBO children."""
|
||||
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
|
||||
|
||||
def start(forward_batch):
|
||||
"""Begin a fresh embedding-side invocation and return its stream."""
|
||||
stream = ResidualStream()
|
||||
forward_batch.residual_stream = stream
|
||||
return stream
|
||||
|
||||
|
||||
def current(forward_batch):
|
||||
"""Return the invocation's authoritative stream while its stack is active."""
|
||||
stream = forward_batch.residual_stream
|
||||
if stream is None:
|
||||
raise RuntimeError("start the layer stack before entering a stage")
|
||||
return stream
|
||||
@@ -143,6 +143,7 @@ class _AttentionUpdate:
|
||||
|
||||
adds_plainly = False
|
||||
at_producer = False
|
||||
can_defer_across_layers = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
@@ -185,6 +186,7 @@ class _FfnUpdate:
|
||||
|
||||
adds_plainly = False
|
||||
at_producer = True
|
||||
can_defer_across_layers = False
|
||||
|
||||
def __init__(self, state: MHCState):
|
||||
self.state = state
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""One forward's residual and its producer's outstanding contribution."""
|
||||
|
||||
from typing import Callable, Optional, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput
|
||||
from sglang.srt.layers.communicator.residual import StageUpdate
|
||||
|
||||
|
||||
class CarriedSum(msgspec.Struct, frozen=True):
|
||||
"""Work left by the producer, including a move back to local token rows."""
|
||||
|
||||
group: Optional[GroupCoordinator] = None
|
||||
reduce_and_redistribute: Optional[Callable] = None
|
||||
|
||||
def complete(self, value):
|
||||
if self.reduce_and_redistribute is not None:
|
||||
return self.reduce_and_redistribute(value)
|
||||
return self.group.all_reduce(value)
|
||||
|
||||
|
||||
class Contribution(msgspec.Struct):
|
||||
"""The sole owner of an output's value, update and remaining work."""
|
||||
|
||||
value: Optional[torch.Tensor]
|
||||
update: StageUpdate
|
||||
owed: Union[CarriedSum, HandoffOutput, None] = None
|
||||
|
||||
def for_boundary(self):
|
||||
# These forms are private inputs to the existing fused-kernel adapters.
|
||||
if isinstance(self.owed, CarriedSum):
|
||||
return UnreducedOutput(
|
||||
self.value, self.owed.group, self.owed.reduce_and_redistribute
|
||||
)
|
||||
if isinstance(self.owed, HandoffOutput):
|
||||
return self.owed
|
||||
return self.value
|
||||
|
||||
def complete(self):
|
||||
if isinstance(self.owed, CarriedSum):
|
||||
value = self.owed.complete(self.value)
|
||||
elif isinstance(self.owed, HandoffOutput):
|
||||
value = self.owed.complete()
|
||||
else:
|
||||
return self.value
|
||||
self.value, self.owed = value, None
|
||||
return value
|
||||
|
||||
|
||||
class OwedOutput(msgspec.Struct, frozen=True):
|
||||
"""Opaque model-facing handle. Only its boundary may read the contribution."""
|
||||
|
||||
contribution: Contribution
|
||||
|
||||
|
||||
class ResidualStream:
|
||||
"""Uninitialized, written, or holding one pending producer contribution.
|
||||
|
||||
Each forward or TBO microbatch owns its stream. A prepare consumes its
|
||||
pending contribution once; completing a sum for capture leaves the update
|
||||
pending. No layout or batch selection is stored on the value.
|
||||
"""
|
||||
|
||||
__slots__ = ("residual", "pending")
|
||||
|
||||
def __init__(self, residual=None):
|
||||
self.residual = residual
|
||||
self.pending = None
|
||||
|
||||
@classmethod
|
||||
def arrive(cls, hidden, residual, update):
|
||||
"""Rebuild an explicit tensor handoff, including each TBO microbatch."""
|
||||
stream = cls(residual)
|
||||
if residual is None:
|
||||
return stream.write(hidden), stream
|
||||
return stream.leave(hidden, update), stream
|
||||
|
||||
def write(self, residual):
|
||||
self.residual = residual
|
||||
self.pending = None
|
||||
return residual
|
||||
|
||||
def leave(self, output, update):
|
||||
if self.pending is not None:
|
||||
raise RuntimeError("cannot replace an unconsumed producer contribution")
|
||||
if isinstance(output, UnreducedOutput):
|
||||
contribution = Contribution(
|
||||
output.partial,
|
||||
update,
|
||||
CarriedSum(output.group, output.reduce_and_redistribute),
|
||||
)
|
||||
elif isinstance(output, HandoffOutput):
|
||||
contribution = Contribution(None, update, output)
|
||||
else:
|
||||
contribution = Contribution(output, update)
|
||||
self.pending = contribution
|
||||
return OwedOutput(contribution) if contribution.owed is not None else output
|
||||
|
||||
def check(self, hidden):
|
||||
pending = self.pending
|
||||
if pending is None:
|
||||
if isinstance(hidden, OwedOutput):
|
||||
raise RuntimeError("an owed handle has no contribution in this stream")
|
||||
if self.residual is not None and hidden is not self.residual:
|
||||
raise RuntimeError(
|
||||
"written residual does not match the supplied output"
|
||||
)
|
||||
return
|
||||
if pending.owed is not None:
|
||||
valid = isinstance(hidden, OwedOutput) and hidden.contribution is pending
|
||||
else:
|
||||
valid = hidden is pending.value
|
||||
if not valid:
|
||||
raise RuntimeError(
|
||||
"output does not belong to this residual stream; "
|
||||
"change layer outputs through boundary accessors"
|
||||
)
|
||||
|
||||
def input(self, hidden):
|
||||
self.check(hidden)
|
||||
if self.pending is not None:
|
||||
return self.pending.for_boundary(), self.residual
|
||||
# An initialized, written stream needs read only. An uninitialized
|
||||
# stream still uses the first stage's enter operation.
|
||||
return hidden, None
|
||||
|
||||
def complete(self, hidden):
|
||||
self.check(hidden)
|
||||
return self.pending.complete() if self.pending is not None else hidden
|
||||
|
||||
def snapshot(self, hidden, *, group=None):
|
||||
self.check(hidden)
|
||||
pending = self.pending
|
||||
if pending is None:
|
||||
return hidden.clone()
|
||||
if pending.owed is None:
|
||||
value = pending.value
|
||||
elif isinstance(pending.owed, CarriedSum):
|
||||
value = pending.owed.complete(pending.value.clone())
|
||||
else:
|
||||
raise NotImplementedError("a finalize handoff requires main-output capture")
|
||||
if group is not None:
|
||||
value = group.all_reduce(value.clone())
|
||||
return value.clone() if self.residual is None else value + self.residual
|
||||
|
||||
def finish(self, hidden, *, takes_handoff=False):
|
||||
self.check(hidden)
|
||||
if self.pending is None:
|
||||
return hidden, None
|
||||
if takes_handoff and isinstance(self.pending.owed, HandoffOutput):
|
||||
return self.pending.owed, self.residual
|
||||
return self.pending.complete(), self.residual
|
||||
@@ -71,6 +71,7 @@ from sglang.srt.utils.common import ceil_align, is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.layers.communicator.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.cp.base import BaseContextParallelMetadata
|
||||
from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
@@ -564,6 +565,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
dp_prefill_cuda_graph_max_prefix_len: int = 0
|
||||
global_forward_mode: Optional[ForwardMode] = None
|
||||
|
||||
# Owned only while this invocation executes its local decoder stack.
|
||||
residual_stream: Optional[ResidualStream] = None
|
||||
|
||||
# For two-batch overlap
|
||||
tbo_split_seq_index: Optional[int] = None
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -814,12 +815,12 @@ class BailingMoEModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -980,12 +981,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = inputs_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
device = inputs_embeds.device if inputs_embeds is not None else input_ids.device
|
||||
|
||||
@@ -26,6 +26,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -161,7 +162,7 @@ class BailingMoEModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
if self.is_hybrid:
|
||||
device = input_ids.device
|
||||
zero_allocator = BumpAllocator(
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -1339,12 +1340,12 @@ class BailingMoELinearModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
device = hidden_states.device
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -215,7 +216,7 @@ class DeepseekModelNextN(nn.Module):
|
||||
else:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual, topk_indices = self.decoder(
|
||||
|
||||
@@ -76,6 +76,7 @@ from sglang.srt.layers.communicator import (
|
||||
from sglang.srt.layers.communicator.adapters.context_parallel import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.dcp.planner import (
|
||||
prepare_decode_context_parallel_metadata,
|
||||
@@ -3027,13 +3028,13 @@ class DeepseekV2Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
initial_topk_indices = None
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices")
|
||||
index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices)
|
||||
if not self.pp_group.is_first_rank:
|
||||
|
||||
@@ -62,6 +62,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -1765,12 +1766,12 @@ class Dots3Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
normal_start_layer = self.start_layer
|
||||
normal_end_layer = self.end_layer
|
||||
|
||||
@@ -8,6 +8,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -113,7 +114,7 @@ class Dot3NoteModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = head.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -421,7 +422,7 @@ class FalconH1Model(nn.Module):
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids) * self.embedding_multiplier
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
|
||||
@@ -24,6 +24,7 @@ from transformers import PretrainedConfig
|
||||
import sglang.srt.models.deepseek_v2 as deepseek_v2
|
||||
from sglang.srt.configs.gigachat35 import GigaChat35Config
|
||||
from sglang.srt.layers.communicator import get_attn_tp_context
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -558,12 +559,12 @@ class GigaChat35Model(nn.Module):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
total_num_layers = self.end_layer - self.start_layer
|
||||
zero_allocator = BumpAllocator(
|
||||
|
||||
@@ -18,6 +18,7 @@ from typing import Iterable, Optional
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
@@ -106,7 +107,7 @@ class GigaChat35ModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
hidden_states, residual = self.decoder(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
@@ -1084,12 +1085,12 @@ class Glm4MoeModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
normal_start_layer = self.start_layer
|
||||
normal_end_layer = self.end_layer
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.layers.communicator import (
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
@@ -776,12 +777,12 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
device = hidden_states.device
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
|
||||
|
||||
@@ -22,6 +22,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -114,7 +115,7 @@ class Glm4MoeLiteModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
|
||||
@@ -22,6 +22,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -100,7 +101,7 @@ class Glm4MoeModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.layers.communicator import (
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelBatchedLinear,
|
||||
@@ -1100,12 +1101,12 @@ class Glm5NextModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
device = hidden_states.device
|
||||
zero_allocator = BumpAllocator(
|
||||
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -700,12 +701,12 @@ class GptOssModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
# Capture hidden-state boundaries: boundary 0 is the embedding output,
|
||||
# and boundary i + 1 is the output after transformer block i.
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.srt.configs.interns2_mobius import (
|
||||
InternS2MobiusTextConfig,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -762,7 +763,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
hidden_states, residual = layer(
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -581,12 +582,12 @@ class LagunaModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -746,12 +747,12 @@ class LLaDA2MoeModel(nn.Module):
|
||||
hidden_states = self.word_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
|
||||
@@ -26,6 +26,7 @@ from torch import nn
|
||||
from transformers import Llama4TextConfig
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -518,7 +519,7 @@ class Llama4Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
for i in range(len(self.layers)):
|
||||
if i in self.layers_to_capture:
|
||||
@@ -561,6 +562,39 @@ class Llama4ForCausalLM(LlamaForCausalLM):
|
||||
):
|
||||
super().__init__(config, quant_config, prefix)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward_split_prefill(
|
||||
self, input_ids, positions, forward_batch, split_interval, input_embeds=None
|
||||
):
|
||||
start, end = split_interval
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
forward_batch.hidden_states = (
|
||||
self.model.embed_tokens(input_ids)
|
||||
if input_embeds is None
|
||||
else input_embeds
|
||||
)
|
||||
for layer in self.model.layers[start:end]:
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
)
|
||||
if end != self.model.config.num_hidden_layers:
|
||||
return None
|
||||
hidden, residual = LayerCommunicator.finish_layer_stack(
|
||||
forward_batch.hidden_states, forward_batch.residual, forward_batch
|
||||
)
|
||||
forward_batch.residual = None
|
||||
if residual is None:
|
||||
forward_batch.hidden_states = self.model.norm(hidden)
|
||||
else:
|
||||
forward_batch.hidden_states, _ = self.model.norm(hidden, residual)
|
||||
return self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.embed_tokens
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -624,7 +625,7 @@ class LongcatFlashModel(nn.Module):
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
topk_indices = None
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.configs import LongcatFlashConfig
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -269,7 +270,7 @@ class LongcatFlashModelNextN(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
hidden_states, residual = self.decoder(
|
||||
positions, hidden_states, forward_batch, residual, zero_allocator
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1037,12 +1038,12 @@ class MiMoV2Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
|
||||
if forward_batch.can_run_tbo and not self.layers_to_capture:
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -223,7 +224,7 @@ class MiMoV2ModelNextN(nn.Module):
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=None,
|
||||
residual=residual_batch.start(forward_batch),
|
||||
)
|
||||
|
||||
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_dp_attention_enabled,
|
||||
@@ -1141,12 +1142,12 @@ class MiniMaxM2Model(nn.Module):
|
||||
hidden_states = self.get_input_embeddings(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
if forward_batch.can_run_tbo:
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1476,12 +1477,12 @@ class MiniMaxM3Model(nn.Module):
|
||||
hidden_states = embeds(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
if forward_batch.can_run_tbo:
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -1109,7 +1110,7 @@ class MossVLTextModel(nn.Module):
|
||||
vision_position_ids: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
|
||||
for decoder_layer in self.layers:
|
||||
if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer):
|
||||
@@ -1126,6 +1127,9 @@ class MossVLTextModel(nn.Module):
|
||||
positions=positions,
|
||||
vision_position_ids=vision_position_ids,
|
||||
)
|
||||
hidden_states, residual = LayerCommunicator.written(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
elif isinstance(decoder_layer, MossVLSelfAttentionDecoderLayer):
|
||||
hidden_states, residual = decoder_layer(
|
||||
positions=positions,
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -478,7 +479,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
hidden_states = self._forward_mixer(
|
||||
hidden_states, forward_batch, mixer_exit.skips_reduction
|
||||
)
|
||||
return mixer_exit.finish(hidden_states), residual
|
||||
return mixer_exit.finish(hidden_states, residual)
|
||||
|
||||
|
||||
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
@@ -732,12 +733,12 @@ class NemotronHModel(nn.Module):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
|
||||
@@ -18,6 +18,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs import NemotronHConfig
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
@@ -304,6 +305,8 @@ class NemotronHMultiTokenPredictor(nn.Module):
|
||||
residual = None
|
||||
|
||||
for i in range(self.pattern_len):
|
||||
if residual is None:
|
||||
residual = residual_batch.start(forward_batch)
|
||||
hidden_states, residual = self.layers[str(i)](
|
||||
inputs_embeds=inputs_embeds,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1175,12 +1176,12 @@ class Qwen2MoeModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
if forward_batch.can_run_tbo:
|
||||
@@ -1306,6 +1307,7 @@ class Qwen2MoeForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -1332,6 +1334,7 @@ class Qwen2MoeForCausalLM(nn.Module):
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
# Adapted from qwen2.py
|
||||
import logging
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import access as residual_access
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -446,6 +448,63 @@ class Qwen3Model(Qwen2Model):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: torch.Tensor = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[torch.Tensor, PPProxyTensors]:
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
if input_embeds is None:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(
|
||||
residual_access.snapshot(hidden_states, residual)
|
||||
)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
|
||||
hidden_states, residual = residual_access.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if hidden_states.shape[0] != 0:
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
|
||||
class Qwen3ForCausalLM(nn.Module):
|
||||
# BitandBytes specific attributes
|
||||
@@ -554,6 +613,7 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -578,6 +638,7 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
|
||||
@@ -30,8 +30,6 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
|
||||
qwen3_5_gdn_prefill_projection_views,
|
||||
)
|
||||
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
|
||||
|
||||
# Configs
|
||||
from sglang.srt.configs.qwen3_5 import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5MoeConfig,
|
||||
@@ -47,6 +45,9 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
|
||||
# Configs
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1817,12 +1818,12 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
# Pass through decoder layers
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -1084,6 +1085,7 @@ class Qwen3MoeForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -1110,6 +1112,7 @@ class Qwen3MoeForCausalLM(nn.Module):
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
|
||||
@@ -16,6 +16,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -929,7 +930,7 @@ class Qwen3NextModel(nn.Module):
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
aux_hidden_states = []
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.layers.attention.vision import (
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv3dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -1204,12 +1205,12 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
for layer_idx, layer in enumerate(
|
||||
|
||||
@@ -24,6 +24,7 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
@@ -94,12 +95,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
aux_hidden_states = []
|
||||
for layer_idx, layer in enumerate(
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1151,12 +1152,12 @@ class SarvamMLAModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
@@ -1267,7 +1268,7 @@ class SarvamMLAForCausalLM(nn.Module):
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
forward_batch.hidden_states = input_embeds
|
||||
forward_batch.residual = None
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
|
||||
for i in range(start, end):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
@@ -1291,6 +1292,7 @@ class SarvamMLAForCausalLM(nn.Module):
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
return self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
@@ -1455,7 +1457,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
|
||||
forward_batch.hidden_states = self.model.word_embeddings(input_ids)
|
||||
else:
|
||||
forward_batch.hidden_states = input_embeds
|
||||
forward_batch.residual = None
|
||||
forward_batch.residual = residual_batch.start(forward_batch)
|
||||
|
||||
for i in range(start, end):
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
@@ -1479,6 +1481,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM):
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
)
|
||||
forward_batch.residual = None
|
||||
forward_batch.hidden_states = hidden_states
|
||||
|
||||
return self.logits_processor(
|
||||
|
||||
@@ -12,6 +12,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -403,12 +404,14 @@ class SDARModel(nn.Module):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, allow_missing_residual=True)
|
||||
].layer_communicator.from_pp(
|
||||
pp_proxy_tensors, forward_batch, allow_missing_residual=True
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerFacts,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -469,12 +470,12 @@ class SDARMoeModel(nn.Module):
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -471,7 +472,7 @@ class Step3TextModel(nn.Module):
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -697,12 +698,12 @@ class Step3p5Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -113,7 +114,7 @@ class Step3p5AMultiTokenPredictor(nn.Module):
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=None,
|
||||
residual=residual_batch.start(forward_batch),
|
||||
)
|
||||
hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack(
|
||||
hidden_states, residual, forward_batch
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerFacts,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.communicator.residual import batch as residual_batch
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -1637,12 +1638,12 @@ class XllmModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual = residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states, residual = self.layers[
|
||||
self.start_layer
|
||||
].layer_communicator.from_pp(pp_proxy_tensors)
|
||||
].layer_communicator.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
ctx = (
|
||||
|
||||
Reference in New Issue
Block a user