[Refactor] inkling: Use short-convolution stage boundaries (#43606)
This commit is contained in:
@@ -65,6 +65,10 @@ def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) ->
|
||||
"""Resolve the token-row modes the model's forward actually enters."""
|
||||
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
|
||||
longcat = model_type == "longcat_flash"
|
||||
inkling = architecture in (
|
||||
"InklingForConditionalGeneration",
|
||||
"InklingForConditionalGenerationMTP",
|
||||
)
|
||||
kimi_k3 = architecture in (
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3LinearForCausalLM",
|
||||
@@ -75,7 +79,9 @@ def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) ->
|
||||
raise ValueError(
|
||||
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
|
||||
)
|
||||
if (nemotron or longcat or kimi_k3) and cfg.enable_attn_tp_input_scattered:
|
||||
if (
|
||||
nemotron or longcat or kimi_k3 or inkling
|
||||
) and cfg.enable_attn_tp_input_scattered:
|
||||
# None of these models enters the input-scattered attention scope.
|
||||
logger.warning(
|
||||
"Disabling input-scattered attention for %s: its forward does not enter that scope",
|
||||
|
||||
@@ -282,6 +282,10 @@ def _bind_stage(
|
||||
raise NotImplementedError(
|
||||
"pipeline boundaries require a producer-written residual"
|
||||
)
|
||||
if declaration.sum_impl is not None and declaration.sum_impl.run is None:
|
||||
raise ValueError("a local stage requires its custom sum implementation")
|
||||
if declaration.output_complete and declaration.sum_impl is not None:
|
||||
raise ValueError("compute-completed output cannot also declare a custom sum")
|
||||
if declaration.terminal and not (
|
||||
update.is_plain_add
|
||||
or update.applied_at_exit
|
||||
|
||||
@@ -21,7 +21,7 @@ import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform, SumImplementation
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
|
||||
|
||||
@@ -105,6 +105,8 @@ class OutputContract(msgspec.Struct, frozen=True):
|
||||
transform: Optional operation on the contribution before residual update.
|
||||
in_token_order: The output covers the whole sequence in the batch's
|
||||
token order; the consumer takes this rank's CP shard of it back.
|
||||
sum_impl: Custom implementation of this declared sum, or None for
|
||||
the ordinary boundary collective. It does not change token rows.
|
||||
|
||||
These are permissions, not a record of a particular output. The exit
|
||||
decision and ResidualStream record what that output actually owes.
|
||||
@@ -125,6 +127,7 @@ class OutputContract(msgspec.Struct, frozen=True):
|
||||
update: Optional[ResidualUpdate] = PLAIN_ADD
|
||||
transform: Optional[OutputTransform] = None
|
||||
in_token_order: bool = False
|
||||
sum_impl: Optional[SumImplementation] = None
|
||||
|
||||
|
||||
class StageContract(msgspec.Struct, frozen=True):
|
||||
|
||||
@@ -281,6 +281,14 @@ class ExitPolicy:
|
||||
# A reduce-scatter completes the sum on its way back.
|
||||
owes_sum=steps.output.group is not None and not sum_in_reduce_scatter,
|
||||
)
|
||||
custom = steps.output.sum_impl
|
||||
custom_can_defer = (
|
||||
custom.can_defer(forward_batch)
|
||||
if custom is not None and custom.can_defer is not None
|
||||
else None
|
||||
)
|
||||
if custom_can_defer is False:
|
||||
return ExitDecision(False, sum_in_reduce_scatter, complete_now)
|
||||
defer_moe_finalize = (
|
||||
self.plan.fusions is not None
|
||||
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
|
||||
@@ -302,6 +310,7 @@ class ExitPolicy:
|
||||
steps,
|
||||
sum_in_reduce_scatter=sum_in_reduce_scatter,
|
||||
dp_step=dp_step,
|
||||
custom_can_defer=custom_can_defer,
|
||||
)
|
||||
if defers:
|
||||
group = self.ffn_reduction_group(steps)
|
||||
@@ -315,7 +324,13 @@ class ExitPolicy:
|
||||
),
|
||||
)
|
||||
else:
|
||||
wrap = partial(UnreducedOutput, group=group)
|
||||
wrap = partial(
|
||||
UnreducedOutput,
|
||||
group=group,
|
||||
complete_sum=partial(custom.run, forward_batch=forward_batch)
|
||||
if custom is not None
|
||||
else None,
|
||||
)
|
||||
complete = partial(_defer, wrap)
|
||||
elif (
|
||||
dp_step is not None
|
||||
@@ -353,7 +368,13 @@ class ExitPolicy:
|
||||
chosen: the sum the output owes, the move back to where the next layer
|
||||
reads it, then the write-back into the residual for a layer that does
|
||||
it itself."""
|
||||
if owes_sum:
|
||||
if owes_sum and steps.output.sum_impl is not None:
|
||||
if steps.output_move_completes_sum or dp_step is not None:
|
||||
raise NotImplementedError(
|
||||
"custom output sums require their declared token rows"
|
||||
)
|
||||
hidden_states = steps.output.sum_impl.run(hidden_states, forward_batch)
|
||||
elif owes_sum:
|
||||
hidden_states = sum_output(
|
||||
hidden_states,
|
||||
steps.output.group,
|
||||
@@ -414,6 +435,7 @@ class ExitPolicy:
|
||||
*,
|
||||
sum_in_reduce_scatter: bool,
|
||||
dp_step: Optional[Callable],
|
||||
custom_can_defer: Optional[bool] = None,
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
||||
input when no fused kernel takes it: when the next layer would run the
|
||||
@@ -422,7 +444,11 @@ class ExitPolicy:
|
||||
the scatter the next input runs."""
|
||||
return (
|
||||
steps.exit.may_defer_sum
|
||||
and _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan)
|
||||
and (
|
||||
custom_can_defer
|
||||
if custom_can_defer is not None
|
||||
else _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan)
|
||||
)
|
||||
and not sum_in_reduce_scatter
|
||||
and dp_step is None
|
||||
)
|
||||
|
||||
@@ -9,6 +9,8 @@ from dataclasses import dataclass, fields, is_dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Callable, Mapping, NamedTuple, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.layer_boundary.adapters.overlap import (
|
||||
@@ -42,7 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform
|
||||
from sglang.srt.layers.layer_boundary.output import OutputTransform, SumImplementation
|
||||
from sglang.srt.layers.layer_boundary.residual import (
|
||||
ResidualReadout,
|
||||
ResidualUpdate,
|
||||
@@ -300,6 +302,9 @@ class StageDeclaration:
|
||||
output_complete: Whether the FFN's compute completes its own output
|
||||
sum, so the exit owes none. For an FFN whose output sum is fused
|
||||
with a reduction its computation needs anyway.
|
||||
sum_impl: Optional implementation of the declared output sum using
|
||||
the model's custom collective. Its batch predicate, when supplied,
|
||||
preserves eligibility for carrying that sum to the consumer.
|
||||
attn_tp_gather: Optional implementation of the gather over attention
|
||||
TP that brings rows into this stage: at its own entry, or at its
|
||||
producer's exit, which runs its consumer's gather. Given this rank's
|
||||
@@ -334,6 +339,7 @@ class StageDeclaration:
|
||||
writes_at_handoff: bool = False
|
||||
output_complete: bool = False
|
||||
attn_tp_gather: Optional[Callable] = None
|
||||
sum_impl: Optional[SumImplementation] = None
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
@@ -374,7 +380,7 @@ def _require(protocol, implementation, facts):
|
||||
|
||||
|
||||
def _check_update(update):
|
||||
_require("ResidualUpdate", update, _UPDATE_FACTS + ("bind",))
|
||||
_require("ResidualUpdate", update, _UPDATE_FACTS + ("bind", "replay_update"))
|
||||
if update.bind is not None and not callable(update.bind):
|
||||
raise TypeError("ResidualUpdate.bind must be callable or None")
|
||||
name = type(update).__name__
|
||||
@@ -429,6 +435,7 @@ def declare_attn(
|
||||
tp_group=SumGroup.ATTN_TP,
|
||||
attn_tp_gather=None,
|
||||
in_token_order=False,
|
||||
sum_impl=None,
|
||||
):
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
@@ -449,6 +456,8 @@ def declare_attn(
|
||||
CP extend its input is gathered into that order, and the next
|
||||
stage's input completes the sum and takes this rank's shard back.
|
||||
Requires ALWAYS_PARTIAL.
|
||||
sum_impl: Model-specific implementation of the declared output sum.
|
||||
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -469,6 +478,7 @@ def declare_attn(
|
||||
tp_group=tp_group,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
in_token_order=in_token_order,
|
||||
sum_impl=sum_impl,
|
||||
)
|
||||
|
||||
|
||||
@@ -482,6 +492,7 @@ def declare_ffn(
|
||||
dense_tp_size=None,
|
||||
exit_rows=None,
|
||||
output_complete=False,
|
||||
sum_impl=None,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
@@ -498,6 +509,9 @@ def declare_ffn(
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
output_complete: Whether the compute completes its own output sum,
|
||||
fused with a reduction it needs anyway; the exit then owes none.
|
||||
sum_impl: Model-specific implementation and deferral eligibility of
|
||||
the output sum. Cannot be combined with output_complete.
|
||||
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -511,6 +525,7 @@ def declare_ffn(
|
||||
dense_tp_size=dense_tp_size,
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
output_complete=output_complete,
|
||||
sum_impl=sum_impl,
|
||||
)
|
||||
|
||||
|
||||
@@ -582,6 +597,24 @@ def _stack_arrival(variant):
|
||||
)
|
||||
|
||||
|
||||
def _custom_sum(stage, variant, group):
|
||||
if stage.sum_impl is None:
|
||||
return None
|
||||
if (
|
||||
variant is not BatchVariant.ORDINARY
|
||||
or get_parallel().attn_dp_size > 1
|
||||
or (
|
||||
stage.kind is StageKind.ATTENTION
|
||||
and stage.reduction is ProducerReduction.EXIT_SCOPED
|
||||
)
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"custom output sums require ordinary rows without attention DP "
|
||||
"and an always-partial attention or an FFN"
|
||||
)
|
||||
return stage.sum_impl if group is not None and get_parallel().tp_size > 1 else None
|
||||
|
||||
|
||||
def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
|
||||
"""``stage``'s data flow for ``variant``, given what arrives at its entry
|
||||
and the stage after it (None when it ends the stack or a branch)."""
|
||||
@@ -600,6 +633,13 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
|
||||
terminal_accepts_update=stage.terminal
|
||||
and takes_pending_update(final_read, stage.update),
|
||||
)
|
||||
declaration = msgspec.structs.replace(
|
||||
declaration,
|
||||
output=msgspec.structs.replace(
|
||||
declaration.output,
|
||||
sum_impl=_custom_sum(stage, variant, declaration.output.group),
|
||||
),
|
||||
)
|
||||
exit_rows = resolve_exit_rows(stage.exit_rows)
|
||||
if exit_rows is ExitRows.ATTENTION or (
|
||||
following is not None and following.read.reads_after_attn_tp_gather
|
||||
@@ -665,6 +705,7 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
|
||||
update=stage.update,
|
||||
transform=stage.output_transform,
|
||||
in_token_order=in_token_order,
|
||||
sum_impl=_custom_sum(stage, variant, group if owes else None),
|
||||
),
|
||||
)
|
||||
# An attention moves no residual: it keeps the rows it arrives on, except
|
||||
|
||||
@@ -29,6 +29,7 @@ class UpdateFacts(msgspec.Struct, frozen=True):
|
||||
"""The capabilities a ResidualUpdate declares (see ResidualUpdate)."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add: bool
|
||||
applied_at_exit: bool
|
||||
@@ -110,6 +111,9 @@ def facts_of(declaration):
|
||||
read=ReadFacts.of(declaration.read),
|
||||
update=UpdateFacts.of(declaration.update),
|
||||
output_transform=TransformFacts.of(declaration.output_transform),
|
||||
sum_impl=declaration.sum_impl.facts()
|
||||
if declaration.sum_impl is not None
|
||||
else None,
|
||||
previous=None,
|
||||
prepared_from=None,
|
||||
)
|
||||
|
||||
@@ -37,6 +37,25 @@ class OutputTransform(msgspec.Struct, frozen=True):
|
||||
before_reduce_scatter: bool = False
|
||||
|
||||
|
||||
class SumImplementation(msgspec.Struct, frozen=True):
|
||||
"""A producer's implementation of its declared output sum.
|
||||
|
||||
``run(partial, forward_batch)`` completes the same group the output
|
||||
contract declares. It may use a model's custom collective; it does not
|
||||
change token rows or perform the residual update. A hidden-dimension
|
||||
scatter may leave a hidden shard for the producer update to consume.
|
||||
``can_defer(batch)`` preserves that implementation's batch eligibility;
|
||||
None uses the ordinary exit eligibility. Neither callback is retained in
|
||||
a neighbouring stage's construction facts.
|
||||
"""
|
||||
|
||||
run: Optional[Callable]
|
||||
can_defer: Optional[Callable] = None
|
||||
|
||||
def facts(self):
|
||||
return SumImplementation(None)
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""Internal adapter value describing an unfinished reduction.
|
||||
|
||||
@@ -45,6 +64,9 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
group: All-reduce group when no redistribution callable is supplied.
|
||||
reduce_to_dp_local: Callable(partial) completing the sum and moving
|
||||
it to destination rows; takes precedence over group.
|
||||
complete_sum: Custom implementation of this group's sum on the same
|
||||
token rows; mutually exclusive with reduce_to_dp_local.
|
||||
|
||||
|
||||
Exits hand this form to ResidualStream.record(), which keeps it as the
|
||||
contribution's owed work and exposes an opaque OwedOutput to models.
|
||||
@@ -58,8 +80,17 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
|
||||
reduce_to_dp_local: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
|
||||
# Custom implementation of the same group, without token redistribution.
|
||||
complete_sum: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.complete_sum is not None and self.reduce_to_dp_local is not None:
|
||||
raise ValueError("choose a custom sum or a DP redistribution")
|
||||
|
||||
def complete(self) -> torch.Tensor:
|
||||
"""Complete the sum, on the destination rows when it moves them."""
|
||||
if self.complete_sum is not None:
|
||||
return self.complete_sum(self.partial)
|
||||
if self.reduce_to_dp_local is not None:
|
||||
return self.reduce_to_dp_local(self.partial)
|
||||
return self.group.all_reduce(self.partial)
|
||||
|
||||
@@ -337,6 +337,27 @@ def _run_entry(
|
||||
if pending.owed is not None:
|
||||
raise RuntimeError("a declared input sum carried another completion")
|
||||
step = completed_step
|
||||
if pending is not None and isinstance(pending.owed, DeclaredSum):
|
||||
owed = pending.owed
|
||||
if owed.complete_sum is not None:
|
||||
result = _run_read_fusions(
|
||||
tuple(
|
||||
f
|
||||
for f in read.completing_fusions
|
||||
if f.completes is owed.group and not f.scatters
|
||||
),
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
norm,
|
||||
read=read,
|
||||
update=update,
|
||||
**call,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
hidden_states = pending.complete()
|
||||
step = completed_step
|
||||
if written and written_step is not None:
|
||||
step = written_step
|
||||
if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)):
|
||||
|
||||
@@ -50,6 +50,8 @@ class ResidualUpdate(Protocol):
|
||||
|
||||
# None explicitly declares a stateless update. Bindings take (batch, context).
|
||||
bind: Optional[Callable] = None
|
||||
# An invocation may export its reusable producer declaration for graph replay.
|
||||
replay_update: Optional["ResidualUpdate"] = None
|
||||
|
||||
# A plain add, which one rank may run before the sum it adds into
|
||||
# completes (the DP partial order, a residual that joins the attention
|
||||
|
||||
@@ -274,6 +274,7 @@ class PlainAdd:
|
||||
"""A plain residual: the output is added into it."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = True
|
||||
applied_at_exit = False
|
||||
@@ -299,6 +300,7 @@ class ReplaceAtExit:
|
||||
boundary complete each part's sum first (StageBoundary.sum_part)."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
|
||||
@@ -13,8 +13,10 @@
|
||||
# ==============================================================================
|
||||
"""Residual access for one layer-stack invocation or TBO microbatch."""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
from sglang.srt.layers.layer_boundary.output import complete_owed
|
||||
from sglang.srt.layers.layer_boundary.residual import access
|
||||
from sglang.srt.layers.layer_boundary.residual import access, bind_update
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
@@ -35,6 +37,13 @@ def stream_of(forward_batch):
|
||||
return stream
|
||||
|
||||
|
||||
class ReplayHandoff(NamedTuple):
|
||||
value: object
|
||||
residual: object
|
||||
update: object
|
||||
written: bool
|
||||
|
||||
|
||||
def clear(forward_batch):
|
||||
"""Release invocation state on completion or an exception."""
|
||||
stream = forward_batch.residual_stream
|
||||
@@ -45,6 +54,50 @@ def clear(forward_batch):
|
||||
forward_batch.residual_stream = None
|
||||
|
||||
|
||||
def export_replay_handoff(hidden_states, forward_batch):
|
||||
"""Export a complete operand and reusable producer state across an eager break.
|
||||
|
||||
The handoff does not complete owed work or apply the residual update.
|
||||
Replay rebinds dynamic producer metadata to the live batch.
|
||||
"""
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
|
||||
stream = stream_of(forward_batch)
|
||||
stream.check(hidden_states)
|
||||
pending = stream.pending
|
||||
if pending is not None and pending.owed is not None:
|
||||
raise RuntimeError("complete the contribution before a graph replay handoff")
|
||||
value, residual = stream.input(hidden_states)
|
||||
update = pending.update if pending is not None else PLAIN_ADD
|
||||
reusable = update.replay_update
|
||||
result = ReplayHandoff(
|
||||
value,
|
||||
residual,
|
||||
update if reusable is None else reusable,
|
||||
pending is None and stream.residual is not None,
|
||||
)
|
||||
clear(forward_batch)
|
||||
return result
|
||||
|
||||
|
||||
def import_replay_handoff(handoff, forward_batch, *, update_context=None):
|
||||
"""Reconstruct the residual state with replay's live producer metadata."""
|
||||
if handoff.written:
|
||||
if handoff.residual is not None:
|
||||
raise ValueError("a written replay handoff has no separate residual")
|
||||
set_written(handoff.value, forward_batch)
|
||||
return handoff.value
|
||||
if handoff.residual is None:
|
||||
start(forward_batch)
|
||||
return handoff.value
|
||||
hidden, forward_batch.residual_stream = ResidualStream.from_handoff(
|
||||
handoff.value,
|
||||
handoff.residual,
|
||||
bind_update(handoff.update, forward_batch, update_context),
|
||||
)
|
||||
return hidden
|
||||
|
||||
|
||||
def written_residual(forward_batch):
|
||||
"""The residual the last prepare wrote, for a stage whose compute reads it
|
||||
too (e.g. to write the next stream at its exit). Borrowed: do not modify."""
|
||||
|
||||
@@ -170,6 +170,7 @@ class _AttnUpdate:
|
||||
only once the sum it writes in is complete."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
@@ -227,6 +228,7 @@ class _FfnUpdate:
|
||||
streams stay widened: the layer stack's terminal read mixes them down."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
|
||||
@@ -132,6 +132,7 @@ class _AttnUpdate:
|
||||
complete."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
@@ -188,6 +189,7 @@ class _FfnUpdate:
|
||||
so it belongs to the final norm."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
|
||||
@@ -167,6 +167,7 @@ class _AttnUpdate:
|
||||
plain add, so it runs only once the sum it writes in is complete."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
@@ -222,6 +223,7 @@ class _FfnUpdate:
|
||||
hidden states the layer stack hands on."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
applied_at_exit = True
|
||||
|
||||
@@ -25,6 +25,7 @@ class PostNormAdd:
|
||||
complete output. ``applied_at_exit`` makes the stage write it itself."""
|
||||
|
||||
bind = None
|
||||
replay_update = None
|
||||
|
||||
is_plain_add = False
|
||||
outlives_layer = True
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Deferred channelwise convolution followed by residual add and normalization."""
|
||||
|
||||
from typing import Callable, NamedTuple, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
|
||||
|
||||
|
||||
class _UpdateFacts:
|
||||
bind = None
|
||||
replay_update = None
|
||||
is_plain_add = False
|
||||
applied_at_exit = False
|
||||
outlives_layer = True
|
||||
writes_stream = False
|
||||
quantized_sum = False
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
raise NotImplementedError("short convolution needs its original token rows")
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
raise NotImplementedError("short convolution needs its original token rows")
|
||||
|
||||
|
||||
class ShortConvUpdate(_UpdateFacts, msgspec.Struct, frozen=True):
|
||||
"""Producer parameters; bind() creates state owned by one contribution.
|
||||
|
||||
A scattered completion produces a hidden shard. The convolution reads
|
||||
that shard and ``gather_hidden`` reconstructs the hidden dimension before
|
||||
the residual add. These operations leave the token layout unchanged.
|
||||
"""
|
||||
|
||||
convolve: Callable
|
||||
group: object
|
||||
sum_group: object
|
||||
gather_hidden: Optional[Callable] = None
|
||||
can_fuse: Optional[Callable] = None
|
||||
|
||||
def bind(self, forward_batch, positions=None):
|
||||
return ShortConvInvocation(
|
||||
self,
|
||||
forward_batch,
|
||||
forward_batch.positions if positions is None else positions,
|
||||
)
|
||||
|
||||
|
||||
class ShortConvInvocation(_UpdateFacts, msgspec.Struct, frozen=True):
|
||||
spec: ShortConvUpdate
|
||||
forward_batch: object
|
||||
positions: object
|
||||
|
||||
@property
|
||||
def replay_update(self):
|
||||
return self.spec
|
||||
|
||||
def contribution(self, hidden_states):
|
||||
batch = self.forward_batch
|
||||
value = self.spec.convolve(
|
||||
hidden_states, self.positions[: hidden_states.shape[0]], batch
|
||||
)
|
||||
if self.spec.gather_hidden is not None:
|
||||
value = self.spec.gather_hidden(value)
|
||||
return value
|
||||
|
||||
def update(self, hidden_states, residual):
|
||||
return self.contribution(hidden_states) + residual
|
||||
|
||||
|
||||
class NormalizationHandoff(NamedTuple):
|
||||
"""Completed producer contribution and residual, before add+norm."""
|
||||
|
||||
value: object
|
||||
residual: object
|
||||
|
||||
|
||||
class ShortConvRead(PendingUpdateRead):
|
||||
"""Read the actual producer's convolution and keep add+norm together."""
|
||||
|
||||
bind = None
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = True
|
||||
reads_after_attn_tp_gather = True
|
||||
gathering_reads = ()
|
||||
|
||||
def __init__(self, completing_fusions=()):
|
||||
self.completing_fusions = completing_fusions
|
||||
|
||||
@staticmethod
|
||||
def accepts_pending_update(update):
|
||||
return update.is_plain_add or isinstance(
|
||||
update, (ShortConvUpdate, ShortConvInvocation)
|
||||
)
|
||||
|
||||
def export_normalization_handoff(self, hidden_states, forward_batch):
|
||||
"""Complete producer work with live metadata, leaving add+norm to capture.
|
||||
|
||||
Piecewise prefill completes the unfused sum and convolution before
|
||||
exporting the unadded pair to the captured normalization.
|
||||
"""
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
|
||||
read = _NORMALIZATION_PAIR_READ
|
||||
return NormalizationHandoff(
|
||||
*residual_batch.read_output(hidden_states, forward_batch, read, _pair)
|
||||
)
|
||||
|
||||
def read_normalization_handoff(self, handoff, norm):
|
||||
"""Consume the complete pair with the original add+norm kernel."""
|
||||
if handoff.residual is None:
|
||||
return self.read(handoff.value, norm)
|
||||
return NORM_READOUT.update_and_read(
|
||||
PLAIN_ADD, handoff.value, handoff.residual, norm
|
||||
)
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
return NORM_READOUT.read(residual, norm, quant_format, post_residual_addition)
|
||||
|
||||
def update_and_read(
|
||||
self,
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format="",
|
||||
post_residual_addition=None,
|
||||
):
|
||||
if update.is_plain_add:
|
||||
return NORM_READOUT.update_and_read(
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format,
|
||||
post_residual_addition,
|
||||
)
|
||||
if not isinstance(update, ShortConvInvocation):
|
||||
raise TypeError("short-conv read requires the producer's bound invocation")
|
||||
value = update.contribution(hidden_states)
|
||||
# Normalize the contribution/residual pair without a separately rounded add.
|
||||
return NORM_READOUT.update_and_read(
|
||||
PLAIN_ADD, value, residual, norm, quant_format, post_residual_addition
|
||||
)
|
||||
|
||||
|
||||
def _pair(value, residual=None, **kwargs):
|
||||
return value if residual is None else (value, residual)
|
||||
|
||||
|
||||
class _NormalizationPairRead(ShortConvRead):
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
return residual, None
|
||||
|
||||
|
||||
_NORMALIZATION_PAIR_READ = _NormalizationPairRead()
|
||||
@@ -13,7 +13,7 @@
|
||||
# ==============================================================================
|
||||
"""One forward's residual and its producer's outstanding contribution."""
|
||||
|
||||
from typing import Optional, Union
|
||||
from typing import Callable, Optional, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -24,11 +24,16 @@ from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
|
||||
|
||||
|
||||
class DeclaredSum(msgspec.Struct, frozen=True):
|
||||
"""A sum every output of this producer owes to its declared input edge."""
|
||||
"""A sum every output owes to its declared input edge. complete_sum is
|
||||
the producer's custom implementation bound to this invocation, or None
|
||||
for the ordinary group all-reduce."""
|
||||
|
||||
group: SumGroup
|
||||
complete_sum: Optional[Callable] = None
|
||||
|
||||
def complete(self, value):
|
||||
if self.complete_sum is not None:
|
||||
return self.complete_sum(value)
|
||||
return _sum_group(self.group).all_reduce(value)
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
# ==============================================================================
|
||||
"""A stage's model-facing input and output boundary."""
|
||||
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
import torch
|
||||
@@ -176,10 +177,21 @@ class StageBoundary:
|
||||
take a MoE finalize handoff) finishes through exit() instead."""
|
||||
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
|
||||
produced = self.plan.path_for(forward_batch).output
|
||||
completion = (
|
||||
partial(produced.sum_impl.run, forward_batch=forward_batch)
|
||||
if produced.sum_impl is not None
|
||||
else None
|
||||
)
|
||||
if produced.always_partial:
|
||||
owed = DeclaredSum(produced.group, completion)
|
||||
else:
|
||||
owed = None
|
||||
if completion is not None:
|
||||
hidden_states = completion(hidden_states)
|
||||
return stream_of(forward_batch).record(
|
||||
hidden_states,
|
||||
bind_update(produced.update, forward_batch, update_context),
|
||||
declared_sum=produced.group if produced.always_partial else None,
|
||||
declared_sum=owed,
|
||||
)
|
||||
if self.kind is StageKind.FFN and self.plan.fusions is not None:
|
||||
raise RuntimeError(
|
||||
|
||||
+444
-513
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user