[Refactor] inkling: Use short-convolution stage boundaries (#43606)

This commit is contained in:
Cheng Wan
2026-10-15 02:04:46 -07:00
committed by GitHub
parent e1f9fce7b3
commit 3a835cf417
25 changed files with 1350 additions and 526 deletions
@@ -65,6 +65,10 @@ def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) ->
"""Resolve the token-row modes the model's forward actually enters."""
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
longcat = model_type == "longcat_flash"
inkling = architecture in (
"InklingForConditionalGeneration",
"InklingForConditionalGenerationMTP",
)
kimi_k3 = architecture in (
"KimiK3ForConditionalGeneration",
"KimiK3LinearForCausalLM",
@@ -75,7 +79,9 @@ def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) ->
raise ValueError(
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
)
if (nemotron or longcat or kimi_k3) and cfg.enable_attn_tp_input_scattered:
if (
nemotron or longcat or kimi_k3 or inkling
) and cfg.enable_attn_tp_input_scattered:
# None of these models enters the input-scattered attention scope.
logger.warning(
"Disabling input-scattered attention for %s: its forward does not enter that scope",
@@ -282,6 +282,10 @@ def _bind_stage(
raise NotImplementedError(
"pipeline boundaries require a producer-written residual"
)
if declaration.sum_impl is not None and declaration.sum_impl.run is None:
raise ValueError("a local stage requires its custom sum implementation")
if declaration.output_complete and declaration.sum_impl is not None:
raise ValueError("compute-completed output cannot also declare a custom sum")
if declaration.terminal and not (
update.is_plain_add
or update.applied_at_exit
@@ -21,7 +21,7 @@ import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.output import OutputTransform, SumImplementation
from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
@@ -105,6 +105,8 @@ class OutputContract(msgspec.Struct, frozen=True):
transform: Optional operation on the contribution before residual update.
in_token_order: The output covers the whole sequence in the batch's
token order; the consumer takes this rank's CP shard of it back.
sum_impl: Custom implementation of this declared sum, or None for
the ordinary boundary collective. It does not change token rows.
These are permissions, not a record of a particular output. The exit
decision and ResidualStream record what that output actually owes.
@@ -125,6 +127,7 @@ class OutputContract(msgspec.Struct, frozen=True):
update: Optional[ResidualUpdate] = PLAIN_ADD
transform: Optional[OutputTransform] = None
in_token_order: bool = False
sum_impl: Optional[SumImplementation] = None
class StageContract(msgspec.Struct, frozen=True):
@@ -281,6 +281,14 @@ class ExitPolicy:
# A reduce-scatter completes the sum on its way back.
owes_sum=steps.output.group is not None and not sum_in_reduce_scatter,
)
custom = steps.output.sum_impl
custom_can_defer = (
custom.can_defer(forward_batch)
if custom is not None and custom.can_defer is not None
else None
)
if custom_can_defer is False:
return ExitDecision(False, sum_in_reduce_scatter, complete_now)
defer_moe_finalize = (
self.plan.fusions is not None
and self.plan.fusions.can_defer_finalize(self.plan, forward_batch)
@@ -302,6 +310,7 @@ class ExitPolicy:
steps,
sum_in_reduce_scatter=sum_in_reduce_scatter,
dp_step=dp_step,
custom_can_defer=custom_can_defer,
)
if defers:
group = self.ffn_reduction_group(steps)
@@ -315,7 +324,13 @@ class ExitPolicy:
),
)
else:
wrap = partial(UnreducedOutput, group=group)
wrap = partial(
UnreducedOutput,
group=group,
complete_sum=partial(custom.run, forward_batch=forward_batch)
if custom is not None
else None,
)
complete = partial(_defer, wrap)
elif (
dp_step is not None
@@ -353,7 +368,13 @@ class ExitPolicy:
chosen: the sum the output owes, the move back to where the next layer
reads it, then the write-back into the residual for a layer that does
it itself."""
if owes_sum:
if owes_sum and steps.output.sum_impl is not None:
if steps.output_move_completes_sum or dp_step is not None:
raise NotImplementedError(
"custom output sums require their declared token rows"
)
hidden_states = steps.output.sum_impl.run(hidden_states, forward_batch)
elif owes_sum:
hidden_states = sum_output(
hidden_states,
steps.output.group,
@@ -414,6 +435,7 @@ class ExitPolicy:
*,
sum_in_reduce_scatter: bool,
dp_step: Optional[Callable],
custom_can_defer: Optional[bool] = None,
) -> bool:
"""Whether the FFN leaves its output's all-reduce to the next layer's
input when no fused kernel takes it: when the next layer would run the
@@ -422,7 +444,11 @@ class ExitPolicy:
the scatter the next input runs."""
return (
steps.exit.may_defer_sum
and _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan)
and (
custom_can_defer
if custom_can_defer is not None
else _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan)
)
and not sum_in_reduce_scatter
and dp_step is None
)
@@ -9,6 +9,8 @@ from dataclasses import dataclass, fields, is_dataclass, replace
from types import MappingProxyType
from typing import Callable, Mapping, NamedTuple, Optional
import msgspec
from sglang.srt.environ import envs
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.layer_boundary.adapters.overlap import (
@@ -42,7 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import (
moe_gathers_over_moe_cp,
token_axis_sizes,
)
from sglang.srt.layers.layer_boundary.output import OutputTransform
from sglang.srt.layers.layer_boundary.output import OutputTransform, SumImplementation
from sglang.srt.layers.layer_boundary.residual import (
ResidualReadout,
ResidualUpdate,
@@ -300,6 +302,9 @@ class StageDeclaration:
output_complete: Whether the FFN's compute completes its own output
sum, so the exit owes none. For an FFN whose output sum is fused
with a reduction its computation needs anyway.
sum_impl: Optional implementation of the declared output sum using
the model's custom collective. Its batch predicate, when supplied,
preserves eligibility for carrying that sum to the consumer.
attn_tp_gather: Optional implementation of the gather over attention
TP that brings rows into this stage: at its own entry, or at its
producer's exit, which runs its consumer's gather. Given this rank's
@@ -334,6 +339,7 @@ class StageDeclaration:
writes_at_handoff: bool = False
output_complete: bool = False
attn_tp_gather: Optional[Callable] = None
sum_impl: Optional[SumImplementation] = None
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
@@ -374,7 +380,7 @@ def _require(protocol, implementation, facts):
def _check_update(update):
_require("ResidualUpdate", update, _UPDATE_FACTS + ("bind",))
_require("ResidualUpdate", update, _UPDATE_FACTS + ("bind", "replay_update"))
if update.bind is not None and not callable(update.bind):
raise TypeError("ResidualUpdate.bind must be callable or None")
name = type(update).__name__
@@ -429,6 +435,7 @@ def declare_attn(
tp_group=SumGroup.ATTN_TP,
attn_tp_gather=None,
in_token_order=False,
sum_impl=None,
):
"""Declare attention or a mixer; construct its executable boundary later.
@@ -449,6 +456,8 @@ def declare_attn(
CP extend its input is gathered into that order, and the next
stage's input completes the sum and takes this rank's shard back.
Requires ALWAYS_PARTIAL.
sum_impl: Model-specific implementation of the declared output sum.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -469,6 +478,7 @@ def declare_attn(
tp_group=tp_group,
attn_tp_gather=attn_tp_gather,
in_token_order=in_token_order,
sum_impl=sum_impl,
)
@@ -482,6 +492,7 @@ def declare_ffn(
dense_tp_size=None,
exit_rows=None,
output_complete=False,
sum_impl=None,
):
"""Declare a dense or MoE FFN independently of its compute module.
@@ -498,6 +509,9 @@ def declare_ffn(
the adjacent FFN kinds and TBO configuration.
output_complete: Whether the compute completes its own output sum,
fused with a reduction it needs anyway; the exit then owes none.
sum_impl: Model-specific implementation and deferral eligibility of
the output sum. Cannot be combined with output_complete.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -511,6 +525,7 @@ def declare_ffn(
dense_tp_size=dense_tp_size,
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
output_complete=output_complete,
sum_impl=sum_impl,
)
@@ -582,6 +597,24 @@ def _stack_arrival(variant):
)
def _custom_sum(stage, variant, group):
if stage.sum_impl is None:
return None
if (
variant is not BatchVariant.ORDINARY
or get_parallel().attn_dp_size > 1
or (
stage.kind is StageKind.ATTENTION
and stage.reduction is ProducerReduction.EXIT_SCOPED
)
):
raise NotImplementedError(
"custom output sums require ordinary rows without attention DP "
"and an always-partial attention or an FFN"
)
return stage.sum_impl if group is not None and get_parallel().tp_size > 1 else None
def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
"""``stage``'s data flow for ``variant``, given what arrives at its entry
and the stage after it (None when it ends the stack or a branch)."""
@@ -600,6 +633,13 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
terminal_accepts_update=stage.terminal
and takes_pending_update(final_read, stage.update),
)
declaration = msgspec.structs.replace(
declaration,
output=msgspec.structs.replace(
declaration.output,
sum_impl=_custom_sum(stage, variant, declaration.output.group),
),
)
exit_rows = resolve_exit_rows(stage.exit_rows)
if exit_rows is ExitRows.ATTENTION or (
following is not None and following.read.reads_after_attn_tp_gather
@@ -665,6 +705,7 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None):
update=stage.update,
transform=stage.output_transform,
in_token_order=in_token_order,
sum_impl=_custom_sum(stage, variant, group if owes else None),
),
)
# An attention moves no residual: it keeps the rows it arrives on, except
@@ -29,6 +29,7 @@ class UpdateFacts(msgspec.Struct, frozen=True):
"""The capabilities a ResidualUpdate declares (see ResidualUpdate)."""
bind = None
replay_update = None
is_plain_add: bool
applied_at_exit: bool
@@ -110,6 +111,9 @@ def facts_of(declaration):
read=ReadFacts.of(declaration.read),
update=UpdateFacts.of(declaration.update),
output_transform=TransformFacts.of(declaration.output_transform),
sum_impl=declaration.sum_impl.facts()
if declaration.sum_impl is not None
else None,
previous=None,
prepared_from=None,
)
@@ -37,6 +37,25 @@ class OutputTransform(msgspec.Struct, frozen=True):
before_reduce_scatter: bool = False
class SumImplementation(msgspec.Struct, frozen=True):
"""A producer's implementation of its declared output sum.
``run(partial, forward_batch)`` completes the same group the output
contract declares. It may use a model's custom collective; it does not
change token rows or perform the residual update. A hidden-dimension
scatter may leave a hidden shard for the producer update to consume.
``can_defer(batch)`` preserves that implementation's batch eligibility;
None uses the ordinary exit eligibility. Neither callback is retained in
a neighbouring stage's construction facts.
"""
run: Optional[Callable]
can_defer: Optional[Callable] = None
def facts(self):
return SumImplementation(None)
class UnreducedOutput(msgspec.Struct, frozen=True):
"""Internal adapter value describing an unfinished reduction.
@@ -45,6 +64,9 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
group: All-reduce group when no redistribution callable is supplied.
reduce_to_dp_local: Callable(partial) completing the sum and moving
it to destination rows; takes precedence over group.
complete_sum: Custom implementation of this group's sum on the same
token rows; mutually exclusive with reduce_to_dp_local.
Exits hand this form to ResidualStream.record(), which keeps it as the
contribution's owed work and exposes an opaque OwedOutput to models.
@@ -58,8 +80,17 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
reduce_to_dp_local: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
# Custom implementation of the same group, without token redistribution.
complete_sum: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
def __post_init__(self):
if self.complete_sum is not None and self.reduce_to_dp_local is not None:
raise ValueError("choose a custom sum or a DP redistribution")
def complete(self) -> torch.Tensor:
"""Complete the sum, on the destination rows when it moves them."""
if self.complete_sum is not None:
return self.complete_sum(self.partial)
if self.reduce_to_dp_local is not None:
return self.reduce_to_dp_local(self.partial)
return self.group.all_reduce(self.partial)
@@ -337,6 +337,27 @@ def _run_entry(
if pending.owed is not None:
raise RuntimeError("a declared input sum carried another completion")
step = completed_step
if pending is not None and isinstance(pending.owed, DeclaredSum):
owed = pending.owed
if owed.complete_sum is not None:
result = _run_read_fusions(
tuple(
f
for f in read.completing_fusions
if f.completes is owed.group and not f.scatters
),
hidden_states,
residual,
forward_batch,
norm,
read=read,
update=update,
**call,
)
if result is not None:
return result
hidden_states = pending.complete()
step = completed_step
if written and written_step is not None:
step = written_step
if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)):
@@ -50,6 +50,8 @@ class ResidualUpdate(Protocol):
# None explicitly declares a stateless update. Bindings take (batch, context).
bind: Optional[Callable] = None
# An invocation may export its reusable producer declaration for graph replay.
replay_update: Optional["ResidualUpdate"] = None
# A plain add, which one rank may run before the sum it adds into
# completes (the DP partial order, a residual that joins the attention
@@ -274,6 +274,7 @@ class PlainAdd:
"""A plain residual: the output is added into it."""
bind = None
replay_update = None
is_plain_add = True
applied_at_exit = False
@@ -299,6 +300,7 @@ class ReplaceAtExit:
boundary complete each part's sum first (StageBoundary.sum_part)."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = True
@@ -13,8 +13,10 @@
# ==============================================================================
"""Residual access for one layer-stack invocation or TBO microbatch."""
from typing import NamedTuple
from sglang.srt.layers.layer_boundary.output import complete_owed
from sglang.srt.layers.layer_boundary.residual import access
from sglang.srt.layers.layer_boundary.residual import access, bind_update
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
@@ -35,6 +37,13 @@ def stream_of(forward_batch):
return stream
class ReplayHandoff(NamedTuple):
value: object
residual: object
update: object
written: bool
def clear(forward_batch):
"""Release invocation state on completion or an exception."""
stream = forward_batch.residual_stream
@@ -45,6 +54,50 @@ def clear(forward_batch):
forward_batch.residual_stream = None
def export_replay_handoff(hidden_states, forward_batch):
"""Export a complete operand and reusable producer state across an eager break.
The handoff does not complete owed work or apply the residual update.
Replay rebinds dynamic producer metadata to the live batch.
"""
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
stream = stream_of(forward_batch)
stream.check(hidden_states)
pending = stream.pending
if pending is not None and pending.owed is not None:
raise RuntimeError("complete the contribution before a graph replay handoff")
value, residual = stream.input(hidden_states)
update = pending.update if pending is not None else PLAIN_ADD
reusable = update.replay_update
result = ReplayHandoff(
value,
residual,
update if reusable is None else reusable,
pending is None and stream.residual is not None,
)
clear(forward_batch)
return result
def import_replay_handoff(handoff, forward_batch, *, update_context=None):
"""Reconstruct the residual state with replay's live producer metadata."""
if handoff.written:
if handoff.residual is not None:
raise ValueError("a written replay handoff has no separate residual")
set_written(handoff.value, forward_batch)
return handoff.value
if handoff.residual is None:
start(forward_batch)
return handoff.value
hidden, forward_batch.residual_stream = ResidualStream.from_handoff(
handoff.value,
handoff.residual,
bind_update(handoff.update, forward_batch, update_context),
)
return hidden
def written_residual(forward_batch):
"""The residual the last prepare wrote, for a stage whose compute reads it
too (e.g. to write the next stream at its exit). Borrowed: do not modify."""
@@ -170,6 +170,7 @@ class _AttnUpdate:
only once the sum it writes in is complete."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = False
@@ -227,6 +228,7 @@ class _FfnUpdate:
streams stay widened: the layer stack's terminal read mixes them down."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = True
@@ -132,6 +132,7 @@ class _AttnUpdate:
complete."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = False
@@ -188,6 +189,7 @@ class _FfnUpdate:
so it belongs to the final norm."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = True
@@ -167,6 +167,7 @@ class _AttnUpdate:
plain add, so it runs only once the sum it writes in is complete."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = False
@@ -222,6 +223,7 @@ class _FfnUpdate:
hidden states the layer stack hands on."""
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = True
@@ -25,6 +25,7 @@ class PostNormAdd:
complete output. ``applied_at_exit`` makes the stage write it itself."""
bind = None
replay_update = None
is_plain_add = False
outlives_layer = True
@@ -0,0 +1,173 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Deferred channelwise convolution followed by residual add and normalization."""
from typing import Callable, NamedTuple, Optional
import msgspec
from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD
class _UpdateFacts:
bind = None
replay_update = None
is_plain_add = False
applied_at_exit = False
outlives_layer = True
writes_stream = False
quantized_sum = False
def slice_residual_attn_tp(self, residual):
raise NotImplementedError("short convolution needs its original token rows")
def gather_residual_attn_tp(self, residual):
raise NotImplementedError("short convolution needs its original token rows")
class ShortConvUpdate(_UpdateFacts, msgspec.Struct, frozen=True):
"""Producer parameters; bind() creates state owned by one contribution.
A scattered completion produces a hidden shard. The convolution reads
that shard and ``gather_hidden`` reconstructs the hidden dimension before
the residual add. These operations leave the token layout unchanged.
"""
convolve: Callable
group: object
sum_group: object
gather_hidden: Optional[Callable] = None
can_fuse: Optional[Callable] = None
def bind(self, forward_batch, positions=None):
return ShortConvInvocation(
self,
forward_batch,
forward_batch.positions if positions is None else positions,
)
class ShortConvInvocation(_UpdateFacts, msgspec.Struct, frozen=True):
spec: ShortConvUpdate
forward_batch: object
positions: object
@property
def replay_update(self):
return self.spec
def contribution(self, hidden_states):
batch = self.forward_batch
value = self.spec.convolve(
hidden_states, self.positions[: hidden_states.shape[0]], batch
)
if self.spec.gather_hidden is not None:
value = self.spec.gather_hidden(value)
return value
def update(self, hidden_states, residual):
return self.contribution(hidden_states) + residual
class NormalizationHandoff(NamedTuple):
"""Completed producer contribution and residual, before add+norm."""
value: object
residual: object
class ShortConvRead(PendingUpdateRead):
"""Read the actual producer's convolution and keep add+norm together."""
bind = None
is_plain_norm = False
reads_before_dp_gather = True
reads_after_attn_tp_gather = True
gathering_reads = ()
def __init__(self, completing_fusions=()):
self.completing_fusions = completing_fusions
@staticmethod
def accepts_pending_update(update):
return update.is_plain_add or isinstance(
update, (ShortConvUpdate, ShortConvInvocation)
)
def export_normalization_handoff(self, hidden_states, forward_batch):
"""Complete producer work with live metadata, leaving add+norm to capture.
Piecewise prefill completes the unfused sum and convolution before
exporting the unadded pair to the captured normalization.
"""
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
read = _NORMALIZATION_PAIR_READ
return NormalizationHandoff(
*residual_batch.read_output(hidden_states, forward_batch, read, _pair)
)
def read_normalization_handoff(self, handoff, norm):
"""Consume the complete pair with the original add+norm kernel."""
if handoff.residual is None:
return self.read(handoff.value, norm)
return NORM_READOUT.update_and_read(
PLAIN_ADD, handoff.value, handoff.residual, norm
)
def init_residual(self, hidden_states):
return hidden_states
def read(self, residual, norm, quant_format="", post_residual_addition=None):
return NORM_READOUT.read(residual, norm, quant_format, post_residual_addition)
def update_and_read(
self,
update,
hidden_states,
residual,
norm,
quant_format="",
post_residual_addition=None,
):
if update.is_plain_add:
return NORM_READOUT.update_and_read(
update,
hidden_states,
residual,
norm,
quant_format,
post_residual_addition,
)
if not isinstance(update, ShortConvInvocation):
raise TypeError("short-conv read requires the producer's bound invocation")
value = update.contribution(hidden_states)
# Normalize the contribution/residual pair without a separately rounded add.
return NORM_READOUT.update_and_read(
PLAIN_ADD, value, residual, norm, quant_format, post_residual_addition
)
def _pair(value, residual=None, **kwargs):
return value if residual is None else (value, residual)
class _NormalizationPairRead(ShortConvRead):
def read(self, residual, norm, quant_format="", post_residual_addition=None):
return residual, None
_NORMALIZATION_PAIR_READ = _NormalizationPairRead()
@@ -13,7 +13,7 @@
# ==============================================================================
"""One forward's residual and its producer's outstanding contribution."""
from typing import Optional, Union
from typing import Callable, Optional, Union
import msgspec
import torch
@@ -24,11 +24,16 @@ from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
class DeclaredSum(msgspec.Struct, frozen=True):
"""A sum every output of this producer owes to its declared input edge."""
"""A sum every output owes to its declared input edge. complete_sum is
the producer's custom implementation bound to this invocation, or None
for the ordinary group all-reduce."""
group: SumGroup
complete_sum: Optional[Callable] = None
def complete(self, value):
if self.complete_sum is not None:
return self.complete_sum(value)
return _sum_group(self.group).all_reduce(value)
@@ -13,6 +13,7 @@
# ==============================================================================
"""A stage's model-facing input and output boundary."""
from functools import partial
from typing import TYPE_CHECKING, Callable, Optional
import torch
@@ -176,10 +177,21 @@ class StageBoundary:
take a MoE finalize handoff) finishes through exit() instead."""
if self.kind is StageKind.ATTENTION and self.plan.finishes_directly:
produced = self.plan.path_for(forward_batch).output
completion = (
partial(produced.sum_impl.run, forward_batch=forward_batch)
if produced.sum_impl is not None
else None
)
if produced.always_partial:
owed = DeclaredSum(produced.group, completion)
else:
owed = None
if completion is not None:
hidden_states = completion(hidden_states)
return stream_of(forward_batch).record(
hidden_states,
bind_update(produced.update, forward_batch, update_context),
declared_sum=produced.group if produced.always_partial else None,
declared_sum=owed,
)
if self.kind is StageKind.FFN and self.plan.fusions is not None:
raise RuntimeError(
File diff suppressed because it is too large Load Diff