From 3a835cf417d5b50ac9ef0e57dee8ffad5c7c4217 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:04:46 -0700 Subject: [PATCH] [Refactor] inkling: Use short-convolution stage boundaries (#43606) --- python/sglang/srt/arg_groups/parallel_hook.py | 8 +- .../srt/layers/layer_boundary/construction.py | 4 + .../srt/layers/layer_boundary/contracts.py | 5 +- .../sglang/srt/layers/layer_boundary/exit.py | 32 +- .../srt/layers/layer_boundary/factories.py | 45 +- .../sglang/srt/layers/layer_boundary/facts.py | 4 + .../srt/layers/layer_boundary/output.py | 31 + .../srt/layers/layer_boundary/prepare.py | 21 + .../layer_boundary/residual/__init__.py | 2 + .../layer_boundary/residual/add_norm.py | 2 + .../layers/layer_boundary/residual/batch.py | 55 +- .../layers/layer_boundary/residual/gated.py | 2 + .../srt/layers/layer_boundary/residual/ihc.py | 2 + .../srt/layers/layer_boundary/residual/mhc.py | 2 + .../layer_boundary/residual/post_norm.py | 1 + .../layer_boundary/residual/short_conv.py | 173 ++++ .../layers/layer_boundary/residual/stream.py | 9 +- .../sglang/srt/layers/layer_boundary/stage.py | 14 +- python/sglang/srt/models/inkling.py | 957 ++++++++---------- .../unit/layer_boundary/test_append_stages.py | 2 + .../test_attn_bank_residual_ops.py | 1 + .../layer_boundary/test_boundary_edges.py | 1 + .../test_producer_completion.py | 5 +- .../layer_boundary/test_residual_stream.py | 5 +- .../test_short_conv_boundary.py | 493 +++++++++ 25 files changed, 1350 insertions(+), 526 deletions(-) create mode 100644 python/sglang/srt/layers/layer_boundary/residual/short_conv.py create mode 100644 test/registered/unit/layer_boundary/test_short_conv_boundary.py diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index bf52160671..fbe3d95de0 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -65,6 +65,10 @@ def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) -> """Resolve the token-row modes the model's forward actually enters.""" nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle") longcat = model_type == "longcat_flash" + inkling = architecture in ( + "InklingForConditionalGeneration", + "InklingForConditionalGenerationMTP", + ) kimi_k3 = architecture in ( "KimiK3ForConditionalGeneration", "KimiK3LinearForCausalLM", @@ -75,7 +79,9 @@ def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) -> raise ValueError( "LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1" ) - if (nemotron or longcat or kimi_k3) and cfg.enable_attn_tp_input_scattered: + if ( + nemotron or longcat or kimi_k3 or inkling + ) and cfg.enable_attn_tp_input_scattered: # None of these models enters the input-scattered attention scope. logger.warning( "Disabling input-scattered attention for %s: its forward does not enter that scope", diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 62e851291c..75e7894223 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -282,6 +282,10 @@ def _bind_stage( raise NotImplementedError( "pipeline boundaries require a producer-written residual" ) + if declaration.sum_impl is not None and declaration.sum_impl.run is None: + raise ValueError("a local stage requires its custom sum implementation") + if declaration.output_complete and declaration.sum_impl is not None: + raise ValueError("compute-completed output cannot also declare a custom sum") if declaration.terminal and not ( update.is_plain_add or update.applied_at_exit diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 44d74082be..7b741d3de5 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -21,7 +21,7 @@ import torch from sglang.srt.distributed import GroupCoordinator from sglang.srt.layers.layer_boundary.layout import Layout, SumGroup, TokenAxis -from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.output import OutputTransform, SumImplementation from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD @@ -105,6 +105,8 @@ class OutputContract(msgspec.Struct, frozen=True): transform: Optional operation on the contribution before residual update. in_token_order: The output covers the whole sequence in the batch's token order; the consumer takes this rank's CP shard of it back. + sum_impl: Custom implementation of this declared sum, or None for + the ordinary boundary collective. It does not change token rows. These are permissions, not a record of a particular output. The exit decision and ResidualStream record what that output actually owes. @@ -125,6 +127,7 @@ class OutputContract(msgspec.Struct, frozen=True): update: Optional[ResidualUpdate] = PLAIN_ADD transform: Optional[OutputTransform] = None in_token_order: bool = False + sum_impl: Optional[SumImplementation] = None class StageContract(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index cc6eca90d3..1ebd9e574d 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -281,6 +281,14 @@ class ExitPolicy: # A reduce-scatter completes the sum on its way back. owes_sum=steps.output.group is not None and not sum_in_reduce_scatter, ) + custom = steps.output.sum_impl + custom_can_defer = ( + custom.can_defer(forward_batch) + if custom is not None and custom.can_defer is not None + else None + ) + if custom_can_defer is False: + return ExitDecision(False, sum_in_reduce_scatter, complete_now) defer_moe_finalize = ( self.plan.fusions is not None and self.plan.fusions.can_defer_finalize(self.plan, forward_batch) @@ -302,6 +310,7 @@ class ExitPolicy: steps, sum_in_reduce_scatter=sum_in_reduce_scatter, dp_step=dp_step, + custom_can_defer=custom_can_defer, ) if defers: group = self.ffn_reduction_group(steps) @@ -315,7 +324,13 @@ class ExitPolicy: ), ) else: - wrap = partial(UnreducedOutput, group=group) + wrap = partial( + UnreducedOutput, + group=group, + complete_sum=partial(custom.run, forward_batch=forward_batch) + if custom is not None + else None, + ) complete = partial(_defer, wrap) elif ( dp_step is not None @@ -353,7 +368,13 @@ class ExitPolicy: chosen: the sum the output owes, the move back to where the next layer reads it, then the write-back into the residual for a layer that does it itself.""" - if owes_sum: + if owes_sum and steps.output.sum_impl is not None: + if steps.output_move_completes_sum or dp_step is not None: + raise NotImplementedError( + "custom output sums require their declared token rows" + ) + hidden_states = steps.output.sum_impl.run(hidden_states, forward_batch) + elif owes_sum: hidden_states = sum_output( hidden_states, steps.output.group, @@ -414,6 +435,7 @@ class ExitPolicy: *, sum_in_reduce_scatter: bool, dp_step: Optional[Callable], + custom_can_defer: Optional[bool] = None, ) -> bool: """Whether the FFN leaves its output's all-reduce to the next layer's input when no fused kernel takes it: when the next layer would run the @@ -422,7 +444,11 @@ class ExitPolicy: the scatter the next input runs.""" return ( steps.exit.may_defer_sum - and _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan) + and ( + custom_can_defer + if custom_can_defer is not None + else _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan) + ) and not sum_in_reduce_scatter and dp_step is None ) diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 1456d0ef3c..93f4fca370 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -9,6 +9,8 @@ from dataclasses import dataclass, fields, is_dataclass, replace from types import MappingProxyType from typing import Callable, Mapping, NamedTuple, Optional +import msgspec + from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp from sglang.srt.layers.layer_boundary.adapters.overlap import ( @@ -42,7 +44,7 @@ from sglang.srt.layers.layer_boundary.layout import ( moe_gathers_over_moe_cp, token_axis_sizes, ) -from sglang.srt.layers.layer_boundary.output import OutputTransform +from sglang.srt.layers.layer_boundary.output import OutputTransform, SumImplementation from sglang.srt.layers.layer_boundary.residual import ( ResidualReadout, ResidualUpdate, @@ -300,6 +302,9 @@ class StageDeclaration: output_complete: Whether the FFN's compute completes its own output sum, so the exit owes none. For an FFN whose output sum is fused with a reduction its computation needs anyway. + sum_impl: Optional implementation of the declared output sum using + the model's custom collective. Its batch predicate, when supplied, + preserves eligibility for carrying that sum to the consumer. attn_tp_gather: Optional implementation of the gather over attention TP that brings rows into this stage: at its own entry, or at its producer's exit, which runs its consumer's gather. Given this rank's @@ -334,6 +339,7 @@ class StageDeclaration: writes_at_handoff: bool = False output_complete: bool = False attn_tp_gather: Optional[Callable] = None + sum_impl: Optional[SumImplementation] = None # Only declarations participate in construction, never executable stages. previous: Optional[StageDeclaration] = None prepared_from: Optional[StageDeclaration] = None @@ -374,7 +380,7 @@ def _require(protocol, implementation, facts): def _check_update(update): - _require("ResidualUpdate", update, _UPDATE_FACTS + ("bind",)) + _require("ResidualUpdate", update, _UPDATE_FACTS + ("bind", "replay_update")) if update.bind is not None and not callable(update.bind): raise TypeError("ResidualUpdate.bind must be callable or None") name = type(update).__name__ @@ -429,6 +435,7 @@ def declare_attn( tp_group=SumGroup.ATTN_TP, attn_tp_gather=None, in_token_order=False, + sum_impl=None, ): """Declare attention or a mixer; construct its executable boundary later. @@ -449,6 +456,8 @@ def declare_attn( CP extend its input is gathered into that order, and the next stage's input completes the sum and takes this rank's shard back. Requires ALWAYS_PARTIAL. + sum_impl: Model-specific implementation of the declared output sum. + Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -469,6 +478,7 @@ def declare_attn( tp_group=tp_group, attn_tp_gather=attn_tp_gather, in_token_order=in_token_order, + sum_impl=sum_impl, ) @@ -482,6 +492,7 @@ def declare_ffn( dense_tp_size=None, exit_rows=None, output_complete=False, + sum_impl=None, ): """Declare a dense or MoE FFN independently of its compute module. @@ -498,6 +509,9 @@ def declare_ffn( the adjacent FFN kinds and TBO configuration. output_complete: Whether the compute completes its own output sum, fused with a reduction it needs anyway; the exit then owes none. + sum_impl: Model-specific implementation and deferral eligibility of + the output sum. Cannot be combined with output_complete. + Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -511,6 +525,7 @@ def declare_ffn( dense_tp_size=dense_tp_size, exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse), output_complete=output_complete, + sum_impl=sum_impl, ) @@ -582,6 +597,24 @@ def _stack_arrival(variant): ) +def _custom_sum(stage, variant, group): + if stage.sum_impl is None: + return None + if ( + variant is not BatchVariant.ORDINARY + or get_parallel().attn_dp_size > 1 + or ( + stage.kind is StageKind.ATTENTION + and stage.reduction is ProducerReduction.EXIT_SCOPED + ) + ): + raise NotImplementedError( + "custom output sums require ordinary rows without attention DP " + "and an always-partial attention or an FFN" + ) + return stage.sum_impl if group is not None and get_parallel().tp_size > 1 else None + + def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None): """``stage``'s data flow for ``variant``, given what arrives at its entry and the stage after it (None when it ends the stack or a branch).""" @@ -600,6 +633,13 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None): terminal_accepts_update=stage.terminal and takes_pending_update(final_read, stage.update), ) + declaration = msgspec.structs.replace( + declaration, + output=msgspec.structs.replace( + declaration.output, + sum_impl=_custom_sum(stage, variant, declaration.output.group), + ), + ) exit_rows = resolve_exit_rows(stage.exit_rows) if exit_rows is ExitRows.ATTENTION or ( following is not None and following.read.reads_after_attn_tp_gather @@ -665,6 +705,7 @@ def _resolve_stage(stage, variant, arrival, following=None, *, final_read=None): update=stage.update, transform=stage.output_transform, in_token_order=in_token_order, + sum_impl=_custom_sum(stage, variant, group if owes else None), ), ) # An attention moves no residual: it keeps the rows it arrives on, except diff --git a/python/sglang/srt/layers/layer_boundary/facts.py b/python/sglang/srt/layers/layer_boundary/facts.py index a5779bb41c..155bdcb452 100644 --- a/python/sglang/srt/layers/layer_boundary/facts.py +++ b/python/sglang/srt/layers/layer_boundary/facts.py @@ -29,6 +29,7 @@ class UpdateFacts(msgspec.Struct, frozen=True): """The capabilities a ResidualUpdate declares (see ResidualUpdate).""" bind = None + replay_update = None is_plain_add: bool applied_at_exit: bool @@ -110,6 +111,9 @@ def facts_of(declaration): read=ReadFacts.of(declaration.read), update=UpdateFacts.of(declaration.update), output_transform=TransformFacts.of(declaration.output_transform), + sum_impl=declaration.sum_impl.facts() + if declaration.sum_impl is not None + else None, previous=None, prepared_from=None, ) diff --git a/python/sglang/srt/layers/layer_boundary/output.py b/python/sglang/srt/layers/layer_boundary/output.py index 08e433b6f0..a96ecd3cc6 100644 --- a/python/sglang/srt/layers/layer_boundary/output.py +++ b/python/sglang/srt/layers/layer_boundary/output.py @@ -37,6 +37,25 @@ class OutputTransform(msgspec.Struct, frozen=True): before_reduce_scatter: bool = False +class SumImplementation(msgspec.Struct, frozen=True): + """A producer's implementation of its declared output sum. + + ``run(partial, forward_batch)`` completes the same group the output + contract declares. It may use a model's custom collective; it does not + change token rows or perform the residual update. A hidden-dimension + scatter may leave a hidden shard for the producer update to consume. + ``can_defer(batch)`` preserves that implementation's batch eligibility; + None uses the ordinary exit eligibility. Neither callback is retained in + a neighbouring stage's construction facts. + """ + + run: Optional[Callable] + can_defer: Optional[Callable] = None + + def facts(self): + return SumImplementation(None) + + class UnreducedOutput(msgspec.Struct, frozen=True): """Internal adapter value describing an unfinished reduction. @@ -45,6 +64,9 @@ class UnreducedOutput(msgspec.Struct, frozen=True): group: All-reduce group when no redistribution callable is supplied. reduce_to_dp_local: Callable(partial) completing the sum and moving it to destination rows; takes precedence over group. + complete_sum: Custom implementation of this group's sum on the same + token rows; mutually exclusive with reduce_to_dp_local. + Exits hand this form to ResidualStream.record(), which keeps it as the contribution's owed work and exposes an opaque OwedOutput to models. @@ -58,8 +80,17 @@ class UnreducedOutput(msgspec.Struct, frozen=True): # rank's tokens (a reduce-scatter, or an all-reduce then a scatter). reduce_to_dp_local: Optional[Callable[[torch.Tensor], torch.Tensor]] = None + # Custom implementation of the same group, without token redistribution. + complete_sum: Optional[Callable[[torch.Tensor], torch.Tensor]] = None + + def __post_init__(self): + if self.complete_sum is not None and self.reduce_to_dp_local is not None: + raise ValueError("choose a custom sum or a DP redistribution") + def complete(self) -> torch.Tensor: """Complete the sum, on the destination rows when it moves them.""" + if self.complete_sum is not None: + return self.complete_sum(self.partial) if self.reduce_to_dp_local is not None: return self.reduce_to_dp_local(self.partial) return self.group.all_reduce(self.partial) diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 3bf9c06cc2..59d60e27d9 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -337,6 +337,27 @@ def _run_entry( if pending.owed is not None: raise RuntimeError("a declared input sum carried another completion") step = completed_step + if pending is not None and isinstance(pending.owed, DeclaredSum): + owed = pending.owed + if owed.complete_sum is not None: + result = _run_read_fusions( + tuple( + f + for f in read.completing_fusions + if f.completes is owed.group and not f.scatters + ), + hidden_states, + residual, + forward_batch, + norm, + read=read, + update=update, + **call, + ) + if result is not None: + return result + hidden_states = pending.complete() + step = completed_step if written and written_step is not None: step = written_step if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)): diff --git a/python/sglang/srt/layers/layer_boundary/residual/__init__.py b/python/sglang/srt/layers/layer_boundary/residual/__init__.py index 039031d228..23d00cfd63 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/residual/__init__.py @@ -50,6 +50,8 @@ class ResidualUpdate(Protocol): # None explicitly declares a stateless update. Bindings take (batch, context). bind: Optional[Callable] = None + # An invocation may export its reusable producer declaration for graph replay. + replay_update: Optional["ResidualUpdate"] = None # A plain add, which one rank may run before the sum it adds into # completes (the DP partial order, a residual that joins the attention diff --git a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py index 2348145828..59e8651a24 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/add_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/add_norm.py @@ -274,6 +274,7 @@ class PlainAdd: """A plain residual: the output is added into it.""" bind = None + replay_update = None is_plain_add = True applied_at_exit = False @@ -299,6 +300,7 @@ class ReplaceAtExit: boundary complete each part's sum first (StageBoundary.sum_part).""" bind = None + replay_update = None is_plain_add = False applied_at_exit = True diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index b34d28e582..417e62b2ed 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -13,8 +13,10 @@ # ============================================================================== """Residual access for one layer-stack invocation or TBO microbatch.""" +from typing import NamedTuple + from sglang.srt.layers.layer_boundary.output import complete_owed -from sglang.srt.layers.layer_boundary.residual import access +from sglang.srt.layers.layer_boundary.residual import access, bind_update from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import PPProxyTensors @@ -35,6 +37,13 @@ def stream_of(forward_batch): return stream +class ReplayHandoff(NamedTuple): + value: object + residual: object + update: object + written: bool + + def clear(forward_batch): """Release invocation state on completion or an exception.""" stream = forward_batch.residual_stream @@ -45,6 +54,50 @@ def clear(forward_batch): forward_batch.residual_stream = None +def export_replay_handoff(hidden_states, forward_batch): + """Export a complete operand and reusable producer state across an eager break. + + The handoff does not complete owed work or apply the residual update. + Replay rebinds dynamic producer metadata to the live batch. + """ + from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD + + stream = stream_of(forward_batch) + stream.check(hidden_states) + pending = stream.pending + if pending is not None and pending.owed is not None: + raise RuntimeError("complete the contribution before a graph replay handoff") + value, residual = stream.input(hidden_states) + update = pending.update if pending is not None else PLAIN_ADD + reusable = update.replay_update + result = ReplayHandoff( + value, + residual, + update if reusable is None else reusable, + pending is None and stream.residual is not None, + ) + clear(forward_batch) + return result + + +def import_replay_handoff(handoff, forward_batch, *, update_context=None): + """Reconstruct the residual state with replay's live producer metadata.""" + if handoff.written: + if handoff.residual is not None: + raise ValueError("a written replay handoff has no separate residual") + set_written(handoff.value, forward_batch) + return handoff.value + if handoff.residual is None: + start(forward_batch) + return handoff.value + hidden, forward_batch.residual_stream = ResidualStream.from_handoff( + handoff.value, + handoff.residual, + bind_update(handoff.update, forward_batch, update_context), + ) + return hidden + + def written_residual(forward_batch): """The residual the last prepare wrote, for a stage whose compute reads it too (e.g. to write the next stream at its exit). Borrowed: do not modify.""" diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py index 0c77d5ba69..7fc314d350 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/gated.py +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -170,6 +170,7 @@ class _AttnUpdate: only once the sum it writes in is complete.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = False @@ -227,6 +228,7 @@ class _FfnUpdate: streams stay widened: the layer stack's terminal read mixes them down.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = True diff --git a/python/sglang/srt/layers/layer_boundary/residual/ihc.py b/python/sglang/srt/layers/layer_boundary/residual/ihc.py index 90b899a5af..8dbf161344 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/ihc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/ihc.py @@ -132,6 +132,7 @@ class _AttnUpdate: complete.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = False @@ -188,6 +189,7 @@ class _FfnUpdate: so it belongs to the final norm.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = True diff --git a/python/sglang/srt/layers/layer_boundary/residual/mhc.py b/python/sglang/srt/layers/layer_boundary/residual/mhc.py index 337989fb40..d617cdb67f 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/mhc.py +++ b/python/sglang/srt/layers/layer_boundary/residual/mhc.py @@ -167,6 +167,7 @@ class _AttnUpdate: plain add, so it runs only once the sum it writes in is complete.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = False @@ -222,6 +223,7 @@ class _FfnUpdate: hidden states the layer stack hands on.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = True diff --git a/python/sglang/srt/layers/layer_boundary/residual/post_norm.py b/python/sglang/srt/layers/layer_boundary/residual/post_norm.py index dbcfebf4bc..516f41a29d 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/post_norm.py +++ b/python/sglang/srt/layers/layer_boundary/residual/post_norm.py @@ -25,6 +25,7 @@ class PostNormAdd: complete output. ``applied_at_exit`` makes the stage write it itself.""" bind = None + replay_update = None is_plain_add = False outlives_layer = True diff --git a/python/sglang/srt/layers/layer_boundary/residual/short_conv.py b/python/sglang/srt/layers/layer_boundary/residual/short_conv.py new file mode 100644 index 0000000000..e0f16c7369 --- /dev/null +++ b/python/sglang/srt/layers/layer_boundary/residual/short_conv.py @@ -0,0 +1,173 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Deferred channelwise convolution followed by residual add and normalization.""" + +from typing import Callable, NamedTuple, Optional + +import msgspec + +from sglang.srt.layers.layer_boundary.residual import PendingUpdateRead +from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_READOUT, PLAIN_ADD + + +class _UpdateFacts: + bind = None + replay_update = None + is_plain_add = False + applied_at_exit = False + outlives_layer = True + writes_stream = False + quantized_sum = False + + def slice_residual_attn_tp(self, residual): + raise NotImplementedError("short convolution needs its original token rows") + + def gather_residual_attn_tp(self, residual): + raise NotImplementedError("short convolution needs its original token rows") + + +class ShortConvUpdate(_UpdateFacts, msgspec.Struct, frozen=True): + """Producer parameters; bind() creates state owned by one contribution. + + A scattered completion produces a hidden shard. The convolution reads + that shard and ``gather_hidden`` reconstructs the hidden dimension before + the residual add. These operations leave the token layout unchanged. + """ + + convolve: Callable + group: object + sum_group: object + gather_hidden: Optional[Callable] = None + can_fuse: Optional[Callable] = None + + def bind(self, forward_batch, positions=None): + return ShortConvInvocation( + self, + forward_batch, + forward_batch.positions if positions is None else positions, + ) + + +class ShortConvInvocation(_UpdateFacts, msgspec.Struct, frozen=True): + spec: ShortConvUpdate + forward_batch: object + positions: object + + @property + def replay_update(self): + return self.spec + + def contribution(self, hidden_states): + batch = self.forward_batch + value = self.spec.convolve( + hidden_states, self.positions[: hidden_states.shape[0]], batch + ) + if self.spec.gather_hidden is not None: + value = self.spec.gather_hidden(value) + return value + + def update(self, hidden_states, residual): + return self.contribution(hidden_states) + residual + + +class NormalizationHandoff(NamedTuple): + """Completed producer contribution and residual, before add+norm.""" + + value: object + residual: object + + +class ShortConvRead(PendingUpdateRead): + """Read the actual producer's convolution and keep add+norm together.""" + + bind = None + + is_plain_norm = False + reads_before_dp_gather = True + reads_after_attn_tp_gather = True + gathering_reads = () + + def __init__(self, completing_fusions=()): + self.completing_fusions = completing_fusions + + @staticmethod + def accepts_pending_update(update): + return update.is_plain_add or isinstance( + update, (ShortConvUpdate, ShortConvInvocation) + ) + + def export_normalization_handoff(self, hidden_states, forward_batch): + """Complete producer work with live metadata, leaving add+norm to capture. + + Piecewise prefill completes the unfused sum and convolution before + exporting the unadded pair to the captured normalization. + """ + from sglang.srt.layers.layer_boundary.residual import batch as residual_batch + + read = _NORMALIZATION_PAIR_READ + return NormalizationHandoff( + *residual_batch.read_output(hidden_states, forward_batch, read, _pair) + ) + + def read_normalization_handoff(self, handoff, norm): + """Consume the complete pair with the original add+norm kernel.""" + if handoff.residual is None: + return self.read(handoff.value, norm) + return NORM_READOUT.update_and_read( + PLAIN_ADD, handoff.value, handoff.residual, norm + ) + + def init_residual(self, hidden_states): + return hidden_states + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + return NORM_READOUT.read(residual, norm, quant_format, post_residual_addition) + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + if update.is_plain_add: + return NORM_READOUT.update_and_read( + update, + hidden_states, + residual, + norm, + quant_format, + post_residual_addition, + ) + if not isinstance(update, ShortConvInvocation): + raise TypeError("short-conv read requires the producer's bound invocation") + value = update.contribution(hidden_states) + # Normalize the contribution/residual pair without a separately rounded add. + return NORM_READOUT.update_and_read( + PLAIN_ADD, value, residual, norm, quant_format, post_residual_addition + ) + + +def _pair(value, residual=None, **kwargs): + return value if residual is None else (value, residual) + + +class _NormalizationPairRead(ShortConvRead): + def read(self, residual, norm, quant_format="", post_residual_addition=None): + return residual, None + + +_NORMALIZATION_PAIR_READ = _NormalizationPairRead() diff --git a/python/sglang/srt/layers/layer_boundary/residual/stream.py b/python/sglang/srt/layers/layer_boundary/residual/stream.py index cd7ffab3b3..4b70a96c4a 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/stream.py +++ b/python/sglang/srt/layers/layer_boundary/residual/stream.py @@ -13,7 +13,7 @@ # ============================================================================== """One forward's residual and its producer's outstanding contribution.""" -from typing import Optional, Union +from typing import Callable, Optional, Union import msgspec import torch @@ -24,11 +24,16 @@ from sglang.srt.layers.layer_boundary.residual import ResidualUpdate class DeclaredSum(msgspec.Struct, frozen=True): - """A sum every output of this producer owes to its declared input edge.""" + """A sum every output owes to its declared input edge. complete_sum is + the producer's custom implementation bound to this invocation, or None + for the ordinary group all-reduce.""" group: SumGroup + complete_sum: Optional[Callable] = None def complete(self, value): + if self.complete_sum is not None: + return self.complete_sum(value) return _sum_group(self.group).all_reduce(value) diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index bf73903f54..86f64ad668 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -13,6 +13,7 @@ # ============================================================================== """A stage's model-facing input and output boundary.""" +from functools import partial from typing import TYPE_CHECKING, Callable, Optional import torch @@ -176,10 +177,21 @@ class StageBoundary: take a MoE finalize handoff) finishes through exit() instead.""" if self.kind is StageKind.ATTENTION and self.plan.finishes_directly: produced = self.plan.path_for(forward_batch).output + completion = ( + partial(produced.sum_impl.run, forward_batch=forward_batch) + if produced.sum_impl is not None + else None + ) + if produced.always_partial: + owed = DeclaredSum(produced.group, completion) + else: + owed = None + if completion is not None: + hidden_states = completion(hidden_states) return stream_of(forward_batch).record( hidden_states, bind_update(produced.update, forward_batch, update_context), - declared_sum=produced.group if produced.always_partial else None, + declared_sum=owed, ) if self.kind is StageKind.FFN and self.plan.fusions is not None: raise RuntimeError( diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index 63944d6197..c81663f26f 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -4,6 +4,7 @@ import copy import logging import re from array import array +from functools import partial from typing import Iterable, Optional, Set, Tuple import torch @@ -17,6 +18,29 @@ from sglang.srt.configs.inkling import ( ) from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.environ import envs +from sglang.srt.layers.layer_boundary import ( + ReadoutFusion, + append_stages, + declare_attn, + declare_ffn, + layer_stack, +) +from sglang.srt.layers.layer_boundary.facts import ReadFacts, UpdateFacts +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.output import ( + OutputTransform, + SumImplementation, + UnreducedOutput, +) +from sglang.srt.layers.layer_boundary.residual import LayerResidualOps +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD +from sglang.srt.layers.layer_boundary.residual.short_conv import ( + NormalizationHandoff, + ShortConvInvocation, + ShortConvRead, + ShortConvUpdate, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import get_moe_runner_backend @@ -59,7 +83,11 @@ from sglang.srt.models.inkling_common.kernels.comm import ( ar_sconv_norm_fused, ensure_inkling_ar_resources, fullwidth_ar_sconv_fusable, + get_ar_buffer, + reduce_scatter_hidden, scattered_ar_sconv_fusable, + symm_mem_all_reduce, + take_ar_shared, ) from sglang.srt.models.inkling_common.moe import InklingMoE from sglang.srt.models.inkling_common.sconv import SconvType, ShortConvolution @@ -143,6 +171,162 @@ def _is_unsupported_mm_weight_name(name: str) -> bool: ) +def _merge_tp1_shared(value): + """A complete single-rank routed output still needs its shared contribution.""" + shared = take_ar_shared(value.shape[0]) + return value if shared is None else value + shared + + +def _inkling_sum(hidden_states, forward_batch, *, group, scattered, shared=False): + """Complete the declared sum using Inkling's existing hidden transport.""" + extra = take_ar_shared(hidden_states.shape[0]) if shared else None + buf = get_ar_buffer(group, *hidden_states.shape, hidden_states.dtype) + in_buffer = buf is not None and hidden_states.data_ptr() == buf.data_ptr() + if extra is not None and scattered: + if buf is not None: + torch.add(hidden_states, extra, out=buf) + hidden_states, in_buffer = buf, True + else: + hidden_states = hidden_states + extra + extra = None + if scattered: + return reduce_scatter_hidden(hidden_states, group, input_is_ar_buffer=in_buffer) + return symm_mem_all_reduce( + hidden_states, group, input_is_ar_buffer=in_buffer, shared=extra + ) + + +def _inkling_fusable(group, forward_batch, hidden, dtype, *, fullwidth=True): + if forward_batch.forward_mode.is_idle(): + return False + n = forward_batch.input_ids.shape[0] + return ( + ar_sconv_norm_fusable(group, forward_batch, n, hidden, dtype) + or scattered_ar_sconv_fusable( + get_parallel().tp_group, forward_batch, n, hidden, dtype + ) + or ( + fullwidth + and fullwidth_ar_sconv_fusable( + get_parallel().tp_group, forward_batch, n, hidden, dtype + ) + ) + ) + + +def _inkling_fused_read(value, residual, forward_batch, norm, *, update): + if not isinstance(update, ShortConvInvocation): + return None + spec = update.spec + positions = forward_batch.positions + if update.positions is not positions and ( + not isinstance(update.positions, torch.Tensor) + or update.positions.data_ptr() != positions.data_ptr() + or update.positions.stride() != positions.stride() + ): + return None + if spec.can_fuse is None or not spec.can_fuse(forward_batch): + return None + if isinstance(value, UnreducedOutput): + if value.reduce_to_dp_local is not None or value.group is not spec.group: + return None + value = value.partial + if not isinstance(value, torch.Tensor) or residual is None: + return None + fm = forward_batch.forward_mode + if spec.gather_hidden is not None: + if fm.is_decode() or fm.is_target_verify(): + return ar_scattered_sconv_fused( + value, + spec.convolve, + forward_batch, + spec.group, + norm=norm, + norm_residual=residual, + ) + value = ar_scattered_sconv_fused( + value, spec.convolve, forward_batch, spec.group + ) + elif fm.is_decode() or fm.is_target_verify(): + return ar_sconv_norm_fused( + value, residual, spec.convolve, norm, forward_batch, spec.group + ) + else: + value = ar_fullwidth_sconv_fused( + value, spec.convolve, forward_batch, spec.group + ) + return norm(value, residual) + + +def _inkling_read(): + def completions(run): + return tuple( + ReadoutFusion( + group, + run, + reads=True, + carried=True, + accepts_update=lambda u, group=group: ( + isinstance(u, ShortConvInvocation) and u.spec.sum_group is group + ), + ) + for group in (SumGroup.ATTN_TP, SumGroup.TP, SumGroup.MOE_OUTPUT) + ) + + return ShortConvRead(completions(_inkling_fused_read)) + + +class _InklingRawRead(ShortConvRead): + """The MTP chain consumes a rounded add before its separate chain norm.""" + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + return residual, residual + + def update_and_read( + self, + update, + hidden_states, + residual, + norm, + quant_format="", + post_residual_addition=None, + ): + value = update.update(hidden_states, residual) + return value, value + + +def inkling_stages(config, layer_id, *, ops=None, sums=None): + sparse = layer_id >= config.dense_mlp_idx + if ops is None: + read = ReadFacts(False, True, True) + update = ( + UpdateFacts(False, False, True, False) if config.use_sconv else PLAIN_ADD + ) + ops = LayerResidualOps(read, update, read, update) + if sums is None: + sums = (SumImplementation(None), SumImplementation(None)) + return ( + declare_attn( + read=ops.attn_readout, + update=ops.attn_update, + sum_impl=sums[0], + gathers_attn_tp_input=False, + ), + declare_ffn( + sparse=sparse, + read=ops.ffn_readout, + update=ops.ffn_update, + dense_tp_size=None if sparse else get_parallel().attn_tp_size, + sum_impl=sums[1], + output_transform=( + OutputTransform(_merge_tp1_shared) + if sparse and get_parallel().tp_size == 1 + else None + ), + ), + ) + + class InklingDecoderLayer(nn.Module): def __init__( self, @@ -152,6 +336,7 @@ class InklingDecoderLayer(nn.Module): quant_config: QuantizationConfig | None = None, prefix: str = "", alt_stream: torch.cuda.Stream | None = None, + allow_fullwidth_fusion: bool = True, ): super().__init__() self.layer_id = layer_id @@ -241,324 +426,192 @@ class InklingDecoderLayer(nn.Module): if config.use_sconv else None ) - # The fused decode path needs an MoE and this layer's MLP convolution; - # scattered convolution disables the fusion separately. + # The fused path needs an MoE and this layer's MLP convolution; + # the per-batch gate selects the matching scattered or full-width kernel. self.mlp_ar_fusable = ( isinstance(self.mlp, InklingMoE) and self.mlp_sconv is not None ) - # Under BCG the short-conv metadata (cu_seqlens/seq_idx) is baked at bs=1 - # during capture, which is wrong for multi-seq prefill. Running every - # sconv (and the attn whose k/v_sconv it wraps) eagerly makes them re-read - # the LIVE per-seq metadata at replay. `_breakable_attn_group` groups the - # prior layer's (deferred) mlp_sconv + attn_norm + attn + attn_sconv into - # ONE eager break; only mlp_norm + MoE stay captured. Outside a capture - # these wrappers just run inline. `_breakable_mlp_sconv` runs the final - # layer's deferred mlp_sconv after the layer loop. + def can_fuse(group, norm): + return lambda batch: _inkling_fusable( + group, + batch, + config.hidden_size, + norm.weight.dtype, + fullwidth=allow_fullwidth_fusion, + ) + + attn_fuse = can_fuse(self.attn_tp_group, self.attn_norm) + sparse_mlp = isinstance(self.mlp, InklingMoE) + mlp_sum_group = SumGroup.MOE_OUTPUT if sparse_mlp else SumGroup.ATTN_TP + mlp_group = get_parallel().tp_group if sparse_mlp else self.attn_tp_group + mlp_fuse = can_fuse(mlp_group, self.mlp_norm) + mlp_eligible = lambda batch: self.mlp_ar_fusable and mlp_fuse(batch) + gather = ( + partial(all_gather_hidden, group=self.attn_tp_group) + if self.scattered_sconv + else None + ) + attn_update = ( + ShortConvUpdate( + self.attn_sconv, self.attn_tp_group, SumGroup.ATTN_TP, gather, attn_fuse + ) + if self.attn_sconv is not None + else PLAIN_ADD + ) + mlp_update = ( + ShortConvUpdate( + self.mlp_sconv, mlp_group, mlp_sum_group, gather, mlp_eligible + ) + if self.mlp_sconv is not None + else PLAIN_ADD + ) + read = _inkling_read() + declarations = inkling_stages( + config, + layer_id, + ops=LayerResidualOps(read, attn_update, read, mlp_update), + sums=( + SumImplementation( + partial( + _inkling_sum, + group=self.attn_tp_group, + scattered=self.scattered_sconv, + ) + ), + SumImplementation( + partial( + _inkling_sum, + group=mlp_group, + scattered=self.scattered_sconv, + shared=sparse_mlp, + ), + mlp_eligible, + ), + ), + ) + self.attn_boundary, self.ffn_boundary = append_stages( + (declarations[0], self.attn_norm), (declarations[1], self.mlp_norm) + ) + # The producer convolution uses live metadata in the eager break; + # its contribution/residual pair keeps the FFN add+norm captured. self._breakable_attn_group = eager_on_graph(True)(self._attn_group_impl) - self._breakable_mlp_sconv = eager_on_graph(True)(self._mlp_sconv_impl) - def _attn_block( - self, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - positions: torch.Tensor, - forward_batch: ForwardBatch, - prev_mlp_sconv: Optional[ShortConvolution], - log_scaling_tau: Optional[torch.Tensor], - *, - eager_attn: bool, - prev_mlp_partial: bool = False, - fuse_attn_ar: bool = False, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """The {deferred prior-layer mlp_sconv -> attn_norm -> attn -> attn_sconv} - region, shared by the inline `forward` path and the BCG eager group. Returns - (hidden_states, residual). - - When ``eager_attn`` is set (the BCG eager break), the attention runs eagerly - and ``forward_batch.out_cache_loc`` is narrowed to the token count for the KV - write: the eager attn path (unlike the custom op) does NOT narrow it, and under - BCG it is a full-bucket buffer, so the KV-write kernel would size-mismatch. The - caller is responsible for narrowing hidden_states/positions/log_scaling_tau to - the real (non-padded) token count before calling with ``eager_attn=True``. - Otherwise the attention runs normally. - - ``prev_mlp_partial``: ``hidden_states`` holds the previous layer's - UNREDUCED MoE partial sums (``reduce=False``); the {all-reduce -> - prev_mlp_sconv -> attn_norm} chain runs as ONE fused kernel.""" - hs, res = hidden_states, residual - if prev_mlp_partial and self.scattered_sconv: - fm = forward_batch.forward_mode - if fm.is_decode() or fm.is_target_verify(): - # Fused decode/verify {AR + scattered sconv + attn_norm}: the - # add+RMSNorm tail is fused in-kernel (residual always live - # here -- partials only ever come from a previous layer's MoE). - hs, res = ar_scattered_sconv_fused( - hs, - prev_mlp_sconv, - forward_batch, - get_parallel().tp_group, - norm=self.attn_norm, - norm_residual=res, - ) - else: - # Fused extend {AR + scattered sconv}: hs holds the previous - # MoE's unreduced partials; the kernel returns the gathered - # post-conv [T, H], and the norm runs unfused below. - hs = ar_scattered_sconv_fused( - hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group - ) - hs, res = self.attn_norm(hs, res) - elif prev_mlp_partial: - fm = forward_batch.forward_mode - if fm.is_decode() or fm.is_target_verify(): - # Fused decode {AR -> sconv -> add+norm}; residual is always - # live here (partials only ever come from a prior layer's MoE). - hs, res = ar_sconv_norm_fused( - hs, - res, - prev_mlp_sconv, - self.attn_norm, - forward_batch, - get_parallel().tp_group, - ) - else: - # Fused extend {AR + full-width sconv + cache update} - # (non-scattered); norm runs unfused on the gathered [T, H]. - hs = ar_fullwidth_sconv_fused( - hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group - ) - hs, res = self.attn_norm(hs, res) + def _attention_input( + self, hidden_states, positions, forward_batch, log_scaling_tau, *, eager=False + ): + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) + if eager: + original = forward_batch.out_cache_loc + forward_batch.out_cache_loc = original[: hidden_states.shape[0]] + try: + with force_eager_attention(): + hidden_states = self.attn( + hidden_states, + positions, + forward_batch, + log_scaling_tau=log_scaling_tau, + reduce=False, + ) + finally: + forward_batch.out_cache_loc = original else: - if prev_mlp_sconv is not None: - hs = prev_mlp_sconv(hs, positions, forward_batch) - if self.scattered_sconv: - # hs was the previous layer's reduce-scattered [T, H/P] MoE - # shard; gather back to [T, H] before the residual add. - hs = all_gather_hidden(hs, self.attn_tp_group) - - # Fused residual-add + norm for attention input. First layer: no prior - # residual yet, so just norm the (post-deferred-sconv) embeddings. - if res is None: - res = hs - hs = self.attn_norm(hs) - else: - hs, res = self.attn_norm(hs, res) - - if eager_attn: - # Routing through the split op here would start a nested break that - # asserts on the already-ended segment, so force the eager attn path; - # narrow out_cache_loc to the real token count for the KV write (see above), - # then restore the full buffer on forward_batch (shared across layers/replays). - orig_out_cache_loc = forward_batch.out_cache_loc - forward_batch.out_cache_loc = orig_out_cache_loc[: hs.shape[0]] - with force_eager_attention(): - hs = self.attn( - hs, positions, forward_batch, log_scaling_tau=log_scaling_tau - ) - forward_batch.out_cache_loc = orig_out_cache_loc - else: - hs = self.attn( - hs, + hidden_states = self.attn( + hidden_states, positions, forward_batch, log_scaling_tau=log_scaling_tau, - reduce=not fuse_attn_ar, + reduce=False, ) - - if fuse_attn_ar: - # hs holds the UNREDUCED wo_ud partials; the caller (forward) fuses - # {AR -> attn_sconv -> mlp_norm} into one kernel. - return hs, res - if self.attn_sconv is not None: - # Under scattered sconv, hs is the attn output's reduce-scattered - # [T, H/P] shard (attn.py routed the reduction); gather after. - hs = self.attn_sconv(hs, positions, forward_batch) - if self.scattered_sconv: - hs = all_gather_hidden(hs, self.attn_tp_group) - return hs, res + hidden_states = self.attn_boundary.finish( + hidden_states, forward_batch, update_context=positions + ) + if eager: + return self.ffn_boundary.declaration.read.export_normalization_handoff( + hidden_states, forward_batch + ) + return self.ffn_boundary.prepare(hidden_states, forward_batch) def _attn_group_impl( self, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - positions: torch.Tensor, - attn_out: torch.Tensor, - residual_out: torch.Tensor, - prev_mlp_sconv: Optional[ShortConvolution], - log_scaling_tau: Optional[torch.Tensor], - ) -> None: - """Eager break: run `_attn_block` on the REAL (non-padded) tokens with the LIVE - forward_batch and write the result into the padded output buffers. Mutates - attn_out / residual_out and returns None (the eager_on_graph copy-back is - per-tensor, not per-tuple, so outputs must be pre-allocated buffers).""" - forward_batch = get_tc_piecewise_forward_context().forward_batch - n = forward_batch.global_num_token_non_padded_cpu - # log_scaling_tau is per-token, so narrow it to match the real tokens too. - hs, res = self._attn_block( - hidden_states[:n], - residual[:n] if residual is not None else None, - positions[:n], - forward_batch, - prev_mlp_sconv, - log_scaling_tau[:n] if log_scaling_tau is not None else None, - eager_attn=True, + hidden_states, + residual, + positions, + mlp_input, + residual_out, + incoming_update, + log_scaling_tau, + written, + ): + """BCG eager input preparation uses replay's live batch metadata.""" + batch = get_tc_piecewise_forward_context().forward_batch + n = batch.global_num_token_non_padded_cpu + hidden = residual_batch.import_replay_handoff( + residual_batch.ReplayHandoff( + hidden_states[:n], + residual[:n] if residual is not None else None, + incoming_update, + written, + ), + batch, + update_context=positions[:n], ) - torch._foreach_copy_((attn_out[:n], residual_out[:n]), (hs, res)) - if attn_out.shape[0] != n: - torch._foreach_zero_((attn_out[n:], residual_out[n:])) + handoff = self._attention_input( + hidden, + positions[:n], + batch, + log_scaling_tau[:n] if log_scaling_tau is not None else None, + eager=True, + ) + torch._foreach_copy_( + (mlp_input[:n], residual_out[:n]), + (handoff.value, handoff.residual), + ) + if mlp_input.shape[0] != n: + torch._foreach_zero_((mlp_input[n:], residual_out[n:])) - def _mlp_sconv_impl( - self, - hidden_states: torch.Tensor, - positions: torch.Tensor, - out: torch.Tensor, - ) -> None: - """Eager break for the final layer's deferred mlp_sconv: run on the real - tokens with the live forward_batch, write the padded output buffer.""" - forward_batch = get_tc_piecewise_forward_context().forward_batch - n = forward_batch.global_num_token_non_padded_cpu - y = self.mlp_sconv(hidden_states[:n], positions[:n], forward_batch) - if self.scattered_sconv: - # y is the [n, H/P] shard; the output buffer is post-gather [n, H]. - y = all_gather_hidden(y, self.attn_tp_group) - out[:n].copy_(y) - if out.shape[0] != n: - out[n:].zero_() - - def forward( - self, - hidden_states: torch.Tensor, - positions: torch.Tensor, - forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - prev_mlp_sconv: Optional[ShortConvolution] = None, - *, - log_scaling_tau: torch.Tensor | None = None, - prev_mlp_partial: bool = False, - fuse_ar_sconv: bool = False, - fuse_attn_ar: bool = False, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """mlp_sconv is DEFERRED: this layer applies the *previous* layer's - mlp_sconv (`prev_mlp_sconv`) at the head of its eager region, and its own - mlp_sconv is applied by the next layer (or, for the last layer, by - InklingCausalLLM after the loop). Deferral is behavior-identical -- attn_norm - consumes the sconv output either way -- and lets the whole - {mlp_sconv, attn_norm, attn, attn_sconv} region be ONE eager break. - - ``prev_mlp_partial``: ``hidden_states`` holds the previous layer's - UNREDUCED MoE partials; the head chain runs as the fused AR kernel. - ``fuse_ar_sconv``: this layer's own MoE runs ``reduce=False`` so the - NEXT consumer (layer or tail) fuses its AR the same way. The caller - (InklingCausalLLM) threads both consistently.""" + def forward(self, hidden_states, positions, forward_batch, *, log_scaling_tau=None): if forward_batch.forward_mode.is_idle(): - return hidden_states, residual - - # The eager group reads the LIVE forward_batch from the tc_piecewise context - # (the only hook evaluated at BCG replay time — the break's captured args are - # frozen at capture-bucket shapes). Only the prefill BCG runner installs that - # context; the decode breakable backend sets is_in_breakable_cuda_graph() but - # NOT the context, so gate on both and otherwise fall through to the inline - # path below (which uses the passed forward_batch — correct for decode). + return hidden_states if ( is_in_breakable_cuda_graph() and get_tc_piecewise_forward_context() is not None ): - # BCG prefill path: the AR fusion is decode-only, so partials never - # reach (or leave) this branch. - assert not prev_mlp_partial and not fuse_ar_sconv and not fuse_attn_ar - # BCG: {prev mlp_sconv, attn_norm, attn, attn_sconv} run eagerly (one - # break under capture); mlp_norm + MoE stay captured. (The live - # forward_batch inside the break is read from the shared tc_piecewise - # context, which the prefill BCG runner populates at capture and replay.) - # Under scattered sconv the group's INPUT can be the previous layer's - # [T, H/P] MoE shard while its OUTPUT is post-all-gather [T, H], so - # size the output buffers explicitly (residual is always [T, H]). - out_shape = (hidden_states.shape[0], self.attn_norm.weight.shape[0]) - attn_out = hidden_states.new_empty(out_shape) - residual_out = hidden_states.new_empty(out_shape) + value, residual, incoming, written = residual_batch.export_replay_handoff( + hidden_states, forward_batch + ) + shape = (value.shape[0], self.attn_norm.weight.shape[0]) + mlp_input, residual_out = value.new_empty(shape), value.new_empty(shape) self._breakable_attn_group( - hidden_states, + value, residual, positions, - attn_out, + mlp_input, residual_out, - prev_mlp_sconv, + incoming, log_scaling_tau, + written, ) - hidden_states, residual = self.mlp_norm(attn_out, residual_out) - del attn_out - del residual_out - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) - return hidden_states, residual - - # Plain eager / decode: run inline (still deferring mlp_sconv so the - # InklingCausalLLM loop threads prev_mlp_sconv uniformly across modes). - fuse_attn = fuse_attn_ar and self.attn_sconv is not None - hidden_states, residual = self._attn_block( - hidden_states, - residual, - positions, - forward_batch, - prev_mlp_sconv, - log_scaling_tau, - eager_attn=False, - prev_mlp_partial=prev_mlp_partial, - fuse_attn_ar=fuse_attn, - ) - if fuse_attn and self.scattered_sconv: - fm = forward_batch.forward_mode - if fm.is_decode() or fm.is_target_verify(): - # Fused decode/verify {wo_ud AR + scattered attn_sconv + - # mlp_norm} (attn-side chain), norm tail in-kernel. - hidden_states, residual = ar_scattered_sconv_fused( - hidden_states, - self.attn_sconv, - forward_batch, - self.attn_tp_group, - norm=self.mlp_norm, - norm_residual=residual, + hidden_states, residual = ( + self.ffn_boundary.declaration.read.read_normalization_handoff( + NormalizationHandoff(mlp_input, residual_out), self.mlp_norm + ) + ) + residual_batch.set_written(residual, forward_batch) + else: + hidden_states = self._attention_input( + hidden_states, positions, forward_batch, log_scaling_tau + ) + with self.ffn_boundary.exit(forward_batch, update_context=positions) as output: + if isinstance(self.mlp, InklingMoE): + hidden_states = self.mlp( + hidden_states, forward_batch=forward_batch, reduce=False ) else: - # Fused extend {AR + scattered sconv} (attn-side chain); the - # norm runs unfused on the gathered [T, H]. - hidden_states = ar_scattered_sconv_fused( - hidden_states, self.attn_sconv, forward_batch, self.attn_tp_group + hidden_states = self.mlp( + hidden_states, forward_batch=forward_batch, use_reduce_scatter=True ) - hidden_states, residual = self.mlp_norm(hidden_states, residual) - elif fuse_attn: - fm = forward_batch.forward_mode - if fm.is_decode() or fm.is_target_verify(): - # Fused {wo_ud AR -> attn_sconv -> mlp_norm} (attn-side chain). - hidden_states, residual = ar_sconv_norm_fused( - hidden_states, - residual, - self.attn_sconv, - self.mlp_norm, - forward_batch, - get_parallel().attn_tp_group, - ) - else: - # Fused extend {AR + full-width attn_sconv + cache update} - # (attn-side chain, non-scattered); norm runs unfused. - hidden_states = ar_fullwidth_sconv_fused( - hidden_states, - self.attn_sconv, - forward_batch, - get_parallel().attn_tp_group, - ) - hidden_states, residual = self.mlp_norm(hidden_states, residual) - else: - hidden_states, residual = self.mlp_norm(hidden_states, residual) - if fuse_ar_sconv and self.mlp_ar_fusable: - # Skip the MoE's own all-reduce; the next layer (or the model tail) - # fuses {AR -> this layer's mlp_sconv -> norm} into one kernel. - hidden_states = self.mlp( - hidden_states, forward_batch=forward_batch, reduce=False - ) - else: - hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) - return hidden_states, residual + return output.finish(hidden_states) class InklingCausalLLM(nn.Module): @@ -596,12 +649,16 @@ class InklingCausalLLM(nn.Module): alt_stream=self.alt_stream, ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self._final_read = _inkling_read() + self._breakable_final_read = eager_on_graph(True)(self._final_read_impl) self.layers = make_layers( config.num_hidden_layers, get_layer, prefix=add_prefix("layers", prefix), + final_read=self._final_read, + stage_facts=partial(inkling_stages, config), ) - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) # Custom-AR resources must exist BEFORE any CUDA-graph capture (with # the prefill graph disabled and --skip-server-warmup there is no eager @@ -739,199 +796,95 @@ class InklingCausalLLM(nn.Module): if self.config.log_scaling_n_floor is not None else None ) - residual = None - # mlp_sconv is deferred one layer: each layer applies the previous layer's - # mlp_sconv at the head of its (eager) attn region, so the whole sconv+attn - # region is one BCG break. prev_mlp_sconv=None for layer 0. - prev_mlp_sconv = None - # Fused decode {MoE AR -> mlp_sconv -> attn_norm}: decided ONCE per - # forward (a pure function of per-forward state, so the producing MoE - # and the consuming layer/tail always agree). When on, an eligible - # layer's MoE returns UNREDUCED partials (reduce=False) and the next - # consumer runs the fused kernel. - fuse_ar_sconv = ( - not forward_batch.forward_mode.is_idle() - and ar_sconv_norm_fusable( - get_parallel().tp_group, - forward_batch, - hidden_states.shape[0], - hidden_states.shape[-1], - hidden_states.dtype, - ) - ) - # The attn-side chain reduces over the ATTENTION TP group (identical to - # the model TP group without DP attention, but gate on it explicitly). - fuse_attn_ar = ( - not forward_batch.forward_mode.is_idle() - and ar_sconv_norm_fusable( - get_parallel().attn_tp_group, - forward_batch, - hidden_states.shape[0], - hidden_states.shape[-1], - hidden_states.dtype, - ) - ) - # Fused extend {AR + scattered sconv} (--enable-scattered-sconv + - # SGLANG_OPT_USE_INKLING_FUSED_AR_SCONV): same producer contract as the - # decode fusion above (reduce=False), consumer runs the scattered - # kernel. Mutually exclusive with ar_sconv_norm_fusable by mode - # (extend vs decode/verify) and by the scattered gate inside it. - if not forward_batch.forward_mode.is_idle() and scattered_ar_sconv_fusable( - get_parallel().tp_group, - forward_batch, - hidden_states.shape[0], - hidden_states.shape[-1], - hidden_states.dtype, - ): - fuse_ar_sconv = True - fuse_attn_ar = True - # Fused extend {AR + full-width sconv + cache update} (NON-scattered): - # same producer contract (reduce=False); the consumer sites dispatch - # by mode -- extend runs the full-width column kernel, decode/verify - # keep ar_sconv_norm_fused. Mutually exclusive with both gates above - # (mode for ar_sconv_norm_fusable, the scattered flag for - # scattered_ar_sconv_fusable). - if not forward_batch.forward_mode.is_idle() and fullwidth_ar_sconv_fusable( - get_parallel().tp_group, - forward_batch, - hidden_states.shape[0], - hidden_states.shape[-1], - hidden_states.dtype, - ): - fuse_ar_sconv = True - fuse_attn_ar = True - prev_mlp_partial = False - aux_hidden_states: Optional[list[torch.Tensor]] = ( + residual_batch.start(forward_batch) + aux_hidden_states = ( [] if self._dflash_layers_to_capture and not forward_batch.forward_mode.is_idle() else None ) - for layer in self.layers: - hidden_states, residual = layer( - hidden_states, - positions, - forward_batch, - residual, - prev_mlp_sconv, - log_scaling_tau=log_scaling_tau, - prev_mlp_partial=prev_mlp_partial, - fuse_ar_sconv=fuse_ar_sconv, - fuse_attn_ar=fuse_attn_ar, - ) - prev_mlp_sconv = layer.mlp_sconv - prev_mlp_partial = fuse_ar_sconv and layer.mlp_ar_fusable - if ( - aux_hidden_states is not None - and layer.layer_id in self._dflash_layers_to_capture - ): - # The trained taps are post-layer and precede the deferred - # mlp_sconv belonging to this layer. - tap_hidden = hidden_states - if layer.scattered_sconv: - tap_hidden = all_gather_hidden(tap_hidden, layer.attn_tp_group) - aux_hidden_states.append( - tap_hidden if residual is None else tap_hidden + residual - ) - # The final layer's mlp_sconv was deferred; run it now — as an eager break - # under BCG (so it re-reads live per-seq metadata at replay), else inline. - if prev_mlp_sconv is not None and not forward_batch.forward_mode.is_idle(): - if prev_mlp_partial and self.layers[-1].scattered_sconv: - fm = forward_batch.forward_mode - if fm.is_decode() or fm.is_target_verify(): - # Fused decode/verify tail: {AR + scattered sconv + final - # norm} in one kernel. - hidden_states, _ = ar_scattered_sconv_fused( - hidden_states, - prev_mlp_sconv, - forward_batch, - get_parallel().tp_group, - norm=self.norm, - norm_residual=residual, - ) - return ( - (hidden_states, aux_hidden_states) - if self._dflash_layers_to_capture - else hidden_states - ) - # Fused extend tail: {AR + scattered sconv}, then the final - # norm unfused on the gathered [T, H]. - hidden_states = ar_scattered_sconv_fused( + completed = False + try: + for layer in self.layers: + hidden_states = layer( hidden_states, - prev_mlp_sconv, + positions, forward_batch, - get_parallel().tp_group, + log_scaling_tau=log_scaling_tau, ) - hidden_states, _ = self.norm(hidden_states, residual) - return ( - (hidden_states, aux_hidden_states) - if self._dflash_layers_to_capture - else hidden_states - ) - if prev_mlp_partial: - fm = forward_batch.forward_mode - if fm.is_decode() or fm.is_target_verify(): - # Fused tail: {AR -> final mlp_sconv -> final norm} in one - # kernel. - hidden_states, _ = ar_sconv_norm_fused( - hidden_states, - residual, - prev_mlp_sconv, - self.norm, - forward_batch, - get_parallel().tp_group, + if ( + aux_hidden_states is not None + and layer.layer_id in self._dflash_layers_to_capture + ): + # The trained tap precedes this layer's deferred convolution. + value = residual_batch.complete_output(hidden_states, forward_batch) + if layer.scattered_sconv: + value = all_gather_hidden(value, layer.attn_tp_group) + residual = residual_batch.written_residual(forward_batch) + aux_hidden_states.append( + value.clone() if residual is None else value + residual ) - return ( - (hidden_states, aux_hidden_states) - if self._dflash_layers_to_capture - else hidden_states - ) - # Fused extend tail: {AR + full-width sconv + cache update} - # (non-scattered), then the final norm unfused. - hidden_states = ar_fullwidth_sconv_fused( - hidden_states, - prev_mlp_sconv, - forward_batch, - get_parallel().tp_group, - ) - hidden_states, _ = self.norm(hidden_states, residual) - return ( - (hidden_states, aux_hidden_states) - if self._dflash_layers_to_capture - else hidden_states - ) - # Same gate as the per-layer group: the eager break needs the tc_piecewise - # context (installed only by the prefill BCG runner) to read the live - # forward_batch at replay; else run inline with the passed forward_batch. - scattered = self.layers[-1].scattered_sconv if ( is_in_breakable_cuda_graph() and get_tc_piecewise_forward_context() is not None + and self.layers[-1].mlp_sconv is not None ): - # Under scattered sconv the input is the last MoE's [T, H/P] - # shard; the break's output buffer is post-all-gather [T, H]. - out_shape = ( - (hidden_states.shape[0], self.norm.weight.shape[0]) - if scattered - else hidden_states.shape + value, residual, incoming, written = ( + residual_batch.export_replay_handoff(hidden_states, forward_batch) ) - mlp_sconv_out = hidden_states.new_empty(out_shape) - self.layers[-1]._breakable_mlp_sconv( - hidden_states, positions, mlp_sconv_out + out = value.new_empty((value.shape[0], self.norm.weight.shape[0])) + residual_out = torch.empty_like(out) + self._breakable_final_read( + value, residual, incoming, positions, out, residual_out, written + ) + hidden_states, _ = self._final_read.read_normalization_handoff( + NormalizationHandoff(out, residual_out), self.norm ) - hidden_states = mlp_sconv_out else: - hidden_states = prev_mlp_sconv(hidden_states, positions, forward_batch) - if scattered: - hidden_states = all_gather_hidden( - hidden_states, self.layers[-1].attn_tp_group - ) - hidden_states, _ = self.norm(hidden_states, residual) - return ( - (hidden_states, aux_hidden_states) - if self._dflash_layers_to_capture - else hidden_states + hidden_states, _ = residual_batch.read_output( + hidden_states, forward_batch, self._final_read, self.norm + ) + completed = True + return ( + (hidden_states, aux_hidden_states) + if self._dflash_layers_to_capture + else hidden_states + ) + finally: + residual_batch.clear(forward_batch) + # Clear producer scratch after a failed consumer too. + leftover = take_ar_shared(forward_batch.input_ids.shape[0]) + if completed and leftover is not None: + raise RuntimeError("unconsumed fused-AR shared partials at model exit") + + def _final_read_impl( + self, + hidden_states, + residual, + incoming_update, + positions, + out, + residual_out, + written, + ): + batch = get_tc_piecewise_forward_context().forward_batch + n = batch.global_num_token_non_padded_cpu + hidden = residual_batch.import_replay_handoff( + residual_batch.ReplayHandoff( + hidden_states[:n], + residual[:n] if residual is not None else None, + incoming_update, + written, + ), + batch, + update_context=positions[:n], ) + handoff = self._final_read.export_normalization_handoff(hidden, batch) + torch._foreach_copy_( + (out[:n], residual_out[:n]), (handoff.value, handoff.residual) + ) + if out.shape[0] != n: + torch._foreach_zero_((out[n:], residual_out[n:])) class InklingAudio(nn.Module): @@ -1747,14 +1700,17 @@ class InklingMTPLayer(nn.Module): mtp_config.swa_num_key_value_heads = config.mtp_swa_num_key_value_heads mtp_config.swa_head_dim = config.mtp_swa_head_dim self.alt_stream = torch.cuda.Stream() if is_cuda() else None - self.transformer_block = InklingDecoderLayer( - config=mtp_config, - layer_id=layer_id if layer_id is not None else 0, - is_local=is_local, - quant_config=quant_config, - prefix=add_prefix("transformer_block", prefix), - alt_stream=alt_stream, - ) + self._final_read = _InklingRawRead() + with layer_stack(final_read=self._final_read): + self.transformer_block = InklingDecoderLayer( + config=mtp_config, + layer_id=layer_id if layer_id is not None else 0, + is_local=is_local, + quant_config=quant_config, + prefix=add_prefix("transformer_block", prefix), + alt_stream=alt_stream, + allow_fullwidth_fusion=False, + ) def forward( self, @@ -1779,46 +1735,21 @@ class InklingMTPLayer(nn.Module): if self.log_scaling_n_floor is not None else None ) - fm_idle = forward_batch.forward_mode.is_idle() - fuse_attn_ar = not fm_idle and ar_sconv_norm_fusable( - get_parallel().attn_tp_group, - forward_batch, - h_inproj.shape[0], - h_inproj.shape[-1], - h_inproj.dtype, - ) - if not fm_idle and scattered_ar_sconv_fusable( - get_parallel().tp_group, - forward_batch, - h_inproj.shape[0], - h_inproj.shape[-1], - h_inproj.dtype, - ): - fuse_attn_ar = True - # MTP is intentionally outside the main-model AttnRes loop. - h, residual = self.transformer_block( - h_inproj, - positions, - forward_batch, - None, - log_scaling_tau=log_scaling_tau, - fuse_attn_ar=fuse_attn_ar, - ) - mlp_sconv = self.transformer_block.mlp_sconv - if mlp_sconv is not None and not fm_idle: - h = mlp_sconv(h, positions, forward_batch) - if self.transformer_block.scattered_sconv: - # h was the [T, H/P] shard; gather before the residual add. - h = all_gather_hidden(h, self.transformer_block.attn_tp_group) - # transformer_block defers the final residual add to the caller. - if residual is not None: - h = h + residual - # chain_norm applies to the raw block output, before the mup division. - if self.chain_norm is not None: - h = self.chain_norm(h) - if self.mup_width_multiplier is not None: - return h / self.mup_width_multiplier, h - return h, h + residual_batch.start(forward_batch) + try: + h = self.transformer_block( + h_inproj, positions, forward_batch, log_scaling_tau=log_scaling_tau + ) + h, _ = residual_batch.read_output(h, forward_batch, self._final_read, None) + # chain_norm applies to the raw block output, before the mup division. + if self.chain_norm is not None: + h = self.chain_norm(h) + if self.mup_width_multiplier is not None: + return h / self.mup_width_multiplier, h + return h, h + finally: + residual_batch.clear(forward_batch) + take_ar_shared(forward_batch.input_ids.shape[0]) class InklingForConditionalGenerationMTP(nn.Module): diff --git a/test/registered/unit/layer_boundary/test_append_stages.py b/test/registered/unit/layer_boundary/test_append_stages.py index 1ac8c60e52..8ebf383107 100644 --- a/test/registered/unit/layer_boundary/test_append_stages.py +++ b/test/registered/unit/layer_boundary/test_append_stages.py @@ -1079,6 +1079,7 @@ class TestRowsTheConsumerReads(CustomTestCase): """Neither a plain add nor applied at the producer's exit.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = False @@ -1168,6 +1169,7 @@ class TestReadKernels(CustomTestCase): class Update: bind = None + replay_update = None is_plain_add = True def slice_residual_attn_tp(self, residual): diff --git a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py index b779b4889b..e21d122053 100644 --- a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py +++ b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py @@ -421,6 +421,7 @@ class TestAttnBankSpMoeStages(CustomTestCase): class Update: bind = None + replay_update = None is_plain_add = True def slice_residual_attn_tp(self, residual): diff --git a/test/registered/unit/layer_boundary/test_boundary_edges.py b/test/registered/unit/layer_boundary/test_boundary_edges.py index 0722ea2737..860b7b1849 100644 --- a/test/registered/unit/layer_boundary/test_boundary_edges.py +++ b/test/registered/unit/layer_boundary/test_boundary_edges.py @@ -185,6 +185,7 @@ class _WrittenIn: residual only once its sum is complete.""" bind = None + replay_update = None is_plain_add = False applied_at_exit = False diff --git a/test/registered/unit/layer_boundary/test_producer_completion.py b/test/registered/unit/layer_boundary/test_producer_completion.py index 009d53c23f..03c670d079 100644 --- a/test/registered/unit/layer_boundary/test_producer_completion.py +++ b/test/registered/unit/layer_boundary/test_producer_completion.py @@ -19,6 +19,7 @@ from sglang.srt.layers.layer_boundary import ( declare_ffn, layer_stack, ) +from sglang.srt.layers.layer_boundary.contracts import SumImplementation from sglang.srt.layers.layer_boundary.facts import ( ReadFacts, TransformFacts, @@ -412,7 +413,9 @@ class TestProducerCompletion(CustomTestCase): update = Nonplain() update.layer = torch.nn.Linear(4, 4) declaration = declare_ffn( - update=update, output_transform=OutputTransform(update.layer.forward) + update=update, + output_transform=OutputTransform(update.layer.forward), + sum_impl=SumImplementation(update.layer.forward), ) facts = facts_of(declaration) self.assertIsInstance(facts.read, ReadFacts) diff --git a/test/registered/unit/layer_boundary/test_residual_stream.py b/test/registered/unit/layer_boundary/test_residual_stream.py index 64f4db4a70..89f987f3c1 100644 --- a/test/registered/unit/layer_boundary/test_residual_stream.py +++ b/test/registered/unit/layer_boundary/test_residual_stream.py @@ -81,6 +81,7 @@ class TestResidualStream(CustomTestCase): def test_consumer_executes_the_carried_update_not_the_bound_producers(self): class Update: bind = None + replay_update = None is_plain_add = False applied_at_exit = False @@ -214,7 +215,9 @@ class TestResidualStream(CustomTestCase): ) def test_snapshot_rejects_stateful_update_without_touching_main_state(self): - update = SimpleNamespace(is_plain_add=False, update=Mock(), bind=None) + update = SimpleNamespace( + is_plain_add=False, update=Mock(), bind=None, replay_update=None + ) stream = ResidualStream(self.residual) hidden = stream.record(UnreducedOutput(self.partial, group=self.group), update) with self.assertRaisesRegex(NotImplementedError, "plain residual update"): diff --git a/test/registered/unit/layer_boundary/test_short_conv_boundary.py b/test/registered/unit/layer_boundary/test_short_conv_boundary.py new file mode 100644 index 0000000000..e090967e1b --- /dev/null +++ b/test/registered/unit/layer_boundary/test_short_conv_boundary.py @@ -0,0 +1,493 @@ +"""A custom output sum, channelwise convolution, and terminal read share one owner.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import test_declared_decoder_boundary as fixture +import torch + +from sglang.srt.layers.layer_boundary import ( + ProducerReduction, + ReadoutFusion, + append_stages, + declare_attn, + declare_ffn, + layer_stack, + prepare, +) +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.output import SumImplementation, UnreducedOutput +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual import bind_update +from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD +from sglang.srt.layers.layer_boundary.residual.short_conv import ( + ShortConvRead, + ShortConvUpdate, +) +from sglang.srt.layers.layer_boundary.residual.stream import ( + OwedOutput, + ResidualStream, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestShortConvBoundary(unittest.TestCase): + def test_sum_conv_gather_add_norm_and_tail(self): + for defer in (False, True): + for scattered in (False, True): + for lora in (False, True): + with self.subTest(defer=defer, scattered=scattered, lora=lora): + self.run_chain(defer, scattered, lora=lora) + + def run_chain(self, defer, scattered, lora=False): + calls = [] + batch = SimpleNamespace( + positions=torch.tensor([3, 8]), + input_ids=torch.tensor([1, 2]), + forward_mode=ForwardMode.DECODE, + residual_stream=None, + ) + + def complete(value, forward_batch): + calls.append("sum") + value = value * 2 + 0.7 + return value[:, :2] if scattered else value + + def conv(value, positions, forward_batch): + self.assertIs(forward_batch, batch) + calls.append("conv") + return value.square() * 0.1 + positions[:, None] + + def gather(value): + calls.append("hidden_gather") + return torch.cat((value, value + 5), dim=-1) + + norm = fixture.Norm() + read = ShortConvRead() + update = ShortConvUpdate(conv, None, SumGroup.TP, gather if scattered else None) + custom = SumImplementation(complete, lambda fb: defer) + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1) + with ( + fixture.planning(parallel, boundary_reduction="ar"), + patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=lora)), + layer_stack(final_read=read), + ): + stages = [ + stage + for _ in range(2) + for stage in append_stages( + (declare_attn(read=read, update=update, sum_impl=custom), norm), + (declare_ffn(read=read, update=update, sum_impl=custom), norm), + ) + ] + initial = torch.tensor([[0.2, -0.4, 0.6, 0.8], [-0.7, 0.9, 0.1, 0.3]]) + hidden, residual = initial, initial + residual_batch.start(batch) + with ( + fixture.planning(parallel, boundary_reduction="ar"), + patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=lora)), + ): + for index, stage in enumerate(stages): + hidden = stage.prepare(hidden, batch) + torch.testing.assert_close(hidden, 2 * residual) + output = hidden * (0.2 + 0.03 * index) + hidden = stage.finish(output, batch) + expected = output * 2 + 0.7 + if scattered: + expected = expected[:, :2] + expected = expected.square() * 0.1 + batch.positions[:, None] + if scattered: + expected = torch.cat((expected, expected + 5), dim=-1) + residual = residual + expected + if defer: + self.assertIsInstance(hidden, OwedOutput) + result, tail = residual_batch.read_output(hidden, batch, read, norm) + torch.testing.assert_close(result, 2 * residual) + torch.testing.assert_close(tail, residual) + self.assertEqual(calls.count("sum"), 4) + self.assertEqual(calls.count("conv"), 4) + self.assertEqual(calls.count("hidden_gather"), 4 if scattered else 0) + self.assertIsNone(batch.residual_stream) + if isinstance(hidden, OwedOutput): + self.assertIsNone(hidden.contribution.value) + self.assertIsNone(hidden.contribution.owed) + + def test_fusion_receives_the_actual_update_and_rejects_before_mutation(self): + update = ShortConvUpdate( + lambda *args: self.fail("unfused conv"), None, SumGroup.TP + ) + actual = bind_update(update, SimpleNamespace(positions=None)) + events = [] + expected = (torch.ones(2, 4), torch.zeros(2, 4)) + + def run(*args, update): + events.append(update) + return expected + + rejected = ReadoutFusion( + SumGroup.TP, + lambda *args, **kwargs: self.fail("rejected fusion ran"), + accepts_update=lambda u: False, + ) + accepted = ReadoutFusion( + SumGroup.TP, run, reads=True, accepts_update=lambda u: u is actual + ) + result = prepare._run_read_fusions( + (rejected, accepted), + torch.ones(2, 4), + torch.zeros(2, 4), + None, + None, + read=ShortConvRead(), + update=actual, + ) + self.assertIs(result, expected) + self.assertEqual(events, [actual]) + + def test_explicit_positions_belong_to_the_producer_invocation(self): + batch = SimpleNamespace(positions=torch.tensor([2, 4])) + positions = torch.tensor([17, 23]) + update = bind_update( + ShortConvUpdate(lambda h, p, fb: h + p[:, None], None, SumGroup.TP), + batch, + positions, + ) + value = torch.zeros(2, 4) + batch.positions = torch.tensor([101, 103]) + torch.testing.assert_close( + update.contribution(value), positions[:, None].expand_as(value).to(value) + ) + + def test_final_read_requires_explicit_acceptance_and_releases_on_failure(self): + update = bind_update( + ShortConvUpdate(lambda *args: self.fail("conv"), None, SumGroup.TP), + SimpleNamespace(positions=None), + ) + batch = SimpleNamespace(residual_stream=ResidualStream(torch.ones(2, 4))) + output = batch.residual_stream.record( + torch.zeros(2, 4), update, declared_sum=SumGroup.TP + ) + with self.assertRaises(NotImplementedError): + residual_batch.read_output(output, batch, SimpleNamespace(), None) + self.assertIsNone(batch.residual_stream) + self.assertIsNone(output.contribution.value) + + def test_terminal_nonplain_update_requires_final_read(self): + update = ShortConvUpdate(lambda *args: None, None, SumGroup.TP) + with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2, pp_size=1)): + with self.assertRaises(NotImplementedError), layer_stack(): + append_stages( + (declare_ffn(read=ShortConvRead(), update=update), fixture.Norm()) + ) + + def test_already_reduced_output_does_not_run_custom_sum(self): + calls = [] + custom = SumImplementation( + lambda value, batch: calls.append("sum") or value * 2 + ) + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) + batch = SimpleNamespace( + input_ids=torch.ones(2, dtype=torch.long), + forward_mode=ForwardMode.DECODE, + residual_stream=None, + ) + value = torch.randn(2, 4) + with fixture.planning(parallel), layer_stack(): + (stage,) = append_stages((declare_ffn(sum_impl=custom), fixture.Norm())) + with fixture.planning(parallel): + residual_batch.start(batch) + stage.prepare(value, batch) + stage.plan.output.complete_now( + value, batch.residual_stream, batch, already_reduced=True + ) + self.assertEqual(calls, []) + torch.testing.assert_close(batch.residual_stream.pending.value, value) + + def test_inkling_keeps_unpadded_batches_on_declared_dense_and_moe_rows(self): + from sglang.srt.arg_groups.parallel_hook import _boundary_parallelism_overrides + from sglang.srt.layers.layer_boundary import BatchVariant + from sglang.srt.layers.layer_boundary.residual.add_norm import ( + PLAIN_RESIDUAL_OPS, + ) + from sglang.srt.models import inkling + + for architecture in ( + "InklingForConditionalGeneration", + "InklingForConditionalGenerationMTP", + ): + parallel = fixture.parallel_of( + attn_dp=1, + attn_tp=2, + moe_dense_tp_size=1, + disable_attn_tp_gather=True, + enable_attn_tp_input_scattered=True, + ) + overrides = _boundary_parallelism_overrides( + parallel, "inkling", architecture + ) + self.assertEqual( + overrides, + {"enable_attn_tp_input_scattered": False}, + ) + parallel = SimpleNamespace(**(vars(parallel) | overrides)) + self.assertTrue(parallel.disable_attn_tp_gather) + self.assertEqual(parallel.moe_dense_tp_size, 1) + config = SimpleNamespace(dense_mlp_idx=1, use_sconv=False) + custom = SumImplementation(lambda value, batch: value) + with ( + fixture.planning(parallel), + patch.object(inkling, "get_parallel", return_value=parallel), + layer_stack(), + ): + stages = [] + for layer_id in (0, 1): + declarations = inkling.inkling_stages( + config, layer_id, ops=PLAIN_RESIDUAL_OPS, sums=(custom, custom) + ) + stages.extend( + append_stages( + *( + (declaration, fixture.Norm()) + for declaration in declarations + ) + ) + ) + batch = SimpleNamespace( + input_ids=torch.ones(3, dtype=torch.long), + forward_mode=ForwardMode.EXTEND, + attn_cp_metadata=None, + ) + with fixture.planning(parallel): + for stage in stages: + self.assertIs(stage.plan.variant_for(batch), BatchVariant.ORDINARY) + + def test_custom_sums_reject_unimplemented_rows_and_mixer_exits(self): + custom = SumImplementation(lambda value, batch: value) + cases = ( + (fixture.parallel_of(attn_dp=2, attn_tp=2), declare_ffn, False), + ( + fixture.parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True + ), + declare_ffn, + False, + ), + (fixture.parallel_of(attn_dp=1, attn_tp=2), declare_ffn, True), + ( + fixture.parallel_of(attn_dp=1, attn_tp=2), + lambda **kw: declare_attn( + reduction=ProducerReduction.EXIT_SCOPED, **kw + ), + False, + ), + ) + from sglang.srt.layers import layernorm_sp + + for parallel, declare, sp in cases: + with self.subTest( + dp=parallel.attn_dp_size, + scattered=parallel.enable_attn_tp_input_scattered, + sp=sp, + declare=declare, + ): + with ( + fixture.planning(parallel), + patch.object(layernorm_sp, "layernorm_sp_enabled", lambda: sp), + ): + with self.assertRaisesRegex( + NotImplementedError, "custom output sums" + ): + with layer_stack(): + append_stages((declare(sum_impl=custom), fixture.Norm())) + + def test_replay_handoff_rejects_tensor_with_declared_debt(self): + batch = SimpleNamespace(residual_stream=ResidualStream(torch.ones(2, 4))) + hidden = batch.residual_stream.record( + torch.ones(2, 4), PLAIN_ADD, declared_sum=SumGroup.TP + ) + with self.assertRaisesRegex(RuntimeError, "complete the contribution"): + residual_batch.export_replay_handoff(hidden, batch) + self.assertIsNotNone(batch.residual_stream.pending.owed) + + def test_replay_handoff_rebinds_to_live_positions_and_releases_capture_state(self): + batch = SimpleNamespace( + positions=torch.tensor([2, 4]), + residual_stream=ResidualStream(torch.ones(2, 4)), + ) + spec = ShortConvUpdate(lambda h, p, fb: h + p[:, None], None, SumGroup.TP) + actual = bind_update(spec, batch) + hidden = batch.residual_stream.record(torch.zeros(2, 4), actual) + pending = batch.residual_stream.pending + handoff = residual_batch.export_replay_handoff(hidden, batch) + self.assertIsNone(batch.residual_stream) + self.assertIsNone(pending.update) + self.assertIs(handoff.update, spec) + live = SimpleNamespace(positions=torch.tensor([7, 11]), residual_stream=None) + restored = residual_batch.import_replay_handoff( + handoff, live, update_context=live.positions + ) + result, _ = residual_batch.read_output( + restored, live, ShortConvRead(), fixture.Norm() + ) + torch.testing.assert_close(result, torch.tensor([[16.0] * 4, [24.0] * 4])) + self.assertIsNone(live.residual_stream) + + def test_replay_handoff_preserves_written_state(self): + value = torch.ones(2, 4) + batch = SimpleNamespace(residual_stream=ResidualStream(value)) + handoff = residual_batch.export_replay_handoff(value, batch) + self.assertTrue(handoff.written) + restored = residual_batch.import_replay_handoff(handoff, batch) + self.assertIs(restored, value) + self.assertIs(batch.residual_stream.residual, value) + self.assertIsNone(batch.residual_stream.pending) + + def test_normalization_handoff_completes_producer_before_captured_add_norm(self): + for convolve in (False, True): + with self.subTest(convolve=convolve): + events = [] + value = torch.full((2, 4), 0.1, dtype=torch.bfloat16) + residual = torch.full((2, 4), 0.0003, dtype=torch.bfloat16) + batch = SimpleNamespace( + positions=torch.tensor([2, 4]), + residual_stream=ResidualStream(residual), + ) + + def conv(value, positions, live): + self.assertIs(live, batch) + events.append("conv") + return value * 3 + + update = ( + bind_update(ShortConvUpdate(conv, None, SumGroup.TP), batch) + if convolve + else PLAIN_ADD + ) + output = batch.residual_stream.record( + UnreducedOutput( + value, + complete_sum=lambda v: events.append("sum") or v * 2, + ), + update, + ) + pending = batch.residual_stream.pending + read = ShortConvRead() + handoff = read.export_normalization_handoff(output, batch) + self.assertEqual(events, ["sum", "conv"] if convolve else ["sum"]) + self.assertIsNone(batch.residual_stream) + self.assertIsNone(pending.update) + self.assertIs(handoff.residual, residual) + expected = value * (6 if convolve else 2) + torch.testing.assert_close(handoff.value, expected) + + def norm(v, r, **kw): + self.assertIs(v, handoff.value) + self.assertIs(r, residual) + events.append("norm") + merged = v.float() + r.float() + return merged, merged.to(v.dtype) + + normalized, written = read.read_normalization_handoff(handoff, norm) + torch.testing.assert_close( + normalized, expected.float() + residual.float() + ) + self.assertEqual(events[-1], "norm") + self.assertFalse(torch.equal(normalized, (expected + residual).float())) + torch.testing.assert_close( + written, (expected.float() + residual.float()).to(value) + ) + + def test_tp1_moe_merges_shared_without_running_custom_sum(self): + from sglang.srt.layers.layer_boundary.residual import LayerResidualOps + from sglang.srt.models import inkling + from sglang.srt.models.inkling_common.kernels.comm import ( + stash_ar_shared, + take_ar_shared, + ) + + parallel = fixture.parallel_of(attn_dp=1, attn_tp=1, pp_size=1) + read = ShortConvRead() + config = SimpleNamespace(dense_mlp_idx=0, use_sconv=False) + custom = SumImplementation(lambda *a: self.fail("single-rank custom sum")) + with ( + fixture.planning(parallel), + patch.object(inkling, "get_parallel", return_value=parallel), + layer_stack(final_read=read), + ): + stages = append_stages( + *[ + (decl, fixture.Norm()) + for decl in inkling.inkling_stages( + config, + 0, + ops=LayerResidualOps(read, PLAIN_ADD, read, PLAIN_ADD), + sums=(custom, custom), + ) + ] + ) + batch = SimpleNamespace( + input_ids=torch.ones(2, dtype=torch.long), + forward_mode=ForwardMode.DECODE, + residual_stream=None, + ) + value = torch.ones(2, 4) + try: + with fixture.planning(parallel): + residual_batch.start(batch) + hidden = stages[0].prepare(value, batch) + hidden = stages[0].finish(hidden * 0.1, batch) + operand = stages[1].prepare(hidden, batch) + routed, shared = operand * 0.2, operand * 0.3 + stash_ar_shared(shared) + output = stages[1].finish(routed, batch) + torch.testing.assert_close( + batch.residual_stream.pending.value, routed + shared + ) + self.assertIsNone(take_ar_shared(value.shape[0])) + residual_batch.read_output(output, batch, read, fixture.Norm()) + self.assertIsNone(batch.residual_stream) + finally: + take_ar_shared(value.shape[0]) + + +class TestMtpInvocationCleanup(unittest.TestCase): + def test_exception_clears_residual_and_shared_stash(self): + from sglang.srt.models import inkling + + model = inkling.InklingMTPLayer.__new__(inkling.InklingMTPLayer) + torch.nn.Module.__init__(model) + model.embed_tokens = torch.nn.Embedding(4, 2) + model.main_model_embed_norm = None + model.hidden_norm = model.embed_norm = torch.nn.Identity() + model.input_proj = torch.nn.Linear(4, 2) + model.log_scaling_n_floor = None + streams = [] + + def fail(value, positions, batch, **kwargs): + stream = residual_batch.stream_of(batch) + stream.residual = value + streams.append(stream) + raise RuntimeError("draft failed") + + model.transformer_block = fail + ids = torch.tensor([1]) + batch = SimpleNamespace( + input_ids=ids, spec_info=SimpleNamespace(hidden_states=torch.ones(1, 2)) + ) + with ( + patch.object(inkling, "take_ar_shared") as take, + self.assertRaisesRegex(RuntimeError, "draft failed"), + ): + model(ids, ids, batch) + self.assertIsNone(batch.residual_stream) + self.assertIsNone(streams[0].residual) + take.assert_called_once_with(1) + + +if __name__ == "__main__": + unittest.main()