diff --git a/python/sglang/srt/layers/attention/dsa/utils.py b/python/sglang/srt/layers/attention/dsa/utils.py index 2ae70ec78e..6da616e9f2 100644 --- a/python/sglang/srt/layers/attention/dsa/utils.py +++ b/python/sglang/srt/layers/attention/dsa/utils.py @@ -1,11 +1,12 @@ from functools import lru_cache -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import torch import triton from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import DpPaddingMode, dp_slot_in +from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( is_in_breakable_cuda_graph, ) @@ -222,6 +223,25 @@ def dsa_use_prefill_cp(forward_batch, dsa_enable_prefill_cp=None): return False +def maybe_prefetch_next_full_attention_kv( + forward_batch: "ForwardBatch", + next_full_attention_layer_id: Optional[int], +) -> None: + """Prefetch (owner-broadcast) the next layer's DSA KV under layer split. + + No-op unless the current batch runs DSA prefill-CP and the active KV pool is + a layer-sharded pool exposing ``prefetch_kv_buffer`` (i.e. + ``LayerSplitDSATokenToKVPool``). Kicking the broadcast off one layer ahead + overlaps it with the current layer's attention compute. + """ + if next_full_attention_layer_id is None or not dsa_use_prefill_cp(forward_batch): + return + + prefetch_kv_buffer = getattr(get_token_to_kv_pool(), "prefetch_kv_buffer", None) + if prefetch_kv_buffer is not None: + prefetch_kv_buffer(next_full_attention_layer_id) + + def fp8_mqa_logits_ceil_to_ue8m0(x: torch.Tensor) -> torch.Tensor: return torch.pow(2.0, torch.ceil(torch.log2(x.abs()))) diff --git a/python/sglang/srt/layers/layer_boundary/adapters/attention.py b/python/sglang/srt/layers/layer_boundary/adapters/attention.py index 798e6d93c2..a53ad65e13 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/attention.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/attention.py @@ -60,12 +60,6 @@ class AttentionInputs: # (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again. self.is_pre_gathered = is_pre_gathered - def tp_all_gather_hidden_states(self, hidden_states, forward_batch): - total_tokens = forward_batch.input_ids.shape[0] - output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) - get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) - return output - def fetch_qkv_latent(self): if self.qkv_latent_ is not None: return self.qkv_latent_ @@ -74,9 +68,7 @@ class AttentionInputs: self.hidden_states_local, self.forward_batch ) if get_attn_tp_context().input_scattered and not self.is_pre_gathered: - self.qkv_latent_ = self.tp_all_gather_hidden_states( - self.qkv_latent_, self.forward_batch - ) + self.qkv_latent_ = tp_gather(self.qkv_latent_, self.forward_batch) return self.qkv_latent_ def fetch_hidden_states(self): @@ -84,9 +76,7 @@ class AttentionInputs: return self.hidden_states_ self.hidden_states_ = self.hidden_states_local if get_attn_tp_context().input_scattered and not self.is_pre_gathered: - self.hidden_states_ = self.tp_all_gather_hidden_states( - self.hidden_states_, self.forward_batch - ) + self.hidden_states_ = tp_gather(self.hidden_states_, self.forward_batch) return self.hidden_states_ @@ -176,6 +166,14 @@ def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor: return gathered +def tp_gather(hidden_states: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: + # Input-scattered attention keeps the same number of tokens on every TP rank. + total_tokens = forward_batch.input_ids.shape[0] + output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) + get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) + return output + + def attn_tp_slice(tensor: torch.Tensor) -> torch.Tensor: parallel = get_parallel() return tensor.tensor_split(parallel.attn_tp_size)[parallel.attn_tp_rank] diff --git a/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py b/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py index 3cae0b0687..2aaba78c8d 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/context_parallel.py @@ -13,42 +13,16 @@ # ============================================================================== -from typing import Optional - import torch -from sglang.srt.layers.attention.dsa.utils import ( - dsa_use_prefill_cp, -) from sglang.srt.layers.dp_attention import ( attn_cp_all_gather_into_tensor, attn_cp_reduce_scatter_tensor, get_local_dp_buffer, ) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.runtime_context import get_parallel -def maybe_prefetch_next_full_attention_kv( - forward_batch: ForwardBatch, - next_full_attention_layer_id: Optional[int], -) -> None: - """Prefetch (owner-broadcast) the next layer's DSA KV under layer split. - - No-op unless the current batch runs DSA prefill-CP and the active KV pool is - a layer-sharded pool exposing ``prefetch_kv_buffer`` (i.e. - ``LayerSplitDSATokenToKVPool``). Kicking the broadcast off one layer ahead - overlaps it with the current layer's attention compute. - """ - if next_full_attention_layer_id is None or not dsa_use_prefill_cp(forward_batch): - return - - prefetch_kv_buffer = getattr(get_token_to_kv_pool(), "prefetch_kv_buffer", None) - if prefetch_kv_buffer is not None: - prefetch_kv_buffer(next_full_attention_layer_id) - - def attn_cp_gather(hidden_states: torch.Tensor): attn_dp_size = get_parallel().attn_dp_size attn_tp_size = get_parallel().attn_tp_size diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index b870f9e792..26464d24fc 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -15,13 +15,14 @@ declarations.""" from functools import partial -from typing import Callable, Optional, Tuple +from typing import Callable, NamedTuple, Optional, Tuple import msgspec from sglang.srt.layers.layer_boundary.contracts import ( CpMoves, EdgeContract, + EntryPath, FfnInputFusion, InputContract, OutputContract, @@ -100,55 +101,38 @@ def _cp_moves() -> CpMoves: ) -class EdgeBinding(msgspec.Struct, frozen=True): - """The steps one layer runs at one boundary, chosen from both sides' - declarations. A layer runs the consumer's half of a boundary into one of - its stages, and the producer's half of the boundary after its last stage; - the neighbouring layer runs the other half of that one.""" +class ExitMove(NamedTuple): + """The producer's half of a boundary, which a layer runs after its last + stage; the neighbouring layer runs the consumer's half.""" - edge: EdgeContract - # The consumer's half: completing what the input owes, the add and the - # norm, and the moves onto the rows it needs that come before the read. - prepare: Optional[Callable] = None - # The move onto the consumer's rows after prepare; None when there is none. - input_move: Optional[Callable] = None - # The producer's half: the postprocess that moves the output onto the rows - # the layer hands on; None when it goes back over attention DP, whose step - # the FFN exit and postprocess choose per batch. + # Moves the output onto the rows the layer hands on; None when it goes + # back over attention DP, whose step the FFN exit and finish choose per + # batch. output_move: Optional[Callable] = None # Whether output_move also completes the sum the producer leaves. output_move_completes_sum: bool = False returns_over_dp: bool = False - preserves_residual: Optional[Callable] = None - - @property - def input_rows(self) -> Layout: - """The rows the consumer is handed: what it needs, still sharded over - the axes it gathers itself as the rows its input is read on are.""" - return input_rows(self.edge) - - @property - def capture_move_allocates(self) -> bool: - """Returning through a gather allocates; returning through a cut aliases.""" - return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded) - - @property - def capture_move(self) -> Optional[Callable]: - if self.edge.residual_to == self.edge.produced.layout: - return None - return partial( - move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout - ) def input_rows(edge: EdgeContract) -> Layout: - """Consumer rows, retaining the axes it gathers inside its computation.""" + """The rows the consumer is handed: what it needs, still sharded over the + axes it gathers itself as the rows its input is read on are.""" return Layout( edge.need.layout.sharded | (edge.residual_to.sharded & edge.need.gathered_by_compute) ) +def _capture_move(edge: EdgeContract) -> Tuple[Optional[Callable], bool]: + """The move of the updated residual back onto the producer's rows for aux + capture, and whether it allocates: returning through a gather allocates; + returning through a cut aliases.""" + if edge.residual_to == edge.produced.layout: + return None, False + move = partial(move_rows, rows=edge.residual_to, to=edge.produced.layout) + return move, bool(edge.residual_to.sharded - edge.produced.layout.sharded) + + def bind_entry( edge: EdgeContract, *, @@ -156,7 +140,8 @@ def bind_entry( carried_fusions: Tuple[Callable, ...] = (), cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, -) -> EdgeBinding: + attn_input_adapter: Optional[Callable] = None, +) -> EntryPath: """Bind the consumer half of an edge at construction time. Args: @@ -165,9 +150,10 @@ def bind_entry( carried_fusions: Ordered candidates accepting dynamically carried work. cp_moves: Context-parallel strategy operations for this edge, if needed. enters_stack: Whether the read must initialize the stack's residual. + attn_input_adapter: Hands an attention its input once it is on its rows. Returns: - A Boundary whose prepare accepts the actual update from the stream. + An EntryPath whose prepare accepts the actual update from the stream. Multiple update capabilities select among preconstructed paths; a single capability needs no runtime dispatcher. """ @@ -184,6 +170,7 @@ def bind_entry( carried_fusions=carried_fusions, cp_moves=cp_moves, enters_stack=enters_stack, + attn_input_adapter=attn_input_adapter, ) for capability in capabilities } @@ -210,7 +197,8 @@ def _bind_entry_path( carried_fusions: Tuple[Callable, ...] = (), cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, -) -> EdgeBinding: + attn_input_adapter: Optional[Callable] = None, +) -> EntryPath: """The consumer's half of ``edge``, chosen from the edge's declarations and the producer's update: what a value carries for a batch (a sum or a handoff its producer left) is completed first, trying ``carried_fusions``; a sum @@ -277,23 +265,29 @@ def _bind_entry_path( preserves_residual = next( f.preserves_residual for f in fusions if f.run == selected[0] ) - return EdgeBinding( - edge, - preserves_residual=preserves_residual, + declared_sum = edge.produced.group if edge.produced.always_partial else None + capture_move, capture_move_allocates = _capture_move(edge) + return EntryPath( prepare=partial( _run_entry, step=step, is_plain_add=is_plain_add, carried_fusions=carried_fusions if plain else (), - expected_sum=edge.produced.group if edge.produced.always_partial else None, + expected_sum=declared_sum, completed_step=completed_step, written_step=written_step, ), + input_rows=input_rows(edge), input_move=input_move, + attn_input_adapter=attn_input_adapter, + capture_move=capture_move, + capture_move_allocates=capture_move_allocates, + declared_sum=declared_sum, + preserves_residual=preserves_residual, ) -def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> EdgeBinding: +def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> ExitMove: """Bind the producer half of a layer or branch exit. Args: @@ -303,7 +297,7 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Edge cp_moves: Context-parallel return operations, when the edge needs them. Returns: - A Boundary with fixed transport or a marker for batch-dependent DP + An ExitMove with fixed transport or a marker for batch-dependent DP transport. The receiver binds its read independently. """ update = edge.produced.update @@ -324,11 +318,10 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Edge to=edge.residual_to, cp_moves=cp_moves, ) - return EdgeBinding( - edge, + return ExitMove( output_move=output_move, - returns_over_dp=returns_over_dp, output_move_completes_sum=completes_sum, + returns_over_dp=returns_over_dp, ) diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index d0045c3429..da41e5ee19 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -20,6 +20,7 @@ from typing import Callable, Optional from sglang.srt.environ import envs from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context from sglang.srt.layers.layer_boundary.boundary import ( + ExitMove, _cp_moves, bind_entry, bind_exit, @@ -194,38 +195,25 @@ class StagePlan: input_rows=edges.incoming.need.layout, ) else: - into = bind_entry( + entry = bind_entry( edges.incoming, fusions=fused, carried_fusions=carried, cp_moves=edges.cp_moves, enters_stack=enters_stack, - ) - entry = EntryPath( - prepare=into.prepare, - input_rows=into.input_rows, - input_move=into.input_move, attn_input_adapter=edges.attn_input_adapter, - capture_move=into.capture_move, - capture_move_allocates=into.capture_move_allocates, - preserves_residual=into.preserves_residual, - declared_sum=edges.incoming.produced.group - if edges.incoming.produced.always_partial - else None, ) out = ( - None + ExitMove() if finishes_directly else bind_exit(edges.outgoing, cp_moves=edges.cp_moves) ) self.paths[variant] = StagePath( entry=entry, output=edges.outgoing.produced, - output_move=None if out is None else out.output_move, - returns_over_dp=out is not None and out.returns_over_dp, - output_move_completes_sum=False - if out is None - else out.output_move_completes_sum, + output_move=out.output_move, + output_move_completes_sum=out.output_move_completes_sum, + returns_over_dp=out.returns_over_dp, ) @property @@ -234,7 +222,8 @@ class StagePlan: @property def input_on_attn_tp_slices(self): - return TokenAxis.ATTN_TP in self.incoming_residual_rows.sharded + entry = self.paths[BatchVariant.ORDINARY].entry + return TokenAxis.ATTN_TP in entry.input_rows.sharded def variant_for(self, forward_batch): # The batch determines its rows; missing paths must not change them. diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index 7d10a80277..fb1a2ea7a7 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -212,14 +212,6 @@ def move_rows( return hidden_states -def tp_gather(hidden_states: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: - # Input-scattered attention keeps the same number of tokens on every TP rank. - total_tokens = forward_batch.input_ids.shape[0] - output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) - get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) - return output - - def _dp_scatter_group() -> GroupCoordinator: parallel = get_parallel() if parallel.tp_size == parallel.attn_dp_size: diff --git a/python/sglang/srt/layers/layer_boundary/output.py b/python/sglang/srt/layers/layer_boundary/output.py index f2c9c15322..935e17d75b 100644 --- a/python/sglang/srt/layers/layer_boundary/output.py +++ b/python/sglang/srt/layers/layer_boundary/output.py @@ -46,9 +46,10 @@ class UnreducedOutput(msgspec.Struct, frozen=True): reduce_to_dp_local: Callable(partial) completing the sum and moving it to destination rows; takes precedence over group. - Exits hand this form to ResidualStream.record(), which exposes an opaque - OwedOutput to models. Low-level adapters use complete_owed() before reading - it. A group is required when reduce_to_dp_local is absent. + Exits hand this form to ResidualStream.record(), which keeps it as the + contribution's owed work and exposes an opaque OwedOutput to models. + Low-level adapters use complete_owed() before reading it. A group is + required when reduce_to_dp_local is absent. """ partial: torch.Tensor @@ -57,6 +58,12 @@ class UnreducedOutput(msgspec.Struct, frozen=True): # rank's tokens (a reduce-scatter, or an all-reduce then a scatter). reduce_to_dp_local: Optional[Callable[[torch.Tensor], torch.Tensor]] = None + def complete(self) -> torch.Tensor: + """Complete the sum, on the destination rows when it moves them.""" + if self.reduce_to_dp_local is not None: + return self.reduce_to_dp_local(self.partial) + return self.group.all_reduce(self.partial) + class DeferredFinalize(msgspec.Struct, frozen=True): """A layer output that still owes work only its producer knows how to do (a @@ -75,10 +82,6 @@ def complete_owed( ) -> Optional[torch.Tensor]: """Run the work an UnreducedOutput or a DeferredFinalize still owes; pass anything else through.""" - if isinstance(hidden_states, UnreducedOutput): - if hidden_states.reduce_to_dp_local is not None: - return hidden_states.reduce_to_dp_local(hidden_states.partial) - return hidden_states.group.all_reduce(hidden_states.partial) - if isinstance(hidden_states, DeferredFinalize): + if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)): return hidden_states.complete() return hidden_states diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 20d35e9b62..e104a022e3 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -35,6 +35,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import ( attn_tp_gather, attn_tp_slice, get_attn_tp_context, + tp_gather, ) from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( attn_cp_gather, @@ -70,7 +71,6 @@ from sglang.srt.layers.layer_boundary.ops import ( dp_gather, dp_gather_sum, moe_cp_gather, - tp_gather, ) diff --git a/python/sglang/srt/layers/layer_boundary/residual/stream.py b/python/sglang/srt/layers/layer_boundary/residual/stream.py index 7a8c479791..586dc55a1e 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/stream.py +++ b/python/sglang/srt/layers/layer_boundary/residual/stream.py @@ -13,35 +13,16 @@ # ============================================================================== """One forward's residual and its producer's outstanding contribution.""" -from typing import Callable, Optional, Union +from typing import Optional, Union import msgspec import torch -from sglang.srt.distributed import GroupCoordinator from sglang.srt.layers.layer_boundary.layout import SumGroup, _sum_group from sglang.srt.layers.layer_boundary.output import DeferredFinalize, UnreducedOutput from sglang.srt.layers.layer_boundary.residual import ResidualUpdate -class DeferredSum(msgspec.Struct, frozen=True): - """Runtime reduction left by an exit decision. - - Fields: - group: All-reduce group when no redistribution callable is supplied. - reduce_to_dp_local: Optional callable(value) returning a reduced - value on destination rows; takes precedence over group. - """ - - group: Optional[GroupCoordinator] = None - reduce_to_dp_local: Optional[Callable] = None - - def complete(self, value): - if self.reduce_to_dp_local is not None: - return self.reduce_to_dp_local(value) - return self.group.all_reduce(value) - - class DeclaredSum(msgspec.Struct, frozen=True): """A sum every output of this producer owes to its declared input edge.""" @@ -55,31 +36,29 @@ class Contribution(msgspec.Struct): """Own a producer's output, residual update and outstanding completion. Fields: - value: Contribution tensor, or None for an opaque finalize handoff. + value: Contribution tensor (an unreduced output's partial), or None + for an opaque finalize handoff. update: Producer operation to apply before the consumer reads input. - owed: Runtime sum, declared sum, producer-specific finalize, or None. + owed: The exit's unreduced output, a declared sum, a producer-specific + finalize, or None. Completing owed work clears owed but does not apply the residual update. """ value: Optional[torch.Tensor] update: ResidualUpdate - owed: Union[DeferredSum, DeclaredSum, DeferredFinalize, None] = None + owed: Union[UnreducedOutput, DeclaredSum, DeferredFinalize, None] = None def for_boundary(self): # These forms are private inputs to the existing fused-kernel adapters. - if isinstance(self.owed, DeferredSum): - return UnreducedOutput( - self.value, self.owed.group, self.owed.reduce_to_dp_local - ) - if isinstance(self.owed, DeferredFinalize): + if isinstance(self.owed, (UnreducedOutput, DeferredFinalize)): return self.owed return self.value def complete(self): - if isinstance(self.owed, (DeferredSum, DeclaredSum)): + if isinstance(self.owed, DeclaredSum): value = self.owed.complete(self.value) - elif isinstance(self.owed, DeferredFinalize): + elif self.owed is not None: value = self.owed.complete() else: return self.value @@ -170,11 +149,7 @@ class ResidualStream: raise RuntimeError("a declared sum cannot carry another completion") contribution = Contribution(output, update, DeclaredSum(declared_sum)) elif isinstance(output, UnreducedOutput): - contribution = Contribution( - output.partial, - update, - DeferredSum(output.group, output.reduce_to_dp_local), - ) + contribution = Contribution(output.partial, update, output) elif isinstance(output, DeferredFinalize): contribution = Contribution(None, update, output) else: @@ -223,8 +198,12 @@ class ResidualStream: raise NotImplementedError("snapshot requires a plain residual update") if pending.owed is None: value = pending.value - elif isinstance(pending.owed, (DeferredSum, DeclaredSum)): + elif isinstance(pending.owed, DeclaredSum): value = pending.owed.complete(pending.value.clone()) + elif isinstance(pending.owed, UnreducedOutput): + value = msgspec.structs.replace( + pending.owed, partial=pending.value.clone() + ).complete() else: raise NotImplementedError("a finalize handoff requires main-output capture") return value.clone() if self.residual is None else value + self.residual diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index fbebff26c9..019173a448 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -62,6 +62,9 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.amx_utils import PackWeightMethod from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool +from sglang.srt.layers.attention.dsa.utils import ( + maybe_prefetch_next_full_attention_kv, +) from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, @@ -79,9 +82,6 @@ from sglang.srt.layers.layer_boundary import ( is_dense_ffn_fully_dp, make_stages, ) -from sglang.srt.layers.layer_boundary.adapters.context_parallel import ( - maybe_prefetch_next_full_attention_kv, -) from sglang.srt.layers.layer_boundary.residual import access as residual_access from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index 2314ff84b4..f4b9a67c2b 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -83,7 +83,6 @@ def sp_region_steps(): from sglang.srt.layers.layer_boundary import ( NORM_QUANT_READOUT, EdgeContract, - EntryPath, InputContract, Layout, OutputContract, @@ -97,10 +96,10 @@ def sp_region_steps(): output = OutputContract(rows) def entry(read, attn_input_adapter=None): - selected = bind_entry( - EdgeContract(output, InputContract(rows, read=read), rows, rows) + return bind_entry( + EdgeContract(output, InputContract(rows, read=read), rows, rows), + attn_input_adapter=attn_input_adapter, ) - return EntryPath(selected.prepare, rows, attn_input_adapter=attn_input_adapter) return StagePath( entry(NORM_QUANT_READOUT, _attn_input_default), output, keep_output diff --git a/test/registered/unit/layer_boundary/test_aux_hidden_states.py b/test/registered/unit/layer_boundary/test_aux_hidden_states.py index 8b836d8938..eeda55a440 100644 --- a/test/registered/unit/layer_boundary/test_aux_hidden_states.py +++ b/test/registered/unit/layer_boundary/test_aux_hidden_states.py @@ -35,7 +35,7 @@ class TestAuxStorage(CustomTestCase): backend._copy_output_to_buffer(source, tuple(buffers), 2) def test_capture_move_owns_gather_but_not_slice(self): - from sglang.srt.layers.layer_boundary.boundary import EdgeBinding + from sglang.srt.layers.layer_boundary.boundary import _capture_move from sglang.srt.layers.layer_boundary.layout import Layout, TokenAxis from sglang.test.communicator_patch import patch_communicator @@ -46,8 +46,8 @@ class TestAuxStorage(CustomTestCase): edge = SimpleNamespace( residual_to=rows, produced=SimpleNamespace(layout=target) ) - boundary = EdgeBinding(edge=edge, prepare=None) - self.assertEqual(boundary.capture_move_allocates, owns) + capture_move, allocates = _capture_move(edge) + self.assertEqual(allocates, owns) with ( patch_communicator( "attn_tp_gather", @@ -58,16 +58,16 @@ class TestAuxStorage(CustomTestCase): return_value=SimpleNamespace(attn_tp_size=2, attn_tp_rank=0), ), ): - value = boundary.capture_move(source, forward_batch=None) + value = capture_move(source, forward_batch=None) outputs = AuxHiddenStateList() if owns: with patch.object( torch.Tensor, "clone", side_effect=AssertionError("extra clone") ): - outputs.capture(value, owned=boundary.capture_move_allocates) + outputs.capture(value, owned=allocates) self.assertIs(outputs[0], value) else: - outputs.capture(value, owned=boundary.capture_move_allocates) + outputs.capture(value, owned=allocates) self.assertIsNot(outputs[0], value) def test_list_snapshots_borrowed_views_and_reused_buffers(self):