[Refactor] Simplify layer boundary internals (#41778)

This commit is contained in:
Cheng Wan
2026-09-29 21:15:58 -07:00
committed by GitHub
parent 7fb85a9d88
commit 113e6a1496
12 changed files with 119 additions and 172 deletions
@@ -1,11 +1,12 @@
from functools import lru_cache
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Optional
import torch
import triton
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import DpPaddingMode, dp_slot_in
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
)
@@ -222,6 +223,25 @@ def dsa_use_prefill_cp(forward_batch, dsa_enable_prefill_cp=None):
return False
def maybe_prefetch_next_full_attention_kv(
forward_batch: "ForwardBatch",
next_full_attention_layer_id: Optional[int],
) -> None:
"""Prefetch (owner-broadcast) the next layer's DSA KV under layer split.
No-op unless the current batch runs DSA prefill-CP and the active KV pool is
a layer-sharded pool exposing ``prefetch_kv_buffer`` (i.e.
``LayerSplitDSATokenToKVPool``). Kicking the broadcast off one layer ahead
overlaps it with the current layer's attention compute.
"""
if next_full_attention_layer_id is None or not dsa_use_prefill_cp(forward_batch):
return
prefetch_kv_buffer = getattr(get_token_to_kv_pool(), "prefetch_kv_buffer", None)
if prefetch_kv_buffer is not None:
prefetch_kv_buffer(next_full_attention_layer_id)
def fp8_mqa_logits_ceil_to_ue8m0(x: torch.Tensor) -> torch.Tensor:
return torch.pow(2.0, torch.ceil(torch.log2(x.abs())))
@@ -60,12 +60,6 @@ class AttentionInputs:
# (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again.
self.is_pre_gathered = is_pre_gathered
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
def fetch_qkv_latent(self):
if self.qkv_latent_ is not None:
return self.qkv_latent_
@@ -74,9 +68,7 @@ class AttentionInputs:
self.hidden_states_local, self.forward_batch
)
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
self.qkv_latent_ = self.tp_all_gather_hidden_states(
self.qkv_latent_, self.forward_batch
)
self.qkv_latent_ = tp_gather(self.qkv_latent_, self.forward_batch)
return self.qkv_latent_
def fetch_hidden_states(self):
@@ -84,9 +76,7 @@ class AttentionInputs:
return self.hidden_states_
self.hidden_states_ = self.hidden_states_local
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
self.hidden_states_ = self.tp_all_gather_hidden_states(
self.hidden_states_, self.forward_batch
)
self.hidden_states_ = tp_gather(self.hidden_states_, self.forward_batch)
return self.hidden_states_
@@ -176,6 +166,14 @@ def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
return gathered
def tp_gather(hidden_states: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
# Input-scattered attention keeps the same number of tokens on every TP rank.
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
def attn_tp_slice(tensor: torch.Tensor) -> torch.Tensor:
parallel = get_parallel()
return tensor.tensor_split(parallel.attn_tp_size)[parallel.attn_tp_rank]
@@ -13,42 +13,16 @@
# ==============================================================================
from typing import Optional
import torch
from sglang.srt.layers.attention.dsa.utils import (
dsa_use_prefill_cp,
)
from sglang.srt.layers.dp_attention import (
attn_cp_all_gather_into_tensor,
attn_cp_reduce_scatter_tensor,
get_local_dp_buffer,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.runtime_context import get_parallel
def maybe_prefetch_next_full_attention_kv(
forward_batch: ForwardBatch,
next_full_attention_layer_id: Optional[int],
) -> None:
"""Prefetch (owner-broadcast) the next layer's DSA KV under layer split.
No-op unless the current batch runs DSA prefill-CP and the active KV pool is
a layer-sharded pool exposing ``prefetch_kv_buffer`` (i.e.
``LayerSplitDSATokenToKVPool``). Kicking the broadcast off one layer ahead
overlaps it with the current layer's attention compute.
"""
if next_full_attention_layer_id is None or not dsa_use_prefill_cp(forward_batch):
return
prefetch_kv_buffer = getattr(get_token_to_kv_pool(), "prefetch_kv_buffer", None)
if prefetch_kv_buffer is not None:
prefetch_kv_buffer(next_full_attention_layer_id)
def attn_cp_gather(hidden_states: torch.Tensor):
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
@@ -15,13 +15,14 @@
declarations."""
from functools import partial
from typing import Callable, Optional, Tuple
from typing import Callable, NamedTuple, Optional, Tuple
import msgspec
from sglang.srt.layers.layer_boundary.contracts import (
CpMoves,
EdgeContract,
EntryPath,
FfnInputFusion,
InputContract,
OutputContract,
@@ -100,55 +101,38 @@ def _cp_moves() -> CpMoves:
)
class EdgeBinding(msgspec.Struct, frozen=True):
"""The steps one layer runs at one boundary, chosen from both sides'
declarations. A layer runs the consumer's half of a boundary into one of
its stages, and the producer's half of the boundary after its last stage;
the neighbouring layer runs the other half of that one."""
class ExitMove(NamedTuple):
"""The producer's half of a boundary, which a layer runs after its last
stage; the neighbouring layer runs the consumer's half."""
edge: EdgeContract
# The consumer's half: completing what the input owes, the add and the
# norm, and the moves onto the rows it needs that come before the read.
prepare: Optional[Callable] = None
# The move onto the consumer's rows after prepare; None when there is none.
input_move: Optional[Callable] = None
# The producer's half: the postprocess that moves the output onto the rows
# the layer hands on; None when it goes back over attention DP, whose step
# the FFN exit and postprocess choose per batch.
# Moves the output onto the rows the layer hands on; None when it goes
# back over attention DP, whose step the FFN exit and finish choose per
# batch.
output_move: Optional[Callable] = None
# Whether output_move also completes the sum the producer leaves.
output_move_completes_sum: bool = False
returns_over_dp: bool = False
preserves_residual: Optional[Callable] = None
@property
def input_rows(self) -> Layout:
"""The rows the consumer is handed: what it needs, still sharded over
the axes it gathers itself as the rows its input is read on are."""
return input_rows(self.edge)
@property
def capture_move_allocates(self) -> bool:
"""Returning through a gather allocates; returning through a cut aliases."""
return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded)
@property
def capture_move(self) -> Optional[Callable]:
if self.edge.residual_to == self.edge.produced.layout:
return None
return partial(
move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout
)
def input_rows(edge: EdgeContract) -> Layout:
"""Consumer rows, retaining the axes it gathers inside its computation."""
"""The rows the consumer is handed: what it needs, still sharded over the
axes it gathers itself as the rows its input is read on are."""
return Layout(
edge.need.layout.sharded
| (edge.residual_to.sharded & edge.need.gathered_by_compute)
)
def _capture_move(edge: EdgeContract) -> Tuple[Optional[Callable], bool]:
"""The move of the updated residual back onto the producer's rows for aux
capture, and whether it allocates: returning through a gather allocates;
returning through a cut aliases."""
if edge.residual_to == edge.produced.layout:
return None, False
move = partial(move_rows, rows=edge.residual_to, to=edge.produced.layout)
return move, bool(edge.residual_to.sharded - edge.produced.layout.sharded)
def bind_entry(
edge: EdgeContract,
*,
@@ -156,7 +140,8 @@ def bind_entry(
carried_fusions: Tuple[Callable, ...] = (),
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> EdgeBinding:
attn_input_adapter: Optional[Callable] = None,
) -> EntryPath:
"""Bind the consumer half of an edge at construction time.
Args:
@@ -165,9 +150,10 @@ def bind_entry(
carried_fusions: Ordered candidates accepting dynamically carried work.
cp_moves: Context-parallel strategy operations for this edge, if needed.
enters_stack: Whether the read must initialize the stack's residual.
attn_input_adapter: Hands an attention its input once it is on its rows.
Returns:
A Boundary whose prepare accepts the actual update from the stream.
An EntryPath whose prepare accepts the actual update from the stream.
Multiple update capabilities select among preconstructed paths; a single
capability needs no runtime dispatcher.
"""
@@ -184,6 +170,7 @@ def bind_entry(
carried_fusions=carried_fusions,
cp_moves=cp_moves,
enters_stack=enters_stack,
attn_input_adapter=attn_input_adapter,
)
for capability in capabilities
}
@@ -210,7 +197,8 @@ def _bind_entry_path(
carried_fusions: Tuple[Callable, ...] = (),
cp_moves: Optional[CpMoves] = None,
enters_stack: bool = False,
) -> EdgeBinding:
attn_input_adapter: Optional[Callable] = None,
) -> EntryPath:
"""The consumer's half of ``edge``, chosen from the edge's declarations and
the producer's update: what a value carries for a batch (a sum or a handoff
its producer left) is completed first, trying ``carried_fusions``; a sum
@@ -277,23 +265,29 @@ def _bind_entry_path(
preserves_residual = next(
f.preserves_residual for f in fusions if f.run == selected[0]
)
return EdgeBinding(
edge,
preserves_residual=preserves_residual,
declared_sum = edge.produced.group if edge.produced.always_partial else None
capture_move, capture_move_allocates = _capture_move(edge)
return EntryPath(
prepare=partial(
_run_entry,
step=step,
is_plain_add=is_plain_add,
carried_fusions=carried_fusions if plain else (),
expected_sum=edge.produced.group if edge.produced.always_partial else None,
expected_sum=declared_sum,
completed_step=completed_step,
written_step=written_step,
),
input_rows=input_rows(edge),
input_move=input_move,
attn_input_adapter=attn_input_adapter,
capture_move=capture_move,
capture_move_allocates=capture_move_allocates,
declared_sum=declared_sum,
preserves_residual=preserves_residual,
)
def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> EdgeBinding:
def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> ExitMove:
"""Bind the producer half of a layer or branch exit.
Args:
@@ -303,7 +297,7 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Edge
cp_moves: Context-parallel return operations, when the edge needs them.
Returns:
A Boundary with fixed transport or a marker for batch-dependent DP
An ExitMove with fixed transport or a marker for batch-dependent DP
transport. The receiver binds its read independently.
"""
update = edge.produced.update
@@ -324,11 +318,10 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Edge
to=edge.residual_to,
cp_moves=cp_moves,
)
return EdgeBinding(
edge,
return ExitMove(
output_move=output_move,
returns_over_dp=returns_over_dp,
output_move_completes_sum=completes_sum,
returns_over_dp=returns_over_dp,
)
@@ -20,6 +20,7 @@ from typing import Callable, Optional
from sglang.srt.environ import envs
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
from sglang.srt.layers.layer_boundary.boundary import (
ExitMove,
_cp_moves,
bind_entry,
bind_exit,
@@ -194,38 +195,25 @@ class StagePlan:
input_rows=edges.incoming.need.layout,
)
else:
into = bind_entry(
entry = bind_entry(
edges.incoming,
fusions=fused,
carried_fusions=carried,
cp_moves=edges.cp_moves,
enters_stack=enters_stack,
)
entry = EntryPath(
prepare=into.prepare,
input_rows=into.input_rows,
input_move=into.input_move,
attn_input_adapter=edges.attn_input_adapter,
capture_move=into.capture_move,
capture_move_allocates=into.capture_move_allocates,
preserves_residual=into.preserves_residual,
declared_sum=edges.incoming.produced.group
if edges.incoming.produced.always_partial
else None,
)
out = (
None
ExitMove()
if finishes_directly
else bind_exit(edges.outgoing, cp_moves=edges.cp_moves)
)
self.paths[variant] = StagePath(
entry=entry,
output=edges.outgoing.produced,
output_move=None if out is None else out.output_move,
returns_over_dp=out is not None and out.returns_over_dp,
output_move_completes_sum=False
if out is None
else out.output_move_completes_sum,
output_move=out.output_move,
output_move_completes_sum=out.output_move_completes_sum,
returns_over_dp=out.returns_over_dp,
)
@property
@@ -234,7 +222,8 @@ class StagePlan:
@property
def input_on_attn_tp_slices(self):
return TokenAxis.ATTN_TP in self.incoming_residual_rows.sharded
entry = self.paths[BatchVariant.ORDINARY].entry
return TokenAxis.ATTN_TP in entry.input_rows.sharded
def variant_for(self, forward_batch):
# The batch determines its rows; missing paths must not change them.
@@ -212,14 +212,6 @@ def move_rows(
return hidden_states
def tp_gather(hidden_states: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
# Input-scattered attention keeps the same number of tokens on every TP rank.
total_tokens = forward_batch.input_ids.shape[0]
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
return output
def _dp_scatter_group() -> GroupCoordinator:
parallel = get_parallel()
if parallel.tp_size == parallel.attn_dp_size:
@@ -46,9 +46,10 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
reduce_to_dp_local: Callable(partial) completing the sum and moving
it to destination rows; takes precedence over group.
Exits hand this form to ResidualStream.record(), which exposes an opaque
OwedOutput to models. Low-level adapters use complete_owed() before reading
it. A group is required when reduce_to_dp_local is absent.
Exits hand this form to ResidualStream.record(), which keeps it as the
contribution's owed work and exposes an opaque OwedOutput to models.
Low-level adapters use complete_owed() before reading it. A group is
required when reduce_to_dp_local is absent.
"""
partial: torch.Tensor
@@ -57,6 +58,12 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
reduce_to_dp_local: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
def complete(self) -> torch.Tensor:
"""Complete the sum, on the destination rows when it moves them."""
if self.reduce_to_dp_local is not None:
return self.reduce_to_dp_local(self.partial)
return self.group.all_reduce(self.partial)
class DeferredFinalize(msgspec.Struct, frozen=True):
"""A layer output that still owes work only its producer knows how to do (a
@@ -75,10 +82,6 @@ def complete_owed(
) -> Optional[torch.Tensor]:
"""Run the work an UnreducedOutput or a DeferredFinalize still owes; pass
anything else through."""
if isinstance(hidden_states, UnreducedOutput):
if hidden_states.reduce_to_dp_local is not None:
return hidden_states.reduce_to_dp_local(hidden_states.partial)
return hidden_states.group.all_reduce(hidden_states.partial)
if isinstance(hidden_states, DeferredFinalize):
if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)):
return hidden_states.complete()
return hidden_states
@@ -35,6 +35,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
attn_tp_gather,
attn_tp_slice,
get_attn_tp_context,
tp_gather,
)
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
attn_cp_gather,
@@ -70,7 +71,6 @@ from sglang.srt.layers.layer_boundary.ops import (
dp_gather,
dp_gather_sum,
moe_cp_gather,
tp_gather,
)
@@ -13,35 +13,16 @@
# ==============================================================================
"""One forward's residual and its producer's outstanding contribution."""
from typing import Callable, Optional, Union
from typing import Optional, Union
import msgspec
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.layer_boundary.layout import SumGroup, _sum_group
from sglang.srt.layers.layer_boundary.output import DeferredFinalize, UnreducedOutput
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
class DeferredSum(msgspec.Struct, frozen=True):
"""Runtime reduction left by an exit decision.
Fields:
group: All-reduce group when no redistribution callable is supplied.
reduce_to_dp_local: Optional callable(value) returning a reduced
value on destination rows; takes precedence over group.
"""
group: Optional[GroupCoordinator] = None
reduce_to_dp_local: Optional[Callable] = None
def complete(self, value):
if self.reduce_to_dp_local is not None:
return self.reduce_to_dp_local(value)
return self.group.all_reduce(value)
class DeclaredSum(msgspec.Struct, frozen=True):
"""A sum every output of this producer owes to its declared input edge."""
@@ -55,31 +36,29 @@ class Contribution(msgspec.Struct):
"""Own a producer's output, residual update and outstanding completion.
Fields:
value: Contribution tensor, or None for an opaque finalize handoff.
value: Contribution tensor (an unreduced output's partial), or None
for an opaque finalize handoff.
update: Producer operation to apply before the consumer reads input.
owed: Runtime sum, declared sum, producer-specific finalize, or None.
owed: The exit's unreduced output, a declared sum, a producer-specific
finalize, or None.
Completing owed work clears owed but does not apply the residual update.
"""
value: Optional[torch.Tensor]
update: ResidualUpdate
owed: Union[DeferredSum, DeclaredSum, DeferredFinalize, None] = None
owed: Union[UnreducedOutput, DeclaredSum, DeferredFinalize, None] = None
def for_boundary(self):
# These forms are private inputs to the existing fused-kernel adapters.
if isinstance(self.owed, DeferredSum):
return UnreducedOutput(
self.value, self.owed.group, self.owed.reduce_to_dp_local
)
if isinstance(self.owed, DeferredFinalize):
if isinstance(self.owed, (UnreducedOutput, DeferredFinalize)):
return self.owed
return self.value
def complete(self):
if isinstance(self.owed, (DeferredSum, DeclaredSum)):
if isinstance(self.owed, DeclaredSum):
value = self.owed.complete(self.value)
elif isinstance(self.owed, DeferredFinalize):
elif self.owed is not None:
value = self.owed.complete()
else:
return self.value
@@ -170,11 +149,7 @@ class ResidualStream:
raise RuntimeError("a declared sum cannot carry another completion")
contribution = Contribution(output, update, DeclaredSum(declared_sum))
elif isinstance(output, UnreducedOutput):
contribution = Contribution(
output.partial,
update,
DeferredSum(output.group, output.reduce_to_dp_local),
)
contribution = Contribution(output.partial, update, output)
elif isinstance(output, DeferredFinalize):
contribution = Contribution(None, update, output)
else:
@@ -223,8 +198,12 @@ class ResidualStream:
raise NotImplementedError("snapshot requires a plain residual update")
if pending.owed is None:
value = pending.value
elif isinstance(pending.owed, (DeferredSum, DeclaredSum)):
elif isinstance(pending.owed, DeclaredSum):
value = pending.owed.complete(pending.value.clone())
elif isinstance(pending.owed, UnreducedOutput):
value = msgspec.structs.replace(
pending.owed, partial=pending.value.clone()
).complete()
else:
raise NotImplementedError("a finalize handoff requires main-output capture")
return value.clone() if self.residual is None else value + self.residual
+3 -3
View File
@@ -62,6 +62,9 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.amx_utils import PackWeightMethod
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool
from sglang.srt.layers.attention.dsa.utils import (
maybe_prefetch_next_full_attention_kv,
)
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -79,9 +82,6 @@ from sglang.srt.layers.layer_boundary import (
is_dense_ffn_fully_dp,
make_stages,
)
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
maybe_prefetch_next_full_attention_kv,
)
from sglang.srt.layers.layer_boundary.residual import access as residual_access
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
+3 -4
View File
@@ -83,7 +83,6 @@ def sp_region_steps():
from sglang.srt.layers.layer_boundary import (
NORM_QUANT_READOUT,
EdgeContract,
EntryPath,
InputContract,
Layout,
OutputContract,
@@ -97,10 +96,10 @@ def sp_region_steps():
output = OutputContract(rows)
def entry(read, attn_input_adapter=None):
selected = bind_entry(
EdgeContract(output, InputContract(rows, read=read), rows, rows)
return bind_entry(
EdgeContract(output, InputContract(rows, read=read), rows, rows),
attn_input_adapter=attn_input_adapter,
)
return EntryPath(selected.prepare, rows, attn_input_adapter=attn_input_adapter)
return StagePath(
entry(NORM_QUANT_READOUT, _attn_input_default), output, keep_output