[Refactor] Simplify layer boundary internals (#41778)
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import DpPaddingMode, dp_slot_in
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
@@ -222,6 +223,25 @@ def dsa_use_prefill_cp(forward_batch, dsa_enable_prefill_cp=None):
|
||||
return False
|
||||
|
||||
|
||||
def maybe_prefetch_next_full_attention_kv(
|
||||
forward_batch: "ForwardBatch",
|
||||
next_full_attention_layer_id: Optional[int],
|
||||
) -> None:
|
||||
"""Prefetch (owner-broadcast) the next layer's DSA KV under layer split.
|
||||
|
||||
No-op unless the current batch runs DSA prefill-CP and the active KV pool is
|
||||
a layer-sharded pool exposing ``prefetch_kv_buffer`` (i.e.
|
||||
``LayerSplitDSATokenToKVPool``). Kicking the broadcast off one layer ahead
|
||||
overlaps it with the current layer's attention compute.
|
||||
"""
|
||||
if next_full_attention_layer_id is None or not dsa_use_prefill_cp(forward_batch):
|
||||
return
|
||||
|
||||
prefetch_kv_buffer = getattr(get_token_to_kv_pool(), "prefetch_kv_buffer", None)
|
||||
if prefetch_kv_buffer is not None:
|
||||
prefetch_kv_buffer(next_full_attention_layer_id)
|
||||
|
||||
|
||||
def fp8_mqa_logits_ceil_to_ue8m0(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.pow(2.0, torch.ceil(torch.log2(x.abs())))
|
||||
|
||||
|
||||
@@ -60,12 +60,6 @@ class AttentionInputs:
|
||||
# (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again.
|
||||
self.is_pre_gathered = is_pre_gathered
|
||||
|
||||
def tp_all_gather_hidden_states(self, hidden_states, forward_batch):
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
def fetch_qkv_latent(self):
|
||||
if self.qkv_latent_ is not None:
|
||||
return self.qkv_latent_
|
||||
@@ -74,9 +68,7 @@ class AttentionInputs:
|
||||
self.hidden_states_local, self.forward_batch
|
||||
)
|
||||
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
||||
self.qkv_latent_ = self.tp_all_gather_hidden_states(
|
||||
self.qkv_latent_, self.forward_batch
|
||||
)
|
||||
self.qkv_latent_ = tp_gather(self.qkv_latent_, self.forward_batch)
|
||||
return self.qkv_latent_
|
||||
|
||||
def fetch_hidden_states(self):
|
||||
@@ -84,9 +76,7 @@ class AttentionInputs:
|
||||
return self.hidden_states_
|
||||
self.hidden_states_ = self.hidden_states_local
|
||||
if get_attn_tp_context().input_scattered and not self.is_pre_gathered:
|
||||
self.hidden_states_ = self.tp_all_gather_hidden_states(
|
||||
self.hidden_states_, self.forward_batch
|
||||
)
|
||||
self.hidden_states_ = tp_gather(self.hidden_states_, self.forward_batch)
|
||||
return self.hidden_states_
|
||||
|
||||
|
||||
@@ -176,6 +166,14 @@ def attn_tp_gather(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return gathered
|
||||
|
||||
|
||||
def tp_gather(hidden_states: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
# Input-scattered attention keeps the same number of tokens on every TP rank.
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
def attn_tp_slice(tensor: torch.Tensor) -> torch.Tensor:
|
||||
parallel = get_parallel()
|
||||
return tensor.tensor_split(parallel.attn_tp_size)[parallel.attn_tp_rank]
|
||||
|
||||
@@ -13,42 +13,16 @@
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
dsa_use_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_all_gather_into_tensor,
|
||||
attn_cp_reduce_scatter_tensor,
|
||||
get_local_dp_buffer,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def maybe_prefetch_next_full_attention_kv(
|
||||
forward_batch: ForwardBatch,
|
||||
next_full_attention_layer_id: Optional[int],
|
||||
) -> None:
|
||||
"""Prefetch (owner-broadcast) the next layer's DSA KV under layer split.
|
||||
|
||||
No-op unless the current batch runs DSA prefill-CP and the active KV pool is
|
||||
a layer-sharded pool exposing ``prefetch_kv_buffer`` (i.e.
|
||||
``LayerSplitDSATokenToKVPool``). Kicking the broadcast off one layer ahead
|
||||
overlaps it with the current layer's attention compute.
|
||||
"""
|
||||
if next_full_attention_layer_id is None or not dsa_use_prefill_cp(forward_batch):
|
||||
return
|
||||
|
||||
prefetch_kv_buffer = getattr(get_token_to_kv_pool(), "prefetch_kv_buffer", None)
|
||||
if prefetch_kv_buffer is not None:
|
||||
prefetch_kv_buffer(next_full_attention_layer_id)
|
||||
|
||||
|
||||
def attn_cp_gather(hidden_states: torch.Tensor):
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
@@ -15,13 +15,14 @@
|
||||
declarations."""
|
||||
|
||||
from functools import partial
|
||||
from typing import Callable, Optional, Tuple
|
||||
from typing import Callable, NamedTuple, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
CpMoves,
|
||||
EdgeContract,
|
||||
EntryPath,
|
||||
FfnInputFusion,
|
||||
InputContract,
|
||||
OutputContract,
|
||||
@@ -100,55 +101,38 @@ def _cp_moves() -> CpMoves:
|
||||
)
|
||||
|
||||
|
||||
class EdgeBinding(msgspec.Struct, frozen=True):
|
||||
"""The steps one layer runs at one boundary, chosen from both sides'
|
||||
declarations. A layer runs the consumer's half of a boundary into one of
|
||||
its stages, and the producer's half of the boundary after its last stage;
|
||||
the neighbouring layer runs the other half of that one."""
|
||||
class ExitMove(NamedTuple):
|
||||
"""The producer's half of a boundary, which a layer runs after its last
|
||||
stage; the neighbouring layer runs the consumer's half."""
|
||||
|
||||
edge: EdgeContract
|
||||
# The consumer's half: completing what the input owes, the add and the
|
||||
# norm, and the moves onto the rows it needs that come before the read.
|
||||
prepare: Optional[Callable] = None
|
||||
# The move onto the consumer's rows after prepare; None when there is none.
|
||||
input_move: Optional[Callable] = None
|
||||
# The producer's half: the postprocess that moves the output onto the rows
|
||||
# the layer hands on; None when it goes back over attention DP, whose step
|
||||
# the FFN exit and postprocess choose per batch.
|
||||
# Moves the output onto the rows the layer hands on; None when it goes
|
||||
# back over attention DP, whose step the FFN exit and finish choose per
|
||||
# batch.
|
||||
output_move: Optional[Callable] = None
|
||||
# Whether output_move also completes the sum the producer leaves.
|
||||
output_move_completes_sum: bool = False
|
||||
returns_over_dp: bool = False
|
||||
preserves_residual: Optional[Callable] = None
|
||||
|
||||
@property
|
||||
def input_rows(self) -> Layout:
|
||||
"""The rows the consumer is handed: what it needs, still sharded over
|
||||
the axes it gathers itself as the rows its input is read on are."""
|
||||
return input_rows(self.edge)
|
||||
|
||||
@property
|
||||
def capture_move_allocates(self) -> bool:
|
||||
"""Returning through a gather allocates; returning through a cut aliases."""
|
||||
return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded)
|
||||
|
||||
@property
|
||||
def capture_move(self) -> Optional[Callable]:
|
||||
if self.edge.residual_to == self.edge.produced.layout:
|
||||
return None
|
||||
return partial(
|
||||
move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout
|
||||
)
|
||||
|
||||
|
||||
def input_rows(edge: EdgeContract) -> Layout:
|
||||
"""Consumer rows, retaining the axes it gathers inside its computation."""
|
||||
"""The rows the consumer is handed: what it needs, still sharded over the
|
||||
axes it gathers itself as the rows its input is read on are."""
|
||||
return Layout(
|
||||
edge.need.layout.sharded
|
||||
| (edge.residual_to.sharded & edge.need.gathered_by_compute)
|
||||
)
|
||||
|
||||
|
||||
def _capture_move(edge: EdgeContract) -> Tuple[Optional[Callable], bool]:
|
||||
"""The move of the updated residual back onto the producer's rows for aux
|
||||
capture, and whether it allocates: returning through a gather allocates;
|
||||
returning through a cut aliases."""
|
||||
if edge.residual_to == edge.produced.layout:
|
||||
return None, False
|
||||
move = partial(move_rows, rows=edge.residual_to, to=edge.produced.layout)
|
||||
return move, bool(edge.residual_to.sharded - edge.produced.layout.sharded)
|
||||
|
||||
|
||||
def bind_entry(
|
||||
edge: EdgeContract,
|
||||
*,
|
||||
@@ -156,7 +140,8 @@ def bind_entry(
|
||||
carried_fusions: Tuple[Callable, ...] = (),
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> EdgeBinding:
|
||||
attn_input_adapter: Optional[Callable] = None,
|
||||
) -> EntryPath:
|
||||
"""Bind the consumer half of an edge at construction time.
|
||||
|
||||
Args:
|
||||
@@ -165,9 +150,10 @@ def bind_entry(
|
||||
carried_fusions: Ordered candidates accepting dynamically carried work.
|
||||
cp_moves: Context-parallel strategy operations for this edge, if needed.
|
||||
enters_stack: Whether the read must initialize the stack's residual.
|
||||
attn_input_adapter: Hands an attention its input once it is on its rows.
|
||||
|
||||
Returns:
|
||||
A Boundary whose prepare accepts the actual update from the stream.
|
||||
An EntryPath whose prepare accepts the actual update from the stream.
|
||||
Multiple update capabilities select among preconstructed paths; a single
|
||||
capability needs no runtime dispatcher.
|
||||
"""
|
||||
@@ -184,6 +170,7 @@ def bind_entry(
|
||||
carried_fusions=carried_fusions,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
)
|
||||
for capability in capabilities
|
||||
}
|
||||
@@ -210,7 +197,8 @@ def _bind_entry_path(
|
||||
carried_fusions: Tuple[Callable, ...] = (),
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
enters_stack: bool = False,
|
||||
) -> EdgeBinding:
|
||||
attn_input_adapter: Optional[Callable] = None,
|
||||
) -> EntryPath:
|
||||
"""The consumer's half of ``edge``, chosen from the edge's declarations and
|
||||
the producer's update: what a value carries for a batch (a sum or a handoff
|
||||
its producer left) is completed first, trying ``carried_fusions``; a sum
|
||||
@@ -277,23 +265,29 @@ def _bind_entry_path(
|
||||
preserves_residual = next(
|
||||
f.preserves_residual for f in fusions if f.run == selected[0]
|
||||
)
|
||||
return EdgeBinding(
|
||||
edge,
|
||||
preserves_residual=preserves_residual,
|
||||
declared_sum = edge.produced.group if edge.produced.always_partial else None
|
||||
capture_move, capture_move_allocates = _capture_move(edge)
|
||||
return EntryPath(
|
||||
prepare=partial(
|
||||
_run_entry,
|
||||
step=step,
|
||||
is_plain_add=is_plain_add,
|
||||
carried_fusions=carried_fusions if plain else (),
|
||||
expected_sum=edge.produced.group if edge.produced.always_partial else None,
|
||||
expected_sum=declared_sum,
|
||||
completed_step=completed_step,
|
||||
written_step=written_step,
|
||||
),
|
||||
input_rows=input_rows(edge),
|
||||
input_move=input_move,
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
capture_move=capture_move,
|
||||
capture_move_allocates=capture_move_allocates,
|
||||
declared_sum=declared_sum,
|
||||
preserves_residual=preserves_residual,
|
||||
)
|
||||
|
||||
|
||||
def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> EdgeBinding:
|
||||
def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> ExitMove:
|
||||
"""Bind the producer half of a layer or branch exit.
|
||||
|
||||
Args:
|
||||
@@ -303,7 +297,7 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Edge
|
||||
cp_moves: Context-parallel return operations, when the edge needs them.
|
||||
|
||||
Returns:
|
||||
A Boundary with fixed transport or a marker for batch-dependent DP
|
||||
An ExitMove with fixed transport or a marker for batch-dependent DP
|
||||
transport. The receiver binds its read independently.
|
||||
"""
|
||||
update = edge.produced.update
|
||||
@@ -324,11 +318,10 @@ def bind_exit(edge: EdgeContract, *, cp_moves: Optional[CpMoves] = None) -> Edge
|
||||
to=edge.residual_to,
|
||||
cp_moves=cp_moves,
|
||||
)
|
||||
return EdgeBinding(
|
||||
edge,
|
||||
return ExitMove(
|
||||
output_move=output_move,
|
||||
returns_over_dp=returns_over_dp,
|
||||
output_move_completes_sum=completes_sum,
|
||||
returns_over_dp=returns_over_dp,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from typing import Callable, Optional
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.boundary import (
|
||||
ExitMove,
|
||||
_cp_moves,
|
||||
bind_entry,
|
||||
bind_exit,
|
||||
@@ -194,38 +195,25 @@ class StagePlan:
|
||||
input_rows=edges.incoming.need.layout,
|
||||
)
|
||||
else:
|
||||
into = bind_entry(
|
||||
entry = bind_entry(
|
||||
edges.incoming,
|
||||
fusions=fused,
|
||||
carried_fusions=carried,
|
||||
cp_moves=edges.cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
entry = EntryPath(
|
||||
prepare=into.prepare,
|
||||
input_rows=into.input_rows,
|
||||
input_move=into.input_move,
|
||||
attn_input_adapter=edges.attn_input_adapter,
|
||||
capture_move=into.capture_move,
|
||||
capture_move_allocates=into.capture_move_allocates,
|
||||
preserves_residual=into.preserves_residual,
|
||||
declared_sum=edges.incoming.produced.group
|
||||
if edges.incoming.produced.always_partial
|
||||
else None,
|
||||
)
|
||||
out = (
|
||||
None
|
||||
ExitMove()
|
||||
if finishes_directly
|
||||
else bind_exit(edges.outgoing, cp_moves=edges.cp_moves)
|
||||
)
|
||||
self.paths[variant] = StagePath(
|
||||
entry=entry,
|
||||
output=edges.outgoing.produced,
|
||||
output_move=None if out is None else out.output_move,
|
||||
returns_over_dp=out is not None and out.returns_over_dp,
|
||||
output_move_completes_sum=False
|
||||
if out is None
|
||||
else out.output_move_completes_sum,
|
||||
output_move=out.output_move,
|
||||
output_move_completes_sum=out.output_move_completes_sum,
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -234,7 +222,8 @@ class StagePlan:
|
||||
|
||||
@property
|
||||
def input_on_attn_tp_slices(self):
|
||||
return TokenAxis.ATTN_TP in self.incoming_residual_rows.sharded
|
||||
entry = self.paths[BatchVariant.ORDINARY].entry
|
||||
return TokenAxis.ATTN_TP in entry.input_rows.sharded
|
||||
|
||||
def variant_for(self, forward_batch):
|
||||
# The batch determines its rows; missing paths must not change them.
|
||||
|
||||
@@ -212,14 +212,6 @@ def move_rows(
|
||||
return hidden_states
|
||||
|
||||
|
||||
def tp_gather(hidden_states: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
# Input-scattered attention keeps the same number of tokens on every TP rank.
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
def _dp_scatter_group() -> GroupCoordinator:
|
||||
parallel = get_parallel()
|
||||
if parallel.tp_size == parallel.attn_dp_size:
|
||||
|
||||
@@ -46,9 +46,10 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
reduce_to_dp_local: Callable(partial) completing the sum and moving
|
||||
it to destination rows; takes precedence over group.
|
||||
|
||||
Exits hand this form to ResidualStream.record(), which exposes an opaque
|
||||
OwedOutput to models. Low-level adapters use complete_owed() before reading
|
||||
it. A group is required when reduce_to_dp_local is absent.
|
||||
Exits hand this form to ResidualStream.record(), which keeps it as the
|
||||
contribution's owed work and exposes an opaque OwedOutput to models.
|
||||
Low-level adapters use complete_owed() before reading it. A group is
|
||||
required when reduce_to_dp_local is absent.
|
||||
"""
|
||||
|
||||
partial: torch.Tensor
|
||||
@@ -57,6 +58,12 @@ class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
# rank's tokens (a reduce-scatter, or an all-reduce then a scatter).
|
||||
reduce_to_dp_local: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
|
||||
|
||||
def complete(self) -> torch.Tensor:
|
||||
"""Complete the sum, on the destination rows when it moves them."""
|
||||
if self.reduce_to_dp_local is not None:
|
||||
return self.reduce_to_dp_local(self.partial)
|
||||
return self.group.all_reduce(self.partial)
|
||||
|
||||
|
||||
class DeferredFinalize(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes work only its producer knows how to do (a
|
||||
@@ -75,10 +82,6 @@ def complete_owed(
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Run the work an UnreducedOutput or a DeferredFinalize still owes; pass
|
||||
anything else through."""
|
||||
if isinstance(hidden_states, UnreducedOutput):
|
||||
if hidden_states.reduce_to_dp_local is not None:
|
||||
return hidden_states.reduce_to_dp_local(hidden_states.partial)
|
||||
return hidden_states.group.all_reduce(hidden_states.partial)
|
||||
if isinstance(hidden_states, DeferredFinalize):
|
||||
if isinstance(hidden_states, (UnreducedOutput, DeferredFinalize)):
|
||||
return hidden_states.complete()
|
||||
return hidden_states
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
attn_tp_gather,
|
||||
attn_tp_slice,
|
||||
get_attn_tp_context,
|
||||
tp_gather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
attn_cp_gather,
|
||||
@@ -70,7 +71,6 @@ from sglang.srt.layers.layer_boundary.ops import (
|
||||
dp_gather,
|
||||
dp_gather_sum,
|
||||
moe_cp_gather,
|
||||
tp_gather,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -13,35 +13,16 @@
|
||||
# ==============================================================================
|
||||
"""One forward's residual and its producer's outstanding contribution."""
|
||||
|
||||
from typing import Callable, Optional, Union
|
||||
from typing import Optional, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import GroupCoordinator
|
||||
from sglang.srt.layers.layer_boundary.layout import SumGroup, _sum_group
|
||||
from sglang.srt.layers.layer_boundary.output import DeferredFinalize, UnreducedOutput
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
|
||||
|
||||
|
||||
class DeferredSum(msgspec.Struct, frozen=True):
|
||||
"""Runtime reduction left by an exit decision.
|
||||
|
||||
Fields:
|
||||
group: All-reduce group when no redistribution callable is supplied.
|
||||
reduce_to_dp_local: Optional callable(value) returning a reduced
|
||||
value on destination rows; takes precedence over group.
|
||||
"""
|
||||
|
||||
group: Optional[GroupCoordinator] = None
|
||||
reduce_to_dp_local: Optional[Callable] = None
|
||||
|
||||
def complete(self, value):
|
||||
if self.reduce_to_dp_local is not None:
|
||||
return self.reduce_to_dp_local(value)
|
||||
return self.group.all_reduce(value)
|
||||
|
||||
|
||||
class DeclaredSum(msgspec.Struct, frozen=True):
|
||||
"""A sum every output of this producer owes to its declared input edge."""
|
||||
|
||||
@@ -55,31 +36,29 @@ class Contribution(msgspec.Struct):
|
||||
"""Own a producer's output, residual update and outstanding completion.
|
||||
|
||||
Fields:
|
||||
value: Contribution tensor, or None for an opaque finalize handoff.
|
||||
value: Contribution tensor (an unreduced output's partial), or None
|
||||
for an opaque finalize handoff.
|
||||
update: Producer operation to apply before the consumer reads input.
|
||||
owed: Runtime sum, declared sum, producer-specific finalize, or None.
|
||||
owed: The exit's unreduced output, a declared sum, a producer-specific
|
||||
finalize, or None.
|
||||
|
||||
Completing owed work clears owed but does not apply the residual update.
|
||||
"""
|
||||
|
||||
value: Optional[torch.Tensor]
|
||||
update: ResidualUpdate
|
||||
owed: Union[DeferredSum, DeclaredSum, DeferredFinalize, None] = None
|
||||
owed: Union[UnreducedOutput, DeclaredSum, DeferredFinalize, None] = None
|
||||
|
||||
def for_boundary(self):
|
||||
# These forms are private inputs to the existing fused-kernel adapters.
|
||||
if isinstance(self.owed, DeferredSum):
|
||||
return UnreducedOutput(
|
||||
self.value, self.owed.group, self.owed.reduce_to_dp_local
|
||||
)
|
||||
if isinstance(self.owed, DeferredFinalize):
|
||||
if isinstance(self.owed, (UnreducedOutput, DeferredFinalize)):
|
||||
return self.owed
|
||||
return self.value
|
||||
|
||||
def complete(self):
|
||||
if isinstance(self.owed, (DeferredSum, DeclaredSum)):
|
||||
if isinstance(self.owed, DeclaredSum):
|
||||
value = self.owed.complete(self.value)
|
||||
elif isinstance(self.owed, DeferredFinalize):
|
||||
elif self.owed is not None:
|
||||
value = self.owed.complete()
|
||||
else:
|
||||
return self.value
|
||||
@@ -170,11 +149,7 @@ class ResidualStream:
|
||||
raise RuntimeError("a declared sum cannot carry another completion")
|
||||
contribution = Contribution(output, update, DeclaredSum(declared_sum))
|
||||
elif isinstance(output, UnreducedOutput):
|
||||
contribution = Contribution(
|
||||
output.partial,
|
||||
update,
|
||||
DeferredSum(output.group, output.reduce_to_dp_local),
|
||||
)
|
||||
contribution = Contribution(output.partial, update, output)
|
||||
elif isinstance(output, DeferredFinalize):
|
||||
contribution = Contribution(None, update, output)
|
||||
else:
|
||||
@@ -223,8 +198,12 @@ class ResidualStream:
|
||||
raise NotImplementedError("snapshot requires a plain residual update")
|
||||
if pending.owed is None:
|
||||
value = pending.value
|
||||
elif isinstance(pending.owed, (DeferredSum, DeclaredSum)):
|
||||
elif isinstance(pending.owed, DeclaredSum):
|
||||
value = pending.owed.complete(pending.value.clone())
|
||||
elif isinstance(pending.owed, UnreducedOutput):
|
||||
value = msgspec.structs.replace(
|
||||
pending.owed, partial=pending.value.clone()
|
||||
).complete()
|
||||
else:
|
||||
raise NotImplementedError("a finalize handoff requires main-output capture")
|
||||
return value.clone() if self.residual is None else value + self.residual
|
||||
|
||||
@@ -62,6 +62,9 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.amx_utils import PackWeightMethod
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
)
|
||||
from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -79,9 +82,6 @@ from sglang.srt.layers.layer_boundary import (
|
||||
is_dense_ffn_fully_dp,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.context_parallel import (
|
||||
maybe_prefetch_next_full_attention_kv,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import access as residual_access
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -83,7 +83,6 @@ def sp_region_steps():
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
NORM_QUANT_READOUT,
|
||||
EdgeContract,
|
||||
EntryPath,
|
||||
InputContract,
|
||||
Layout,
|
||||
OutputContract,
|
||||
@@ -97,10 +96,10 @@ def sp_region_steps():
|
||||
output = OutputContract(rows)
|
||||
|
||||
def entry(read, attn_input_adapter=None):
|
||||
selected = bind_entry(
|
||||
EdgeContract(output, InputContract(rows, read=read), rows, rows)
|
||||
return bind_entry(
|
||||
EdgeContract(output, InputContract(rows, read=read), rows, rows),
|
||||
attn_input_adapter=attn_input_adapter,
|
||||
)
|
||||
return EntryPath(selected.prepare, rows, attn_input_adapter=attn_input_adapter)
|
||||
|
||||
return StagePath(
|
||||
entry(NORM_QUANT_READOUT, _attn_input_default), output, keep_output
|
||||
|
||||
Reference in New Issue
Block a user