[CP] GLM-5.3-Flash Prefill CP implemented on layer boundaries (#42044)
This commit is contained in:
@@ -17,7 +17,7 @@ from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_pha
|
||||
# MoE and linear attention keep their TP partition (the linear attention
|
||||
# gathers the sequence where its recurrence needs it). Model ports register
|
||||
# here; every other model keeps the ordinary prefill CP path.
|
||||
_SUPPORTED_MODELS: set[str] = set()
|
||||
_SUPPORTED_MODELS: set[str] = {"Glm5NextForConditionalGeneration"}
|
||||
|
||||
|
||||
def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
@@ -34,11 +34,27 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
"CP-TP group sharing requires --attn-cp-size == --tp-size, got "
|
||||
f"attn_cp_size={view.attn_cp_size}, tp_size={cfg.tp_size}."
|
||||
)
|
||||
if cfg.cp_strategy != "zigzag":
|
||||
raise ValueError("CP-TP group sharing requires --cp-strategy zigzag.")
|
||||
strategy = (
|
||||
"interleave"
|
||||
if model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration"
|
||||
else "zigzag"
|
||||
)
|
||||
if cfg.cp_strategy != strategy:
|
||||
raise ValueError(f"CP-TP group sharing requires --cp-strategy {strategy}.")
|
||||
|
||||
# This hook runs before speculative defaults and NEXTN alias resolution.
|
||||
# Native MTP keeps the target's CP/TP placement; other draft architectures
|
||||
# have their own topology and model-boundary contracts.
|
||||
if cfg.speculative_algorithm is not None and (
|
||||
model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration"
|
||||
or cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
|
||||
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
|
||||
or cfg.speculative_eagle_topk not in (None, 1)
|
||||
):
|
||||
raise ValueError("CP/TP group sharing only supports native MTP with top-k 1.")
|
||||
|
||||
unsupported = {
|
||||
"speculative-algorithm": cfg.speculative_algorithm is not None,
|
||||
"enable-dsa-cache-layer-split": cfg.enable_dsa_cache_layer_split,
|
||||
"dcp-size > 1": cfg.dcp_size > 1,
|
||||
"enable-mixed-chunk": cfg.enable_mixed_chunk,
|
||||
"moe-dp-size > 1": cfg.moe_dp_size > 1,
|
||||
|
||||
@@ -4,7 +4,6 @@ from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.models.glm_ocr.configuration_glm_ocr import GlmOcrVisionConfig
|
||||
|
||||
from sglang.srt.configs.mamba_utils import KimiLinearCacheParams, KimiLinearStateShape
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
_GLM5_NEXT_TOP_LEVEL_CONFIG_KEYS = (
|
||||
"architectures",
|
||||
@@ -262,8 +261,10 @@ class Glm5NextTextConfig(PretrainedConfig):
|
||||
|
||||
@property
|
||||
def mamba2_cache_params(self) -> KimiLinearCacheParams:
|
||||
from sglang.srt.runtime_context import linear_attn_tp_size
|
||||
|
||||
shape = KimiLinearStateShape.create(
|
||||
tp_world_size=get_parallel().attn_tp_size,
|
||||
tp_world_size=linear_attn_tp_size(),
|
||||
num_heads=self.linear_attn_config["num_heads"],
|
||||
head_dim=self.linear_attn_config["head_dim"],
|
||||
conv_kernel_size=self.linear_attn_config["short_conv_kernel_size"],
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.srt.layers.attention.dsa.kpool_plan import (
|
||||
update_kpool_write_plan,
|
||||
update_pooled_paged_mqa_metadata,
|
||||
)
|
||||
from sglang.srt.layers.cp.interleave_kpool import local_query_inputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import (
|
||||
@@ -100,6 +101,7 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
topk_transform_method=topk_transform_method,
|
||||
full_real_page_table=kpool_inputs.full_real_page_table,
|
||||
full_seqlens_expanded=kpool_inputs.full_seqlens_expanded,
|
||||
**local_query_inputs(forward_batch, metadata),
|
||||
)
|
||||
|
||||
metadata = init_pooled_paged_mqa_metadata(
|
||||
|
||||
@@ -600,12 +600,17 @@ class IndexerKPool(MultiPlatformOp):
|
||||
return_compressed: bool = False,
|
||||
write_cache: bool = True,
|
||||
):
|
||||
if key.shape[0] == 0:
|
||||
return None
|
||||
|
||||
if gate_score is None:
|
||||
gate_score = F.linear(x, self.index_kpool_compress_gate)
|
||||
|
||||
from sglang.srt.layers.cp.interleave_kpool import materialize_compression_inputs
|
||||
|
||||
key, gate_score, positions = materialize_compression_inputs(
|
||||
key, gate_score, positions, forward_batch
|
||||
)
|
||||
if key.shape[0] == 0:
|
||||
return None
|
||||
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
self._compress_write_decode(
|
||||
key=key,
|
||||
|
||||
@@ -78,6 +78,61 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
|
||||
moe_local_token_count: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def is_interleave_extend(forward_batch):
|
||||
return forward_batch.forward_mode.is_context_parallel_extend() and isinstance(
|
||||
forward_batch.attn_cp_metadata, InterleaveContextParallelMetadata
|
||||
)
|
||||
|
||||
|
||||
def cp_interleave_to_sequence_order(hidden_states, forward_batch):
|
||||
"""Restore sequence order after CP attention and before linear attention.
|
||||
|
||||
With CP2 and seven tokens, the boundary all-gathers the interleaved shards
|
||||
as ``[t0, t2, t4, t6, t1, t3, t5, pad]``. This returns
|
||||
``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute.
|
||||
|
||||
The residual and mHC coefficients do not move: only the already-read mixer
|
||||
input is reordered. Remove physical CP padding before the recurrent kernel.
|
||||
"""
|
||||
if not is_interleave_extend(forward_batch):
|
||||
return hidden_states
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
if metadata.gather_index is not None:
|
||||
return hidden_states.index_select(0, metadata.gather_index)
|
||||
size = get_parallel().attn_cp_size
|
||||
rows = hidden_states.shape[0] // size
|
||||
return (
|
||||
hidden_states.reshape(size, rows, *hidden_states.shape[1:])
|
||||
.transpose(0, 1)
|
||||
.flatten(0, 1)[: metadata.total_seq_lens]
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
def cp_sequence_to_interleave_order(hidden_states, forward_batch, gathered_rows):
|
||||
"""Restore interleave order after linear attention and before CP attention.
|
||||
|
||||
With CP2 and seven tokens, ``[t0, t1, t2, t3, t4, t5, t6]`` becomes
|
||||
``[t0, t2, t4, t6, t1, t3, t5, 0]``. These padded rank-major rows match
|
||||
the boundary's reduce-scatter and the local residual/mHC rows. The reorder
|
||||
itself performs no communication or reduction.
|
||||
"""
|
||||
if not is_interleave_extend(forward_batch):
|
||||
return hidden_states
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
padded = hidden_states.new_zeros((gathered_rows, *hidden_states.shape[1:]))
|
||||
if metadata.gather_index is not None:
|
||||
return padded.index_copy_(0, metadata.gather_index, hidden_states)
|
||||
padded[: hidden_states.shape[0]] = hidden_states
|
||||
size = get_parallel().attn_cp_size
|
||||
return (
|
||||
padded.reshape(-1, size, *hidden_states.shape[1:])
|
||||
.transpose(0, 1)
|
||||
.flatten(0, 1)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
class InterleaveCPStrategy(ContextParallelStrategy):
|
||||
name = "interleave"
|
||||
kind = ContextParallelStrategyKind.INTERLEAVE
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""KPool compression and query metadata for interleave context parallelism."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.cp.base import get_cp_strategy
|
||||
from sglang.srt.layers.cp.interleave import (
|
||||
interleave_rows_per_request,
|
||||
is_interleave_extend,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def local_query_inputs(forward_batch, metadata):
|
||||
"""Compression covers full requests; logits cover only this rank's queries."""
|
||||
if not is_interleave_extend(forward_batch):
|
||||
return {}
|
||||
parallel = get_parallel()
|
||||
counts = interleave_rows_per_request(
|
||||
forward_batch.extend_seq_lens_cpu, parallel.attn_cp_rank, parallel.attn_cp_size
|
||||
)
|
||||
indices = torch.tensor(
|
||||
[i for i, count in enumerate(counts) if count],
|
||||
dtype=torch.long,
|
||||
device=forward_batch.req_pool_indices.device,
|
||||
)
|
||||
return dict(
|
||||
local_real_page_table=metadata.real_page_table,
|
||||
local_seqlens_expanded=metadata.dsa_seqlens_expanded,
|
||||
local_extend_seq_lens_cpu=metadata.dsa_extend_seq_lens_list,
|
||||
local_seq_lens_cpu=metadata.indexer_seq_lens_cpu.tolist(),
|
||||
local_req_pool_indices=forward_batch.req_pool_indices.index_select(0, indices),
|
||||
)
|
||||
|
||||
|
||||
def materialize_compression_inputs(key, score, positions, forward_batch):
|
||||
"""Replicate ordered keys/scores before pooling across adjacent tokens.
|
||||
|
||||
Queries and head gates stay local. Every rank writes its replicated index
|
||||
cache and tail state, so subsequent decode needs no layer-owner protocol.
|
||||
"""
|
||||
if not is_interleave_extend(forward_batch):
|
||||
return key, score, positions
|
||||
width = key.shape[-1]
|
||||
packed = torch.cat((key, score), dim=-1)
|
||||
packed = get_cp_strategy().gather_kv_cache(packed, forward_batch)
|
||||
return (
|
||||
packed[..., :width].contiguous(),
|
||||
packed[..., width:].contiguous(),
|
||||
forward_batch.positions,
|
||||
)
|
||||
@@ -278,8 +278,10 @@ def cp_shard_model_inputs(
|
||||
):
|
||||
"""Restore the shared batch so logits processing keeps full-batch metadata."""
|
||||
assert is_cp_active(forward_batch)
|
||||
sharded_hidden_states = cp_shard_hidden_states(
|
||||
complete_hidden_states, forward_batch
|
||||
sharded_hidden_states = (
|
||||
cp_shard_hidden_states(complete_hidden_states, forward_batch)
|
||||
if complete_hidden_states is not None
|
||||
else None
|
||||
)
|
||||
sharded_positions = cp_shard_position_ids(complete_position_ids, forward_batch)
|
||||
model_input_ids = (
|
||||
@@ -300,6 +302,7 @@ def cp_shard_model_inputs(
|
||||
spec_hidden_states_backup = None
|
||||
if (
|
||||
spec_hidden_states is not None
|
||||
and complete_hidden_states is not None
|
||||
and spec_hidden_states.shape[0] == complete_hidden_states.shape[0]
|
||||
):
|
||||
spec_hidden_states_backup = spec_hidden_states
|
||||
|
||||
@@ -57,6 +57,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_attn_tp_slice_update_read,
|
||||
_dispatch_by_update,
|
||||
_dp_gather_sum_read,
|
||||
_move_before_read,
|
||||
_reduce_update_read,
|
||||
_reduce_update_read_dp_gather,
|
||||
_run_entry,
|
||||
@@ -404,6 +405,36 @@ def _select_entry_step(
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded - need.gathered_by_compute
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_CP}:
|
||||
# A head-parallel mixer consumes full context, but its residual and
|
||||
# readout coefficients remain on the CP shard. Complete the producer's
|
||||
# sum onto that shard before the (possibly nonlinear) update/read.
|
||||
if (
|
||||
residual != residual_to
|
||||
or cp_moves is None
|
||||
or owes not in (None, SumGroup.ATTN_CP)
|
||||
or (owes is not None and cp_moves.reduce_scatter is None)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
local = msgspec.structs.replace(
|
||||
produced, layout=residual_to, group=None, always_partial=False
|
||||
)
|
||||
read_local, input_move = _select_entry_step(
|
||||
local,
|
||||
residual=residual,
|
||||
residual_to=residual_to,
|
||||
need=need,
|
||||
is_plain_add=is_plain_add,
|
||||
fusions=(),
|
||||
residual_joins_sum=False,
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return partial(
|
||||
_move_before_read,
|
||||
move=cp_moves.reduce_scatter if owes is not None else cp_moves.take_back,
|
||||
read=read_local,
|
||||
), input_move
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
# completes the attention-TP sum and slices in one collective; a
|
||||
@@ -536,7 +567,7 @@ def _select_entry_step(
|
||||
return (partial(_tp_sum_with_residual_read, read=read), None)
|
||||
# A sum over TP completes only on rows every TP rank holds: the TP group
|
||||
# spans attention DP and CP.
|
||||
if owes is not SumGroup.ATTN_TP and (
|
||||
if owes not in (SumGroup.ATTN_TP, SumGroup.ATTN_CP) and (
|
||||
owes is not SumGroup.TP or produced.layout.sharded
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
@@ -551,7 +582,7 @@ def _select_entry_step(
|
||||
),
|
||||
None,
|
||||
)
|
||||
if owes not in (None, SumGroup.ATTN_TP):
|
||||
if owes not in (None, SumGroup.ATTN_TP, SumGroup.ATTN_CP):
|
||||
raise NotImplementedError(f"{produced=} {need=}")
|
||||
owes_attention_tp = owes is SumGroup.ATTN_TP
|
||||
# The partial order adds the residual on attention-TP rank 0 before the DP
|
||||
@@ -579,7 +610,8 @@ def _select_entry_step(
|
||||
partial(
|
||||
_reduce_update_read_dp_gather,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
reduces_attention_tp=owes is not None,
|
||||
group=owes,
|
||||
places_cp_shards=places_cp_shards,
|
||||
read=read,
|
||||
),
|
||||
|
||||
@@ -282,6 +282,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
|
||||
declaration.update.applied_at_exit
|
||||
and TokenAxis.ATTN_CP
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
and not _cp_gathers_over_attn_cp()
|
||||
):
|
||||
raise NotImplementedError("MHC with a gather over attention CP")
|
||||
attn_input_adapter = None
|
||||
|
||||
@@ -209,6 +209,8 @@ class StageDeclaration:
|
||||
reduction: Whether the next stage's input always completes the sum
|
||||
(an attention's), or the exit decides (an FFN's or a mixer's).
|
||||
gathers_attn_tp_input: Whether attention gathers TP-sharded input itself.
|
||||
tp_group: Group partitioning attention heads. Full TP consumers require
|
||||
full token rows, including when TP shares the prefill CP group.
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, and the full TP size means TP compute.
|
||||
exit_rows: Required FFN output rows at the layer or branch exit.
|
||||
@@ -228,6 +230,7 @@ class StageDeclaration:
|
||||
output_transform: Optional[OutputTransform] = None
|
||||
reduction: ProducerReduction = ProducerReduction.EXIT_SCOPED
|
||||
gathers_attn_tp_input: bool = False
|
||||
tp_group: SumGroup = SumGroup.ATTN_TP
|
||||
dense_tp_size: Optional[int] = None
|
||||
exit_rows: Optional[ExitRows] = None
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
@@ -273,6 +276,7 @@ def declare_attn(
|
||||
reduction=ProducerReduction.ALWAYS_PARTIAL,
|
||||
gathers_attn_tp_input=True,
|
||||
output_transform=None,
|
||||
tp_group=SumGroup.ATTN_TP,
|
||||
):
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
@@ -286,6 +290,7 @@ def declare_attn(
|
||||
complete, before the residual update (a sandwich norm). The next
|
||||
stage's input runs it, so no fused add + norm takes that input.
|
||||
Requires ALWAYS_PARTIAL.
|
||||
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -302,6 +307,7 @@ def declare_attn(
|
||||
output_transform=output_transform,
|
||||
reduction=reduction,
|
||||
gathers_attn_tp_input=gathers_attn_tp_input,
|
||||
tp_group=tp_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -375,11 +381,30 @@ def _resolve_stage(stage, variant, following=None):
|
||||
else frozenset()
|
||||
)
|
||||
owes = not sp and axes[TokenAxis.ATTN_TP] > 1
|
||||
group = SumGroup.ATTN_TP
|
||||
compute_rows = attention
|
||||
if stage.tp_group is SumGroup.TP:
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_dp_size != 1 or sp or scattered:
|
||||
raise NotImplementedError(
|
||||
"full-TP attention requires unscattered input without attention DP"
|
||||
)
|
||||
compute_rows = full
|
||||
owes = parallel.tp_size > 1
|
||||
# Canonicalize equal groups to the CP transport's reduction contract.
|
||||
# initialize_model_parallel aliases ATTN_CP to TP at this topology.
|
||||
group = (
|
||||
SumGroup.ATTN_CP
|
||||
if parallel.attn_cp_size == parallel.tp_size and parallel.attn_cp_size > 1
|
||||
else SumGroup.TP
|
||||
)
|
||||
elif stage.tp_group is not SumGroup.ATTN_TP:
|
||||
raise ValueError("attention heads must be partitioned over ATTN_TP or TP")
|
||||
declaration = StageContract(
|
||||
InputContract(attention, gathered_by_compute=gathers, read=stage.read),
|
||||
InputContract(compute_rows, gathered_by_compute=gathers, read=stage.read),
|
||||
OutputContract(
|
||||
local if sp else attention,
|
||||
group=SumGroup.ATTN_TP if owes else None,
|
||||
local if sp else compute_rows,
|
||||
group=group if owes else None,
|
||||
always_partial=owes
|
||||
and (
|
||||
stage.reduction is ProducerReduction.ALWAYS_PARTIAL
|
||||
|
||||
@@ -73,6 +73,7 @@ class SumGroup(Enum):
|
||||
from get_parallel() when the sum runs."""
|
||||
|
||||
ATTN_TP = auto()
|
||||
ATTN_CP = auto()
|
||||
TP = auto()
|
||||
# The group one all-reduce of a MoE output runs over.
|
||||
MOE_OUTPUT = auto()
|
||||
@@ -145,6 +146,8 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
parallel = get_parallel()
|
||||
if group is SumGroup.ATTN_TP:
|
||||
return parallel.attn_tp_group
|
||||
if group is SumGroup.ATTN_CP:
|
||||
return parallel.attn_cp_group
|
||||
if group is SumGroup.TP:
|
||||
return parallel.tp_group
|
||||
if group is SumGroup.MOE_OUTPUT:
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
_cp_shard_token_rows,
|
||||
_sum_group,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.output import (
|
||||
DeferredFinalize,
|
||||
@@ -105,6 +106,8 @@ def _reduce_update_read(
|
||||
)
|
||||
elif group is SumGroup.TP:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
elif group is not None:
|
||||
hidden_states = _sum_group(group).all_reduce(hidden_states)
|
||||
if _is_npu and cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, cache)
|
||||
return read.update_and_read(
|
||||
@@ -127,6 +130,7 @@ def _reduce_update_read_dp_gather(
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
):
|
||||
@@ -138,7 +142,11 @@ def _reduce_update_read_dp_gather(
|
||||
residual = update.gather_residual_attn_tp(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = (
|
||||
attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
if group is SumGroup.ATTN_TP
|
||||
else _sum_group(group).all_reduce(hidden_states)
|
||||
)
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
@@ -417,16 +425,25 @@ def _then_attn_cp_gather(
|
||||
cache=None,
|
||||
gather: Callable,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
**call,
|
||||
):
|
||||
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
|
||||
equal length, over the attention-CP group. The residual stays on the
|
||||
shard."""
|
||||
hidden_states, residual = gather(
|
||||
hidden_states, residual, forward_batch, norm, update=update, cache=cache
|
||||
hidden_states, residual, forward_batch, norm, update=update, cache=cache, **call
|
||||
)
|
||||
return attn_cp_interleave_gather(hidden_states), residual
|
||||
|
||||
|
||||
def _move_before_read(
|
||||
hidden_states, residual, forward_batch, norm, *, move, read, **call
|
||||
):
|
||||
"""Complete and place a producer contribution before its residual update."""
|
||||
hidden_states, residual = move(hidden_states, residual, forward_batch)
|
||||
return read(hidden_states, residual, forward_batch, norm, **call)
|
||||
|
||||
|
||||
def _then_moe_cp_gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
@@ -654,32 +654,22 @@ def _embed_mm_inputs_with_split(
|
||||
return input_embeds, other_info
|
||||
|
||||
|
||||
def general_mm_embed_routine(
|
||||
def prepare_mm_inputs(
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
language_model: nn.Module,
|
||||
kwargs: dict,
|
||||
multimodal_model: Optional[nn.Module] = None,
|
||||
data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: Optional[dict[Modality, List[int]]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Process multimodal inputs and forward through language model.
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Prepare full-sequence embeddings without running the language model.
|
||||
|
||||
Args:
|
||||
input_ids: Input token IDs tensor
|
||||
forward_batch: Batch information for model forward pass
|
||||
language_model: Base language model to use
|
||||
data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function.
|
||||
placeholder_tokens: Token IDs for multimodal placeholders
|
||||
use_deepstack: Whether to use deepstack embeddings for each modality, default False
|
||||
feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup.
|
||||
**kwargs: Additional arguments passed to language model
|
||||
|
||||
Returns:
|
||||
Hidden states from language model forward pass
|
||||
Preserve multimodal cache/offload bookkeeping and the MTP embedding copy.
|
||||
Additional language-model inputs, such as DeepStack features, update kwargs.
|
||||
Non-first pipeline ranks consume proxy tensors and return no embeddings.
|
||||
"""
|
||||
assert hasattr(language_model, "get_input_embeddings")
|
||||
embed_tokens = language_model.get_input_embeddings()
|
||||
@@ -808,6 +798,47 @@ def general_mm_embed_routine(
|
||||
input_embeds = forward_batch.input_embeds
|
||||
else:
|
||||
input_embeds = None
|
||||
return input_embeds
|
||||
|
||||
|
||||
def general_mm_embed_routine(
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
language_model: nn.Module,
|
||||
multimodal_model: Optional[nn.Module] = None,
|
||||
data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: Optional[dict[Modality, List[int]]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Process multimodal inputs and forward through language model.
|
||||
|
||||
Args:
|
||||
input_ids: Input token IDs tensor
|
||||
forward_batch: Batch information for model forward pass
|
||||
language_model: Base language model to use
|
||||
data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function.
|
||||
placeholder_tokens: Token IDs for multimodal placeholders
|
||||
use_deepstack: Whether to use deepstack embeddings for each modality, default False
|
||||
feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup.
|
||||
**kwargs: Additional arguments passed to language model
|
||||
|
||||
Returns:
|
||||
Hidden states from language model forward pass
|
||||
"""
|
||||
input_embeds = prepare_mm_inputs(
|
||||
input_ids,
|
||||
forward_batch,
|
||||
language_model,
|
||||
kwargs,
|
||||
multimodal_model,
|
||||
data_embedding_funcs,
|
||||
placeholder_tokens,
|
||||
use_deepstack,
|
||||
feature_dtypes,
|
||||
)
|
||||
|
||||
with torch.profiler.record_function("sglang.vlm.language_model_prefill"):
|
||||
hidden_states = language_model(
|
||||
|
||||
@@ -370,8 +370,12 @@ class EagerRunner(BaseRunner):
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
# Under CP-TP group sharing the model owns the CP row layout.
|
||||
elif cp_active and not get_parallel().enable_cp_tp_group_sharing:
|
||||
elif cp_active and (
|
||||
not get_parallel().enable_cp_tp_group_sharing
|
||||
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
|
||||
):
|
||||
# GLM delegates its CP boundary to this runner via prepare_cp_inputs.
|
||||
# Other group-sharing models keep owning the layout in forward().
|
||||
ret = self._execute_extend_cp(forward_batch, kwargs)
|
||||
else:
|
||||
ret = model_runner.model.forward(
|
||||
@@ -390,18 +394,25 @@ class EagerRunner(BaseRunner):
|
||||
"""
|
||||
model = self.model_runner.model
|
||||
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
|
||||
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
|
||||
input_embeds, positions, model_kwargs = prepare_inputs(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
else:
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
|
||||
positions = forward_batch.positions
|
||||
model_kwargs = {}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds,
|
||||
forward_batch.positions,
|
||||
positions,
|
||||
forward_batch,
|
||||
forward_batch.input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs = {"input_embeds": sharded_input_embeds}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
|
||||
model_kwargs["input_embeds"] = sharded_input_embeds
|
||||
hidden_states = model.model(
|
||||
model_input_ids,
|
||||
sharded_positions,
|
||||
|
||||
@@ -137,6 +137,28 @@ class DeepseekModelNextN(nn.Module):
|
||||
self.shared_head = nn.Module()
|
||||
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def embed_input_ids(
|
||||
self, input_ids: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
# MM positions in input_ids hold MM_PAD_SHIFT_VALUE+hash sentinels
|
||||
# (far above vocab_size). Use target-produced mm_input_embeds for
|
||||
# these positions and only call embed_tokens on the appended
|
||||
# next-token to avoid embed OOB.
|
||||
input_embeds = forward_batch.mm_input_embeds
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and forward_batch.contains_mm_inputs()
|
||||
and not forward_batch.forward_mode.is_draft_extend_v2()
|
||||
):
|
||||
assert input_embeds is not None
|
||||
last_indices = (
|
||||
forward_batch.extend_start_loc + forward_batch.extend_seq_lens - 1
|
||||
).long()
|
||||
input_embeds[last_indices] = self.embed_tokens(input_ids[last_indices])
|
||||
if input_embeds is None:
|
||||
input_embeds = self.embed_tokens(input_ids)
|
||||
return input_embeds
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -168,27 +190,7 @@ class DeepseekModelNextN(nn.Module):
|
||||
)
|
||||
|
||||
if input_embeds is None:
|
||||
# MM positions in input_ids hold MM_PAD_SHIFT_VALUE+hash sentinels
|
||||
# (far above vocab_size). Use target-produced mm_input_embeds for
|
||||
# these positions and only call embed_tokens on the appended
|
||||
# next-token to avoid embed OOB.
|
||||
input_embeds = forward_batch.mm_input_embeds
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and forward_batch.contains_mm_inputs()
|
||||
and not forward_batch.forward_mode.is_draft_extend_v2()
|
||||
):
|
||||
assert input_embeds is not None
|
||||
last_indices = (
|
||||
forward_batch.extend_start_loc
|
||||
+ forward_batch.extend_seq_lens
|
||||
- 1
|
||||
).long()
|
||||
input_embeds[last_indices] = self.embed_tokens(
|
||||
input_ids[last_indices]
|
||||
)
|
||||
if input_embeds is None:
|
||||
input_embeds = self.embed_tokens(input_ids)
|
||||
input_embeds = self.embed_input_ids(input_ids, forward_batch)
|
||||
hidden_states = input_embeds
|
||||
|
||||
if hidden_states.shape[0] > 0:
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
MHCState,
|
||||
SumGroup,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
@@ -68,6 +69,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
prepare_mm_inputs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
@@ -395,7 +397,13 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
head_shard_size = get_parallel().attn_tp_size
|
||||
from sglang.srt.runtime_context import (
|
||||
linear_attn_parallel_group,
|
||||
linear_attn_tp_size,
|
||||
)
|
||||
|
||||
head_shard_size = linear_attn_tp_size()
|
||||
head_parallel_group = linear_attn_parallel_group()
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.config = config
|
||||
@@ -434,7 +442,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.fg_sizes,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.fused_qkvbfg_a_proj",
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
self.split_sizes = [
|
||||
3 * projection_size // head_shard_size,
|
||||
@@ -446,7 +454,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
dtype=self.fused_qkvbfg_a_proj.params_dtype,
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
else:
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
@@ -456,7 +464,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.num_k_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
|
||||
@@ -467,7 +475,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
[self.head_dim, self.head_dim],
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.fused_bfg_a_proj",
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
self.bfg_split_sizes = [self.local_num_heads, 2 * self.head_dim]
|
||||
self.fused_fg_b_proj = ColumnParallelBatchedLinear(
|
||||
@@ -475,7 +483,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
dtype=self.fused_bfg_a_proj.params_dtype,
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
else:
|
||||
self.f_a_proj = ReplicatedLinear(
|
||||
@@ -492,7 +500,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
@@ -501,7 +509,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
@@ -517,7 +525,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
|
||||
self.dt_bias = nn.Parameter(
|
||||
@@ -526,7 +534,11 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
|
||||
set_weight_attrs(
|
||||
self.dt_bias,
|
||||
{"weight_loader": sharded_weight_loader(0, parallel_group="attn_tp")},
|
||||
{
|
||||
"weight_loader": sharded_weight_loader(
|
||||
0, parallel_group=head_parallel_group
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.qkv_conv1d = MergedColumnParallelLinear(
|
||||
@@ -535,7 +547,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.qkv_conv1d",
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
# ColumnParallelLinear's loader cannot reshape conv1d weights, so add the
|
||||
# singleton dimension after construction.
|
||||
@@ -546,7 +558,11 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
)
|
||||
set_weight_attrs(
|
||||
self.A_log,
|
||||
{"weight_loader": sharded_weight_loader(2, parallel_group="attn_tp")},
|
||||
{
|
||||
"weight_loader": sharded_weight_loader(
|
||||
2, parallel_group=head_parallel_group
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(
|
||||
@@ -559,7 +575,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
reduce_results=reduce_results,
|
||||
parallel_group="attn_tp",
|
||||
parallel_group=head_parallel_group,
|
||||
)
|
||||
|
||||
conv_weights = self.qkv_conv1d.weight.squeeze(1)
|
||||
@@ -629,6 +645,14 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states
|
||||
|
||||
from sglang.srt.layers.cp.interleave import (
|
||||
cp_interleave_to_sequence_order,
|
||||
cp_sequence_to_interleave_order,
|
||||
)
|
||||
|
||||
gathered_rows = hidden_states.shape[0]
|
||||
hidden_states = cp_interleave_to_sequence_order(hidden_states, forward_batch)
|
||||
|
||||
if self.do_fuse_qkvbfg:
|
||||
mixed_qkv, beta, forget_gate, g_proj_states = self.forward_qkvbfg_fused(
|
||||
hidden_states, forward_batch
|
||||
@@ -653,7 +677,9 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
core_attn_out = self.o_norm(core_attn_out, norm_gate)
|
||||
core_attn_out = core_attn_out.squeeze(0).flatten(-2)
|
||||
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
return cp_sequence_to_interleave_order(
|
||||
self.o_proj(core_attn_out)[0], forward_batch, gathered_rows
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextDecoderLayer(nn.Module):
|
||||
@@ -785,7 +811,13 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
).residual_ops()
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
declare_attn(
|
||||
read=residual.attn_readout,
|
||||
update=residual.attn_update,
|
||||
tp_group=SumGroup.TP
|
||||
if self.is_linear_attn and get_parallel().enable_cp_tp_group_sharing
|
||||
else SumGroup.ATTN_TP,
|
||||
),
|
||||
self.input_layernorm,
|
||||
{
|
||||
"qkv_latent_func": self.self_attn.prepare_qkv_latent
|
||||
@@ -939,6 +971,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
isinstance(self.mlp, Glm5NextMoE)
|
||||
and not self.mlp.experts.moe_runner_config.inplace
|
||||
and not torch.compiler.is_compiling()
|
||||
and hidden_states_orig.shape[0] == hidden_states.shape[0]
|
||||
):
|
||||
from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx
|
||||
|
||||
@@ -1564,6 +1597,24 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
video_embeds = self.visual(pixel_values, grid_thw=flattened_video_grid_thw)
|
||||
return video_embeds
|
||||
|
||||
def prepare_cp_inputs(self, forward_batch, **kwargs):
|
||||
"""Finish multimodal embedding injection before the runner shards tokens."""
|
||||
input_embeds = kwargs.pop("input_embeds", None)
|
||||
positions = (
|
||||
forward_batch.mrope_positions
|
||||
if self.is_mrope_enabled
|
||||
else forward_batch.positions
|
||||
)
|
||||
if input_embeds is None or forward_batch.contains_mm_inputs():
|
||||
input_embeds = prepare_mm_inputs(
|
||||
forward_batch.input_ids,
|
||||
forward_batch,
|
||||
self.model,
|
||||
kwargs,
|
||||
multimodal_model=self,
|
||||
)
|
||||
return input_embeds, positions, kwargs
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
@@ -1577,14 +1628,25 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
positions = forward_batch.mrope_positions
|
||||
|
||||
with get_attn_tp_context().maybe_input_scattered(forward_batch):
|
||||
hidden_states = general_mm_embed_routine(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
language_model=self.model,
|
||||
multimodal_model=self,
|
||||
positions=positions,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
if input_embeds is not None and not forward_batch.contains_mm_inputs():
|
||||
# Preserve input_embeds / replace_embeds supplied by the runner.
|
||||
# The MM routine otherwise performs a fresh token lookup.
|
||||
hidden_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=input_embeds,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
else:
|
||||
hidden_states = general_mm_embed_routine(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
language_model=self.model,
|
||||
multimodal_model=self,
|
||||
positions=positions,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
|
||||
aux_hidden_states = None
|
||||
if self.capture_aux_hidden_states:
|
||||
|
||||
@@ -22,6 +22,15 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN):
|
||||
def prepare_cp_inputs(self, forward_batch, **kwargs):
|
||||
# The worker has rotated the target's multimodal embeddings per request.
|
||||
# Fill each appended token while indices still address full sequences,
|
||||
# then let the runner shard embeddings and target states together.
|
||||
input_embeds = self.model.embed_input_ids(
|
||||
forward_batch.input_ids, forward_batch
|
||||
)
|
||||
return input_embeds, forward_batch.positions, {}
|
||||
|
||||
@classmethod
|
||||
def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper:
|
||||
text_config = getattr(config, "text_config", config)
|
||||
|
||||
Reference in New Issue
Block a user