[CP] GLM-5.3-Flash Prefill CP implemented on layer boundaries (#42044)

This commit is contained in:
Baizhou Zhang
2026-10-08 20:42:11 -07:00
committed by GitHub
parent 7859eff6ca
commit 33bffcd490
20 changed files with 587 additions and 95 deletions
@@ -17,7 +17,7 @@ from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_pha
# MoE and linear attention keep their TP partition (the linear attention
# gathers the sequence where its recurrence needs it). Model ports register
# here; every other model keeps the ordinary prefill CP path.
_SUPPORTED_MODELS: set[str] = set()
_SUPPORTED_MODELS: set[str] = {"Glm5NextForConditionalGeneration"}
def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
@@ -34,11 +34,27 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
"CP-TP group sharing requires --attn-cp-size == --tp-size, got "
f"attn_cp_size={view.attn_cp_size}, tp_size={cfg.tp_size}."
)
if cfg.cp_strategy != "zigzag":
raise ValueError("CP-TP group sharing requires --cp-strategy zigzag.")
strategy = (
"interleave"
if model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration"
else "zigzag"
)
if cfg.cp_strategy != strategy:
raise ValueError(f"CP-TP group sharing requires --cp-strategy {strategy}.")
# This hook runs before speculative defaults and NEXTN alias resolution.
# Native MTP keeps the target's CP/TP placement; other draft architectures
# have their own topology and model-boundary contracts.
if cfg.speculative_algorithm is not None and (
model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration"
or cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
or cfg.speculative_eagle_topk not in (None, 1)
):
raise ValueError("CP/TP group sharing only supports native MTP with top-k 1.")
unsupported = {
"speculative-algorithm": cfg.speculative_algorithm is not None,
"enable-dsa-cache-layer-split": cfg.enable_dsa_cache_layer_split,
"dcp-size > 1": cfg.dcp_size > 1,
"enable-mixed-chunk": cfg.enable_mixed_chunk,
"moe-dp-size > 1": cfg.moe_dp_size > 1,
+3 -2
View File
@@ -4,7 +4,6 @@ from transformers.configuration_utils import PretrainedConfig
from transformers.models.glm_ocr.configuration_glm_ocr import GlmOcrVisionConfig
from sglang.srt.configs.mamba_utils import KimiLinearCacheParams, KimiLinearStateShape
from sglang.srt.runtime_context import get_parallel
_GLM5_NEXT_TOP_LEVEL_CONFIG_KEYS = (
"architectures",
@@ -262,8 +261,10 @@ class Glm5NextTextConfig(PretrainedConfig):
@property
def mamba2_cache_params(self) -> KimiLinearCacheParams:
from sglang.srt.runtime_context import linear_attn_tp_size
shape = KimiLinearStateShape.create(
tp_world_size=get_parallel().attn_tp_size,
tp_world_size=linear_attn_tp_size(),
num_heads=self.linear_attn_config["num_heads"],
head_dim=self.linear_attn_config["head_dim"],
conv_kernel_size=self.linear_attn_config["short_conv_kernel_size"],
@@ -13,6 +13,7 @@ from sglang.srt.layers.attention.dsa.kpool_plan import (
update_kpool_write_plan,
update_pooled_paged_mqa_metadata,
)
from sglang.srt.layers.cp.interleave_kpool import local_query_inputs
if TYPE_CHECKING:
from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import (
@@ -100,6 +101,7 @@ class DeepseekSparseAttnBackendKPoolMixin:
topk_transform_method=topk_transform_method,
full_real_page_table=kpool_inputs.full_real_page_table,
full_seqlens_expanded=kpool_inputs.full_seqlens_expanded,
**local_query_inputs(forward_batch, metadata),
)
metadata = init_pooled_paged_mqa_metadata(
@@ -600,12 +600,17 @@ class IndexerKPool(MultiPlatformOp):
return_compressed: bool = False,
write_cache: bool = True,
):
if key.shape[0] == 0:
return None
if gate_score is None:
gate_score = F.linear(x, self.index_kpool_compress_gate)
from sglang.srt.layers.cp.interleave_kpool import materialize_compression_inputs
key, gate_score, positions = materialize_compression_inputs(
key, gate_score, positions, forward_batch
)
if key.shape[0] == 0:
return None
if forward_batch.forward_mode.is_decode_or_idle():
self._compress_write_decode(
key=key,
+55
View File
@@ -78,6 +78,61 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
moe_local_token_count: Optional[torch.Tensor] = None
def is_interleave_extend(forward_batch):
return forward_batch.forward_mode.is_context_parallel_extend() and isinstance(
forward_batch.attn_cp_metadata, InterleaveContextParallelMetadata
)
def cp_interleave_to_sequence_order(hidden_states, forward_batch):
"""Restore sequence order after CP attention and before linear attention.
With CP2 and seven tokens, the boundary all-gathers the interleaved shards
as ``[t0, t2, t4, t6, t1, t3, t5, pad]``. This returns
``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute.
The residual and mHC coefficients do not move: only the already-read mixer
input is reordered. Remove physical CP padding before the recurrent kernel.
"""
if not is_interleave_extend(forward_batch):
return hidden_states
metadata = forward_batch.attn_cp_metadata
if metadata.gather_index is not None:
return hidden_states.index_select(0, metadata.gather_index)
size = get_parallel().attn_cp_size
rows = hidden_states.shape[0] // size
return (
hidden_states.reshape(size, rows, *hidden_states.shape[1:])
.transpose(0, 1)
.flatten(0, 1)[: metadata.total_seq_lens]
.contiguous()
)
def cp_sequence_to_interleave_order(hidden_states, forward_batch, gathered_rows):
"""Restore interleave order after linear attention and before CP attention.
With CP2 and seven tokens, ``[t0, t1, t2, t3, t4, t5, t6]`` becomes
``[t0, t2, t4, t6, t1, t3, t5, 0]``. These padded rank-major rows match
the boundary's reduce-scatter and the local residual/mHC rows. The reorder
itself performs no communication or reduction.
"""
if not is_interleave_extend(forward_batch):
return hidden_states
metadata = forward_batch.attn_cp_metadata
padded = hidden_states.new_zeros((gathered_rows, *hidden_states.shape[1:]))
if metadata.gather_index is not None:
return padded.index_copy_(0, metadata.gather_index, hidden_states)
padded[: hidden_states.shape[0]] = hidden_states
size = get_parallel().attn_cp_size
return (
padded.reshape(-1, size, *hidden_states.shape[1:])
.transpose(0, 1)
.flatten(0, 1)
.contiguous()
)
class InterleaveCPStrategy(ContextParallelStrategy):
name = "interleave"
kind = ContextParallelStrategyKind.INTERLEAVE
@@ -0,0 +1,50 @@
"""KPool compression and query metadata for interleave context parallelism."""
import torch
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.cp.interleave import (
interleave_rows_per_request,
is_interleave_extend,
)
from sglang.srt.runtime_context import get_parallel
def local_query_inputs(forward_batch, metadata):
"""Compression covers full requests; logits cover only this rank's queries."""
if not is_interleave_extend(forward_batch):
return {}
parallel = get_parallel()
counts = interleave_rows_per_request(
forward_batch.extend_seq_lens_cpu, parallel.attn_cp_rank, parallel.attn_cp_size
)
indices = torch.tensor(
[i for i, count in enumerate(counts) if count],
dtype=torch.long,
device=forward_batch.req_pool_indices.device,
)
return dict(
local_real_page_table=metadata.real_page_table,
local_seqlens_expanded=metadata.dsa_seqlens_expanded,
local_extend_seq_lens_cpu=metadata.dsa_extend_seq_lens_list,
local_seq_lens_cpu=metadata.indexer_seq_lens_cpu.tolist(),
local_req_pool_indices=forward_batch.req_pool_indices.index_select(0, indices),
)
def materialize_compression_inputs(key, score, positions, forward_batch):
"""Replicate ordered keys/scores before pooling across adjacent tokens.
Queries and head gates stay local. Every rank writes its replicated index
cache and tail state, so subsequent decode needs no layer-owner protocol.
"""
if not is_interleave_extend(forward_batch):
return key, score, positions
width = key.shape[-1]
packed = torch.cat((key, score), dim=-1)
packed = get_cp_strategy().gather_kv_cache(packed, forward_batch)
return (
packed[..., :width].contiguous(),
packed[..., width:].contiguous(),
forward_batch.positions,
)
+5 -2
View File
@@ -278,8 +278,10 @@ def cp_shard_model_inputs(
):
"""Restore the shared batch so logits processing keeps full-batch metadata."""
assert is_cp_active(forward_batch)
sharded_hidden_states = cp_shard_hidden_states(
complete_hidden_states, forward_batch
sharded_hidden_states = (
cp_shard_hidden_states(complete_hidden_states, forward_batch)
if complete_hidden_states is not None
else None
)
sharded_positions = cp_shard_position_ids(complete_position_ids, forward_batch)
model_input_ids = (
@@ -300,6 +302,7 @@ def cp_shard_model_inputs(
spec_hidden_states_backup = None
if (
spec_hidden_states is not None
and complete_hidden_states is not None
and spec_hidden_states.shape[0] == complete_hidden_states.shape[0]
):
spec_hidden_states_backup = spec_hidden_states
@@ -57,6 +57,7 @@ from sglang.srt.layers.layer_boundary.prepare import (
_attn_tp_slice_update_read,
_dispatch_by_update,
_dp_gather_sum_read,
_move_before_read,
_reduce_update_read,
_reduce_update_read_dp_gather,
_run_entry,
@@ -404,6 +405,36 @@ def _select_entry_step(
raise NotImplementedError(f"{produced=}")
gathered = produced.layout.sharded - need.layout.sharded - need.gathered_by_compute
sliced = need.layout.sharded - produced.layout.sharded
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_CP}:
# A head-parallel mixer consumes full context, but its residual and
# readout coefficients remain on the CP shard. Complete the producer's
# sum onto that shard before the (possibly nonlinear) update/read.
if (
residual != residual_to
or cp_moves is None
or owes not in (None, SumGroup.ATTN_CP)
or (owes is not None and cp_moves.reduce_scatter is None)
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
local = msgspec.structs.replace(
produced, layout=residual_to, group=None, always_partial=False
)
read_local, input_move = _select_entry_step(
local,
residual=residual,
residual_to=residual_to,
need=need,
is_plain_add=is_plain_add,
fusions=(),
residual_joins_sum=False,
cp_moves=cp_moves,
enters_stack=enters_stack,
)
return partial(
_move_before_read,
move=cp_moves.reduce_scatter if owes is not None else cp_moves.take_back,
read=read_local,
), input_move
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
# completes the attention-TP sum and slices in one collective; a
@@ -536,7 +567,7 @@ def _select_entry_step(
return (partial(_tp_sum_with_residual_read, read=read), None)
# A sum over TP completes only on rows every TP rank holds: the TP group
# spans attention DP and CP.
if owes is not SumGroup.ATTN_TP and (
if owes not in (SumGroup.ATTN_TP, SumGroup.ATTN_CP) and (
owes is not SumGroup.TP or produced.layout.sharded
):
raise NotImplementedError(f"{produced=} {need=}")
@@ -551,7 +582,7 @@ def _select_entry_step(
),
None,
)
if owes not in (None, SumGroup.ATTN_TP):
if owes not in (None, SumGroup.ATTN_TP, SumGroup.ATTN_CP):
raise NotImplementedError(f"{produced=} {need=}")
owes_attention_tp = owes is SumGroup.ATTN_TP
# The partial order adds the residual on attention-TP rank 0 before the DP
@@ -579,7 +610,8 @@ def _select_entry_step(
partial(
_reduce_update_read_dp_gather,
gathers_residual=gathers_residual,
reduces_attention_tp=owes_attention_tp,
reduces_attention_tp=owes is not None,
group=owes,
places_cp_shards=places_cp_shards,
read=read,
),
@@ -282,6 +282,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options):
declaration.update.applied_at_exit
and TokenAxis.ATTN_CP
in edge.produced.layout.sharded - edge.need.layout.sharded
and not _cp_gathers_over_attn_cp()
):
raise NotImplementedError("MHC with a gather over attention CP")
attn_input_adapter = None
@@ -209,6 +209,8 @@ class StageDeclaration:
reduction: Whether the next stage's input always completes the sum
(an attention's), or the exit decides (an FFN's or a mixer's).
gathers_attn_tp_input: Whether attention gathers TP-sharded input itself.
tp_group: Group partitioning attention heads. Full TP consumers require
full token rows, including when TP shares the prefill CP group.
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, and the full TP size means TP compute.
exit_rows: Required FFN output rows at the layer or branch exit.
@@ -228,6 +230,7 @@ class StageDeclaration:
output_transform: Optional[OutputTransform] = None
reduction: ProducerReduction = ProducerReduction.EXIT_SCOPED
gathers_attn_tp_input: bool = False
tp_group: SumGroup = SumGroup.ATTN_TP
dense_tp_size: Optional[int] = None
exit_rows: Optional[ExitRows] = None
# Only declarations participate in construction, never executable stages.
@@ -273,6 +276,7 @@ def declare_attn(
reduction=ProducerReduction.ALWAYS_PARTIAL,
gathers_attn_tp_input=True,
output_transform=None,
tp_group=SumGroup.ATTN_TP,
):
"""Declare attention or a mixer; construct its executable boundary later.
@@ -286,6 +290,7 @@ def declare_attn(
complete, before the residual update (a sandwich norm). The next
stage's input runs it, so no fused add + norm takes that input.
Requires ALWAYS_PARTIAL.
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -302,6 +307,7 @@ def declare_attn(
output_transform=output_transform,
reduction=reduction,
gathers_attn_tp_input=gathers_attn_tp_input,
tp_group=tp_group,
)
@@ -375,11 +381,30 @@ def _resolve_stage(stage, variant, following=None):
else frozenset()
)
owes = not sp and axes[TokenAxis.ATTN_TP] > 1
group = SumGroup.ATTN_TP
compute_rows = attention
if stage.tp_group is SumGroup.TP:
parallel = get_parallel()
if parallel.attn_dp_size != 1 or sp or scattered:
raise NotImplementedError(
"full-TP attention requires unscattered input without attention DP"
)
compute_rows = full
owes = parallel.tp_size > 1
# Canonicalize equal groups to the CP transport's reduction contract.
# initialize_model_parallel aliases ATTN_CP to TP at this topology.
group = (
SumGroup.ATTN_CP
if parallel.attn_cp_size == parallel.tp_size and parallel.attn_cp_size > 1
else SumGroup.TP
)
elif stage.tp_group is not SumGroup.ATTN_TP:
raise ValueError("attention heads must be partitioned over ATTN_TP or TP")
declaration = StageContract(
InputContract(attention, gathered_by_compute=gathers, read=stage.read),
InputContract(compute_rows, gathered_by_compute=gathers, read=stage.read),
OutputContract(
local if sp else attention,
group=SumGroup.ATTN_TP if owes else None,
local if sp else compute_rows,
group=group if owes else None,
always_partial=owes
and (
stage.reduction is ProducerReduction.ALWAYS_PARTIAL
@@ -73,6 +73,7 @@ class SumGroup(Enum):
from get_parallel() when the sum runs."""
ATTN_TP = auto()
ATTN_CP = auto()
TP = auto()
# The group one all-reduce of a MoE output runs over.
MOE_OUTPUT = auto()
@@ -145,6 +146,8 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
parallel = get_parallel()
if group is SumGroup.ATTN_TP:
return parallel.attn_tp_group
if group is SumGroup.ATTN_CP:
return parallel.attn_cp_group
if group is SumGroup.TP:
return parallel.tp_group
if group is SumGroup.MOE_OUTPUT:
@@ -43,6 +43,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
from sglang.srt.layers.layer_boundary.layout import (
SumGroup,
_cp_shard_token_rows,
_sum_group,
)
from sglang.srt.layers.layer_boundary.output import (
DeferredFinalize,
@@ -105,6 +106,8 @@ def _reduce_update_read(
)
elif group is SumGroup.TP:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
elif group is not None:
hidden_states = _sum_group(group).all_reduce(hidden_states)
if _is_npu and cache is not None:
_ = prepare_weight_cache(hidden_states, cache)
return read.update_and_read(
@@ -127,6 +130,7 @@ def _reduce_update_read_dp_gather(
gathers_residual: bool,
reduces_attention_tp: bool,
places_cp_shards: bool = False,
group: SumGroup = SumGroup.ATTN_TP,
read: ResidualReadout = NORM_READOUT,
update: ResidualUpdate = PLAIN_ADD,
):
@@ -138,7 +142,11 @@ def _reduce_update_read_dp_gather(
residual = update.gather_residual_attn_tp(residual)
if hidden_states.shape[0] != 0:
if reduces_attention_tp:
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
hidden_states = (
attention_tensor_model_parallel_all_reduce(hidden_states)
if group is SumGroup.ATTN_TP
else _sum_group(group).all_reduce(hidden_states)
)
with use_symmetric_memory(
get_parallel().tp_group,
disabled=not is_allocation_symmetric(),
@@ -417,16 +425,25 @@ def _then_attn_cp_gather(
cache=None,
gather: Callable,
update: ResidualUpdate = PLAIN_ADD,
**call,
):
"""DSA and MLA CP: complete this rank's shard, then gather the shards, of
equal length, over the attention-CP group. The residual stays on the
shard."""
hidden_states, residual = gather(
hidden_states, residual, forward_batch, norm, update=update, cache=cache
hidden_states, residual, forward_batch, norm, update=update, cache=cache, **call
)
return attn_cp_interleave_gather(hidden_states), residual
def _move_before_read(
hidden_states, residual, forward_batch, norm, *, move, read, **call
):
"""Complete and place a producer contribution before its residual update."""
hidden_states, residual = move(hidden_states, residual, forward_batch)
return read(hidden_states, residual, forward_batch, norm, **call)
def _then_moe_cp_gather(
hidden_states: torch.Tensor,
residual: torch.Tensor,
+48 -17
View File
@@ -654,32 +654,22 @@ def _embed_mm_inputs_with_split(
return input_embeds, other_info
def general_mm_embed_routine(
def prepare_mm_inputs(
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
language_model: nn.Module,
kwargs: dict,
multimodal_model: Optional[nn.Module] = None,
data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None,
placeholder_tokens: Optional[dict[Modality, List[int]]] = None,
use_deepstack: Dict[Modality, bool] = {},
feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None,
**kwargs,
) -> torch.Tensor:
"""
Process multimodal inputs and forward through language model.
) -> Optional[torch.Tensor]:
"""Prepare full-sequence embeddings without running the language model.
Args:
input_ids: Input token IDs tensor
forward_batch: Batch information for model forward pass
language_model: Base language model to use
data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function.
placeholder_tokens: Token IDs for multimodal placeholders
use_deepstack: Whether to use deepstack embeddings for each modality, default False
feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup.
**kwargs: Additional arguments passed to language model
Returns:
Hidden states from language model forward pass
Preserve multimodal cache/offload bookkeeping and the MTP embedding copy.
Additional language-model inputs, such as DeepStack features, update kwargs.
Non-first pipeline ranks consume proxy tensors and return no embeddings.
"""
assert hasattr(language_model, "get_input_embeddings")
embed_tokens = language_model.get_input_embeddings()
@@ -808,6 +798,47 @@ def general_mm_embed_routine(
input_embeds = forward_batch.input_embeds
else:
input_embeds = None
return input_embeds
def general_mm_embed_routine(
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
language_model: nn.Module,
multimodal_model: Optional[nn.Module] = None,
data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None,
placeholder_tokens: Optional[dict[Modality, List[int]]] = None,
use_deepstack: Dict[Modality, bool] = {},
feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None,
**kwargs,
) -> torch.Tensor:
"""
Process multimodal inputs and forward through language model.
Args:
input_ids: Input token IDs tensor
forward_batch: Batch information for model forward pass
language_model: Base language model to use
data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function.
placeholder_tokens: Token IDs for multimodal placeholders
use_deepstack: Whether to use deepstack embeddings for each modality, default False
feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup.
**kwargs: Additional arguments passed to language model
Returns:
Hidden states from language model forward pass
"""
input_embeds = prepare_mm_inputs(
input_ids,
forward_batch,
language_model,
kwargs,
multimodal_model,
data_embedding_funcs,
placeholder_tokens,
use_deepstack,
feature_dtypes,
)
with torch.profiler.record_function("sglang.vlm.language_model_prefill"):
hidden_states = language_model(
@@ -370,8 +370,12 @@ class EagerRunner(BaseRunner):
forward_batch,
**kwargs,
)
# Under CP-TP group sharing the model owns the CP row layout.
elif cp_active and not get_parallel().enable_cp_tp_group_sharing:
elif cp_active and (
not get_parallel().enable_cp_tp_group_sharing
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
):
# GLM delegates its CP boundary to this runner via prepare_cp_inputs.
# Other group-sharing models keep owning the layout in forward().
ret = self._execute_extend_cp(forward_batch, kwargs)
else:
ret = model_runner.model.forward(
@@ -390,18 +394,25 @@ class EagerRunner(BaseRunner):
"""
model = self.model_runner.model
input_embeds = kwargs.get("input_embeds")
if input_embeds is None:
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
input_embeds, positions, model_kwargs = prepare_inputs(
forward_batch, **kwargs
)
else:
input_embeds = kwargs.get("input_embeds")
if input_embeds is None:
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
positions = forward_batch.positions
model_kwargs = {}
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
with cp_shard_model_inputs(
input_embeds,
forward_batch.positions,
positions,
forward_batch,
forward_batch.input_ids,
) as (sharded_input_embeds, sharded_positions, model_input_ids):
model_kwargs = {"input_embeds": sharded_input_embeds}
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
model_kwargs["input_embeds"] = sharded_input_embeds
hidden_states = model.model(
model_input_ids,
sharded_positions,
+23 -21
View File
@@ -137,6 +137,28 @@ class DeepseekModelNextN(nn.Module):
self.shared_head = nn.Module()
self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def embed_input_ids(
self, input_ids: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
# MM positions in input_ids hold MM_PAD_SHIFT_VALUE+hash sentinels
# (far above vocab_size). Use target-produced mm_input_embeds for
# these positions and only call embed_tokens on the appended
# next-token to avoid embed OOB.
input_embeds = forward_batch.mm_input_embeds
if (
forward_batch.forward_mode.is_extend()
and forward_batch.contains_mm_inputs()
and not forward_batch.forward_mode.is_draft_extend_v2()
):
assert input_embeds is not None
last_indices = (
forward_batch.extend_start_loc + forward_batch.extend_seq_lens - 1
).long()
input_embeds[last_indices] = self.embed_tokens(input_ids[last_indices])
if input_embeds is None:
input_embeds = self.embed_tokens(input_ids)
return input_embeds
def forward(
self,
input_ids: torch.Tensor,
@@ -168,27 +190,7 @@ class DeepseekModelNextN(nn.Module):
)
if input_embeds is None:
# MM positions in input_ids hold MM_PAD_SHIFT_VALUE+hash sentinels
# (far above vocab_size). Use target-produced mm_input_embeds for
# these positions and only call embed_tokens on the appended
# next-token to avoid embed OOB.
input_embeds = forward_batch.mm_input_embeds
if (
forward_batch.forward_mode.is_extend()
and forward_batch.contains_mm_inputs()
and not forward_batch.forward_mode.is_draft_extend_v2()
):
assert input_embeds is not None
last_indices = (
forward_batch.extend_start_loc
+ forward_batch.extend_seq_lens
- 1
).long()
input_embeds[last_indices] = self.embed_tokens(
input_ids[last_indices]
)
if input_embeds is None:
input_embeds = self.embed_tokens(input_ids)
input_embeds = self.embed_input_ids(input_ids, forward_batch)
hidden_states = input_embeds
if hidden_states.shape[0] > 0:
+85 -23
View File
@@ -30,6 +30,7 @@ from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
PLAIN_RESIDUAL_OPS,
MHCState,
SumGroup,
append_stages,
declare_attn,
declare_ffn,
@@ -68,6 +69,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
general_mm_embed_routine,
prepare_mm_inputs,
)
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
from sglang.srt.model_executor.cuda_graph_config import (
@@ -395,7 +397,13 @@ class Glm5NextLinearAttention(nn.Module):
**kwargs,
) -> None:
super().__init__()
head_shard_size = get_parallel().attn_tp_size
from sglang.srt.runtime_context import (
linear_attn_parallel_group,
linear_attn_tp_size,
)
head_shard_size = linear_attn_tp_size()
head_parallel_group = linear_attn_parallel_group()
self.hidden_size = hidden_size
self.config = config
@@ -434,7 +442,7 @@ class Glm5NextLinearAttention(nn.Module):
self.fg_sizes,
quant_config=None,
prefix=f"{prefix}.fused_qkvbfg_a_proj",
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
self.split_sizes = [
3 * projection_size // head_shard_size,
@@ -446,7 +454,7 @@ class Glm5NextLinearAttention(nn.Module):
self.head_dim,
projection_size,
dtype=self.fused_qkvbfg_a_proj.params_dtype,
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
else:
self.qkv_proj = QKVParallelLinear(
@@ -456,7 +464,7 @@ class Glm5NextLinearAttention(nn.Module):
self.num_k_heads,
bias=False,
quant_config=quant_config,
parallel_group="attn_tp",
parallel_group=head_parallel_group,
prefix=f"{prefix}.qkv_proj",
)
@@ -467,7 +475,7 @@ class Glm5NextLinearAttention(nn.Module):
[self.head_dim, self.head_dim],
quant_config=None,
prefix=f"{prefix}.fused_bfg_a_proj",
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
self.bfg_split_sizes = [self.local_num_heads, 2 * self.head_dim]
self.fused_fg_b_proj = ColumnParallelBatchedLinear(
@@ -475,7 +483,7 @@ class Glm5NextLinearAttention(nn.Module):
self.head_dim,
projection_size,
dtype=self.fused_bfg_a_proj.params_dtype,
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
else:
self.f_a_proj = ReplicatedLinear(
@@ -492,7 +500,7 @@ class Glm5NextLinearAttention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.f_b_proj",
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
self.b_proj = ColumnParallelLinear(
@@ -501,7 +509,7 @@ class Glm5NextLinearAttention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.b_proj",
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
self.g_a_proj = ReplicatedLinear(
@@ -517,7 +525,7 @@ class Glm5NextLinearAttention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.g_b_proj",
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
self.dt_bias = nn.Parameter(
@@ -526,7 +534,11 @@ class Glm5NextLinearAttention(nn.Module):
set_weight_attrs(
self.dt_bias,
{"weight_loader": sharded_weight_loader(0, parallel_group="attn_tp")},
{
"weight_loader": sharded_weight_loader(
0, parallel_group=head_parallel_group
)
},
)
self.qkv_conv1d = MergedColumnParallelLinear(
@@ -535,7 +547,7 @@ class Glm5NextLinearAttention(nn.Module):
bias=False,
params_dtype=torch.float32,
prefix=f"{prefix}.qkv_conv1d",
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
# ColumnParallelLinear's loader cannot reshape conv1d weights, so add the
# singleton dimension after construction.
@@ -546,7 +558,11 @@ class Glm5NextLinearAttention(nn.Module):
)
set_weight_attrs(
self.A_log,
{"weight_loader": sharded_weight_loader(2, parallel_group="attn_tp")},
{
"weight_loader": sharded_weight_loader(
2, parallel_group=head_parallel_group
)
},
)
self.o_norm = FusedRMSNormGated(
@@ -559,7 +575,7 @@ class Glm5NextLinearAttention(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
reduce_results=reduce_results,
parallel_group="attn_tp",
parallel_group=head_parallel_group,
)
conv_weights = self.qkv_conv1d.weight.squeeze(1)
@@ -629,6 +645,14 @@ class Glm5NextLinearAttention(nn.Module):
if forward_batch.forward_mode.is_idle():
return hidden_states
from sglang.srt.layers.cp.interleave import (
cp_interleave_to_sequence_order,
cp_sequence_to_interleave_order,
)
gathered_rows = hidden_states.shape[0]
hidden_states = cp_interleave_to_sequence_order(hidden_states, forward_batch)
if self.do_fuse_qkvbfg:
mixed_qkv, beta, forget_gate, g_proj_states = self.forward_qkvbfg_fused(
hidden_states, forward_batch
@@ -653,7 +677,9 @@ class Glm5NextLinearAttention(nn.Module):
core_attn_out = self.o_norm(core_attn_out, norm_gate)
core_attn_out = core_attn_out.squeeze(0).flatten(-2)
return self.o_proj(core_attn_out)[0]
return cp_sequence_to_interleave_order(
self.o_proj(core_attn_out)[0], forward_batch, gathered_rows
)
class Glm5NextDecoderLayer(nn.Module):
@@ -785,7 +811,13 @@ class Glm5NextDecoderLayer(nn.Module):
).residual_ops()
self.attn_boundary, self.ffn_boundary = append_stages(
(
declare_attn(read=residual.attn_readout, update=residual.attn_update),
declare_attn(
read=residual.attn_readout,
update=residual.attn_update,
tp_group=SumGroup.TP
if self.is_linear_attn and get_parallel().enable_cp_tp_group_sharing
else SumGroup.ATTN_TP,
),
self.input_layernorm,
{
"qkv_latent_func": self.self_attn.prepare_qkv_latent
@@ -939,6 +971,7 @@ class Glm5NextDecoderLayer(nn.Module):
isinstance(self.mlp, Glm5NextMoE)
and not self.mlp.experts.moe_runner_config.inplace
and not torch.compiler.is_compiling()
and hidden_states_orig.shape[0] == hidden_states.shape[0]
):
from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx
@@ -1564,6 +1597,24 @@ class Glm5NextForConditionalGeneration(nn.Module):
video_embeds = self.visual(pixel_values, grid_thw=flattened_video_grid_thw)
return video_embeds
def prepare_cp_inputs(self, forward_batch, **kwargs):
"""Finish multimodal embedding injection before the runner shards tokens."""
input_embeds = kwargs.pop("input_embeds", None)
positions = (
forward_batch.mrope_positions
if self.is_mrope_enabled
else forward_batch.positions
)
if input_embeds is None or forward_batch.contains_mm_inputs():
input_embeds = prepare_mm_inputs(
forward_batch.input_ids,
forward_batch,
self.model,
kwargs,
multimodal_model=self,
)
return input_embeds, positions, kwargs
@torch.no_grad()
def forward(
self,
@@ -1577,14 +1628,25 @@ class Glm5NextForConditionalGeneration(nn.Module):
positions = forward_batch.mrope_positions
with get_attn_tp_context().maybe_input_scattered(forward_batch):
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
multimodal_model=self,
positions=positions,
pp_proxy_tensors=pp_proxy_tensors,
)
if input_embeds is not None and not forward_batch.contains_mm_inputs():
# Preserve input_embeds / replace_embeds supplied by the runner.
# The MM routine otherwise performs a fresh token lookup.
hidden_states = self.model(
input_ids=input_ids,
positions=positions,
forward_batch=forward_batch,
input_embeds=input_embeds,
pp_proxy_tensors=pp_proxy_tensors,
)
else:
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
multimodal_model=self,
positions=positions,
pp_proxy_tensors=pp_proxy_tensors,
)
aux_hidden_states = None
if self.capture_aux_hidden_states:
@@ -22,6 +22,15 @@ logger = logging.getLogger(__name__)
class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN):
def prepare_cp_inputs(self, forward_batch, **kwargs):
# The worker has rotated the target's multimodal embeddings per request.
# Fill each appended token while indices still address full sequences,
# then let the runner shard embeddings and target states together.
input_embeds = self.model.embed_input_ids(
forward_batch.input_ids, forward_batch
)
return input_embeds, forward_batch.positions, {}
@classmethod
def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper:
text_config = getattr(config, "text_config", config)