[Feat] glm5.3flash: Support padded CP BCG and prefix replay with PD/DFlash2 (#43774)

This commit is contained in:
Baizhou Zhang
2026-10-23 14:21:31 -07:00
committed by GitHub
parent 5f7542d762
commit ea226cf782
17 changed files with 398 additions and 195 deletions
@@ -81,8 +81,19 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
# Prefill only: a decode node serves plain TP.
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
}
else:
unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null"
elif model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration":
if cfg.disaggregation_mode != "null":
unsupported |= {
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
"disaggregation-transfer-backend != mooncake": (
cfg.disaggregation_transfer_backend != "mooncake"
),
"speculative-algorithm != DFLASH": cfg.speculative_algorithm
!= "DFLASH",
"pp-size > 1": cfg.pp_size > 1,
"dp-size > 1": cfg.dp_size > 1,
"ep-size > 1": view.ep_size > 1,
}
for flag, enabled in unsupported.items():
if enabled:
raise ValueError(f"--{flag} is not supported with CP-TP group sharing.")
@@ -77,6 +77,7 @@ from sglang.srt.runtime_context import (
get_memory,
get_observability,
get_schedule,
get_spec,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.network import NetworkAddress
@@ -951,6 +952,20 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
a reason rejects the peer, ``fully_checked`` retires the per-transfer
check.
"""
if (
get_spec().speculative_algorithm == "DFLASH"
# Bootstrap runs concurrently with temporary draft TP scopes.
# Use the manager's captured target layout, not scoped CP flags.
and self.state_tp_size != self.attn_tp_size
and registration_info.dst_attn_tp_size != self.state_tp_size
):
# Appended draft MHA KV uses physical TP shards, while target MLA
# KV is replicated. The draft transfer cannot reslice those shards.
return False, (
"CP/TP-sharing DFLASH PD requires the same physical TP size: "
f"prefill={self.state_tp_size}, "
f"decode={registration_info.dst_attn_tp_size}"
)
state_types = getattr(self.kv_args, "state_types", [])
fully_checked = True
for i, st in enumerate(state_types):
@@ -3958,6 +3958,10 @@ class DeepseekSparseAttnBackend(
kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim)
block_tables = page_table_1.unsqueeze(1)
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
if is_prefill:
# CP graph metadata includes the physical shard's padding; its
# eager attention bridge can narrow Q to the live local rows.
seq_lens = seq_lens[:batch_size]
attention_kwargs = dict(
kv_cache=kv,
@@ -32,8 +32,23 @@ def _zero_padded_tokens(output, actual_tokens):
output[actual_tokens:].zero_()
def attention_query_tokens(forward_batch, tokens):
# Import at call time: the CP package also imports RadixAttention.
from sglang.srt.layers.cp.padding import attention_query_tokens as cp_query_tokens
return cp_query_tokens(forward_batch, tokens)
def attention_output_tokens(forward_batch, tokens):
from sglang.srt.layers.cp.padding import attention_output_tokens as cp_output_tokens
return cp_output_tokens(forward_batch, tokens)
@contextmanager
def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs):
def attention_input_scope(
forward_batch, output, query_tokens, kv_tokens, kwargs, *, cache_tokens=None
):
"""Narrow per-token operands and restore the caller's batch even on failure."""
kwargs = dict(kwargs)
for name in (
@@ -56,9 +71,11 @@ def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs
]
cache_loc, positions = forward_batch.out_cache_loc, forward_batch.positions
previous_output = forward_batch._attn_output
forward_batch.out_cache_loc = cache_loc[:query_tokens]
if cache_tokens is None:
cache_tokens = query_tokens
forward_batch.out_cache_loc = cache_loc[:cache_tokens]
if positions is not None:
forward_batch.positions = positions[:query_tokens]
forward_batch.positions = positions[:cache_tokens]
forward_batch._attn_output = output[:query_tokens]
try:
yield kwargs
+17
View File
@@ -25,6 +25,7 @@ The strategy implementation is split across:
from __future__ import annotations
from abc import ABC, abstractmethod
from contextlib import contextmanager
from dataclasses import dataclass
from enum import IntEnum
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple
@@ -247,6 +248,22 @@ def _is_dsa_active() -> bool:
_STRATEGY: Optional[ContextParallelStrategy] = None
@contextmanager
def disable_cp_strategy():
"""Suspend the cached target strategy while a TP-only draft runs.
Pair with a runtime topology scope that disables prefill CP so lazy lookup
cannot reconstruct the target strategy. Restore it even if draft work fails.
"""
global _STRATEGY
strategy = _STRATEGY
_STRATEGY = None
try:
yield
finally:
_STRATEGY = strategy
def init_cp_strategy(
*, enable_prefill_cp: bool, cp_size: int, cp_strategy: str
) -> None:
@@ -1,8 +1,7 @@
"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash.
"""Interleave BCG adapter for GLM-5.3 Flash text prefills.
Keep full-sequence KPool positions separate from the local DSA positions.
Prefix-free text batches replay when their total token count matches a capture,
independently of request boundaries. Other batches retain eager CP, including MM.
Graph row capacity is fixed while live token counts and prefix state can change.
"""
import torch
@@ -27,16 +26,14 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
result.input_ids_global = torch.zeros(
capacity, dtype=torch.int64, device=runner.device
)
# Graphs must read the live count from the same address on each replay.
result.moe_local_token_count = torch.zeros(
(), dtype=torch.int32, device=runner.device
)
return result
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
return (
batch_size > 0
and num_tokens in self.bucket_local_tokens
and not contains_mm_inputs
and prefix_lens is not None
and not any(prefix_lens)
)
return batch_size > 0 and not contains_mm_inputs and prefix_lens is not None
@staticmethod
def supports(server_args):
@@ -63,17 +60,21 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture
)
metadata = forward_batch.attn_cp_metadata
cp_size = len(metadata.per_rank_actual_token)
base, remainder = divmod(static_num_tokens, cp_size)
metadata.total_seq_lens = static_num_tokens
metadata.per_rank_logical_token = [
base + int(rank < remainder) for rank in range(cp_size)
]
# KDA projections/reorders retain the captured shape. Eager KPool
# compression, KV writes and the output gather use the live lengths
# produced by prepare_cp_forward, never the bucket's padded tail.
metadata.graph_num_tokens = static_num_tokens
self.moe_local_token_count.fill_(
metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
)
metadata.moe_local_token_count = self.moe_local_token_count
forward_batch.out_cache_loc = out_cache_loc
# KPool compresses consecutive global tokens; only the model argument
# (DSA RoPE) is local. Do not replace ForwardBatch's global positions.
forward_batch.positions = global_positions
ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch)
ids = cp_interleave_input_ids(
forward_batch.input_ids[: forward_batch.extend_num_tokens], forward_batch
)
self.input_ids_global[: ids.shape[0]].copy_(ids)
forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]]
+12 -2
View File
@@ -76,6 +76,9 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
gather_index: Optional[torch.Tensor] = None
local_index: Optional[torch.Tensor] = None
moe_local_token_count: Optional[torch.Tensor] = None
# Only sequence-dependent graph segments use this capacity. Eager cache
# operations and gathers must continue using total_seq_lens (live tokens).
graph_num_tokens: Optional[int] = None
def is_interleave_extend(forward_batch):
@@ -92,7 +95,9 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute.
The residual and mHC coefficients do not move: only the already-read mixer
input is reordered. Remove physical CP padding before the recurrent kernel.
input is reordered. Eager execution removes physical CP padding. Graph
execution keeps the capture shape; KDA's refreshed sequence table masks
padded rows out of its recurrent state updates.
"""
if not is_interleave_extend(forward_batch):
return hidden_states
@@ -101,10 +106,15 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
return hidden_states.index_select(0, metadata.gather_index)
size = get_parallel().attn_cp_size
rows = hidden_states.shape[0] // size
num_tokens = (
metadata.graph_num_tokens
if metadata.graph_num_tokens is not None
else metadata.total_seq_lens
)
return (
hidden_states.reshape(size, rows, *hidden_states.shape[1:])
.transpose(0, 1)
.flatten(0, 1)[: metadata.total_seq_lens]
.flatten(0, 1)[:num_tokens]
.contiguous()
)
+19
View File
@@ -60,3 +60,22 @@ def pad_local_rows(x: torch.Tensor, metadata: Any, dim: int) -> torch.Tensor:
pad_shape = list(x.shape)
pad_shape[dim] = pad_size
return torch.cat([x, x.new_zeros(pad_shape)], dim=dim)
def attention_query_tokens(forward_batch, n):
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
if getattr(cp_metadata, "graph_num_tokens", None) is not None:
# Interleave BCG gathers KV in global sequence order but Q stays
# local. Empty padded queries must not enter sparse attention.
n = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
return n
def attention_output_tokens(forward_batch, raw):
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
if cp_metadata is not None and cp_metadata.per_rank_logical_token is not None:
# Q/output rows are CP-local; cache locations and positions
# above remain global. Zero every invalid local row before
# captured projections/MoE can consume an empty-attention NaN.
raw = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
return raw
+6 -1
View File
@@ -26,6 +26,8 @@ from sglang.srt.layers.attention.graph_utils import (
_zero_padded_tokens,
allocate_attention_outputs,
attention_input_scope,
attention_output_tokens,
attention_query_tokens,
padded_extend_real_tokens,
)
from sglang.srt.model_executor.forward_context import (
@@ -190,6 +192,8 @@ class RadixAttention(nn.Module):
rows = output.shape[0]
n = forward_batch.global_num_token_non_padded_cpu
kv_n = n if key_value_num_tokens is None else key_value_num_tokens
cache_n = n
n = attention_query_tokens(forward_batch, n)
return_lse = bool(kwargs.get("return_lse") or forward_batch.mha_return_lse)
lse = None
if n == 0:
@@ -200,7 +204,7 @@ class RadixAttention(nn.Module):
lse = q.new_zeros((rows, q.shape[1]), dtype=torch.float32)
else:
with attention_input_scope(
forward_batch, output, n, kv_n, kwargs
forward_batch, output, n, kv_n, kwargs, cache_tokens=cache_n
) as kwargs:
result = get_attn_backend().forward(
q[:n],
@@ -220,6 +224,7 @@ class RadixAttention(nn.Module):
if result.data_ptr() != output.data_ptr():
output[:n].view(result.shape).copy_(result)
raw = get_prefill_raw_num_tokens()
raw = attention_output_tokens(forward_batch, raw)
_zero_padded_tokens(output, raw)
if idx_output is not None:
_zero_padded_tokens(idx_output, raw)
@@ -1985,6 +1985,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
mamba_track_indices=mamba_track_indices,
mamba_track_mask=mamba_track_mask,
mamba_track_seqlens=mamba_track_seqlens,
# CP replay plans attention against this static view, including
# KDA's host-side checkpoint plan for the current request.
mamba_prefill_track_mask_cpu=forward_batch.mamba_prefill_track_mask_cpu,
mamba_track_seqlens_cpu=forward_batch.mamba_track_seqlens_cpu,
encoder_lens=forward_batch.encoder_lens,
return_logprob=(
forward_batch.return_logprob
@@ -103,7 +103,7 @@ from sglang.srt.speculative.spec_utils import (
draft_pp_context,
draft_tp_context,
)
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
from sglang.srt.utils import async_h2d, is_cuda, is_hip, is_npu, is_xpu
_is_npu = is_npu()
@@ -2375,8 +2375,10 @@ class DFlashWorkerV2(BaseSpecWorker):
# Materialize prompt tokens into the draft KV cache immediately. This is required
# for radix cache safety (the scheduler may update radix after prefill returns).
device = next_token_ids.device
ctx_lens = torch.tensor(batch.extend_lens, dtype=torch.int32, device=device)
draft_seq_lens = torch.tensor(
# Enqueue metadata copies without waiting for the target prefill's
# outstanding GPU work; the same stream orders their consumers.
ctx_lens = async_h2d(batch.extend_lens, dtype=torch.int32, device=device)
draft_seq_lens = async_h2d(
batch.prefix_lens, dtype=torch.int32, device=device
)
+19 -2
View File
@@ -34,6 +34,7 @@ from sglang.srt.distributed.parallel_state import (
)
from sglang.srt.environ import envs
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.layers.cp.base import disable_cp_strategy
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
from sglang.srt.managers.utils import _async_d2h
from sglang.srt.mem_cache.allocation import (
@@ -819,8 +820,24 @@ def draft_tp_context(owns_attention: bool):
"""
if not owns_attention:
return contextlib.nullcontext()
group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group")
return patch_tensor_parallel_group(group, owns_attention=True)
parallel = get_parallel()
group = getattr(parallel, f"{draft_attn_parallel_group()}_group")
tp_context = patch_tensor_parallel_group(group, owns_attention=True)
if parallel.enable_prefill_cp:
return _draft_without_cp(tp_context)
return tp_context
@contextmanager
def _draft_without_cp(tp_context):
with (
tp_context,
get_parallel().override(
enable_prefill_cp=False, enable_cp_tp_group_sharing=False
),
disable_cp_strategy(),
):
yield
def spec_stage_span(name: str):