[Feat] glm5.3flash: Support padded CP BCG and prefix replay with PD/DFlash2 (#43774)
This commit is contained in:
@@ -81,8 +81,19 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
# Prefill only: a decode node serves plain TP.
|
||||
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
|
||||
}
|
||||
else:
|
||||
unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null"
|
||||
elif model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration":
|
||||
if cfg.disaggregation_mode != "null":
|
||||
unsupported |= {
|
||||
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
|
||||
"disaggregation-transfer-backend != mooncake": (
|
||||
cfg.disaggregation_transfer_backend != "mooncake"
|
||||
),
|
||||
"speculative-algorithm != DFLASH": cfg.speculative_algorithm
|
||||
!= "DFLASH",
|
||||
"pp-size > 1": cfg.pp_size > 1,
|
||||
"dp-size > 1": cfg.dp_size > 1,
|
||||
"ep-size > 1": view.ep_size > 1,
|
||||
}
|
||||
for flag, enabled in unsupported.items():
|
||||
if enabled:
|
||||
raise ValueError(f"--{flag} is not supported with CP-TP group sharing.")
|
||||
|
||||
@@ -77,6 +77,7 @@ from sglang.srt.runtime_context import (
|
||||
get_memory,
|
||||
get_observability,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.network import NetworkAddress
|
||||
@@ -951,6 +952,20 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
a reason rejects the peer, ``fully_checked`` retires the per-transfer
|
||||
check.
|
||||
"""
|
||||
if (
|
||||
get_spec().speculative_algorithm == "DFLASH"
|
||||
# Bootstrap runs concurrently with temporary draft TP scopes.
|
||||
# Use the manager's captured target layout, not scoped CP flags.
|
||||
and self.state_tp_size != self.attn_tp_size
|
||||
and registration_info.dst_attn_tp_size != self.state_tp_size
|
||||
):
|
||||
# Appended draft MHA KV uses physical TP shards, while target MLA
|
||||
# KV is replicated. The draft transfer cannot reslice those shards.
|
||||
return False, (
|
||||
"CP/TP-sharing DFLASH PD requires the same physical TP size: "
|
||||
f"prefill={self.state_tp_size}, "
|
||||
f"decode={registration_info.dst_attn_tp_size}"
|
||||
)
|
||||
state_types = getattr(self.kv_args, "state_types", [])
|
||||
fully_checked = True
|
||||
for i, st in enumerate(state_types):
|
||||
|
||||
@@ -3958,6 +3958,10 @@ class DeepseekSparseAttnBackend(
|
||||
kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim)
|
||||
block_tables = page_table_1.unsqueeze(1)
|
||||
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
|
||||
if is_prefill:
|
||||
# CP graph metadata includes the physical shard's padding; its
|
||||
# eager attention bridge can narrow Q to the live local rows.
|
||||
seq_lens = seq_lens[:batch_size]
|
||||
|
||||
attention_kwargs = dict(
|
||||
kv_cache=kv,
|
||||
|
||||
@@ -32,8 +32,23 @@ def _zero_padded_tokens(output, actual_tokens):
|
||||
output[actual_tokens:].zero_()
|
||||
|
||||
|
||||
def attention_query_tokens(forward_batch, tokens):
|
||||
# Import at call time: the CP package also imports RadixAttention.
|
||||
from sglang.srt.layers.cp.padding import attention_query_tokens as cp_query_tokens
|
||||
|
||||
return cp_query_tokens(forward_batch, tokens)
|
||||
|
||||
|
||||
def attention_output_tokens(forward_batch, tokens):
|
||||
from sglang.srt.layers.cp.padding import attention_output_tokens as cp_output_tokens
|
||||
|
||||
return cp_output_tokens(forward_batch, tokens)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs):
|
||||
def attention_input_scope(
|
||||
forward_batch, output, query_tokens, kv_tokens, kwargs, *, cache_tokens=None
|
||||
):
|
||||
"""Narrow per-token operands and restore the caller's batch even on failure."""
|
||||
kwargs = dict(kwargs)
|
||||
for name in (
|
||||
@@ -56,9 +71,11 @@ def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs
|
||||
]
|
||||
cache_loc, positions = forward_batch.out_cache_loc, forward_batch.positions
|
||||
previous_output = forward_batch._attn_output
|
||||
forward_batch.out_cache_loc = cache_loc[:query_tokens]
|
||||
if cache_tokens is None:
|
||||
cache_tokens = query_tokens
|
||||
forward_batch.out_cache_loc = cache_loc[:cache_tokens]
|
||||
if positions is not None:
|
||||
forward_batch.positions = positions[:query_tokens]
|
||||
forward_batch.positions = positions[:cache_tokens]
|
||||
forward_batch._attn_output = output[:query_tokens]
|
||||
try:
|
||||
yield kwargs
|
||||
|
||||
@@ -25,6 +25,7 @@ The strategy implementation is split across:
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple
|
||||
@@ -247,6 +248,22 @@ def _is_dsa_active() -> bool:
|
||||
_STRATEGY: Optional[ContextParallelStrategy] = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def disable_cp_strategy():
|
||||
"""Suspend the cached target strategy while a TP-only draft runs.
|
||||
|
||||
Pair with a runtime topology scope that disables prefill CP so lazy lookup
|
||||
cannot reconstruct the target strategy. Restore it even if draft work fails.
|
||||
"""
|
||||
global _STRATEGY
|
||||
strategy = _STRATEGY
|
||||
_STRATEGY = None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_STRATEGY = strategy
|
||||
|
||||
|
||||
def init_cp_strategy(
|
||||
*, enable_prefill_cp: bool, cp_size: int, cp_strategy: str
|
||||
) -> None:
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash.
|
||||
"""Interleave BCG adapter for GLM-5.3 Flash text prefills.
|
||||
|
||||
Keep full-sequence KPool positions separate from the local DSA positions.
|
||||
Prefix-free text batches replay when their total token count matches a capture,
|
||||
independently of request boundaries. Other batches retain eager CP, including MM.
|
||||
Graph row capacity is fixed while live token counts and prefix state can change.
|
||||
"""
|
||||
|
||||
import torch
|
||||
@@ -27,16 +26,14 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
|
||||
result.input_ids_global = torch.zeros(
|
||||
capacity, dtype=torch.int64, device=runner.device
|
||||
)
|
||||
# Graphs must read the live count from the same address on each replay.
|
||||
result.moe_local_token_count = torch.zeros(
|
||||
(), dtype=torch.int32, device=runner.device
|
||||
)
|
||||
return result
|
||||
|
||||
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
|
||||
return (
|
||||
batch_size > 0
|
||||
and num_tokens in self.bucket_local_tokens
|
||||
and not contains_mm_inputs
|
||||
and prefix_lens is not None
|
||||
and not any(prefix_lens)
|
||||
)
|
||||
return batch_size > 0 and not contains_mm_inputs and prefix_lens is not None
|
||||
|
||||
@staticmethod
|
||||
def supports(server_args):
|
||||
@@ -63,17 +60,21 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
|
||||
runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture
|
||||
)
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
cp_size = len(metadata.per_rank_actual_token)
|
||||
base, remainder = divmod(static_num_tokens, cp_size)
|
||||
metadata.total_seq_lens = static_num_tokens
|
||||
metadata.per_rank_logical_token = [
|
||||
base + int(rank < remainder) for rank in range(cp_size)
|
||||
]
|
||||
# KDA projections/reorders retain the captured shape. Eager KPool
|
||||
# compression, KV writes and the output gather use the live lengths
|
||||
# produced by prepare_cp_forward, never the bucket's padded tail.
|
||||
metadata.graph_num_tokens = static_num_tokens
|
||||
self.moe_local_token_count.fill_(
|
||||
metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
|
||||
)
|
||||
metadata.moe_local_token_count = self.moe_local_token_count
|
||||
forward_batch.out_cache_loc = out_cache_loc
|
||||
# KPool compresses consecutive global tokens; only the model argument
|
||||
# (DSA RoPE) is local. Do not replace ForwardBatch's global positions.
|
||||
forward_batch.positions = global_positions
|
||||
ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch)
|
||||
ids = cp_interleave_input_ids(
|
||||
forward_batch.input_ids[: forward_batch.extend_num_tokens], forward_batch
|
||||
)
|
||||
self.input_ids_global[: ids.shape[0]].copy_(ids)
|
||||
forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]]
|
||||
|
||||
|
||||
@@ -76,6 +76,9 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
|
||||
gather_index: Optional[torch.Tensor] = None
|
||||
local_index: Optional[torch.Tensor] = None
|
||||
moe_local_token_count: Optional[torch.Tensor] = None
|
||||
# Only sequence-dependent graph segments use this capacity. Eager cache
|
||||
# operations and gathers must continue using total_seq_lens (live tokens).
|
||||
graph_num_tokens: Optional[int] = None
|
||||
|
||||
|
||||
def is_interleave_extend(forward_batch):
|
||||
@@ -92,7 +95,9 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
|
||||
``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute.
|
||||
|
||||
The residual and mHC coefficients do not move: only the already-read mixer
|
||||
input is reordered. Remove physical CP padding before the recurrent kernel.
|
||||
input is reordered. Eager execution removes physical CP padding. Graph
|
||||
execution keeps the capture shape; KDA's refreshed sequence table masks
|
||||
padded rows out of its recurrent state updates.
|
||||
"""
|
||||
if not is_interleave_extend(forward_batch):
|
||||
return hidden_states
|
||||
@@ -101,10 +106,15 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
|
||||
return hidden_states.index_select(0, metadata.gather_index)
|
||||
size = get_parallel().attn_cp_size
|
||||
rows = hidden_states.shape[0] // size
|
||||
num_tokens = (
|
||||
metadata.graph_num_tokens
|
||||
if metadata.graph_num_tokens is not None
|
||||
else metadata.total_seq_lens
|
||||
)
|
||||
return (
|
||||
hidden_states.reshape(size, rows, *hidden_states.shape[1:])
|
||||
.transpose(0, 1)
|
||||
.flatten(0, 1)[: metadata.total_seq_lens]
|
||||
.flatten(0, 1)[:num_tokens]
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
@@ -60,3 +60,22 @@ def pad_local_rows(x: torch.Tensor, metadata: Any, dim: int) -> torch.Tensor:
|
||||
pad_shape = list(x.shape)
|
||||
pad_shape[dim] = pad_size
|
||||
return torch.cat([x, x.new_zeros(pad_shape)], dim=dim)
|
||||
|
||||
|
||||
def attention_query_tokens(forward_batch, n):
|
||||
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
|
||||
if getattr(cp_metadata, "graph_num_tokens", None) is not None:
|
||||
# Interleave BCG gathers KV in global sequence order but Q stays
|
||||
# local. Empty padded queries must not enter sparse attention.
|
||||
n = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
|
||||
return n
|
||||
|
||||
|
||||
def attention_output_tokens(forward_batch, raw):
|
||||
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
|
||||
if cp_metadata is not None and cp_metadata.per_rank_logical_token is not None:
|
||||
# Q/output rows are CP-local; cache locations and positions
|
||||
# above remain global. Zero every invalid local row before
|
||||
# captured projections/MoE can consume an empty-attention NaN.
|
||||
raw = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
|
||||
return raw
|
||||
|
||||
@@ -26,6 +26,8 @@ from sglang.srt.layers.attention.graph_utils import (
|
||||
_zero_padded_tokens,
|
||||
allocate_attention_outputs,
|
||||
attention_input_scope,
|
||||
attention_output_tokens,
|
||||
attention_query_tokens,
|
||||
padded_extend_real_tokens,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
@@ -190,6 +192,8 @@ class RadixAttention(nn.Module):
|
||||
rows = output.shape[0]
|
||||
n = forward_batch.global_num_token_non_padded_cpu
|
||||
kv_n = n if key_value_num_tokens is None else key_value_num_tokens
|
||||
cache_n = n
|
||||
n = attention_query_tokens(forward_batch, n)
|
||||
return_lse = bool(kwargs.get("return_lse") or forward_batch.mha_return_lse)
|
||||
lse = None
|
||||
if n == 0:
|
||||
@@ -200,7 +204,7 @@ class RadixAttention(nn.Module):
|
||||
lse = q.new_zeros((rows, q.shape[1]), dtype=torch.float32)
|
||||
else:
|
||||
with attention_input_scope(
|
||||
forward_batch, output, n, kv_n, kwargs
|
||||
forward_batch, output, n, kv_n, kwargs, cache_tokens=cache_n
|
||||
) as kwargs:
|
||||
result = get_attn_backend().forward(
|
||||
q[:n],
|
||||
@@ -220,6 +224,7 @@ class RadixAttention(nn.Module):
|
||||
if result.data_ptr() != output.data_ptr():
|
||||
output[:n].view(result.shape).copy_(result)
|
||||
raw = get_prefill_raw_num_tokens()
|
||||
raw = attention_output_tokens(forward_batch, raw)
|
||||
_zero_padded_tokens(output, raw)
|
||||
if idx_output is not None:
|
||||
_zero_padded_tokens(idx_output, raw)
|
||||
|
||||
@@ -1985,6 +1985,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
mamba_track_indices=mamba_track_indices,
|
||||
mamba_track_mask=mamba_track_mask,
|
||||
mamba_track_seqlens=mamba_track_seqlens,
|
||||
# CP replay plans attention against this static view, including
|
||||
# KDA's host-side checkpoint plan for the current request.
|
||||
mamba_prefill_track_mask_cpu=forward_batch.mamba_prefill_track_mask_cpu,
|
||||
mamba_track_seqlens_cpu=forward_batch.mamba_track_seqlens_cpu,
|
||||
encoder_lens=forward_batch.encoder_lens,
|
||||
return_logprob=(
|
||||
forward_batch.return_logprob
|
||||
|
||||
@@ -103,7 +103,7 @@ from sglang.srt.speculative.spec_utils import (
|
||||
draft_pp_context,
|
||||
draft_tp_context,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.srt.utils import async_h2d, is_cuda, is_hip, is_npu, is_xpu
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
@@ -2375,8 +2375,10 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# Materialize prompt tokens into the draft KV cache immediately. This is required
|
||||
# for radix cache safety (the scheduler may update radix after prefill returns).
|
||||
device = next_token_ids.device
|
||||
ctx_lens = torch.tensor(batch.extend_lens, dtype=torch.int32, device=device)
|
||||
draft_seq_lens = torch.tensor(
|
||||
# Enqueue metadata copies without waiting for the target prefill's
|
||||
# outstanding GPU work; the same stream orders their consumers.
|
||||
ctx_lens = async_h2d(batch.extend_lens, dtype=torch.int32, device=device)
|
||||
draft_seq_lens = async_h2d(
|
||||
batch.prefix_lens, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.distributed.parallel_state import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.layers.cp.base import disable_cp_strategy
|
||||
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
|
||||
from sglang.srt.managers.utils import _async_d2h
|
||||
from sglang.srt.mem_cache.allocation import (
|
||||
@@ -819,8 +820,24 @@ def draft_tp_context(owns_attention: bool):
|
||||
"""
|
||||
if not owns_attention:
|
||||
return contextlib.nullcontext()
|
||||
group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group")
|
||||
return patch_tensor_parallel_group(group, owns_attention=True)
|
||||
parallel = get_parallel()
|
||||
group = getattr(parallel, f"{draft_attn_parallel_group()}_group")
|
||||
tp_context = patch_tensor_parallel_group(group, owns_attention=True)
|
||||
if parallel.enable_prefill_cp:
|
||||
return _draft_without_cp(tp_context)
|
||||
return tp_context
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _draft_without_cp(tp_context):
|
||||
with (
|
||||
tp_context,
|
||||
get_parallel().override(
|
||||
enable_prefill_cp=False, enable_cp_tp_group_sharing=False
|
||||
),
|
||||
disable_cp_strategy(),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def spec_stage_span(name: str):
|
||||
|
||||
Reference in New Issue
Block a user