[BCG][DSV4-Flash][CP] Support DeepSeek V4 prefill with interleave CP and breakable CUDA graphs (#38603)

This commit is contained in:
BingjiaWang
2026-10-12 01:03:10 -07:00
committed by GitHub
parent 8d491fbc3e
commit 54409c2fa0
4 changed files with 520 additions and 49 deletions
@@ -1394,6 +1394,10 @@ class DeepseekV4AttnBackend(
(0, padded_num_tokens - swa_replay_start.shape[0]),
)
num_write_tokens = (
out_cache_loc.shape[0] if use_prefill_cuda_graph else num_tokens
)
seq_lens_casual, req_pool_indices_repeated = self.expand_prefill_casually(
num_tokens=num_tokens,
seq_lens=seq_lens_cpu,
@@ -1414,13 +1418,13 @@ class DeepseekV4AttnBackend(
is_prefill=True,
dspark_block_size=dspark_block_size,
dspark_swa_buffers=dspark_swa_buffers,
num_tokens=num_tokens if cp_active else None,
num_tokens=num_write_tokens if cp_active else None,
swa_replay_start=swa_replay_start,
num_groups=len(extend_seq_lens_cpu),
)
if cp_active:
core_attn_metadata.apply_cp_reindex(
num_tokens=num_tokens, local_index=cp_metadata.local_index
num_tokens=num_write_tokens, local_index=cp_metadata.local_index
)
core_attn_metadata.init_flashmla_related(is_prefill=True)
indexer_metadata = (
@@ -2364,6 +2368,19 @@ class DeepseekV4AttnBackend(
self.forward_metadata.core_attn_metadata.request_window_layout
)
def _use_sparse_prefill(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> bool:
return (
forward_batch.forward_mode.is_extend_without_speculative()
and not is_cp_active(forward_batch)
and not get_platform().is_sm120
and (
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
)
)
def prepare_prefill_shared_read_snapshot(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> None:
@@ -2386,15 +2403,9 @@ class DeepseekV4AttnBackend(
assert isinstance(metadata, DSV4Metadata)
# The tail never takes the sparse path, so it carries no chunk cache.
use_sparse_prefill = (
not get_platform().is_sm120
and metadata.late_layer_tail is None
and (
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
)
)
if use_sparse_prefill:
if metadata.late_layer_tail is None and self._use_sparse_prefill(
forward_batch, num_qo_tokens=num_qo_tokens
):
metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache(
forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
)
@@ -2413,14 +2424,21 @@ class DeepseekV4AttnBackend(
assert seq_lens_cpu is not None
# Decoder tails require explicit SWA indices to exclude unwritten history.
tail = self.forward_metadata.late_layer_tail
request_layout = core_attn_metadata.request_window_layout
assert tail is None or request_layout is not None
extend_seq_lens = forward_batch.extend_seq_lens
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
if tail is not None:
extend_seq_lens = tail.extend_seq_lens
extend_seq_lens_cpu = tail.extend_seq_lens_cpu
assert extend_seq_lens_cpu is not None
num_global_queries = sum(extend_seq_lens_cpu)
assert num_qo_tokens >= num_global_queries, (
"DSV4 sparse prefill requires global query rows: "
f"allocated {num_qo_tokens}, but request offsets cover {num_global_queries}. "
"CP-local queries must use paged attention."
)
# Reject undersized query buffers before accessing attention metadata.
request_layout = core_attn_metadata.request_window_layout
assert tail is None or request_layout is not None
seq_lens_cpu_list = seq_lens_cpu.tolist()
total_swa = sum(
min(int(seq_len), int(extend_len) + SWA_WINDOW - 1)
@@ -3480,24 +3498,12 @@ class DeepseekV4AttnBackend(
f"{extra_indices.shape=}'s last dimension is not aligned to 64"
)
# RequestWindow sparse gathering does not support CP yet.
# The tail stays dense: its window floor lives in swa_page_indices,
# which the sparse chunk cache ignores.
if (
forward_batch.forward_mode.is_extend_without_speculative()
and not get_platform().is_sm120
and (
(
token_to_kv_pool.request_window is None
and self.forward_metadata.late_layer_tail is None
)
or (
token_to_kv_pool.request_window is not None
and not is_cp_active(forward_batch)
)
)
and (
q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
)
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
and self.forward_metadata.late_layer_tail is None
and token_to_kv_pool.request_window is None
):
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
return self._forward_prefill_sparse_q8kv8(
+65 -19
View File
@@ -27,6 +27,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
from sglang.srt.layers.cp.padding import get_cp_padding_align_size
from sglang.srt.layers.cp.utils import (
cp_gather_after_forward,
@@ -54,8 +55,10 @@ def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
cfg.enable_prefill_cp
and cfg.pp_size == 1
and resolved.attn_cp_size == cfg.tp_size
and cfg.cp_strategy == "zigzag"
and prefill_attention_backend == "trtllm_mha"
and (
(cfg.cp_strategy == "zigzag" and prefill_attention_backend == "trtllm_mha")
or (cfg.cp_strategy == "interleave" and prefill_attention_backend == "dsv4")
)
)
@@ -67,8 +70,10 @@ def enable_cp_bcg_capture(server_args: ServerArgs) -> bool:
def filter_prefill_cp_bcg_capture_num_tokens(
capture_num_tokens: list[int], server_args: ServerArgs
) -> list[int]:
"""Keep only token buckets where the zigzag CP strategy can run."""
min_num_tokens = resolved_view(server_args).attn_cp_size * 2
"""Keep only token buckets where the selected CP strategy can run."""
min_num_tokens = resolved_view(server_args).attn_cp_size
if resolving_view(server_args).cp_strategy == "zigzag":
min_num_tokens *= 2
filtered = [size for size in capture_num_tokens if size >= min_num_tokens]
if not filtered:
raise ValueError(
@@ -117,24 +122,30 @@ class PrefillCPBCGInput:
)
def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]:
"""Return the aligned CP-local rows required by a live zigzag layout."""
"""Return the largest aligned local shard, uniformly across CP ranks."""
strategy = get_cp_strategy()
if not isinstance(strategy, ZigzagCPStrategy) or extend_seq_lens is None:
if extend_seq_lens is None:
return None
if isinstance(strategy, InterleaveCPStrategy):
num_tokens = sum(int(length) for length in extend_seq_lens)
max_local_tokens = (num_tokens + strategy.cp_size - 1) // strategy.cp_size
elif isinstance(strategy, ZigzagCPStrategy):
cp_segment_num = strategy.cp_size * 2
per_rank_logical_tokens = [0] * strategy.cp_size
for raw_length in extend_seq_lens:
base, remainder = divmod(int(raw_length), cp_segment_num)
for rank in range(strategy.cp_size):
opposite_rank = cp_segment_num - 1 - rank
per_rank_logical_tokens[rank] += (
base * 2
+ int(rank < remainder)
+ int(opposite_rank < remainder)
)
max_local_tokens = max(per_rank_logical_tokens)
else:
return None
cp_segment_num = strategy.cp_size * 2
per_rank_logical_tokens = [0] * strategy.cp_size
for raw_length in extend_seq_lens:
base, remainder = divmod(int(raw_length), cp_segment_num)
for rank in range(strategy.cp_size):
opposite_rank = cp_segment_num - 1 - rank
per_rank_logical_tokens[rank] += (
base * 2 + int(rank < remainder) + int(opposite_rank < remainder)
)
align_size = get_cp_padding_align_size()
return (
(max(per_rank_logical_tokens) + align_size - 1) // align_size * align_size
)
return (max_local_tokens + align_size - 1) // align_size * align_size
def select_replay_bucket(
self,
@@ -189,6 +200,7 @@ class PrefillCPBCGInput:
# Replay batches may reuse a ForwardBatch object whose metadata was
# built for a different request layout. Always rebuild before sharding.
forward_batch.attn_cp_metadata = None
out_cache_loc = forward_batch.out_cache_loc
prepare_cp_forward(forward_batch)
captured_local_tokens = None
@@ -257,6 +269,16 @@ class PrefillCPBCGInput:
forward_batch.positions = positions
self.live_local_tokens = live_local_tokens
if isinstance(get_cp_strategy(), InterleaveCPStrategy):
metadata = forward_batch.attn_cp_metadata
cp_size = len(metadata.per_rank_actual_token)
base, remainder = divmod(static_num_tokens, cp_size)
metadata.total_seq_lens = static_num_tokens
metadata.per_rank_logical_token = [
base + int(rank < remainder) for rank in range(cp_size)
]
forward_batch.out_cache_loc = out_cache_loc
def execute_prefill_cp_bcg(
runner: PrefillCudaGraphRunner,
@@ -307,10 +329,34 @@ def execute_prefill_cp_bcg(
static_forward_batch,
torch.cuda.current_stream(),
)
hidden_states = _slice_output_rows(hidden_states, raw_num_tokens)
if aux_hidden_states is not None:
def gather_aux(aux):
return _slice_output_rows(
cp_gather_after_forward(
aux, static_forward_batch, torch.cuda.current_stream()
),
raw_num_tokens,
)
if torch.is_tensor(aux_hidden_states):
aux_hidden_states = gather_aux(aux_hidden_states)
else:
aux_hidden_states = [gather_aux(aux) for aux in aux_hidden_states]
# Mirror the DSV4 outer forward and the eager CP runner: the body
# returns (normalized hidden states, pre-mHC-head hidden states).
logits_kwargs = {}
if isinstance(hidden_states, tuple):
hidden_states, hidden_states_before_norm = hidden_states
if aux_hidden_states is None:
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
return model.logits_processor(
forward_batch.input_ids,
hidden_states,
model.lm_head,
forward_batch,
aux_hidden_states,
**logits_kwargs,
)
@@ -593,6 +593,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.capture_num_tokens, server_args
)
self.prefill_cp_bcg_input = PrefillCPBCGInput.create(self)
logger.info(
"Prefill CP breakable CUDA graph enabled: strategy=%s, cp_size=%d",
get_parallel().cp_strategy,
get_parallel().attn_cp_size,
)
if self.max_context_size is not None and not (
model_runner.attn_backend.supports_prefill_cuda_graph_max_context_size
):
@@ -1370,6 +1376,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
if self._has_inactive_dp_rank(forward_batch):
return False
# A batch can be too small to activate CP. Its unsharded inputs must
# not replay a model body captured with CP-local rows.
if getattr(self, "enable_cp_bcg_capture", False) and not is_cp_active(
forward_batch
):
return False
# Non-DP local check (sole decision for tp-only).
batch_max_context_len = (
self._batch_max_context_len(forward_batch)