[BCG][DSV4-Flash][CP] Support DeepSeek V4 prefill with interleave CP and breakable CUDA graphs (#38603)
This commit is contained in:
@@ -1394,6 +1394,10 @@ class DeepseekV4AttnBackend(
|
||||
(0, padded_num_tokens - swa_replay_start.shape[0]),
|
||||
)
|
||||
|
||||
num_write_tokens = (
|
||||
out_cache_loc.shape[0] if use_prefill_cuda_graph else num_tokens
|
||||
)
|
||||
|
||||
seq_lens_casual, req_pool_indices_repeated = self.expand_prefill_casually(
|
||||
num_tokens=num_tokens,
|
||||
seq_lens=seq_lens_cpu,
|
||||
@@ -1414,13 +1418,13 @@ class DeepseekV4AttnBackend(
|
||||
is_prefill=True,
|
||||
dspark_block_size=dspark_block_size,
|
||||
dspark_swa_buffers=dspark_swa_buffers,
|
||||
num_tokens=num_tokens if cp_active else None,
|
||||
num_tokens=num_write_tokens if cp_active else None,
|
||||
swa_replay_start=swa_replay_start,
|
||||
num_groups=len(extend_seq_lens_cpu),
|
||||
)
|
||||
if cp_active:
|
||||
core_attn_metadata.apply_cp_reindex(
|
||||
num_tokens=num_tokens, local_index=cp_metadata.local_index
|
||||
num_tokens=num_write_tokens, local_index=cp_metadata.local_index
|
||||
)
|
||||
core_attn_metadata.init_flashmla_related(is_prefill=True)
|
||||
indexer_metadata = (
|
||||
@@ -2364,6 +2368,19 @@ class DeepseekV4AttnBackend(
|
||||
self.forward_metadata.core_attn_metadata.request_window_layout
|
||||
)
|
||||
|
||||
def _use_sparse_prefill(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> bool:
|
||||
return (
|
||||
forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and not is_cp_active(forward_batch)
|
||||
and not get_platform().is_sm120
|
||||
and (
|
||||
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
)
|
||||
|
||||
def prepare_prefill_shared_read_snapshot(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> None:
|
||||
@@ -2386,15 +2403,9 @@ class DeepseekV4AttnBackend(
|
||||
|
||||
assert isinstance(metadata, DSV4Metadata)
|
||||
# The tail never takes the sparse path, so it carries no chunk cache.
|
||||
use_sparse_prefill = (
|
||||
not get_platform().is_sm120
|
||||
and metadata.late_layer_tail is None
|
||||
and (
|
||||
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
)
|
||||
if use_sparse_prefill:
|
||||
if metadata.late_layer_tail is None and self._use_sparse_prefill(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
):
|
||||
metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache(
|
||||
forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
@@ -2413,14 +2424,21 @@ class DeepseekV4AttnBackend(
|
||||
assert seq_lens_cpu is not None
|
||||
# Decoder tails require explicit SWA indices to exclude unwritten history.
|
||||
tail = self.forward_metadata.late_layer_tail
|
||||
request_layout = core_attn_metadata.request_window_layout
|
||||
assert tail is None or request_layout is not None
|
||||
extend_seq_lens = forward_batch.extend_seq_lens
|
||||
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
if tail is not None:
|
||||
extend_seq_lens = tail.extend_seq_lens
|
||||
extend_seq_lens_cpu = tail.extend_seq_lens_cpu
|
||||
assert extend_seq_lens_cpu is not None
|
||||
num_global_queries = sum(extend_seq_lens_cpu)
|
||||
assert num_qo_tokens >= num_global_queries, (
|
||||
"DSV4 sparse prefill requires global query rows: "
|
||||
f"allocated {num_qo_tokens}, but request offsets cover {num_global_queries}. "
|
||||
"CP-local queries must use paged attention."
|
||||
)
|
||||
# Reject undersized query buffers before accessing attention metadata.
|
||||
request_layout = core_attn_metadata.request_window_layout
|
||||
assert tail is None or request_layout is not None
|
||||
seq_lens_cpu_list = seq_lens_cpu.tolist()
|
||||
total_swa = sum(
|
||||
min(int(seq_len), int(extend_len) + SWA_WINDOW - 1)
|
||||
@@ -3480,24 +3498,12 @@ class DeepseekV4AttnBackend(
|
||||
f"{extra_indices.shape=}'s last dimension is not aligned to 64"
|
||||
)
|
||||
|
||||
# RequestWindow sparse gathering does not support CP yet.
|
||||
# The tail stays dense: its window floor lives in swa_page_indices,
|
||||
# which the sparse chunk cache ignores.
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and not get_platform().is_sm120
|
||||
and (
|
||||
(
|
||||
token_to_kv_pool.request_window is None
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
)
|
||||
or (
|
||||
token_to_kv_pool.request_window is not None
|
||||
and not is_cp_active(forward_batch)
|
||||
)
|
||||
)
|
||||
and (
|
||||
q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and token_to_kv_pool.request_window is None
|
||||
):
|
||||
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
|
||||
return self._forward_prefill_sparse_q8kv8(
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.layers.cp.base import get_cp_strategy
|
||||
from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
|
||||
from sglang.srt.layers.cp.padding import get_cp_padding_align_size
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_after_forward,
|
||||
@@ -54,8 +55,10 @@ def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
|
||||
cfg.enable_prefill_cp
|
||||
and cfg.pp_size == 1
|
||||
and resolved.attn_cp_size == cfg.tp_size
|
||||
and cfg.cp_strategy == "zigzag"
|
||||
and prefill_attention_backend == "trtllm_mha"
|
||||
and (
|
||||
(cfg.cp_strategy == "zigzag" and prefill_attention_backend == "trtllm_mha")
|
||||
or (cfg.cp_strategy == "interleave" and prefill_attention_backend == "dsv4")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -67,8 +70,10 @@ def enable_cp_bcg_capture(server_args: ServerArgs) -> bool:
|
||||
def filter_prefill_cp_bcg_capture_num_tokens(
|
||||
capture_num_tokens: list[int], server_args: ServerArgs
|
||||
) -> list[int]:
|
||||
"""Keep only token buckets where the zigzag CP strategy can run."""
|
||||
min_num_tokens = resolved_view(server_args).attn_cp_size * 2
|
||||
"""Keep only token buckets where the selected CP strategy can run."""
|
||||
min_num_tokens = resolved_view(server_args).attn_cp_size
|
||||
if resolving_view(server_args).cp_strategy == "zigzag":
|
||||
min_num_tokens *= 2
|
||||
filtered = [size for size in capture_num_tokens if size >= min_num_tokens]
|
||||
if not filtered:
|
||||
raise ValueError(
|
||||
@@ -117,24 +122,30 @@ class PrefillCPBCGInput:
|
||||
)
|
||||
|
||||
def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]:
|
||||
"""Return the aligned CP-local rows required by a live zigzag layout."""
|
||||
"""Return the largest aligned local shard, uniformly across CP ranks."""
|
||||
strategy = get_cp_strategy()
|
||||
if not isinstance(strategy, ZigzagCPStrategy) or extend_seq_lens is None:
|
||||
if extend_seq_lens is None:
|
||||
return None
|
||||
if isinstance(strategy, InterleaveCPStrategy):
|
||||
num_tokens = sum(int(length) for length in extend_seq_lens)
|
||||
max_local_tokens = (num_tokens + strategy.cp_size - 1) // strategy.cp_size
|
||||
elif isinstance(strategy, ZigzagCPStrategy):
|
||||
cp_segment_num = strategy.cp_size * 2
|
||||
per_rank_logical_tokens = [0] * strategy.cp_size
|
||||
for raw_length in extend_seq_lens:
|
||||
base, remainder = divmod(int(raw_length), cp_segment_num)
|
||||
for rank in range(strategy.cp_size):
|
||||
opposite_rank = cp_segment_num - 1 - rank
|
||||
per_rank_logical_tokens[rank] += (
|
||||
base * 2
|
||||
+ int(rank < remainder)
|
||||
+ int(opposite_rank < remainder)
|
||||
)
|
||||
max_local_tokens = max(per_rank_logical_tokens)
|
||||
else:
|
||||
return None
|
||||
|
||||
cp_segment_num = strategy.cp_size * 2
|
||||
per_rank_logical_tokens = [0] * strategy.cp_size
|
||||
for raw_length in extend_seq_lens:
|
||||
base, remainder = divmod(int(raw_length), cp_segment_num)
|
||||
for rank in range(strategy.cp_size):
|
||||
opposite_rank = cp_segment_num - 1 - rank
|
||||
per_rank_logical_tokens[rank] += (
|
||||
base * 2 + int(rank < remainder) + int(opposite_rank < remainder)
|
||||
)
|
||||
align_size = get_cp_padding_align_size()
|
||||
return (
|
||||
(max(per_rank_logical_tokens) + align_size - 1) // align_size * align_size
|
||||
)
|
||||
return (max_local_tokens + align_size - 1) // align_size * align_size
|
||||
|
||||
def select_replay_bucket(
|
||||
self,
|
||||
@@ -189,6 +200,7 @@ class PrefillCPBCGInput:
|
||||
# Replay batches may reuse a ForwardBatch object whose metadata was
|
||||
# built for a different request layout. Always rebuild before sharding.
|
||||
forward_batch.attn_cp_metadata = None
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
prepare_cp_forward(forward_batch)
|
||||
|
||||
captured_local_tokens = None
|
||||
@@ -257,6 +269,16 @@ class PrefillCPBCGInput:
|
||||
forward_batch.positions = positions
|
||||
self.live_local_tokens = live_local_tokens
|
||||
|
||||
if isinstance(get_cp_strategy(), InterleaveCPStrategy):
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
cp_size = len(metadata.per_rank_actual_token)
|
||||
base, remainder = divmod(static_num_tokens, cp_size)
|
||||
metadata.total_seq_lens = static_num_tokens
|
||||
metadata.per_rank_logical_token = [
|
||||
base + int(rank < remainder) for rank in range(cp_size)
|
||||
]
|
||||
forward_batch.out_cache_loc = out_cache_loc
|
||||
|
||||
|
||||
def execute_prefill_cp_bcg(
|
||||
runner: PrefillCudaGraphRunner,
|
||||
@@ -307,10 +329,34 @@ def execute_prefill_cp_bcg(
|
||||
static_forward_batch,
|
||||
torch.cuda.current_stream(),
|
||||
)
|
||||
hidden_states = _slice_output_rows(hidden_states, raw_num_tokens)
|
||||
if aux_hidden_states is not None:
|
||||
|
||||
def gather_aux(aux):
|
||||
return _slice_output_rows(
|
||||
cp_gather_after_forward(
|
||||
aux, static_forward_batch, torch.cuda.current_stream()
|
||||
),
|
||||
raw_num_tokens,
|
||||
)
|
||||
|
||||
if torch.is_tensor(aux_hidden_states):
|
||||
aux_hidden_states = gather_aux(aux_hidden_states)
|
||||
else:
|
||||
aux_hidden_states = [gather_aux(aux) for aux in aux_hidden_states]
|
||||
|
||||
# Mirror the DSV4 outer forward and the eager CP runner: the body
|
||||
# returns (normalized hidden states, pre-mHC-head hidden states).
|
||||
logits_kwargs = {}
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states, hidden_states_before_norm = hidden_states
|
||||
if aux_hidden_states is None:
|
||||
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
|
||||
return model.logits_processor(
|
||||
forward_batch.input_ids,
|
||||
hidden_states,
|
||||
model.lm_head,
|
||||
forward_batch,
|
||||
aux_hidden_states,
|
||||
**logits_kwargs,
|
||||
)
|
||||
|
||||
@@ -593,6 +593,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.capture_num_tokens, server_args
|
||||
)
|
||||
self.prefill_cp_bcg_input = PrefillCPBCGInput.create(self)
|
||||
logger.info(
|
||||
"Prefill CP breakable CUDA graph enabled: strategy=%s, cp_size=%d",
|
||||
get_parallel().cp_strategy,
|
||||
get_parallel().attn_cp_size,
|
||||
)
|
||||
|
||||
if self.max_context_size is not None and not (
|
||||
model_runner.attn_backend.supports_prefill_cuda_graph_max_context_size
|
||||
):
|
||||
@@ -1370,6 +1376,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if self._has_inactive_dp_rank(forward_batch):
|
||||
return False
|
||||
|
||||
# A batch can be too small to activate CP. Its unsharded inputs must
|
||||
# not replay a model body captured with CP-local rows.
|
||||
if getattr(self, "enable_cp_bcg_capture", False) and not is_cp_active(
|
||||
forward_batch
|
||||
):
|
||||
return False
|
||||
|
||||
# Non-DP local check (sole decision for tp-only).
|
||||
batch_max_context_len = (
|
||||
self._batch_max_context_len(forward_batch)
|
||||
|
||||
Reference in New Issue
Block a user