[Perf] deepseek-v4.1: Replay the breakable prefill CUDA graph at any context length (#43733)

Co-authored-by: Oasis-Git <ayw.sirius19@gmail.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Yizheng Jiao
2026-10-19 01:29:14 -07:00
committed by GitHub
co-authored by Oasis-Git hnyls2002
parent 33dd8f6a2e
commit 8111841103
11 changed files with 389 additions and 92 deletions
@@ -16,7 +16,7 @@ from sglang.srt.arg_groups.overrides import (
run_post_process_pass,
)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import is_gfx95_supported, is_npu
@@ -364,30 +364,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"both servers use the same block size and target/draft KV layout."
)
prefill_graph = cfg.cuda_graph_config.prefill
cp_breakable_prefill = (
cfg.enable_prefill_cp
and cfg.cp_strategy == "interleave"
and cfg.tp_size > 1
and prefill_graph.backend == Backend.BREAKABLE
)
if (
prefill_graph.backend != Backend.DISABLED
and prefill_graph.max_seq_len is None
and not cp_breakable_prefill
):
declare_resolution(
server_args,
"validate_deepseek_v41_features",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, max_seq_len=16 * 1024
),
)
logger.warning(
"Setting cuda_graph_config[prefill].max_seq_len to 16384 for "
"DeepSeek-V4.1; longer contexts run eager prefill."
)
incompatible = (
(
"the prefill CUDA graph",
@@ -171,6 +171,9 @@ class AttentionBackend(ABC):
# True when prefill graph metadata can use ForwardBatch.max_seq_len_override.
supports_prefill_cuda_graph_max_context_size: bool = False
# Replayed context limit when the user sets none; None when nothing the backend
# captures depends on the context length.
prefill_cuda_graph_default_max_seq_len: Optional[int] = None
dllm_attention = None
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
@@ -90,6 +90,9 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
make_candidate_indexer,
make_full_topk_indexer,
)
from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import (
SparseTableBackend,
)
from sglang.srt.layers.attention.verify_mask import (
VerifyMask,
maybe_create_verify_mask,
@@ -114,6 +117,14 @@ from sglang.srt.model_executor.forward_batch_info import (
ForwardMode,
get_server_return_hidden_states_mode,
)
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import (
eager_on_graph,
is_capturing,
)
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
get_tc_piecewise_forward_context,
)
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
@@ -262,10 +273,6 @@ def _maybe_precompute_flashmla_sched_meta(
def _low_ratio_source_projections(
layer, x, q_lora, positions, bufs, run_compressor, run_indexer
):
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
get_tc_piecewise_forward_context,
)
real = (
get_tc_piecewise_forward_context().forward_batch.global_num_token_non_padded_cpu
)
@@ -292,25 +299,22 @@ def _low_ratio_source_projections(
if score is not None:
put("score", score)
if run_indexer and layer.indexer is not None:
# Only the captured consumers read these; a full top-k layer projects
# once inside its eager break (_index_q_and_weights), so skip them there.
indexer = layer.indexer
put("q", indexer.queries(q_lora[:real], layer.freqs_cis[positions[:real]]))
put("w", indexer.head_weights(x[:real]))
def _bcg_low_ratio_source_projections(*args):
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import (
eager_on_graph,
def _prefill_graph_full_topk(layer, x, q_lora, pos):
forward_batch = get_tc_piecewise_forward_context().forward_batch
return get_attn_backend()._prefill_graph_full_topk_break(
layer, x, q_lora, pos, forward_batch
)
global _bcg_low_ratio_source_projections_fn
if _bcg_low_ratio_source_projections_fn is None:
_bcg_low_ratio_source_projections_fn = eager_on_graph(True)(
_low_ratio_source_projections
)
return _bcg_low_ratio_source_projections_fn(*args)
_bcg_low_ratio_source_projections_fn = None
_bcg_prefill_graph_full_topk = eager_on_graph(True)(_prefill_graph_full_topk)
_bcg_low_ratio_source_projections = eager_on_graph(True)(_low_ratio_source_projections)
def _as_int_list(values) -> Optional[List[int]]:
@@ -1009,11 +1013,16 @@ def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
return torch.nn.functional.pad(t, (0, num_rows - t.shape[0]))
# Rows per logits chunk for the ratio-1/2 indexer inside the prefill CUDA graph;
# its width is the graph's max_seq_len, and longer contexts replay eagerly.
# Rows per logits chunk of the ratio-1/2 indexer metadata under the prefill CUDA
# graph; its width is _prefill_graph_context_width.
_PREFILL_GRAPH_INDEXER_ROW_CHUNK = 2048
# The candidate window (2,048 blocks of 8 tokens): inside it the captured full
# top-k equals the two-level selection.
_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024
def _prefill_graph_max_seq_len() -> Optional[int]:
from sglang.srt.runtime_context import get_exec
@@ -1071,17 +1080,30 @@ class DSV4Metadata:
self.sparse_prefill_cache = None
self.prefill_shared_reads_snapshotted = False
def refresh_for_breakable_cuda_graph_replay_(self, static_metadata: DSV4Metadata):
def refresh_for_breakable_cuda_graph_replay_(
self,
static_metadata: DSV4Metadata,
*,
two_level: bool,
):
self.core_attn_metadata.refresh_for_breakable_cuda_graph_replay_(
static_metadata.core_attn_metadata
)
maybe_copy_inplace(self.indexer_metadata, src=static_metadata.indexer_metadata)
maybe_copy_inplace(
self.c1_indexer_metadata, src=static_metadata.c1_indexer_metadata
)
maybe_copy_inplace(
self.c2_indexer_metadata, src=static_metadata.c2_indexer_metadata
)
if not two_level:
# The captured full top-k fallback reads the ratio-1/2 lengths and
# page tables by address: refresh them in place.
maybe_copy_inplace(
self.c1_indexer_metadata, src=static_metadata.c1_indexer_metadata
)
maybe_copy_inplace(
self.c2_indexer_metadata, src=static_metadata.c2_indexer_metadata
)
else:
# Two-level candidates: only the eager full top-k breaks read these, so
# they take the live objects; captured consumers read the bridge tensors.
self.c1_indexer_metadata = static_metadata.c1_indexer_metadata
self.c2_indexer_metadata = static_metadata.c2_indexer_metadata
maybe_copy_inplace(
self.low_ratio_req_indices, src=static_metadata.low_ratio_req_indices
)
@@ -1263,6 +1285,8 @@ class DeepseekV4AttnBackend(
self._bcg_tail_buffers: Optional[tuple] = None
self._bcg_tail_metadata: Dict[int, DSV4Metadata] = {}
self._bcg_tail_num_token_non_padded: Optional[torch.Tensor] = None
# Prefill CUDA graph source projection buffers per ratio, newest first.
self._source_proj_bufs: Dict[int, list] = {}
self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner)
self.dsv4_prefill_backend = getattr(kernel, "dsv4_prefill_backend", "auto")
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
@@ -1601,7 +1625,7 @@ class DeepseekV4AttnBackend(
self, core: DSV4AttnMetadata, compress_ratio: int
) -> PagedIndexerMetadata:
num_pages = core.page_table.shape[1]
max_seq_len = _prefill_graph_max_seq_len()
max_seq_len = self._prefill_graph_seq_limit()
if max_seq_len is not None:
num_pages = min(max_seq_len // self.page_size, num_pages)
index_page_size = self.token_to_kv_pool.get_index_k_page_size(compress_ratio)
@@ -1636,11 +1660,11 @@ class DeepseekV4AttnBackend(
)
def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
max_seq_len = _prefill_graph_max_seq_len()
max_seq_len = self._prefill_graph_seq_limit()
seq_lens_cpu = forward_batch.seq_lens_cpu
if max_seq_len is None or seq_lens_cpu is None or seq_lens_cpu.numel() == 0:
if seq_lens_cpu is None or seq_lens_cpu.numel() == 0:
return True
if int(seq_lens_cpu.max().item()) > max_seq_len:
if max_seq_len is not None and int(seq_lens_cpu.max().item()) > max_seq_len:
return False
if self.enable_decoder_swa_bounded_replay:
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
@@ -2950,11 +2974,7 @@ class DeepseekV4AttnBackend(
def init_forward_metadata_for_breakable_cuda_graph_capture(
self, forward_batch: ForwardBatch
):
max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
if self.low_ratios == (1, 2):
graph_context_limit = _prefill_graph_max_seq_len()
if graph_context_limit is not None:
max_seq_len = min(max_seq_len, graph_context_limit)
max_seq_len = self._prefill_graph_context_width(forward_batch)
self.forward_metadata = self._build_forward_metadata(
forward_batch,
max_seq_len_override=max_seq_len,
@@ -2980,6 +3000,19 @@ class DeepseekV4AttnBackend(
)
return self.forward_metadata
def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int:
"""One context width for the full rows' and the late-layer tail's metadata."""
if self._prefill_graph_replays_any_context():
seq_lens_cpu = forward_batch.seq_lens_cpu
if seq_lens_cpu is not None and len(seq_lens_cpu) > 0:
return int(seq_lens_cpu.max().item())
max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
if self.low_ratios == (1, 2):
graph_context_limit = self._prefill_graph_seq_limit()
if graph_context_limit is not None:
max_seq_len = min(max_seq_len, graph_context_limit)
return max_seq_len
def _share_bcg_low_ratio_index_buffers(self, core) -> None:
# BCG buckets replay serially. Share only full-metadata output buffers;
# both ratios must survive the later full-to-tail metadata transition.
@@ -3002,7 +3035,7 @@ class DeepseekV4AttnBackend(
def _source_projection_buffers(self, num_tokens: int, ratio: int) -> dict:
cfg = self.model_runner.model_config.hf_text_config
heads, dim = int(cfg.index_n_heads), int(cfg.index_head_dim)
sets = getattr(self, "_source_proj_bufs", None) or {}
sets = self._source_proj_bufs
have = sets.get(ratio)
if have is None or have[0]["q"].shape[0] < num_tokens:
latent = self.model_runner.model_config.head_dim
@@ -3022,7 +3055,6 @@ class DeepseekV4AttnBackend(
bufs["score"] = zeros(num_tokens, latent, dtype=torch.float32)
# Keep the previous allocations alive: captured graphs still hold them.
sets[ratio] = [bufs] + (have or [])
self._source_proj_bufs = sets
bufs = sets[ratio][0]
return {name: buf[:num_tokens] for name, buf in bufs.items()}
@@ -3038,20 +3070,17 @@ class DeepseekV4AttnBackend(
metadata_batch = (
static_forward_batch if static_forward_batch is not None else forward_batch
)
max_seq_len = (
metadata_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
)
if self.low_ratios == (1, 2):
graph_context_limit = _prefill_graph_max_seq_len()
if graph_context_limit is not None:
max_seq_len = min(max_seq_len, graph_context_limit)
max_seq_len = self._prefill_graph_context_width(metadata_batch)
static_metadata = self._build_forward_metadata(
metadata_batch,
max_seq_len_override=max_seq_len,
use_prefill_cuda_graph=True,
)
assert isinstance(capture_metadata, DSV4Metadata)
capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata)
capture_metadata.refresh_for_breakable_cuda_graph_replay_(
static_metadata,
two_level=self._prefill_graph_two_level(),
)
self.forward_metadata = capture_metadata
self._activate_request_window()
self.tail_forward_metadata = (
@@ -3075,7 +3104,10 @@ class DeepseekV4AttnBackend(
if held is None:
self._bcg_tail_metadata[rows] = fresh
return fresh
held.refresh_for_breakable_cuda_graph_replay_(fresh)
held.refresh_for_breakable_cuda_graph_replay_(
fresh,
two_level=self._prefill_graph_two_level(),
)
held.core_attn_metadata.swa_out_cache_loc = (
fresh.core_attn_metadata.swa_out_cache_loc
)
@@ -3212,15 +3244,32 @@ class DeepseekV4AttnBackend(
and self._low_ratio_in_prefill_graph()
):
bufs = self._source_projection_buffers(x.shape[0], layer.compress_ratio)
two_level = self._prefill_graph_two_level()
captured_index = (
run_indexer
and layer.indexer is not None
and (layer.indexer.uses_candidates or not two_level)
)
_bcg_low_ratio_source_projections(
layer, x, q_lora, pos, bufs, run_compressor, run_indexer
layer, x, q_lora, pos, bufs, run_compressor, captured_index
)
if run_compressor and layer.compressor is not None:
self._low_ratio_compress_torch(
layer, x, req, pos, projected=(bufs["kv"], bufs.get("score"))
)
if run_indexer and layer.indexer is not None:
self._low_ratio_index_topk_captured(layer, bufs["q"], bufs["w"])
if captured_index:
# Captured: sparse scores of the published blocks only, so
# nothing here depends on the context length.
self._low_ratio_index_topk_captured(layer, bufs["q"], bufs["w"])
else:
# Eager break: dense scoring over the live context; the
# candidate source hands its block table to the consumers.
out = _bcg_prefill_graph_full_topk(layer, x, q_lora, pos)
if layer.indexer.is_candidate_source:
self.forward_metadata.candidate_metadata = (
self.prefill_candidates.bind_prefill_graph_table(out)
)
return
if run_compressor and layer.compressor is not None:
self._low_ratio_compress(layer, x, req, pos, forward_batch)
@@ -3660,12 +3709,6 @@ class DeepseekV4AttnBackend(
else self.forward_metadata.c2_indexer_metadata
)
assert metadata is not None, f"no prefill graph indexer metadata for {ratio = }"
if indexer.uses_candidates or indexer.is_candidate_source:
# Inside the window every block is a candidate: the full top-k applies.
width = metadata.max_compressed_seq_len
assert (
width <= indexer.candidate_topk_blocks * indexer.candidate_block_size
), f"prefill graph indexer width {width} exceeds the candidate window"
inputs = CapturedPrefillInputs(
indexer=indexer,
layer_id=layer.layer_id,
@@ -3675,6 +3718,19 @@ class DeepseekV4AttnBackend(
out_page_indices=core.sparse_page_indices(ratio),
out_raw_indices=core.sparse_raw_indices(ratio),
)
# At any width: inside the candidate window the published table is the
# identity, and past it the full top-k would be wrong.
if indexer.uses_candidates and self._prefill_graph_two_level():
self.prefill_candidates.consume_prefill_captured(
inputs, self.forward_metadata.candidate_metadata
)
return
if indexer.uses_candidates or indexer.is_candidate_source:
# Inside the window every block is a candidate: the full top-k applies.
width = metadata.max_compressed_seq_len
assert (
width <= indexer.candidate_topk_blocks * indexer.candidate_block_size
), f"prefill graph indexer width {width} exceeds the candidate window"
self.full_topk_indexer.topk_prefill_captured(inputs)
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
@@ -3763,6 +3819,72 @@ class DeepseekV4AttnBackend(
return
self.forward_metadata.candidate_metadata = published
def _prefill_graph_two_level(self) -> bool:
"""Whether the full top-k layers score at eager breaks and the consumers score
the published candidate blocks inside the prefill graph."""
return isinstance(self.prefill_candidates, SparseTableBackend)
def _prefill_graph_replays_any_context(self) -> bool:
"""Whether a prefill graph replay admits any context length: with two-level
candidates only c4/c128 compress metadata would be captured at a width."""
return (
self.low_ratio_prefill_graph
and self._prefill_graph_two_level()
and not self.has_c4
and not self.has_c128
)
@property
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
"""DeepSeek-V4.1's captured indexer width; prefill context parallelism builds
its own metadata and needs none."""
if (
self.low_ratios != (1, 2)
or get_parallel().attn_cp_size > 1
or self._prefill_graph_replays_any_context()
):
return None
return _PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN
def _prefill_graph_seq_limit(self) -> Optional[int]:
"""The replayed context limit: cuda_graph_config[prefill].max_seq_len when
the user sets it, else this backend's default."""
max_seq_len = _prefill_graph_max_seq_len()
if max_seq_len is not None:
return max_seq_len
return self.prefill_cuda_graph_default_max_seq_len
def _prefill_graph_full_topk_break(self, layer, x, q_lora, pos, forward_batch):
"""One full top-k layer at an eager break of the prefill graph, over the live
batch at any context length: writes the sparse indices the attention break
reads, and the candidate source returns its table for the captured consumers."""
indexer = layer.indexer
real = forward_batch.global_num_token_non_padded_cpu
if real is None:
real = x.shape[0]
if real == 0:
# An idle DP-attention rank replays on one fabricated request of the
# padded token count (ForwardBatch idle materialization); nothing reads it.
real = forward_batch.extend_num_tokens
# The metadata in use holds the rows x has: the late-layer tail's when the
# source is the tail's first layer, which are not the extend's leading rows.
req = self.forward_metadata.low_ratio_req_indices[:real]
inputs = self._make_low_ratio_prefill_indexer_inputs(
layer,
x[:real],
q_lora[:real],
req,
pos[:real],
forward_batch,
None,
)
if not indexer.is_candidate_source:
self.full_topk_indexer.topk_prefill(inputs)
return ()
return self.prefill_candidates.publish_prefill_graph(
inputs, x.shape[0], shared=is_capturing()
)
def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor:
"""Idle always re-translates at store time: its metadata may be stale, and
translating the zero-padded out_cache_loc writes to the dummy slot."""
@@ -66,7 +66,6 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4AttnBackend,
LateLayerTail,
_as_int_list,
_prefill_graph_max_seq_len,
_tail_rows,
late_layer_num_tokens,
late_layer_whole_extend,
@@ -890,6 +889,13 @@ class DeepseekV4HipRadixBackend(
# This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py.
# PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager.
can_run_prefill_cuda_graph = DeepseekV4AttnBackend.can_run_prefill_cuda_graph
prefill_cuda_graph_default_max_seq_len = (
DeepseekV4AttnBackend.prefill_cuda_graph_default_max_seq_len
)
_prefill_graph_replays_any_context = (
DeepseekV4AttnBackend._prefill_graph_replays_any_context
)
_prefill_graph_seq_limit = DeepseekV4AttnBackend._prefill_graph_seq_limit
_low_ratio_in_prefill_graph = DeepseekV4AttnBackend._low_ratio_in_prefill_graph
_low_ratio_compress = DeepseekV4AttnBackend._low_ratio_compress
_low_ratio_compress_decode = DeepseekV4AttnBackend._low_ratio_compress_decode
@@ -2392,14 +2398,13 @@ class DeepseekV4HipRadixBackend(
seq_lens_cpu=seq_lens_cpu,
)
@staticmethod
def _graph_metadata_max_seq_len(forward_batch: ForwardBatch) -> Optional[int]:
def _graph_metadata_max_seq_len(self, forward_batch: ForwardBatch) -> Optional[int]:
"""
Capture and replay build the metadata at this fixed context length, so a
replay's metadata is never larger than the captured one. Longer contexts run
eager.
"""
return forward_batch.max_seq_len_override or _prefill_graph_max_seq_len()
return forward_batch.max_seq_len_override or self._prefill_graph_seq_limit()
def init_forward_metadata_for_breakable_cuda_graph_capture(
self, forward_batch: ForwardBatch
@@ -41,11 +41,13 @@ from .scoring import (
get_deep_gemm_decode_data,
get_deep_gemm_prefill_data,
get_index_k_cache,
quantize_index_q,
score_tiles,
topk_ragged,
)
from .types import (
CandidateMetadata,
CapturedPrefillInputs,
DecodeInputs,
PrefillInputs,
get_tail_row_indices,
@@ -80,14 +82,25 @@ class _SparsePrefillTable(_SparseTable):
request_ids: torch.Tensor # [rows] int32 row-pair ids, see _row_pair_ids
q_dtype: torch.dtype
page_size: int # index-K pool page size (slots)
rows_per_request: List[int] # rows of each request, in row order
# rows of each request, in row order; None for rows gathered without them
rows_per_request: Optional[List[int]]
def tail(self, rows_per_request: List[int]) -> _SparsePrefillTable:
assert self.rows_per_request is not None, (
"a table of gathered rows has no requests"
)
rows = get_tail_row_indices(
full_rows_per_request=self.rows_per_request,
tail_rows_per_request=rows_per_request,
device=self.blocks.device,
)
return self.gather_rows(rows, rows_per_request)
def gather_rows(
self, rows: torch.Tensor, rows_per_request: Optional[List[int]] = None
) -> _SparsePrefillTable:
"""The table restricted to ``rows``: a gather, graph-safe for a persistent
index buffer."""
compress_lens = self.compress_lens[rows].contiguous()
_, valid_lens = candidate_row_lens(compress_lens, self.blocks.shape[1])
return _build_prefill_table(
@@ -121,6 +134,8 @@ class SparseTableBackend:
self._cached_row_ids: Optional[torch.Tensor] = None
# captured graphs keep reading the buffers they saw
self._retired_cached_row_ids: list = []
# per layer: the table tensors every captured prefill bucket reads
self._graph_table_bufs: dict = {}
def publish_prefill(self, inputs: PrefillInputs):
inputs.reset_outputs()
@@ -172,6 +187,80 @@ class SparseTableBackend:
)
data.write_page_indices(inputs)
def consume_prefill_captured(
self, inputs: CapturedPrefillInputs, published: Optional[_SparseTable]
) -> None:
"""A consumer under the prefill CUDA graph: sparse scores of the published
blocks, top-k as pool slots and as compressed positions."""
assert published is not None, (
"prefill graph candidate table missing: the candidate source layer "
"must run under the same graph before its consumers"
)
indexer = inputs.indexer
q_fp4, q_sf = quantize_index_q(inputs.q)
rows = q_fp4.shape[0]
k_cache = get_index_k_cache(
token_to_kv_pool=self.token_to_kv_pool,
layer_id=inputs.layer_id,
page_size=inputs.paged_metadata.compressed_page_size,
)
logits = _prefill_table_logits(published, q_fp4, q_sf, inputs.weights, k_cache)
topk = min(indexer.index_topk, published.blocks.shape[1] * CANDIDATE_BLOCK_SIZE)
def transform(blocks: torch.Tensor, out: torch.Tensor) -> None:
target = out[:rows, :topk]
if target.is_contiguous():
topk_transform_sparse(logits, published.valid_lens, blocks, target)
return
tmp = torch.empty((rows, topk), dtype=torch.int32, device=out.device)
topk_transform_sparse(logits, published.valid_lens, blocks, tmp)
target.copy_(tmp)
if inputs.out_page_indices is not None:
transform(published.phys_blocks, inputs.out_page_indices)
transform(published.blocks, inputs.out_raw_indices)
def publish_prefill_graph(
self, inputs: PrefillInputs, num_rows: int, *, shared: bool
) -> tuple:
"""The candidate source's table over the graph's ``num_rows`` rows. Buckets may
share one buffer: graphs replay one at a time, and the shapes, DeepGEMM's
schedule included, depend only on the row and block counts."""
published = self.publish_prefill(inputs)
assert published is not None, "candidate source published no sparse table"
live = published.blocks.shape[0]
rows = torch.arange(num_rows, device=published.blocks.device).clamp_max(
live - 1
)
table = published.gather_rows(rows)
out = (table.blocks, table.schedule, table.phys_blocks, table.valid_lens)
return self._shared_graph_table(inputs.layer_id, out) if shared else out
def bind_prefill_graph_table(self, tensors: tuple) -> _SparseTable:
"""Capture time: the table the captured consumers read, over the tensors the
candidate source's break refreshes at every replay."""
blocks, schedule, phys_blocks, valid_lens = tensors
ready = torch.cuda.Event()
ready.record()
return _SparseTable(
blocks=blocks,
schedule=schedule,
phys_blocks=phys_blocks,
valid_lens=valid_lens,
ready=ready,
)
def _shared_graph_table(self, layer_id: int, tensors: tuple) -> tuple:
bufs = self._graph_table_bufs.setdefault(layer_id, [None] * len(tensors))
views = []
for i, t in enumerate(tensors):
buf = bufs[i]
if buf is None or buf.dtype != t.dtype or buf.numel() < t.numel():
# Graphs captured on a smaller buffer keep it alive themselves.
buf = bufs[i] = torch.empty(t.numel(), dtype=t.dtype, device=t.device)
views.append(buf[: t.numel()].view(t.shape).copy_(t))
return tuple(views)
def publish_decode(self, inputs: DecodeInputs):
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
metadata = inputs.paged_metadata
@@ -352,7 +441,7 @@ def _build_prefill_table(
page_table: torch.Tensor,
page_size: int,
request_ids: torch.Tensor,
rows_per_request: List[int],
rows_per_request: Optional[List[int]],
q_dtype: torch.dtype,
valid_lens: torch.Tensor,
) -> _SparsePrefillTable:
@@ -446,6 +535,25 @@ def publish_prefill_table(
)
def _prefill_table_logits(
table: _SparseTable,
q_fp4: torch.Tensor,
q_sf: torch.Tensor,
weights: torch.Tensor,
k_cache: torch.Tensor,
) -> torch.Tensor:
"""Sparse logits of prefill rows (one query row each) over a table's blocks."""
rows, heads = q_sf.shape[0], q_sf.shape[1]
return sparse_logits(
q_fp4.view(rows, 1, heads, 64),
q_sf.view(rows, 1, heads),
k_cache,
weights.to(torch.bfloat16),
table.schedule,
table.blocks.shape[1],
)
def select_prefill_table(
*,
table: _SparsePrefillTable,
@@ -453,14 +561,6 @@ def select_prefill_table(
k_cache: torch.Tensor,
out_positions: torch.Tensor,
) -> None:
rows, heads = data.q_sf.shape
logits = sparse_logits(
data.q_fp4.view(rows, 1, heads, 64),
data.q_sf.view(rows, 1, heads),
k_cache,
data.weights.to(torch.bfloat16),
table.schedule,
table.blocks.shape[1],
)
logits = _prefill_table_logits(table, data.q_fp4, data.q_sf, data.weights, k_cache)
# request-relative compressed positions, -1 padded
topk_transform_sparse(logits, table.valid_lens, table.blocks, out_positions)
@@ -107,6 +107,10 @@ class HybridAttnBackend(AttentionBackend):
def supports_prefill_cuda_graph_max_context_size(self) -> bool:
return self.prefill_backend.supports_prefill_cuda_graph_max_context_size
@property
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
return self.prefill_backend.prefill_cuda_graph_default_max_seq_len
def prepare_full_cuda_graph_chunked_prefix(
self,
forward_batch: ForwardBatch,
@@ -32,6 +32,10 @@ class TboAttnBackend(AttentionBackend):
def supports_prefill_cuda_graph_max_context_size(self) -> bool:
return self.primary.supports_prefill_cuda_graph_max_context_size
@property
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
return self.primary.prefill_cuda_graph_default_max_seq_len
@property
def dllm_attention(self):
return self.primary.dllm_attention
@@ -375,6 +375,16 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.max_bs = model_runner.req_to_token_pool.size
self.max_context_size = prefill_config.max_context_size
self.max_seq_len = prefill_config.max_seq_len
if self.max_seq_len is None:
self.max_seq_len = (
model_runner.attn_backend.prefill_cuda_graph_default_max_seq_len
)
if self.max_seq_len is not None:
logger.info(
"Prefill CUDA graph replays contexts up to %d tokens, the "
"attention backend's default; longer contexts run eager prefill.",
self.max_seq_len,
)
self._validate_max_context_capacity(
max_context_size=self.max_context_size,
table_width=model_runner.req_to_token_pool.req_to_token.shape[1],
@@ -47,6 +47,7 @@ __all__ = [
"BreakableCUDAGraph",
"BreakableCUDAGraphCapture",
"break_graph",
"is_capturing",
]
@@ -67,6 +68,13 @@ _current_capture_var: ContextVar["BreakableCUDAGraphCapture | None"] = ContextVa
_current_stream_var: ContextVar[torch.Stream | None] = ContextVar(
"current_stream", default=None
)
def is_capturing() -> bool:
"""Whether a breakable CUDA graph is being captured; False during replay."""
return _current_capture_var.get() is not None
_forked_streams_var: ContextVar[set[torch.Stream] | None] = ContextVar(
"forked_streams", default=None
)