diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index ec1d9830bd..75af46e40c 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -2907,6 +2907,10 @@ class DeepseekV4AttnBackend( self, forward_batch: ForwardBatch ): max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE + if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2): + graph_context_limit = _prefill_graph_max_seq_len() + if graph_context_limit is not None: + max_seq_len = min(max_seq_len, graph_context_limit) self.forward_metadata = self._build_forward_metadata( forward_batch, max_seq_len_override=max_seq_len, @@ -2916,6 +2920,8 @@ class DeepseekV4AttnBackend( self.enable_decoder_swa_bounded_replay and forward_batch.forward_mode.is_extend_without_speculative() ) + if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2): + self._share_bcg_low_ratio_index_buffers(self.forward_metadata.core_metadata) self.forward_metadata.prefill_graph_tail = with_tail self.tail_forward_metadata = ( self._prefill_graph_tail_metadata(forward_batch, max_seq_len) @@ -2929,6 +2935,25 @@ class DeepseekV4AttnBackend( ) return self.forward_metadata + def _share_bcg_low_ratio_index_buffers(self, core) -> None: + # BCG buckets replay serially. Share only full-metadata output buffers; + # both ratios must survive the later full-to-tail metadata transition. + # Keep old allocations alive if capture order ever grows the capacity. + pools = getattr(self, "_bcg_low_ratio_index_buffers", None) + if pools is None: + pools = self._bcg_low_ratio_index_buffers = {} + for ratio in (1, 2): + for kind in ("page", "raw"): + name = f"c{ratio}_sparse_{kind}_indices" + value = getattr(core, name) + if value is None: + continue + key = (name, value.shape[1], value.dtype, value.device) + retained = pools.setdefault(key, []) + if not retained or retained[0].shape[0] < value.shape[0]: + retained.insert(0, value) + setattr(core, name, retained[0][: value.shape[0]]) + def _source_projection_buffers(self, num_tokens: int, ratio: int) -> dict: cfg = self.model_runner.model_config.hf_text_config heads, dim = int(cfg.index_n_heads), int(cfg.index_head_dim) @@ -2971,6 +2996,10 @@ class DeepseekV4AttnBackend( max_seq_len = ( metadata_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE ) + if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2): + graph_context_limit = _prefill_graph_max_seq_len() + if graph_context_limit is not None: + max_seq_len = min(max_seq_len, graph_context_limit) static_metadata = self._build_forward_metadata( metadata_batch, max_seq_len_override=max_seq_len, diff --git a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py index de0ee7ac86..241d2ba09e 100644 --- a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py +++ b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py @@ -337,6 +337,9 @@ class SparsePrefillChunkCache: # Compressed caches keyed by compress ratio: c128 (every block, combined once # per chunk) and the top-k ratios (combined per layer). compressed: Dict[int, CompressedGather] = field(default_factory=dict) + # Per-layer scratch only: combine overwrites all entries before attention. + # c0/c128 are cached across layers and must retain separate storage. + low_ratio_combined: dict = field(default_factory=dict) swa_indices: Optional[torch.Tensor] = None swa_lengths: Optional[torch.Tensor] = None # Host lengths avoid a device synchronization when sizing ragged gathers. @@ -602,6 +605,11 @@ class SparsePrefillChunkCache: """ gather = self.compressed[compress_ratio] topk = sparse_raw_indices.shape[-1] + width = combined_topk_width(topk, self.swa_window_size) + if compress_ratio in (1, 2) and width in self.low_ratio_combined: + gather.combined_indices, gather.combined_lens = self.low_ratio_combined[ + width + ] if gather.combined_indices is None: device = self.seq_lens.device gather.combined_indices = torch.full( @@ -613,6 +621,11 @@ class SparsePrefillChunkCache: gather.combined_lens = torch.zeros( self.num_qo_tokens, dtype=torch.int32, device=device ) + if compress_ratio in (1, 2): + self.low_ratio_combined[width] = ( + gather.combined_indices, + gather.combined_lens, + ) return combine_topk_swa_indices( topk_indices=sparse_raw_indices, query_start_loc=self.query_start_loc,