diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index b1872cce3c..07ecf41ad6 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -88,6 +88,7 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import ( make_candidate_indexer, make_full_topk_indexer, ) +from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import prefill_k_slots from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import ( SparseTableBackend, ) @@ -1042,6 +1043,12 @@ class DSV4Metadata: # Shared by all low-ratio source layers; graph replay refreshes them live. low_ratio_req_indices: Optional[torch.Tensor] = None low_ratio_pos_i64: Optional[torch.Tensor] = None + # Eager extend: each low ratio's index-K slots (scoring.prefill_k_slots), + # shared with the late-layer tail metadata. + low_ratio_k_slots: Optional[Dict[int, torch.Tensor]] = None + # Eager extend: the indexer's batch-only plan per low ratio, filled by its first + # layer (scoring.prefill_index_plan); one per metadata, as the rows differ. + low_ratio_index_plans: Optional[Dict[int, object]] = None # Per-step scratch for TP-padded query heads, zeroed by the first user. # Later layers overwrite real heads and preserve the zero padding. @@ -2717,9 +2724,44 @@ class DeepseekV4AttnBackend( and forward_batch.forward_mode.is_extend_without_speculative() else None ) + if ( + self.low_ratios + and forward_batch.forward_mode.is_extend_without_speculative() + ): + self._snapshot_low_ratio_k_slots(forward_batch) self._activate_request_window() + def _snapshot_low_ratio_k_slots(self, forward_batch: ForwardBatch) -> None: + # Gathered once with the metadata instead of in every indexer layer, so no + # indexer layer reads req_to_token. + seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu) + k_slots = { + ratio: prefill_k_slots( + req_to_token=self.req_to_token, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens_cpu=seq_lens_cpu, + ratio=ratio, + ) + for ratio in self.low_ratios + } + metadata = self.forward_metadata + metadata.low_ratio_k_slots = k_slots + metadata.low_ratio_index_plans = {} + if metadata.low_ratio_req_indices is None: + # Every source layer would rebuild these rows; the tail and the + # prefill graph publish theirs at metadata init the same way. + num_tokens = sum(forward_batch.extend_seq_lens_cpu) + metadata.low_ratio_req_indices = token_req_indices( + forward_batch, num_tokens=num_tokens + ) + metadata.low_ratio_pos_i64 = forward_batch.positions[:num_tokens].to( + torch.int64 + ) + if self.tail_forward_metadata is not None: + self.tail_forward_metadata.low_ratio_k_slots = k_slots + self.tail_forward_metadata.low_ratio_index_plans = {} + def _activate_request_window(self) -> None: if self.token_to_kv_pool.request_window is not None: self.token_to_kv_pool.request_window.activate( @@ -3782,6 +3824,18 @@ class DeepseekV4AttnBackend( else: rows_per_request = _as_int_list(forward_batch.extend_seq_lens_cpu) rows_per_request_device = forward_batch.extend_seq_lens + seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu) + k_slots = self.forward_metadata.low_ratio_k_slots + if k_slots is not None: + k_slots = k_slots[layer.compress_ratio] + else: + # Graph-captured metadata carries no snapshot. + k_slots = prefill_k_slots( + req_to_token=self.req_to_token, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens_cpu=seq_lens_cpu, + ratio=layer.compress_ratio, + ) return PrefillInputs( indexer=layer.indexer, layer_id=layer.layer_id, @@ -3792,8 +3846,10 @@ class DeepseekV4AttnBackend( positions=pos, req_rows=req, req_pool_indices=forward_batch.req_pool_indices, + k_slots=k_slots, + index_plans=self.forward_metadata.low_ratio_index_plans, kv_page_table=core.page_table, - seq_lens_cpu=_as_int_list(forward_batch.seq_lens_cpu), + seq_lens_cpu=seq_lens_cpu, rows_per_request=rows_per_request, rows_per_request_device=rows_per_request_device, out_raw_indices=core.sparse_raw_indices(layer.compress_ratio), diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py index dd03f26d51..59e9e54ac6 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py @@ -123,7 +123,6 @@ def make_candidate_indexer( sparse_table = SparseTableBackend( token_to_kv_pool=token_to_kv_pool, - req_to_token=req_to_token, page_size=page_size, candidate_topk_blocks=candidate_topk_blocks, candidate_block_size=candidate_block_size, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py index eda27cf9f9..647ce5a45e 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/dense_blocks.py @@ -146,7 +146,7 @@ class DenseBlocksBackend: def _deep_gemm_publish_prefill(self, inputs: PrefillInputs): inputs.reset_outputs() - data = get_deep_gemm_prefill_data(inputs, self.req_to_token) + data = get_deep_gemm_prefill_data(inputs) if data is None: return None selected, blocks = _publish_prefill_blocks( @@ -167,7 +167,7 @@ class DenseBlocksBackend: published: Optional[BlockIds], ) -> None: inputs.reset_outputs() - data = get_deep_gemm_prefill_data(inputs, self.req_to_token) + data = get_deep_gemm_prefill_data(inputs) if data is None: return assert published is not None @@ -189,7 +189,6 @@ class DenseBlocksBackend: for request, chunks in prefill_requests( inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, - req_to_token=self.req_to_token, ): for chunk in chunks: picked.append( @@ -225,7 +224,6 @@ class DenseBlocksBackend: for request, chunks in prefill_requests( inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, - req_to_token=self.req_to_token, ): for chunk in chunks: idx = topk_among_blocks( diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py index 04e19803b4..ec0a973928 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/full_topk.py @@ -72,7 +72,7 @@ class FullTopKIndexer: def _deep_gemm_prefill(self, inputs: PrefillInputs) -> None: inputs.reset_outputs() - data = get_deep_gemm_prefill_data(inputs, self.req_to_token) + data = get_deep_gemm_prefill_data(inputs) if data is None: return kv = self.token_to_kv_pool.get_low_ratio_index_k_fp4( @@ -85,7 +85,6 @@ class FullTopKIndexer: requests = prefill_requests( inputs=inputs, token_to_kv_pool=self.token_to_kv_pool, - req_to_token=self.req_to_token, ) for request, chunks in requests: for chunk in chunks: diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index 9b65fc25c3..76c229fcb6 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -173,9 +173,39 @@ def get_deep_gemm_decode_data( return DeepGEMMDecodeData(q_fp4, q_sf, weights, k_cache) -def get_deep_gemm_prefill_data( - inputs: PrefillInputs, req_to_token: torch.Tensor -) -> Optional[DeepGEMMPrefillData]: +def prefill_k_slots( + *, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens_cpu: List[int], + ratio: int, +) -> torch.Tensor: + """[columns] int64 index-K pool slot of each flattened-K column: request by + request, the compressed positions visible at its newest token.""" + device = req_pool_indices.device + lens_per_request = [s // ratio for s in seq_lens_cpu] + columns = sum(lens_per_request) + starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1] + lens = async_h2d(lens_per_request, dtype=torch.int64, device=device) + starts_dev = async_h2d(starts, dtype=torch.int64, device=device) + request = torch.repeat_interleave( + torch.arange(len(lens_per_request), device=device), lens, output_size=columns + ) + position = torch.arange(columns, device=device) - starts_dev[request] + pool_rows = req_pool_indices.to(torch.int64)[request] + return req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio + + +class PrefillIndexPlan(msgspec.Struct, frozen=True): + """The batch-only part of DeepGEMMPrefillData, the same in every layer of one + compress ratio.""" + + request_starts: torch.Tensor # [rows] int32, first column of the row's request + lens_per_request: List[int] # compressed positions at each request's newest token + compress_lens: torch.Tensor # [rows] int32, compressed positions the row sees + + +def prefill_index_plan(inputs: PrefillInputs) -> Optional[PrefillIndexPlan]: """None when the chunk has no row or no visible compressed position.""" ratio = inputs.compress_ratio pos = inputs.positions @@ -192,30 +222,43 @@ def get_deep_gemm_prefill_data( if columns == 0 or num_tokens == 0: return None starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1] - lens = async_h2d(lens_per_request, dtype=torch.int64, device=device) - starts_dev = async_h2d(starts, dtype=torch.int64, device=device) - request = torch.repeat_interleave( - torch.arange(len(lens_per_request), device=device), lens, output_size=columns + starts_dev = async_h2d(starts, dtype=torch.int32, device=device) + return PrefillIndexPlan( + request_starts=torch.repeat_interleave( + starts_dev, + inputs.rows_per_request_device.to(torch.int64), + output_size=num_tokens, + ), + lens_per_request=lens_per_request, + compress_lens=((pos + 1) // ratio).to(torch.int32), ) - position = torch.arange(columns, device=device) - starts_dev[request] - pool_rows = inputs.req_pool_indices.to(torch.int64)[request] + + +def get_deep_gemm_prefill_data(inputs: PrefillInputs) -> Optional[DeepGEMMPrefillData]: + """None when the chunk has no row or no visible compressed position.""" + ratio = inputs.compress_ratio + plans = inputs.index_plans + if plans is not None and ratio in plans: + plan = plans[ratio] + else: + plan = prefill_index_plan(inputs) + if plans is not None: + plans[ratio] = plan + if plan is None: + return None q_fp4, q_sf, weights = _index_q_and_weights( indexer=inputs.indexer, x=inputs.x, q_lora=inputs.q_lora, freqs_cis=inputs.freqs_cis, - positions=pos, + positions=inputs.positions, ) return DeepGEMMPrefillData( - k_slots=req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio, - request_starts=torch.repeat_interleave( - starts_dev.to(torch.int32), - inputs.rows_per_request_device.to(torch.int64), - output_size=num_tokens, - ), - lens_per_request=lens_per_request, + k_slots=inputs.k_slots, + request_starts=plan.request_starts, + lens_per_request=plan.lens_per_request, rows_per_request=inputs.rows_per_request, - compress_lens=((pos + 1) // ratio).to(torch.int32), + compress_lens=plan.compress_lens, q_fp4=q_fp4, q_sf=q_sf, weights=weights, @@ -312,7 +355,6 @@ def prefill_requests( *, inputs: PrefillInputs, token_to_kv_pool: DeepSeekV4TokenToKVPool, - req_to_token: torch.Tensor, ) -> Iterator[tuple[RequestScores, Iterator[ChunkScores]]]: """Requests with nothing visible yet are skipped.""" pool = token_to_kv_pool @@ -320,6 +362,14 @@ def prefill_requests( indexer = inputs.indexer req, pos = inputs.req_rows, inputs.positions assert req is not None, "the torch indexer needs the batch's rows" + assert inputs.seq_lens_cpu is not None + # First flattened-K column of each request row, as inputs.k_slots lays them out. + k_starts = dict( + zip( + inputs.req_pool_indices.tolist(), + itertools.accumulate((s // ratio for s in inputs.seq_lens_cpu), initial=0), + ) + ) inputs.reset_outputs() q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos]) weights = indexer.head_weights(inputs.x) @@ -333,7 +383,7 @@ def prefill_requests( if lc == 0: continue j = torch.arange(lc, device=pos.device) - slots = req_to_token[r, j * ratio].to(torch.int64) // ratio + slots = inputs.k_slots[k_starts[r] : k_starts[r] + lc] # Dequantize only this request's visible K rows; the table is pool-sized. index_k = pool.get_low_ratio_index_k_dequant(inputs.layer_id, slots) request = RequestScores( diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py index 382da22b22..34a48c314c 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/sparse_table.py @@ -120,14 +120,12 @@ class SparseTableBackend: self, *, token_to_kv_pool: DeepSeekV4TokenToKVPool, - req_to_token: torch.Tensor, page_size: int, candidate_topk_blocks: int, candidate_block_size: int, ) -> None: assert candidate_block_size == CANDIDATE_BLOCK_SIZE self.token_to_kv_pool = token_to_kv_pool - self.req_to_token = req_to_token self.page_size = page_size self.topk_blocks = candidate_topk_blocks self.alt_stream = torch.cuda.Stream() @@ -139,7 +137,7 @@ class SparseTableBackend: def publish_prefill(self, inputs: PrefillInputs): inputs.reset_outputs() - data = get_deep_gemm_prefill_data(inputs, self.req_to_token) + data = get_deep_gemm_prefill_data(inputs) if data is None: return None index_page_size = self.token_to_kv_pool.get_index_k_page_size( @@ -169,7 +167,7 @@ class SparseTableBackend: published: Optional[_SparsePrefillTable], ) -> None: inputs.reset_outputs() - data = get_deep_gemm_prefill_data(inputs, self.req_to_token) + data = get_deep_gemm_prefill_data(inputs) if data is None: return assert published is not None diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py index 7800b366ac..36487d083b 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional, Protocol, TypeVar +from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, TypeVar import msgspec import torch @@ -47,6 +47,13 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True): # rank's rows are an interleaved subset of the batch's. req_rows: Optional[torch.Tensor] req_pool_indices: torch.Tensor # [requests] req_to_token row of each request + # [columns] int64 index-K pool slot of each flattened-K column, from + # scoring.prefill_k_slots; gathered with the metadata, never per layer. + k_slots: torch.Tensor + # ratio -> scoring.PrefillIndexPlan, filled by the first layer of each ratio and + # reused by the rest of the forward; None when the metadata outlives a forward + # (captured graphs), so every layer builds its own. + index_plans: Optional[Dict[int, object]] # [rows, pages] int32 at the KV page size; the index-K pool pages it with # expand_index_page_table. kv_page_table: torch.Tensor