[DSv4.1] Prefill indexer: raw selection, row-pair schedule ids, no host syncs (#41659)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
DarkSharpness
2026-10-05 14:33:37 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 BBuf
parent 503b898540
commit 520d6d0229
14 changed files with 441 additions and 293 deletions
@@ -82,7 +82,6 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
CapturedPrefillInputs,
DecodeInputs,
PrefillInputs,
Selection,
has_dense_fp4_indexer,
is_sm100_or_newer,
make_candidate_indexer,
@@ -122,6 +121,7 @@ from sglang.srt.speculative.ragged_verify import (
resolve_ragged_verify_layout,
)
from sglang.srt.utils import ceil_align, is_cuda, is_xpu
from sglang.srt.utils.common import async_h2d
if TYPE_CHECKING:
from sgl_kernel.flash_mla import FlashMLASchedMeta
@@ -3136,7 +3136,6 @@ class DeepseekV4AttnBackend(
is_source = layer.indexer.is_candidate_source
is_consumer = layer.indexer.uses_candidates
ratio = layer.compress_ratio
out = self._get_low_ratio_selection(ratio)
published = self.forward_metadata.candidate_metadata
mode = forward_batch.forward_mode
if mode.is_decode() or mode.is_target_verify():
@@ -3154,27 +3153,28 @@ class DeepseekV4AttnBackend(
layer, x, q_lora, req, pos, mode
)
if is_source:
published = self.decode_candidates.publish_decode(inputs, out)
published = self.decode_candidates.publish_decode(inputs)
self._publish_candidate_metadata(published)
elif is_consumer:
self.decode_candidates.consume_decode(inputs, published, out)
self.decode_candidates.consume_decode(inputs, published)
else:
self.full_topk_indexer.topk_decode(inputs, out)
self.full_topk_indexer.topk_decode(inputs)
else:
inputs = self._make_low_ratio_prefill_indexer_inputs(
layer, x, q_lora, req, pos, forward_batch, rows_per_request
)
if is_source:
published = self.prefill_candidates.publish_prefill(inputs, out)
published = self.prefill_candidates.publish_prefill(inputs)
self._publish_candidate_metadata(published)
elif is_consumer:
self.prefill_candidates.consume_prefill(inputs, published, out)
self.prefill_candidates.consume_prefill(inputs, published)
else:
self.full_topk_indexer.topk_prefill(inputs, out)
self.full_topk_indexer.topk_prefill(inputs)
def _low_ratio_index_topk_captured(self, layer, projected_q, projected_w) -> None:
ratio = layer.compress_ratio
indexer = layer.indexer
core = self.forward_metadata.core_metadata
metadata = (
self.forward_metadata.c1_indexer_metadata
if ratio == 1
@@ -3193,16 +3193,10 @@ class DeepseekV4AttnBackend(
q=projected_q,
weights=projected_w,
paged_metadata=metadata,
out_page_indices=core.sparse_page_indices(ratio),
out_raw_indices=core.sparse_raw_indices(ratio),
)
out = self._get_low_ratio_selection(ratio)
self.full_topk_indexer.topk_prefill_captured(inputs, out)
def _get_low_ratio_selection(self, compress_ratio: int) -> Selection:
core = self.forward_metadata.core_metadata
return Selection(
page_indices=core.sparse_page_indices(compress_ratio),
raw_indices=core.sparse_raw_indices(compress_ratio),
)
self.full_topk_indexer.topk_prefill_captured(inputs)
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
ratio = layer.compress_ratio
@@ -3223,6 +3217,9 @@ class DeepseekV4AttnBackend(
req_rows=req,
paged_metadata=metadata,
is_verify=mode.is_target_verify(),
out_page_indices=self.forward_metadata.core_metadata.sparse_page_indices(
ratio
),
)
def _make_low_ratio_prefill_indexer_inputs(
@@ -3235,9 +3232,10 @@ class DeepseekV4AttnBackend(
forward_batch,
rows_per_request,
) -> PrefillInputs:
core = self.forward_metadata.core_metadata
tail = self.forward_metadata.late_layer_tail
if rows_per_request is not None:
rows_per_request_device = torch.tensor(
rows_per_request_device = async_h2d(
rows_per_request, dtype=torch.int32, device=x.device
)
elif tail is not None:
@@ -3256,10 +3254,29 @@ class DeepseekV4AttnBackend(
positions=pos,
req_rows=req,
req_pool_indices=forward_batch.req_pool_indices,
kv_page_table=self.forward_metadata.core_metadata.page_table,
kv_page_table=core.page_table,
seq_lens_cpu=_as_int_list(forward_batch.seq_lens_cpu),
rows_per_request=rows_per_request,
rows_per_request_device=rows_per_request_device,
out_raw_indices=core.sparse_raw_indices(layer.compress_ratio),
out_page_indices=(
core.sparse_page_indices(layer.compress_ratio)
if self._low_ratio_prefill_reads_page_indices(forward_batch)
else None
),
)
def _low_ratio_prefill_reads_page_indices(self, forward_batch: ForwardBatch):
# The negation of the sparse-prefill gate in forward(), which reads only
# raw_indices; the query-count clause is dropped, so this errs to True.
return (
self.trtllm_attn
or not forward_batch.forward_mode.is_extend_without_speculative()
or get_platform().is_sm120
or self.tail_forward_metadata is not None
or self.forward_metadata.late_layer_tail is not None
or self.token_to_kv_pool.request_window is not None
or not envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
)
def _publish_candidate_metadata(self, published: Optional[CandidateMetadata]):
@@ -17,7 +17,6 @@ from .types import (
DecodeInputs,
PrefillCandidates,
PrefillInputs,
Selection,
)
if TYPE_CHECKING:
@@ -31,7 +30,6 @@ __all__ = [
"FullTopKIndexer",
"PrefillCandidates",
"PrefillInputs",
"Selection",
"has_dense_fp4_indexer",
"is_sm100_or_newer",
"make_candidate_indexer",
@@ -30,7 +30,6 @@ from .types import (
CandidateMetadata,
DecodeInputs,
PrefillInputs,
Selection,
get_tail_row_indices,
)
@@ -73,26 +72,24 @@ class DenseBlocksBackend:
self.block_size = candidate_block_size
self.use_deep_gemm_prefill = use_deep_gemm_prefill
def publish_prefill(self, inputs: PrefillInputs, out: Selection):
def publish_prefill(self, inputs: PrefillInputs):
if self.use_deep_gemm_prefill:
return self._deep_gemm_publish_prefill(inputs, out)
return self._torch_publish_prefill(inputs, out)
return self._deep_gemm_publish_prefill(inputs)
return self._torch_publish_prefill(inputs)
def consume_prefill(
self,
inputs: PrefillInputs,
published: Optional[BlockIds],
out: Selection,
) -> None:
if self.use_deep_gemm_prefill:
self._deep_gemm_consume_prefill(inputs, published, out)
self._deep_gemm_consume_prefill(inputs, published)
else:
self._torch_consume_prefill(inputs, published, out)
self._torch_consume_prefill(inputs, published)
def publish_decode(self, inputs: DecodeInputs, out: Selection):
def publish_decode(self, inputs: DecodeInputs):
d = decode_scores(
inputs=inputs,
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
)
@@ -116,18 +113,16 @@ class DenseBlocksBackend:
block_size=self.block_size,
)
)
select_decode(out, d, inputs.indexer.index_topk)
select_decode(inputs, d, inputs.indexer.index_topk)
return published
def consume_decode(
self,
inputs: DecodeInputs,
published: Optional[BlockIds],
out: Selection,
) -> None:
d = decode_scores(
inputs=inputs,
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
candidate_mask=published.decode_mask if published is not None else None,
@@ -137,18 +132,18 @@ class DenseBlocksBackend:
assert published is not None and published.blocks.shape[0] == d.bs
if isinstance(d, PagedDecodeScores):
assert published.decode_mask is not None
select_decode(out, d, inputs.indexer.index_topk)
select_decode(inputs, d, inputs.indexer.index_topk)
return
k = min(inputs.indexer.index_topk, d.lmax)
idx = topk_among_blocks(
d.scores, d.lens, published.blocks, k, block_size=self.block_size
)
write_decode(out, d, idx.masked_fill(idx < 0, d.lmax))
write_decode(inputs, d, idx.masked_fill(idx < 0, d.lmax))
# ---------- DeepGEMM: flattened-K scores ----------
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs, out: Selection):
out.reset()
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs):
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
if data is None:
return None
@@ -161,16 +156,15 @@ class DenseBlocksBackend:
topk_blocks=self.topk_blocks,
block_size=self.block_size,
)
data.write_selection(selected=selected, out=out)
data.write_selection(selected, inputs)
return BlockIds(blocks=blocks, rows_per_request=data.rows_per_request)
def _deep_gemm_consume_prefill(
self,
inputs: PrefillInputs,
published: Optional[BlockIds],
out: Selection,
) -> None:
out.reset()
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
if data is None:
return
@@ -184,15 +178,14 @@ class DenseBlocksBackend:
blocks=published.blocks,
block_size=self.block_size,
)
data.write_selection(selected=selected, out=out)
data.write_selection(selected, inputs)
# ---------- torch: per-request scores ----------
def _torch_publish_prefill(self, inputs: PrefillInputs, out: Selection):
def _torch_publish_prefill(self, inputs: PrefillInputs):
picked = [] # (query rows, their block ids) per scored chunk
for request, chunks in prefill_requests(
inputs=inputs,
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
):
@@ -209,7 +202,7 @@ class DenseBlocksBackend:
)
)
idx = chunk.scores.topk(request.k, dim=-1, sorted=False).indices
write_prefill(out, request, chunk, idx)
write_prefill(inputs, request, chunk, idx)
width = max((ids.shape[1] for _, ids in picked), default=0)
blocks = torch.full(
(inputs.positions.shape[0], width),
@@ -225,12 +218,10 @@ class DenseBlocksBackend:
self,
inputs: PrefillInputs,
published: Optional[BlockIds],
out: Selection,
) -> None:
assert published is not None
for request, chunks in prefill_requests(
inputs=inputs,
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
):
@@ -242,7 +233,9 @@ class DenseBlocksBackend:
request.k,
block_size=self.block_size,
)
write_prefill(out, request, chunk, idx.masked_fill(idx < 0, request.lc))
write_prefill(
inputs, request, chunk, idx.masked_fill(idx < 0, request.lc)
)
def _publish_prefill_blocks(
@@ -28,7 +28,6 @@ from .types import (
CapturedPrefillInputs,
DecodeInputs,
PrefillInputs,
Selection,
)
if TYPE_CHECKING:
@@ -49,45 +48,44 @@ class FullTopKIndexer:
self.use_deep_gemm_prefill = use_deep_gemm_prefill
self.use_deep_gemm_decode = use_deep_gemm_decode
def topk_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
def topk_prefill(self, inputs: PrefillInputs) -> None:
if self.use_deep_gemm_prefill:
self._deep_gemm_prefill(inputs, out)
self._deep_gemm_prefill(inputs)
else:
self._torch_prefill(inputs, out)
self._torch_prefill(inputs)
def topk_prefill_captured(self, inputs: CapturedPrefillInputs, out: Selection):
self._deep_gemm_prefill_captured(inputs, out)
def topk_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
self._deep_gemm_prefill_captured(inputs)
def topk_decode(self, inputs: DecodeInputs, out: Selection) -> None:
def topk_decode(self, inputs: DecodeInputs) -> None:
if self.use_deep_gemm_decode:
self._deep_gemm_decode(inputs, out)
self._deep_gemm_decode(inputs)
else:
self._torch_decode(inputs, out)
self._torch_decode(inputs)
def _deep_gemm_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
out.reset()
def _deep_gemm_prefill(self, inputs: PrefillInputs) -> None:
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
if data is None:
return
kv = self.token_to_kv_pool.get_low_ratio_index_k_fp4(
inputs.layer_id, data.k_slots
)
selected = dense_prefill_topk(data, kv, topk=inputs.indexer.index_topk)
data.write_selection(selected=selected, out=out)
dense_prefill_topk(data, kv, out=inputs.out_raw_indices[: data.num_rows])
data.write_page_indices(inputs)
def _torch_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
def _torch_prefill(self, inputs: PrefillInputs) -> None:
requests = prefill_requests(
inputs=inputs,
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
)
for request, chunks in requests:
for chunk in chunks:
idx = chunk.scores.topk(request.k, dim=-1, sorted=False).indices
write_prefill(out, request, chunk, idx)
write_prefill(inputs, request, chunk, idx)
def _deep_gemm_decode(self, inputs: DecodeInputs, out: Selection) -> None:
def _deep_gemm_decode(self, inputs: DecodeInputs) -> None:
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
metadata = inputs.paged_metadata
if isinstance(metadata.deep_gemm_metadata, list):
@@ -107,8 +105,7 @@ class FullTopKIndexer:
topk_transform_paged_from_metadata(
logits,
metadata,
out.page_indices,
out.raw_indices,
inputs.out_page_indices,
rows=rows,
topk_metadata=(
topk_plans[chunk_idx] if topk_plans is not None else None
@@ -126,23 +123,20 @@ class FullTopKIndexer:
)
# TODO(dark): add bf16 topk
topk_transform_paged_from_metadata(
logits, metadata, out.page_indices, out.raw_indices
logits, metadata, inputs.out_page_indices, None
)
def _torch_decode(self, inputs: DecodeInputs, out: Selection) -> None:
def _torch_decode(self, inputs: DecodeInputs) -> None:
d = decode_scores(
inputs=inputs,
out=out,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
)
if d is None:
return
select_decode(out, d, inputs.indexer.index_topk)
select_decode(inputs, d, inputs.indexer.index_topk)
def _deep_gemm_prefill_captured(
self, inputs: CapturedPrefillInputs, out: Selection
) -> None:
def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
indexer = inputs.indexer
metadata = inputs.paged_metadata
assert indexer.n_local_heads == indexer.n_heads
@@ -176,7 +170,7 @@ class FullTopKIndexer:
logits,
lens[rows],
page_table[rows],
out.page_indices[rows, :topk],
inputs.out_page_indices[rows, :topk],
page_size,
out.raw_indices[rows, :topk] if out.raw_indices is not None else None,
inputs.out_raw_indices[rows, :topk],
)
@@ -4,29 +4,27 @@ a selection back into pool slots."""
from __future__ import annotations
import itertools
from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple
import msgspec
import torch
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
finish_flat_indexer_topk,
finish_paged_indexer_topk,
fp4_index_logits_decode,
fp4_index_logits_paged,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import index_q_rope_pack_weights
from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles
from sglang.kernels.ops.attention.dsv4.topk import (
plan_topk_v2,
topk_transform_paged_v2,
)
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils.common import async_h2d
from .types import (
DecodeInputs,
PrefillInputs,
Selection,
)
from .types import DecodeInputs, PrefillInputs
if TYPE_CHECKING:
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
@@ -66,31 +64,27 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
(self.num_rows, topk), -1, dtype=torch.int32
)
def write_selection(self, selected: torch.Tensor, out: Selection) -> None:
if (
selected.is_cuda
and get_platform().is_sm100
and selected.dtype == torch.int32
):
finish_flat_indexer_topk(
selected,
self.k_slots,
self.request_starts,
out.page_indices,
out.raw_indices,
)
def write_page_indices(self, inputs: PrefillInputs) -> None:
"""The pool slots of the request-local ``inputs.out_raw_indices`` rows."""
if inputs.out_page_indices is None:
return
raw = inputs.out_raw_indices[: self.num_rows]
columns = (raw + self.request_starts[:, None]).clamp_min(0)
inputs.out_page_indices[: self.num_rows, : raw.shape[1]] = torch.where(
raw >= 0, self.k_slots[columns], -1
)
def write_selection(self, selected: torch.Tensor, inputs: PrefillInputs) -> None:
"""Write flattened-K columns ``selected`` into the selection buffers."""
num_tokens, topk = selected.shape
unselected = torch.iinfo(torch.int32).max
selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values
chosen = selected != unselected
out.page_indices[:num_tokens, :topk] = torch.where(
chosen, self.k_slots[selected.clamp_max(self.k_slots.shape[0] - 1)], -1
).to(torch.int32)
if out.raw_indices is not None:
out.raw_indices[:num_tokens, :topk] = torch.where(
chosen, selected - self.request_starts[:, None], -1
)
chosen = selected >= 0
if inputs.out_page_indices is not None:
inputs.out_page_indices[:num_tokens, :topk] = torch.where(
chosen, self.k_slots[selected.clamp_min(0)], -1
).to(torch.int32)
inputs.out_raw_indices[:num_tokens, :topk] = torch.where(
chosen, selected - self.request_starts[:, None], -1
)
def quantize_index_q(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -116,38 +110,42 @@ def get_index_k_cache(
return k_cache.view(k_cache.shape[0], page_size, 1, 68)
def _index_q_and_weights(
*,
indexer: DeepseekV41Indexer,
x: torch.Tensor,
q_lora: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""fp4 index queries ``[rows, heads, 64]`` int8 with ``[rows, heads]`` packed
ue8m0 scales, and fp32 head weights ``[rows, heads]``."""
rows, heads = x.shape[0], indexer.n_local_heads
q, _ = indexer.wq_b(q_lora)
q_fp4, q_sf, weights = index_q_rope_pack_weights(
q.view(rows, heads, indexer.index_head_dim),
torch.view_as_real(freqs_cis).flatten(-2),
positions,
indexer.head_weights_raw(x),
indexer.head_weight_scale,
)
return q_fp4.view(rows, heads, 64), q_sf.view(rows, heads), weights
def get_deep_gemm_decode_data(
inputs: DecodeInputs, token_to_kv_pool: DeepSeekV4TokenToKVPool
) -> DeepGEMMDecodeData:
indexer = inputs.indexer
x, q_lora, pos = inputs.x, inputs.q_lora, inputs.positions
# The kernel sums head scores locally, so the indexer heads must be replicated.
assert indexer.n_local_heads == indexer.n_heads
if (
x.is_cuda
and torch.version.cuda is not None
and x.dtype == torch.bfloat16
and indexer.index_head_dim == 128
):
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
index_q_rope_pack_weights,
)
q, _ = indexer.wq_b(q_lora)
q = q.view(q.shape[0], indexer.n_local_heads, indexer.index_head_dim)
# The fused pack also computes head_weights(x).float(), same rounding.
q_fp4, q_sf, weights = index_q_rope_pack_weights(
q,
torch.view_as_real(inputs.freqs_cis).flatten(-2),
pos,
indexer.head_weights_raw(x), # [bs, n_local] bf16, n32k5120
indexer.head_weight_scale,
)
else:
q = indexer.queries(q_lora, inputs.freqs_cis[pos])
q_fp4, q_sf = quantize_index_q(q)
weights = indexer.head_weights(x).float() # [bs, n_local]
bs = q.shape[0]
q_fp4, q_sf, weights = _index_q_and_weights(
indexer=indexer,
x=inputs.x,
q_lora=inputs.q_lora,
freqs_cis=inputs.freqs_cis,
positions=inputs.positions,
)
bs = q_fp4.shape[0]
q_fp4 = q_fp4.view(bs, 1, indexer.n_local_heads, 64)
q_sf = q_sf.view(bs, 1, indexer.n_local_heads)
@@ -165,47 +163,44 @@ def get_deep_gemm_prefill_data(
) -> Optional[DeepGEMMPrefillData]:
"""None when the chunk has no row or no visible compressed position."""
ratio = inputs.compress_ratio
indexer = inputs.indexer
pos = inputs.positions
seq_lens_cpu = inputs.seq_lens_cpu
assert (
seq_lens_cpu is not None
inputs.seq_lens_cpu is not None
and inputs.rows_per_request is not None
and inputs.rows_per_request_device is not None
), "the DeepGEMM prefill indexer needs the CPU length vectors"
device = pos.device
# Visible compressed positions per request; (pos + 1) // ratio bounds each row.
lc_per_req = [s // ratio for s in seq_lens_cpu]
req_pool_indices = inputs.req_pool_indices.to(torch.int64)
slot_chunks, starts, start = [], [], 0
for r, lc in enumerate(lc_per_req):
starts.append(start)
if lc == 0:
continue
j = torch.arange(lc, device=device)
slot_chunks.append(
req_to_token[req_pool_indices[r], j * ratio].to(torch.int64) // ratio
)
start += lc
lens_per_request = [s // ratio for s in inputs.seq_lens_cpu]
columns = sum(lens_per_request)
num_tokens = pos.shape[0]
if not slot_chunks or num_tokens == 0:
if columns == 0 or num_tokens == 0:
return None
k_slots = torch.cat(slot_chunks)
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
q_fp4, q_sf = quantize_index_q(q)
weights = indexer.head_weights(inputs.x).float()
compress_lens = ((pos + 1) // ratio).to(torch.int32)
request_starts = torch.repeat_interleave(
torch.tensor(starts, dtype=torch.int32, device=device),
inputs.rows_per_request_device.to(torch.int64),
output_size=num_tokens,
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
request = torch.repeat_interleave(
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
)
position = torch.arange(columns, device=device) - starts_dev[request]
pool_rows = inputs.req_pool_indices.to(torch.int64)[request]
q_fp4, q_sf, weights = _index_q_and_weights(
indexer=inputs.indexer,
x=inputs.x,
q_lora=inputs.q_lora,
freqs_cis=inputs.freqs_cis,
positions=pos,
)
return DeepGEMMPrefillData(
k_slots=k_slots,
request_starts=request_starts,
lens_per_request=lc_per_req,
k_slots=req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio,
request_starts=torch.repeat_interleave(
starts_dev.to(torch.int32),
inputs.rows_per_request_device.to(torch.int64),
output_size=num_tokens,
),
lens_per_request=lens_per_request,
rows_per_request=inputs.rows_per_request,
compress_lens=compress_lens,
compress_lens=((pos + 1) // ratio).to(torch.int32),
q_fp4=q_fp4,
q_sf=q_sf,
weights=weights,
@@ -234,22 +229,22 @@ def dense_prefill_topk(
data: DeepGEMMPrefillData,
kv: Tuple[torch.Tensor, torch.Tensor],
*,
topk: int,
) -> torch.Tensor:
"""Flattened-K columns, ``-1`` padded, unordered."""
out: torch.Tensor,
) -> None:
"""Request-local compressed positions into ``out``, ``-1`` padded, unordered."""
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
selected = data.empty_selection(topk)
# Score columns are request-relative, so a zero offset emits request-local positions.
zero_offsets = torch.zeros_like(data.request_starts)
for tile, logits in score_tiles(data, kv, width_align=4):
topk_transform_ragged_v2(
logits,
data.compress_lens[tile],
out_offsets=data.request_starts[tile],
out_indices=selected[tile],
out_offsets=zero_offsets[tile],
out_indices=out[tile],
)
# Free this tile's logits before the generator scores the next one.
del logits
return selected
# ---------- torch ----------
@@ -286,7 +281,6 @@ class DecodeScores(msgspec.Struct, frozen=True):
def prefill_requests(
*,
inputs: PrefillInputs,
out: Selection,
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
) -> Iterator[tuple[RequestScores, Iterator[ChunkScores]]]:
@@ -296,7 +290,7 @@ def prefill_requests(
indexer = inputs.indexer
req, pos = inputs.req_rows, inputs.positions
assert req is not None, "the torch indexer needs the batch's rows"
out.reset()
inputs.reset_outputs()
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
weights = indexer.head_weights(inputs.x)
# A compressed position is visible once the query has passed its last token.
@@ -343,16 +337,19 @@ def _score_chunks(
def write_prefill(
out: Selection, request: RequestScores, chunk: ChunkScores, idx: torch.Tensor
inputs: PrefillInputs,
request: RequestScores,
chunk: ChunkScores,
idx: torch.Tensor,
) -> None:
k, lc = request.k, request.lc
idx = idx.sort(dim=-1).values
reach = idx < chunk.lens[:, None]
out.page_indices[chunk.tok, :k] = torch.where(
reach, request.slots[idx.clamp_max(lc - 1)], -1
).to(torch.int32)
if out.raw_indices is not None:
out.raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
if inputs.out_page_indices is not None:
inputs.out_page_indices[chunk.tok, :k] = torch.where(
reach, request.slots[idx.clamp_max(lc - 1)], -1
).to(torch.int32)
inputs.out_raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
class PagedDecodeScores(msgspec.Struct, frozen=True):
@@ -377,7 +374,6 @@ class PagedDecodeScores(msgspec.Struct, frozen=True):
def decode_scores(
*,
inputs: DecodeInputs,
out: Selection,
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
candidate_mask: Optional[torch.Tensor] = None,
@@ -393,7 +389,7 @@ def decode_scores(
f"decode expects one token per request, {pos.shape=} {bs=}"
)
if bs == 0:
out.reset()
inputs.reset_outputs()
return None
lens = (pos + 1) // ratio
metadata = inputs.paged_metadata
@@ -401,7 +397,7 @@ def decode_scores(
# A capture-time length read would both synchronize and truncate replay.
lmax = min(metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio)
if lmax == 0:
out.reset()
inputs.reset_outputs()
return None
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
weights = indexer.head_weights(inputs.x)
@@ -433,7 +429,7 @@ def decode_scores(
plan=plan,
has_candidate_mask=candidate_mask is not None,
)
out.reset()
inputs.reset_outputs()
j = torch.arange(lmax, device=pos.device)
valid = j[None, :] < lens[:, None]
slots = req_to_token[req[:, None], (j * ratio)[None, :]].to(torch.int64) // ratio
@@ -445,7 +441,7 @@ def decode_scores(
def select_decode(
out: Selection, d: DecodeScores | PagedDecodeScores, topk: int
inputs: DecodeInputs, d: DecodeScores | PagedDecodeScores, topk: int
) -> None:
k = min(topk, d.lmax)
if isinstance(d, PagedDecodeScores):
@@ -457,21 +453,19 @@ def select_decode(
d.lens,
d.req,
d.req_to_token,
out.page_indices,
out.raw_indices,
inputs.out_page_indices,
None,
d.ratio,
d.has_candidate_mask,
)
else:
write_decode(out, d, d.scores.topk(k, dim=-1, sorted=False).indices)
write_decode(inputs, d, d.scores.topk(k, dim=-1, sorted=False).indices)
def write_decode(out: Selection, d: DecodeScores, idx: torch.Tensor) -> None:
def write_decode(inputs: DecodeInputs, d: DecodeScores, idx: torch.Tensor) -> None:
k = idx.shape[1]
idx = idx.sort(dim=-1).values
reach = idx < d.lens[:, None]
out.page_indices[: d.bs, :k] = torch.where(
inputs.out_page_indices[: d.bs, :k] = torch.where(
reach, d.slots.gather(1, idx.clamp_max(d.lmax - 1)), -1
).to(torch.int32)
if out.raw_indices is not None:
out.raw_indices[: d.bs, :k] = torch.where(reach, idx, -1).to(torch.int32)
@@ -30,6 +30,7 @@ from sglang.kernels.ops.attention.dsv4.topk import (
)
from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata
from sglang.srt.layers.attention.dsv4.metadata import expand_index_page_table
from sglang.srt.utils.common import async_h2d
from .scoring import (
DeepGEMMDecodeData,
@@ -43,7 +44,6 @@ from .types import (
CandidateMetadata,
DecodeInputs,
PrefillInputs,
Selection,
get_tail_row_indices,
)
@@ -73,7 +73,7 @@ class _SparsePrefillTable(_SparseTable):
compress_lens: torch.Tensor # [rows] int32
page_table: torch.Tensor # [rows, index pages] int32
request_ids: torch.Tensor # [rows] int32
request_ids: torch.Tensor # [rows] int32 row-pair ids, see _row_pair_ids
q_dtype: torch.dtype
page_size: int # index-K pool page size (slots)
rows_per_request: List[int] # rows of each request, in row order
@@ -118,12 +118,11 @@ class SparseTableBackend:
# captured graphs keep reading the buffers they saw
self._retired_cached_row_ids: list = []
def publish_prefill(self, inputs: PrefillInputs, out: Selection):
out.reset()
def publish_prefill(self, inputs: PrefillInputs):
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
if data is None:
return None
selected = data.empty_selection(inputs.indexer.index_topk)
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
inputs.compress_ratio
)
@@ -140,23 +139,21 @@ class SparseTableBackend:
).contiguous(),
index_page_size=index_page_size,
topk_blocks=self.topk_blocks,
out_positions=selected,
out_positions=inputs.out_raw_indices[: data.num_rows],
)
data.write_selection(selected=selected, out=out)
data.write_page_indices(inputs)
return table
def consume_prefill(
self,
inputs: PrefillInputs,
published: Optional[_SparsePrefillTable],
out: Selection,
) -> None:
out.reset()
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
if data is None:
return
assert published is not None
selected = data.empty_selection(inputs.indexer.index_topk)
select_prefill_table(
table=published,
data=data,
@@ -167,16 +164,16 @@ class SparseTableBackend:
inputs.compress_ratio
),
),
out_positions=selected,
out_positions=inputs.out_raw_indices[: data.num_rows],
)
data.write_selection(selected=selected, out=out)
data.write_page_indices(inputs)
def publish_decode(self, inputs: DecodeInputs, out: Selection):
def publish_decode(self, inputs: DecodeInputs):
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
metadata = inputs.paged_metadata
seq_lens = metadata.compressed_seq_lens.reshape(-1)
if isinstance(metadata.deep_gemm_metadata, list):
return self._publish_decode_chunked(inputs, data, seq_lens, out)
return self._publish_decode_chunked(inputs, data, seq_lens)
logits = deep_gemm_fp4_paged_mqa_logits(
(data.q_fp4, data.q_sf),
data.k_cache,
@@ -195,10 +192,9 @@ class SparseTableBackend:
logits,
seq_lens,
metadata.page_table,
out.page_indices,
inputs.out_page_indices,
metadata.compressed_page_size,
metadata.topk_metadata,
out_raw_indices=out.raw_indices,
)
# per row: block count for the block top-k, sparse-row length for the consumers
with torch.cuda.stream(self.alt_stream):
@@ -237,7 +233,6 @@ class SparseTableBackend:
inputs: DecodeInputs,
data: DeepGEMMDecodeData,
seq_lens: torch.Tensor,
out: Selection,
) -> _SparseTable:
"""On the current stream, so each chunk's logits are released before the next."""
metadata = inputs.paged_metadata
@@ -259,8 +254,8 @@ class SparseTableBackend:
topk_transform_paged_from_metadata(
logits,
metadata,
out.page_indices,
out.raw_indices,
inputs.out_page_indices,
None,
rows=rows,
topk_metadata=(
topk_plans[chunk_idx] if topk_plans is not None else None
@@ -304,7 +299,6 @@ class SparseTableBackend:
self,
inputs: DecodeInputs,
published: Optional[_SparseTable],
out: Selection,
) -> None:
assert published is not None
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
@@ -317,9 +311,8 @@ class SparseTableBackend:
published.schedule,
published.blocks.shape[1],
)
assert out.raw_indices is None
topk_transform_sparse(
logits, published.valid_lens, published.phys_blocks, out.page_indices
logits, published.valid_lens, published.phys_blocks, inputs.out_page_indices
)
def _get_request_ids(self, inputs: DecodeInputs, rows: int, device: torch.device):
@@ -372,6 +365,25 @@ def _build_prefill_table(
)
def _row_pair_ids(rows_per_request: List[int], *, device: torch.device) -> torch.Tensor:
"""[rows] int32: one id per consecutive row pair of a request, never across two.
DeepGEMM's schedule walks back over equal ids to a row's request start once per
row, quadratic in a request's rows; per-pair ids bound the walk to one step.
Pairs, not larger groups, because DeepGEMM's paged scheduler has BLOCK_Q == 2:
the ids only have to delimit what it already groups, and its page table is
indexed by Q row rather than by the id value, so the ids never address anything.
A wider BLOCK_Q would need the pair width here to follow it."""
counts = async_h2d(rows_per_request, dtype=torch.int64, device=device)
rows = sum(rows_per_request)
row = torch.arange(rows, device=device)
request = torch.repeat_interleave(
torch.arange(counts.numel(), device=device), counts, output_size=rows
)
first_row = torch.cumsum(counts, 0) - counts
return (row - ((row - first_row[request]) & 1)).to(torch.int32)
# TODO(dark): support fusion of publish + topk of publish layer
def publish_prefill_table(
*,
@@ -382,8 +394,9 @@ def publish_prefill_table(
topk_blocks: int,
out_positions: torch.Tensor,
) -> _SparsePrefillTable:
"""The source's own top-k into ``out_positions`` and the chunk's table, from one
pass over its dense scores; ``index_page_table`` is at ``index_page_size`` slots."""
"""The source's own top-k into ``out_positions`` (request-local compressed
positions) and the chunk's table, from one pass over its dense scores;
``index_page_table`` is at ``index_page_size`` slots."""
rows = data.num_rows
device = data.q_fp4.device
nblocks, valid_lens = candidate_row_lens(data.compress_lens, topk_blocks)
@@ -395,7 +408,7 @@ def publish_prefill_table(
topk_transform_ragged_v2(
logits,
lens,
out_offsets=data.request_starts[tile],
out_offsets=zero_offsets[tile],
out_indices=out_positions[tile],
)
# keys past a row's block count stay unset; the top-k reads a row
@@ -411,17 +424,12 @@ def publish_prefill_table(
out_offsets=zero_offsets[: logits.shape[0]],
out_indices=blocks[tile],
)
request_ids = torch.repeat_interleave(
torch.arange(len(data.rows_per_request), dtype=torch.int32, device=device),
torch.tensor(data.rows_per_request, dtype=torch.int64, device=device),
output_size=rows,
)
return _build_prefill_table(
blocks=blocks,
compress_lens=data.compress_lens,
page_table=index_page_table,
page_size=index_page_size,
request_ids=request_ids,
request_ids=_row_pair_ids(data.rows_per_request, device=device),
rows_per_request=data.rows_per_request,
q_dtype=data.q_fp4.dtype,
valid_lens=valid_lens,
@@ -446,4 +454,3 @@ def select_prefill_table(
)
# request-relative compressed positions, -1 padded
topk_transform_sparse(logits, table.valid_lens, table.blocks, out_positions)
out_positions.add_(torch.where(out_positions >= 0, data.request_starts[:, None], 0))
@@ -1,4 +1,4 @@
"""The inputs and outputs of the V4.1 indexer backends."""
"""The inputs of the V4.1 indexer backends, including the selection buffers they fill."""
from __future__ import annotations
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, List, Optional, Protocol, TypeVar
import msgspec
import torch
from sglang.srt.utils.common import async_h2d
if TYPE_CHECKING:
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
@@ -22,11 +24,7 @@ def get_tail_row_indices(
for n, t in zip(full_rows_per_request, tail_rows_per_request):
rows.extend(range(start + n - t, start + n))
start += n
device = torch.device(device)
if device.type != "cuda":
return torch.tensor(rows, dtype=torch.int64, device=device)
staged = torch.tensor(rows, dtype=torch.int64, pin_memory=True)
return staged.to(device, non_blocking=True)
return async_h2d(rows, dtype=torch.int64, device=device)
class CandidateMetadata:
@@ -34,16 +32,6 @@ class CandidateMetadata:
raise NotImplementedError(f"{type(self).__name__} is not a prefill publish")
class Selection(msgspec.Struct, frozen=True):
page_indices: torch.Tensor # index-K pool slots, ascending
raw_indices: Optional[torch.Tensor] # the same picks as compressed positions
def reset(self) -> None:
self.page_indices.fill_(-1)
if self.raw_indices is not None:
self.raw_indices.fill_(-1)
class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
"""An eager extend: the backend projects the queries and gathers the index K."""
@@ -68,6 +56,17 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
rows_per_request: Optional[List[int]]
rows_per_request_device: Optional[torch.Tensor]
# The selection, unordered, -1 padded: [rows, topk] int32 request-local
# compressed positions, and the same picks as index-K pool slots, which is
# None when this forward's attention reads only the positions (sparse prefill).
out_raw_indices: torch.Tensor
out_page_indices: Optional[torch.Tensor]
def reset_outputs(self) -> None:
self.out_raw_indices.fill_(-1)
if self.out_page_indices is not None:
self.out_page_indices.fill_(-1)
class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
"""Decode (one row per request) or verify (one row per draft token)."""
@@ -83,6 +82,12 @@ class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
paged_metadata: PagedIndexerMetadata
is_verify: bool
# [rows, topk] int32 index-K pool slots of the selection, unordered, -1 padded.
out_page_indices: torch.Tensor
def reset_outputs(self) -> None:
self.out_page_indices.fill_(-1)
class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
"""An extend under the prefill graph: projected queries, paged index K."""
@@ -92,6 +97,9 @@ class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
q: torch.Tensor # [rows, heads, 128] bf16, roped
weights: torch.Tensor # [rows, heads] bf16 head weights
paged_metadata: PagedIndexerMetadata
# The selection as pool slots and as request-local compressed positions.
out_page_indices: torch.Tensor
out_raw_indices: torch.Tensor
Metadata = TypeVar("Metadata", bound=CandidateMetadata)
@@ -100,30 +108,16 @@ Metadata = TypeVar("Metadata", bound=CandidateMetadata)
class PrefillCandidates(Protocol[Metadata]):
"""The attention backend keeps a publish alive and hands it back unread."""
def publish_prefill(
self,
inputs: PrefillInputs,
out: Selection,
) -> Optional[Metadata]: ...
def publish_prefill(self, inputs: PrefillInputs) -> Optional[Metadata]: ...
def consume_prefill(
self,
inputs: PrefillInputs,
published: Optional[Metadata],
out: Selection,
self, inputs: PrefillInputs, published: Optional[Metadata]
) -> None: ...
class DecodeCandidates(Protocol[Metadata]):
def publish_decode(
self,
inputs: DecodeInputs,
out: Selection,
) -> Optional[Metadata]: ...
def publish_decode(self, inputs: DecodeInputs) -> Optional[Metadata]: ...
def consume_decode(
self,
inputs: DecodeInputs,
published: Optional[Metadata],
out: Selection,
self, inputs: DecodeInputs, published: Optional[Metadata]
) -> None: ...
+7 -1
View File
@@ -313,7 +313,13 @@ class EngramHasher(nn.Module):
)
lens = forward_batch.extend_seq_lens.to(torch.int64)
starts = forward_batch.extend_start_loc.to(torch.int64)
row = torch.repeat_interleave(torch.arange(bs, device=device), lens)
lens_cpu = forward_batch.extend_seq_lens_cpu
row = torch.repeat_interleave(
torch.arange(bs, device=device),
lens,
# The host total skips the device sum's sync.
output_size=sum(lens_cpu) if lens_cpu is not None else None,
)
num_real = row.shape[0]
kmode = MODE_EXTEND
if forward_batch.engram_history is not None:
@@ -12,6 +12,7 @@ from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.managers.schedule_batch import ForwardMode
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.runtime_context import get_schedule
from sglang.srt.utils.common import async_h2d
if TYPE_CHECKING:
from sglang.srt.layers.engram import EngramHasher
@@ -200,7 +201,8 @@ class NgramEmbeddingManager:
lo = max(0, start - n1)
ids = req.full_untruncated_fill_ids[lo:start]
rows.append([0] * (n1 - len(ids)) + list(ids))
batch.engram_history = torch.tensor(
# The forward stream waits on the schedule stream, which orders this copy.
batch.engram_history = async_h2d(
rows, dtype=torch.int32, device=self.engram_hasher.history.device
).view(len(rows), n1)
+7
View File
@@ -577,6 +577,13 @@ def is_pin_memory_available(device=None) -> bool:
return current_platform.is_pin_memory_available(device)
def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
"""A host list on ``device``; staged in pinned memory so the copy is enqueued
on the current stream instead of synchronizing like a pageable copy."""
pin = is_pin_memory_available(device)
return torch.tensor(args, dtype=dtype, pin_memory=pin).to(device, non_blocking=pin)
def async_d2h(tensor: torch.Tensor) -> torch.Tensor:
"""Enqueue a CUDA-to-pinned-host copy on the current stream."""
if not tensor.is_cuda: