[DSv4.1] Prefill indexer: raw selection, row-pair schedule ids, no host syncs (#41659)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
BBuf
parent
503b898540
commit
520d6d0229
@@ -82,7 +82,6 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
has_dense_fp4_indexer,
|
||||
is_sm100_or_newer,
|
||||
make_candidate_indexer,
|
||||
@@ -122,6 +121,7 @@ from sglang.srt.speculative.ragged_verify import (
|
||||
resolve_ragged_verify_layout,
|
||||
)
|
||||
from sglang.srt.utils import ceil_align, is_cuda, is_xpu
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
@@ -3136,7 +3136,6 @@ class DeepseekV4AttnBackend(
|
||||
is_source = layer.indexer.is_candidate_source
|
||||
is_consumer = layer.indexer.uses_candidates
|
||||
ratio = layer.compress_ratio
|
||||
out = self._get_low_ratio_selection(ratio)
|
||||
published = self.forward_metadata.candidate_metadata
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_decode() or mode.is_target_verify():
|
||||
@@ -3154,27 +3153,28 @@ class DeepseekV4AttnBackend(
|
||||
layer, x, q_lora, req, pos, mode
|
||||
)
|
||||
if is_source:
|
||||
published = self.decode_candidates.publish_decode(inputs, out)
|
||||
published = self.decode_candidates.publish_decode(inputs)
|
||||
self._publish_candidate_metadata(published)
|
||||
elif is_consumer:
|
||||
self.decode_candidates.consume_decode(inputs, published, out)
|
||||
self.decode_candidates.consume_decode(inputs, published)
|
||||
else:
|
||||
self.full_topk_indexer.topk_decode(inputs, out)
|
||||
self.full_topk_indexer.topk_decode(inputs)
|
||||
else:
|
||||
inputs = self._make_low_ratio_prefill_indexer_inputs(
|
||||
layer, x, q_lora, req, pos, forward_batch, rows_per_request
|
||||
)
|
||||
if is_source:
|
||||
published = self.prefill_candidates.publish_prefill(inputs, out)
|
||||
published = self.prefill_candidates.publish_prefill(inputs)
|
||||
self._publish_candidate_metadata(published)
|
||||
elif is_consumer:
|
||||
self.prefill_candidates.consume_prefill(inputs, published, out)
|
||||
self.prefill_candidates.consume_prefill(inputs, published)
|
||||
else:
|
||||
self.full_topk_indexer.topk_prefill(inputs, out)
|
||||
self.full_topk_indexer.topk_prefill(inputs)
|
||||
|
||||
def _low_ratio_index_topk_captured(self, layer, projected_q, projected_w) -> None:
|
||||
ratio = layer.compress_ratio
|
||||
indexer = layer.indexer
|
||||
core = self.forward_metadata.core_metadata
|
||||
metadata = (
|
||||
self.forward_metadata.c1_indexer_metadata
|
||||
if ratio == 1
|
||||
@@ -3193,16 +3193,10 @@ class DeepseekV4AttnBackend(
|
||||
q=projected_q,
|
||||
weights=projected_w,
|
||||
paged_metadata=metadata,
|
||||
out_page_indices=core.sparse_page_indices(ratio),
|
||||
out_raw_indices=core.sparse_raw_indices(ratio),
|
||||
)
|
||||
out = self._get_low_ratio_selection(ratio)
|
||||
self.full_topk_indexer.topk_prefill_captured(inputs, out)
|
||||
|
||||
def _get_low_ratio_selection(self, compress_ratio: int) -> Selection:
|
||||
core = self.forward_metadata.core_metadata
|
||||
return Selection(
|
||||
page_indices=core.sparse_page_indices(compress_ratio),
|
||||
raw_indices=core.sparse_raw_indices(compress_ratio),
|
||||
)
|
||||
self.full_topk_indexer.topk_prefill_captured(inputs)
|
||||
|
||||
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
|
||||
ratio = layer.compress_ratio
|
||||
@@ -3223,6 +3217,9 @@ class DeepseekV4AttnBackend(
|
||||
req_rows=req,
|
||||
paged_metadata=metadata,
|
||||
is_verify=mode.is_target_verify(),
|
||||
out_page_indices=self.forward_metadata.core_metadata.sparse_page_indices(
|
||||
ratio
|
||||
),
|
||||
)
|
||||
|
||||
def _make_low_ratio_prefill_indexer_inputs(
|
||||
@@ -3235,9 +3232,10 @@ class DeepseekV4AttnBackend(
|
||||
forward_batch,
|
||||
rows_per_request,
|
||||
) -> PrefillInputs:
|
||||
core = self.forward_metadata.core_metadata
|
||||
tail = self.forward_metadata.late_layer_tail
|
||||
if rows_per_request is not None:
|
||||
rows_per_request_device = torch.tensor(
|
||||
rows_per_request_device = async_h2d(
|
||||
rows_per_request, dtype=torch.int32, device=x.device
|
||||
)
|
||||
elif tail is not None:
|
||||
@@ -3256,10 +3254,29 @@ class DeepseekV4AttnBackend(
|
||||
positions=pos,
|
||||
req_rows=req,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
kv_page_table=self.forward_metadata.core_metadata.page_table,
|
||||
kv_page_table=core.page_table,
|
||||
seq_lens_cpu=_as_int_list(forward_batch.seq_lens_cpu),
|
||||
rows_per_request=rows_per_request,
|
||||
rows_per_request_device=rows_per_request_device,
|
||||
out_raw_indices=core.sparse_raw_indices(layer.compress_ratio),
|
||||
out_page_indices=(
|
||||
core.sparse_page_indices(layer.compress_ratio)
|
||||
if self._low_ratio_prefill_reads_page_indices(forward_batch)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _low_ratio_prefill_reads_page_indices(self, forward_batch: ForwardBatch):
|
||||
# The negation of the sparse-prefill gate in forward(), which reads only
|
||||
# raw_indices; the query-count clause is dropped, so this errs to True.
|
||||
return (
|
||||
self.trtllm_attn
|
||||
or not forward_batch.forward_mode.is_extend_without_speculative()
|
||||
or get_platform().is_sm120
|
||||
or self.tail_forward_metadata is not None
|
||||
or self.forward_metadata.late_layer_tail is not None
|
||||
or self.token_to_kv_pool.request_window is not None
|
||||
or not envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
|
||||
def _publish_candidate_metadata(self, published: Optional[CandidateMetadata]):
|
||||
|
||||
@@ -17,7 +17,6 @@ from .types import (
|
||||
DecodeInputs,
|
||||
PrefillCandidates,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -31,7 +30,6 @@ __all__ = [
|
||||
"FullTopKIndexer",
|
||||
"PrefillCandidates",
|
||||
"PrefillInputs",
|
||||
"Selection",
|
||||
"has_dense_fp4_indexer",
|
||||
"is_sm100_or_newer",
|
||||
"make_candidate_indexer",
|
||||
|
||||
@@ -30,7 +30,6 @@ from .types import (
|
||||
CandidateMetadata,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
get_tail_row_indices,
|
||||
)
|
||||
|
||||
@@ -73,26 +72,24 @@ class DenseBlocksBackend:
|
||||
self.block_size = candidate_block_size
|
||||
self.use_deep_gemm_prefill = use_deep_gemm_prefill
|
||||
|
||||
def publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
def publish_prefill(self, inputs: PrefillInputs):
|
||||
if self.use_deep_gemm_prefill:
|
||||
return self._deep_gemm_publish_prefill(inputs, out)
|
||||
return self._torch_publish_prefill(inputs, out)
|
||||
return self._deep_gemm_publish_prefill(inputs)
|
||||
return self._torch_publish_prefill(inputs)
|
||||
|
||||
def consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
if self.use_deep_gemm_prefill:
|
||||
self._deep_gemm_consume_prefill(inputs, published, out)
|
||||
self._deep_gemm_consume_prefill(inputs, published)
|
||||
else:
|
||||
self._torch_consume_prefill(inputs, published, out)
|
||||
self._torch_consume_prefill(inputs, published)
|
||||
|
||||
def publish_decode(self, inputs: DecodeInputs, out: Selection):
|
||||
def publish_decode(self, inputs: DecodeInputs):
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
@@ -116,18 +113,16 @@ class DenseBlocksBackend:
|
||||
block_size=self.block_size,
|
||||
)
|
||||
)
|
||||
select_decode(out, d, inputs.indexer.index_topk)
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
return published
|
||||
|
||||
def consume_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
candidate_mask=published.decode_mask if published is not None else None,
|
||||
@@ -137,18 +132,18 @@ class DenseBlocksBackend:
|
||||
assert published is not None and published.blocks.shape[0] == d.bs
|
||||
if isinstance(d, PagedDecodeScores):
|
||||
assert published.decode_mask is not None
|
||||
select_decode(out, d, inputs.indexer.index_topk)
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
return
|
||||
k = min(inputs.indexer.index_topk, d.lmax)
|
||||
idx = topk_among_blocks(
|
||||
d.scores, d.lens, published.blocks, k, block_size=self.block_size
|
||||
)
|
||||
write_decode(out, d, idx.masked_fill(idx < 0, d.lmax))
|
||||
write_decode(inputs, d, idx.masked_fill(idx < 0, d.lmax))
|
||||
|
||||
# ---------- DeepGEMM: flattened-K scores ----------
|
||||
|
||||
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
out.reset()
|
||||
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs):
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return None
|
||||
@@ -161,16 +156,15 @@ class DenseBlocksBackend:
|
||||
topk_blocks=self.topk_blocks,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_selection(selected, inputs)
|
||||
return BlockIds(blocks=blocks, rows_per_request=data.rows_per_request)
|
||||
|
||||
def _deep_gemm_consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return
|
||||
@@ -184,15 +178,14 @@ class DenseBlocksBackend:
|
||||
blocks=published.blocks,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_selection(selected, inputs)
|
||||
|
||||
# ---------- torch: per-request scores ----------
|
||||
|
||||
def _torch_publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
def _torch_publish_prefill(self, inputs: PrefillInputs):
|
||||
picked = [] # (query rows, their block ids) per scored chunk
|
||||
for request, chunks in prefill_requests(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
):
|
||||
@@ -209,7 +202,7 @@ class DenseBlocksBackend:
|
||||
)
|
||||
)
|
||||
idx = chunk.scores.topk(request.k, dim=-1, sorted=False).indices
|
||||
write_prefill(out, request, chunk, idx)
|
||||
write_prefill(inputs, request, chunk, idx)
|
||||
width = max((ids.shape[1] for _, ids in picked), default=0)
|
||||
blocks = torch.full(
|
||||
(inputs.positions.shape[0], width),
|
||||
@@ -225,12 +218,10 @@ class DenseBlocksBackend:
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
assert published is not None
|
||||
for request, chunks in prefill_requests(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
):
|
||||
@@ -242,7 +233,9 @@ class DenseBlocksBackend:
|
||||
request.k,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
write_prefill(out, request, chunk, idx.masked_fill(idx < 0, request.lc))
|
||||
write_prefill(
|
||||
inputs, request, chunk, idx.masked_fill(idx < 0, request.lc)
|
||||
)
|
||||
|
||||
|
||||
def _publish_prefill_blocks(
|
||||
|
||||
@@ -28,7 +28,6 @@ from .types import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -49,45 +48,44 @@ class FullTopKIndexer:
|
||||
self.use_deep_gemm_prefill = use_deep_gemm_prefill
|
||||
self.use_deep_gemm_decode = use_deep_gemm_decode
|
||||
|
||||
def topk_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
|
||||
def topk_prefill(self, inputs: PrefillInputs) -> None:
|
||||
if self.use_deep_gemm_prefill:
|
||||
self._deep_gemm_prefill(inputs, out)
|
||||
self._deep_gemm_prefill(inputs)
|
||||
else:
|
||||
self._torch_prefill(inputs, out)
|
||||
self._torch_prefill(inputs)
|
||||
|
||||
def topk_prefill_captured(self, inputs: CapturedPrefillInputs, out: Selection):
|
||||
self._deep_gemm_prefill_captured(inputs, out)
|
||||
def topk_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
self._deep_gemm_prefill_captured(inputs)
|
||||
|
||||
def topk_decode(self, inputs: DecodeInputs, out: Selection) -> None:
|
||||
def topk_decode(self, inputs: DecodeInputs) -> None:
|
||||
if self.use_deep_gemm_decode:
|
||||
self._deep_gemm_decode(inputs, out)
|
||||
self._deep_gemm_decode(inputs)
|
||||
else:
|
||||
self._torch_decode(inputs, out)
|
||||
self._torch_decode(inputs)
|
||||
|
||||
def _deep_gemm_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
|
||||
out.reset()
|
||||
def _deep_gemm_prefill(self, inputs: PrefillInputs) -> None:
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return
|
||||
kv = self.token_to_kv_pool.get_low_ratio_index_k_fp4(
|
||||
inputs.layer_id, data.k_slots
|
||||
)
|
||||
selected = dense_prefill_topk(data, kv, topk=inputs.indexer.index_topk)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
dense_prefill_topk(data, kv, out=inputs.out_raw_indices[: data.num_rows])
|
||||
data.write_page_indices(inputs)
|
||||
|
||||
def _torch_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
|
||||
def _torch_prefill(self, inputs: PrefillInputs) -> None:
|
||||
requests = prefill_requests(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
for request, chunks in requests:
|
||||
for chunk in chunks:
|
||||
idx = chunk.scores.topk(request.k, dim=-1, sorted=False).indices
|
||||
write_prefill(out, request, chunk, idx)
|
||||
write_prefill(inputs, request, chunk, idx)
|
||||
|
||||
def _deep_gemm_decode(self, inputs: DecodeInputs, out: Selection) -> None:
|
||||
def _deep_gemm_decode(self, inputs: DecodeInputs) -> None:
|
||||
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
|
||||
metadata = inputs.paged_metadata
|
||||
if isinstance(metadata.deep_gemm_metadata, list):
|
||||
@@ -107,8 +105,7 @@ class FullTopKIndexer:
|
||||
topk_transform_paged_from_metadata(
|
||||
logits,
|
||||
metadata,
|
||||
out.page_indices,
|
||||
out.raw_indices,
|
||||
inputs.out_page_indices,
|
||||
rows=rows,
|
||||
topk_metadata=(
|
||||
topk_plans[chunk_idx] if topk_plans is not None else None
|
||||
@@ -126,23 +123,20 @@ class FullTopKIndexer:
|
||||
)
|
||||
# TODO(dark): add bf16 topk
|
||||
topk_transform_paged_from_metadata(
|
||||
logits, metadata, out.page_indices, out.raw_indices
|
||||
logits, metadata, inputs.out_page_indices, None
|
||||
)
|
||||
|
||||
def _torch_decode(self, inputs: DecodeInputs, out: Selection) -> None:
|
||||
def _torch_decode(self, inputs: DecodeInputs) -> None:
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
if d is None:
|
||||
return
|
||||
select_decode(out, d, inputs.indexer.index_topk)
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
|
||||
def _deep_gemm_prefill_captured(
|
||||
self, inputs: CapturedPrefillInputs, out: Selection
|
||||
) -> None:
|
||||
def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
indexer = inputs.indexer
|
||||
metadata = inputs.paged_metadata
|
||||
assert indexer.n_local_heads == indexer.n_heads
|
||||
@@ -176,7 +170,7 @@ class FullTopKIndexer:
|
||||
logits,
|
||||
lens[rows],
|
||||
page_table[rows],
|
||||
out.page_indices[rows, :topk],
|
||||
inputs.out_page_indices[rows, :topk],
|
||||
page_size,
|
||||
out.raw_indices[rows, :topk] if out.raw_indices is not None else None,
|
||||
inputs.out_raw_indices[rows, :topk],
|
||||
)
|
||||
|
||||
@@ -4,29 +4,27 @@ a selection back into pool slots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
finish_flat_indexer_topk,
|
||||
finish_paged_indexer_topk,
|
||||
fp4_index_logits_decode,
|
||||
fp4_index_logits_paged,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import index_q_rope_pack_weights
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles
|
||||
from sglang.kernels.ops.attention.dsv4.topk import (
|
||||
plan_topk_v2,
|
||||
topk_transform_paged_v2,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
from .types import (
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
)
|
||||
from .types import DecodeInputs, PrefillInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
|
||||
@@ -66,31 +64,27 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
|
||||
(self.num_rows, topk), -1, dtype=torch.int32
|
||||
)
|
||||
|
||||
def write_selection(self, selected: torch.Tensor, out: Selection) -> None:
|
||||
if (
|
||||
selected.is_cuda
|
||||
and get_platform().is_sm100
|
||||
and selected.dtype == torch.int32
|
||||
):
|
||||
finish_flat_indexer_topk(
|
||||
selected,
|
||||
self.k_slots,
|
||||
self.request_starts,
|
||||
out.page_indices,
|
||||
out.raw_indices,
|
||||
)
|
||||
def write_page_indices(self, inputs: PrefillInputs) -> None:
|
||||
"""The pool slots of the request-local ``inputs.out_raw_indices`` rows."""
|
||||
if inputs.out_page_indices is None:
|
||||
return
|
||||
raw = inputs.out_raw_indices[: self.num_rows]
|
||||
columns = (raw + self.request_starts[:, None]).clamp_min(0)
|
||||
inputs.out_page_indices[: self.num_rows, : raw.shape[1]] = torch.where(
|
||||
raw >= 0, self.k_slots[columns], -1
|
||||
)
|
||||
|
||||
def write_selection(self, selected: torch.Tensor, inputs: PrefillInputs) -> None:
|
||||
"""Write flattened-K columns ``selected`` into the selection buffers."""
|
||||
num_tokens, topk = selected.shape
|
||||
unselected = torch.iinfo(torch.int32).max
|
||||
selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values
|
||||
chosen = selected != unselected
|
||||
out.page_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, self.k_slots[selected.clamp_max(self.k_slots.shape[0] - 1)], -1
|
||||
).to(torch.int32)
|
||||
if out.raw_indices is not None:
|
||||
out.raw_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, selected - self.request_starts[:, None], -1
|
||||
)
|
||||
chosen = selected >= 0
|
||||
if inputs.out_page_indices is not None:
|
||||
inputs.out_page_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, self.k_slots[selected.clamp_min(0)], -1
|
||||
).to(torch.int32)
|
||||
inputs.out_raw_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, selected - self.request_starts[:, None], -1
|
||||
)
|
||||
|
||||
|
||||
def quantize_index_q(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -116,38 +110,42 @@ def get_index_k_cache(
|
||||
return k_cache.view(k_cache.shape[0], page_size, 1, 68)
|
||||
|
||||
|
||||
def _index_q_and_weights(
|
||||
*,
|
||||
indexer: DeepseekV41Indexer,
|
||||
x: torch.Tensor,
|
||||
q_lora: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""fp4 index queries ``[rows, heads, 64]`` int8 with ``[rows, heads]`` packed
|
||||
ue8m0 scales, and fp32 head weights ``[rows, heads]``."""
|
||||
rows, heads = x.shape[0], indexer.n_local_heads
|
||||
q, _ = indexer.wq_b(q_lora)
|
||||
q_fp4, q_sf, weights = index_q_rope_pack_weights(
|
||||
q.view(rows, heads, indexer.index_head_dim),
|
||||
torch.view_as_real(freqs_cis).flatten(-2),
|
||||
positions,
|
||||
indexer.head_weights_raw(x),
|
||||
indexer.head_weight_scale,
|
||||
)
|
||||
return q_fp4.view(rows, heads, 64), q_sf.view(rows, heads), weights
|
||||
|
||||
|
||||
def get_deep_gemm_decode_data(
|
||||
inputs: DecodeInputs, token_to_kv_pool: DeepSeekV4TokenToKVPool
|
||||
) -> DeepGEMMDecodeData:
|
||||
indexer = inputs.indexer
|
||||
x, q_lora, pos = inputs.x, inputs.q_lora, inputs.positions
|
||||
# The kernel sums head scores locally, so the indexer heads must be replicated.
|
||||
assert indexer.n_local_heads == indexer.n_heads
|
||||
if (
|
||||
x.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and x.dtype == torch.bfloat16
|
||||
and indexer.index_head_dim == 128
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_q_rope_pack_weights,
|
||||
)
|
||||
|
||||
q, _ = indexer.wq_b(q_lora)
|
||||
q = q.view(q.shape[0], indexer.n_local_heads, indexer.index_head_dim)
|
||||
# The fused pack also computes head_weights(x).float(), same rounding.
|
||||
q_fp4, q_sf, weights = index_q_rope_pack_weights(
|
||||
q,
|
||||
torch.view_as_real(inputs.freqs_cis).flatten(-2),
|
||||
pos,
|
||||
indexer.head_weights_raw(x), # [bs, n_local] bf16, n32k5120
|
||||
indexer.head_weight_scale,
|
||||
)
|
||||
else:
|
||||
q = indexer.queries(q_lora, inputs.freqs_cis[pos])
|
||||
q_fp4, q_sf = quantize_index_q(q)
|
||||
weights = indexer.head_weights(x).float() # [bs, n_local]
|
||||
bs = q.shape[0]
|
||||
q_fp4, q_sf, weights = _index_q_and_weights(
|
||||
indexer=indexer,
|
||||
x=inputs.x,
|
||||
q_lora=inputs.q_lora,
|
||||
freqs_cis=inputs.freqs_cis,
|
||||
positions=inputs.positions,
|
||||
)
|
||||
bs = q_fp4.shape[0]
|
||||
q_fp4 = q_fp4.view(bs, 1, indexer.n_local_heads, 64)
|
||||
q_sf = q_sf.view(bs, 1, indexer.n_local_heads)
|
||||
|
||||
@@ -165,47 +163,44 @@ def get_deep_gemm_prefill_data(
|
||||
) -> Optional[DeepGEMMPrefillData]:
|
||||
"""None when the chunk has no row or no visible compressed position."""
|
||||
ratio = inputs.compress_ratio
|
||||
indexer = inputs.indexer
|
||||
pos = inputs.positions
|
||||
seq_lens_cpu = inputs.seq_lens_cpu
|
||||
assert (
|
||||
seq_lens_cpu is not None
|
||||
inputs.seq_lens_cpu is not None
|
||||
and inputs.rows_per_request is not None
|
||||
and inputs.rows_per_request_device is not None
|
||||
), "the DeepGEMM prefill indexer needs the CPU length vectors"
|
||||
device = pos.device
|
||||
# Visible compressed positions per request; (pos + 1) // ratio bounds each row.
|
||||
lc_per_req = [s // ratio for s in seq_lens_cpu]
|
||||
req_pool_indices = inputs.req_pool_indices.to(torch.int64)
|
||||
slot_chunks, starts, start = [], [], 0
|
||||
for r, lc in enumerate(lc_per_req):
|
||||
starts.append(start)
|
||||
if lc == 0:
|
||||
continue
|
||||
j = torch.arange(lc, device=device)
|
||||
slot_chunks.append(
|
||||
req_to_token[req_pool_indices[r], j * ratio].to(torch.int64) // ratio
|
||||
)
|
||||
start += lc
|
||||
lens_per_request = [s // ratio for s in inputs.seq_lens_cpu]
|
||||
columns = sum(lens_per_request)
|
||||
num_tokens = pos.shape[0]
|
||||
if not slot_chunks or num_tokens == 0:
|
||||
if columns == 0 or num_tokens == 0:
|
||||
return None
|
||||
k_slots = torch.cat(slot_chunks)
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
q_fp4, q_sf = quantize_index_q(q)
|
||||
weights = indexer.head_weights(inputs.x).float()
|
||||
compress_lens = ((pos + 1) // ratio).to(torch.int32)
|
||||
request_starts = torch.repeat_interleave(
|
||||
torch.tensor(starts, dtype=torch.int32, device=device),
|
||||
inputs.rows_per_request_device.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
|
||||
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
|
||||
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
|
||||
request = torch.repeat_interleave(
|
||||
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
|
||||
)
|
||||
position = torch.arange(columns, device=device) - starts_dev[request]
|
||||
pool_rows = inputs.req_pool_indices.to(torch.int64)[request]
|
||||
q_fp4, q_sf, weights = _index_q_and_weights(
|
||||
indexer=inputs.indexer,
|
||||
x=inputs.x,
|
||||
q_lora=inputs.q_lora,
|
||||
freqs_cis=inputs.freqs_cis,
|
||||
positions=pos,
|
||||
)
|
||||
return DeepGEMMPrefillData(
|
||||
k_slots=k_slots,
|
||||
request_starts=request_starts,
|
||||
lens_per_request=lc_per_req,
|
||||
k_slots=req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio,
|
||||
request_starts=torch.repeat_interleave(
|
||||
starts_dev.to(torch.int32),
|
||||
inputs.rows_per_request_device.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
),
|
||||
lens_per_request=lens_per_request,
|
||||
rows_per_request=inputs.rows_per_request,
|
||||
compress_lens=compress_lens,
|
||||
compress_lens=((pos + 1) // ratio).to(torch.int32),
|
||||
q_fp4=q_fp4,
|
||||
q_sf=q_sf,
|
||||
weights=weights,
|
||||
@@ -234,22 +229,22 @@ def dense_prefill_topk(
|
||||
data: DeepGEMMPrefillData,
|
||||
kv: Tuple[torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
"""Flattened-K columns, ``-1`` padded, unordered."""
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Request-local compressed positions into ``out``, ``-1`` padded, unordered."""
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
|
||||
|
||||
selected = data.empty_selection(topk)
|
||||
# Score columns are request-relative, so a zero offset emits request-local positions.
|
||||
zero_offsets = torch.zeros_like(data.request_starts)
|
||||
for tile, logits in score_tiles(data, kv, width_align=4):
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
data.compress_lens[tile],
|
||||
out_offsets=data.request_starts[tile],
|
||||
out_indices=selected[tile],
|
||||
out_offsets=zero_offsets[tile],
|
||||
out_indices=out[tile],
|
||||
)
|
||||
# Free this tile's logits before the generator scores the next one.
|
||||
del logits
|
||||
return selected
|
||||
|
||||
|
||||
# ---------- torch ----------
|
||||
@@ -286,7 +281,6 @@ class DecodeScores(msgspec.Struct, frozen=True):
|
||||
def prefill_requests(
|
||||
*,
|
||||
inputs: PrefillInputs,
|
||||
out: Selection,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
) -> Iterator[tuple[RequestScores, Iterator[ChunkScores]]]:
|
||||
@@ -296,7 +290,7 @@ def prefill_requests(
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
assert req is not None, "the torch indexer needs the batch's rows"
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
# A compressed position is visible once the query has passed its last token.
|
||||
@@ -343,16 +337,19 @@ def _score_chunks(
|
||||
|
||||
|
||||
def write_prefill(
|
||||
out: Selection, request: RequestScores, chunk: ChunkScores, idx: torch.Tensor
|
||||
inputs: PrefillInputs,
|
||||
request: RequestScores,
|
||||
chunk: ChunkScores,
|
||||
idx: torch.Tensor,
|
||||
) -> None:
|
||||
k, lc = request.k, request.lc
|
||||
idx = idx.sort(dim=-1).values
|
||||
reach = idx < chunk.lens[:, None]
|
||||
out.page_indices[chunk.tok, :k] = torch.where(
|
||||
reach, request.slots[idx.clamp_max(lc - 1)], -1
|
||||
).to(torch.int32)
|
||||
if out.raw_indices is not None:
|
||||
out.raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
|
||||
if inputs.out_page_indices is not None:
|
||||
inputs.out_page_indices[chunk.tok, :k] = torch.where(
|
||||
reach, request.slots[idx.clamp_max(lc - 1)], -1
|
||||
).to(torch.int32)
|
||||
inputs.out_raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
|
||||
|
||||
|
||||
class PagedDecodeScores(msgspec.Struct, frozen=True):
|
||||
@@ -377,7 +374,6 @@ class PagedDecodeScores(msgspec.Struct, frozen=True):
|
||||
def decode_scores(
|
||||
*,
|
||||
inputs: DecodeInputs,
|
||||
out: Selection,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
candidate_mask: Optional[torch.Tensor] = None,
|
||||
@@ -393,7 +389,7 @@ def decode_scores(
|
||||
f"decode expects one token per request, {pos.shape=} {bs=}"
|
||||
)
|
||||
if bs == 0:
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
return None
|
||||
lens = (pos + 1) // ratio
|
||||
metadata = inputs.paged_metadata
|
||||
@@ -401,7 +397,7 @@ def decode_scores(
|
||||
# A capture-time length read would both synchronize and truncate replay.
|
||||
lmax = min(metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio)
|
||||
if lmax == 0:
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
return None
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
@@ -433,7 +429,7 @@ def decode_scores(
|
||||
plan=plan,
|
||||
has_candidate_mask=candidate_mask is not None,
|
||||
)
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
j = torch.arange(lmax, device=pos.device)
|
||||
valid = j[None, :] < lens[:, None]
|
||||
slots = req_to_token[req[:, None], (j * ratio)[None, :]].to(torch.int64) // ratio
|
||||
@@ -445,7 +441,7 @@ def decode_scores(
|
||||
|
||||
|
||||
def select_decode(
|
||||
out: Selection, d: DecodeScores | PagedDecodeScores, topk: int
|
||||
inputs: DecodeInputs, d: DecodeScores | PagedDecodeScores, topk: int
|
||||
) -> None:
|
||||
k = min(topk, d.lmax)
|
||||
if isinstance(d, PagedDecodeScores):
|
||||
@@ -457,21 +453,19 @@ def select_decode(
|
||||
d.lens,
|
||||
d.req,
|
||||
d.req_to_token,
|
||||
out.page_indices,
|
||||
out.raw_indices,
|
||||
inputs.out_page_indices,
|
||||
None,
|
||||
d.ratio,
|
||||
d.has_candidate_mask,
|
||||
)
|
||||
else:
|
||||
write_decode(out, d, d.scores.topk(k, dim=-1, sorted=False).indices)
|
||||
write_decode(inputs, d, d.scores.topk(k, dim=-1, sorted=False).indices)
|
||||
|
||||
|
||||
def write_decode(out: Selection, d: DecodeScores, idx: torch.Tensor) -> None:
|
||||
def write_decode(inputs: DecodeInputs, d: DecodeScores, idx: torch.Tensor) -> None:
|
||||
k = idx.shape[1]
|
||||
idx = idx.sort(dim=-1).values
|
||||
reach = idx < d.lens[:, None]
|
||||
out.page_indices[: d.bs, :k] = torch.where(
|
||||
inputs.out_page_indices[: d.bs, :k] = torch.where(
|
||||
reach, d.slots.gather(1, idx.clamp_max(d.lmax - 1)), -1
|
||||
).to(torch.int32)
|
||||
if out.raw_indices is not None:
|
||||
out.raw_indices[: d.bs, :k] = torch.where(reach, idx, -1).to(torch.int32)
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.kernels.ops.attention.dsv4.topk import (
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata
|
||||
from sglang.srt.layers.attention.dsv4.metadata import expand_index_page_table
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
from .scoring import (
|
||||
DeepGEMMDecodeData,
|
||||
@@ -43,7 +44,6 @@ from .types import (
|
||||
CandidateMetadata,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
get_tail_row_indices,
|
||||
)
|
||||
|
||||
@@ -73,7 +73,7 @@ class _SparsePrefillTable(_SparseTable):
|
||||
|
||||
compress_lens: torch.Tensor # [rows] int32
|
||||
page_table: torch.Tensor # [rows, index pages] int32
|
||||
request_ids: torch.Tensor # [rows] int32
|
||||
request_ids: torch.Tensor # [rows] int32 row-pair ids, see _row_pair_ids
|
||||
q_dtype: torch.dtype
|
||||
page_size: int # index-K pool page size (slots)
|
||||
rows_per_request: List[int] # rows of each request, in row order
|
||||
@@ -118,12 +118,11 @@ class SparseTableBackend:
|
||||
# captured graphs keep reading the buffers they saw
|
||||
self._retired_cached_row_ids: list = []
|
||||
|
||||
def publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
out.reset()
|
||||
def publish_prefill(self, inputs: PrefillInputs):
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return None
|
||||
selected = data.empty_selection(inputs.indexer.index_topk)
|
||||
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
|
||||
inputs.compress_ratio
|
||||
)
|
||||
@@ -140,23 +139,21 @@ class SparseTableBackend:
|
||||
).contiguous(),
|
||||
index_page_size=index_page_size,
|
||||
topk_blocks=self.topk_blocks,
|
||||
out_positions=selected,
|
||||
out_positions=inputs.out_raw_indices[: data.num_rows],
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_page_indices(inputs)
|
||||
return table
|
||||
|
||||
def consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[_SparsePrefillTable],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return
|
||||
assert published is not None
|
||||
selected = data.empty_selection(inputs.indexer.index_topk)
|
||||
select_prefill_table(
|
||||
table=published,
|
||||
data=data,
|
||||
@@ -167,16 +164,16 @@ class SparseTableBackend:
|
||||
inputs.compress_ratio
|
||||
),
|
||||
),
|
||||
out_positions=selected,
|
||||
out_positions=inputs.out_raw_indices[: data.num_rows],
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_page_indices(inputs)
|
||||
|
||||
def publish_decode(self, inputs: DecodeInputs, out: Selection):
|
||||
def publish_decode(self, inputs: DecodeInputs):
|
||||
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
|
||||
metadata = inputs.paged_metadata
|
||||
seq_lens = metadata.compressed_seq_lens.reshape(-1)
|
||||
if isinstance(metadata.deep_gemm_metadata, list):
|
||||
return self._publish_decode_chunked(inputs, data, seq_lens, out)
|
||||
return self._publish_decode_chunked(inputs, data, seq_lens)
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(data.q_fp4, data.q_sf),
|
||||
data.k_cache,
|
||||
@@ -195,10 +192,9 @@ class SparseTableBackend:
|
||||
logits,
|
||||
seq_lens,
|
||||
metadata.page_table,
|
||||
out.page_indices,
|
||||
inputs.out_page_indices,
|
||||
metadata.compressed_page_size,
|
||||
metadata.topk_metadata,
|
||||
out_raw_indices=out.raw_indices,
|
||||
)
|
||||
# per row: block count for the block top-k, sparse-row length for the consumers
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
@@ -237,7 +233,6 @@ class SparseTableBackend:
|
||||
inputs: DecodeInputs,
|
||||
data: DeepGEMMDecodeData,
|
||||
seq_lens: torch.Tensor,
|
||||
out: Selection,
|
||||
) -> _SparseTable:
|
||||
"""On the current stream, so each chunk's logits are released before the next."""
|
||||
metadata = inputs.paged_metadata
|
||||
@@ -259,8 +254,8 @@ class SparseTableBackend:
|
||||
topk_transform_paged_from_metadata(
|
||||
logits,
|
||||
metadata,
|
||||
out.page_indices,
|
||||
out.raw_indices,
|
||||
inputs.out_page_indices,
|
||||
None,
|
||||
rows=rows,
|
||||
topk_metadata=(
|
||||
topk_plans[chunk_idx] if topk_plans is not None else None
|
||||
@@ -304,7 +299,6 @@ class SparseTableBackend:
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[_SparseTable],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
assert published is not None
|
||||
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
|
||||
@@ -317,9 +311,8 @@ class SparseTableBackend:
|
||||
published.schedule,
|
||||
published.blocks.shape[1],
|
||||
)
|
||||
assert out.raw_indices is None
|
||||
topk_transform_sparse(
|
||||
logits, published.valid_lens, published.phys_blocks, out.page_indices
|
||||
logits, published.valid_lens, published.phys_blocks, inputs.out_page_indices
|
||||
)
|
||||
|
||||
def _get_request_ids(self, inputs: DecodeInputs, rows: int, device: torch.device):
|
||||
@@ -372,6 +365,25 @@ def _build_prefill_table(
|
||||
)
|
||||
|
||||
|
||||
def _row_pair_ids(rows_per_request: List[int], *, device: torch.device) -> torch.Tensor:
|
||||
"""[rows] int32: one id per consecutive row pair of a request, never across two.
|
||||
DeepGEMM's schedule walks back over equal ids to a row's request start once per
|
||||
row, quadratic in a request's rows; per-pair ids bound the walk to one step.
|
||||
|
||||
Pairs, not larger groups, because DeepGEMM's paged scheduler has BLOCK_Q == 2:
|
||||
the ids only have to delimit what it already groups, and its page table is
|
||||
indexed by Q row rather than by the id value, so the ids never address anything.
|
||||
A wider BLOCK_Q would need the pair width here to follow it."""
|
||||
counts = async_h2d(rows_per_request, dtype=torch.int64, device=device)
|
||||
rows = sum(rows_per_request)
|
||||
row = torch.arange(rows, device=device)
|
||||
request = torch.repeat_interleave(
|
||||
torch.arange(counts.numel(), device=device), counts, output_size=rows
|
||||
)
|
||||
first_row = torch.cumsum(counts, 0) - counts
|
||||
return (row - ((row - first_row[request]) & 1)).to(torch.int32)
|
||||
|
||||
|
||||
# TODO(dark): support fusion of publish + topk of publish layer
|
||||
def publish_prefill_table(
|
||||
*,
|
||||
@@ -382,8 +394,9 @@ def publish_prefill_table(
|
||||
topk_blocks: int,
|
||||
out_positions: torch.Tensor,
|
||||
) -> _SparsePrefillTable:
|
||||
"""The source's own top-k into ``out_positions`` and the chunk's table, from one
|
||||
pass over its dense scores; ``index_page_table`` is at ``index_page_size`` slots."""
|
||||
"""The source's own top-k into ``out_positions`` (request-local compressed
|
||||
positions) and the chunk's table, from one pass over its dense scores;
|
||||
``index_page_table`` is at ``index_page_size`` slots."""
|
||||
rows = data.num_rows
|
||||
device = data.q_fp4.device
|
||||
nblocks, valid_lens = candidate_row_lens(data.compress_lens, topk_blocks)
|
||||
@@ -395,7 +408,7 @@ def publish_prefill_table(
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
lens,
|
||||
out_offsets=data.request_starts[tile],
|
||||
out_offsets=zero_offsets[tile],
|
||||
out_indices=out_positions[tile],
|
||||
)
|
||||
# keys past a row's block count stay unset; the top-k reads a row
|
||||
@@ -411,17 +424,12 @@ def publish_prefill_table(
|
||||
out_offsets=zero_offsets[: logits.shape[0]],
|
||||
out_indices=blocks[tile],
|
||||
)
|
||||
request_ids = torch.repeat_interleave(
|
||||
torch.arange(len(data.rows_per_request), dtype=torch.int32, device=device),
|
||||
torch.tensor(data.rows_per_request, dtype=torch.int64, device=device),
|
||||
output_size=rows,
|
||||
)
|
||||
return _build_prefill_table(
|
||||
blocks=blocks,
|
||||
compress_lens=data.compress_lens,
|
||||
page_table=index_page_table,
|
||||
page_size=index_page_size,
|
||||
request_ids=request_ids,
|
||||
request_ids=_row_pair_ids(data.rows_per_request, device=device),
|
||||
rows_per_request=data.rows_per_request,
|
||||
q_dtype=data.q_fp4.dtype,
|
||||
valid_lens=valid_lens,
|
||||
@@ -446,4 +454,3 @@ def select_prefill_table(
|
||||
)
|
||||
# request-relative compressed positions, -1 padded
|
||||
topk_transform_sparse(logits, table.valid_lens, table.blocks, out_positions)
|
||||
out_positions.add_(torch.where(out_positions >= 0, data.request_starts[:, None], 0))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The inputs and outputs of the V4.1 indexer backends."""
|
||||
"""The inputs of the V4.1 indexer backends, including the selection buffers they fill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, List, Optional, Protocol, TypeVar
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
|
||||
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
|
||||
@@ -22,11 +24,7 @@ def get_tail_row_indices(
|
||||
for n, t in zip(full_rows_per_request, tail_rows_per_request):
|
||||
rows.extend(range(start + n - t, start + n))
|
||||
start += n
|
||||
device = torch.device(device)
|
||||
if device.type != "cuda":
|
||||
return torch.tensor(rows, dtype=torch.int64, device=device)
|
||||
staged = torch.tensor(rows, dtype=torch.int64, pin_memory=True)
|
||||
return staged.to(device, non_blocking=True)
|
||||
return async_h2d(rows, dtype=torch.int64, device=device)
|
||||
|
||||
|
||||
class CandidateMetadata:
|
||||
@@ -34,16 +32,6 @@ class CandidateMetadata:
|
||||
raise NotImplementedError(f"{type(self).__name__} is not a prefill publish")
|
||||
|
||||
|
||||
class Selection(msgspec.Struct, frozen=True):
|
||||
page_indices: torch.Tensor # index-K pool slots, ascending
|
||||
raw_indices: Optional[torch.Tensor] # the same picks as compressed positions
|
||||
|
||||
def reset(self) -> None:
|
||||
self.page_indices.fill_(-1)
|
||||
if self.raw_indices is not None:
|
||||
self.raw_indices.fill_(-1)
|
||||
|
||||
|
||||
class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""An eager extend: the backend projects the queries and gathers the index K."""
|
||||
|
||||
@@ -68,6 +56,17 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
rows_per_request: Optional[List[int]]
|
||||
rows_per_request_device: Optional[torch.Tensor]
|
||||
|
||||
# The selection, unordered, -1 padded: [rows, topk] int32 request-local
|
||||
# compressed positions, and the same picks as index-K pool slots, which is
|
||||
# None when this forward's attention reads only the positions (sparse prefill).
|
||||
out_raw_indices: torch.Tensor
|
||||
out_page_indices: Optional[torch.Tensor]
|
||||
|
||||
def reset_outputs(self) -> None:
|
||||
self.out_raw_indices.fill_(-1)
|
||||
if self.out_page_indices is not None:
|
||||
self.out_page_indices.fill_(-1)
|
||||
|
||||
|
||||
class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Decode (one row per request) or verify (one row per draft token)."""
|
||||
@@ -83,6 +82,12 @@ class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
paged_metadata: PagedIndexerMetadata
|
||||
is_verify: bool
|
||||
|
||||
# [rows, topk] int32 index-K pool slots of the selection, unordered, -1 padded.
|
||||
out_page_indices: torch.Tensor
|
||||
|
||||
def reset_outputs(self) -> None:
|
||||
self.out_page_indices.fill_(-1)
|
||||
|
||||
|
||||
class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""An extend under the prefill graph: projected queries, paged index K."""
|
||||
@@ -92,6 +97,9 @@ class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
q: torch.Tensor # [rows, heads, 128] bf16, roped
|
||||
weights: torch.Tensor # [rows, heads] bf16 head weights
|
||||
paged_metadata: PagedIndexerMetadata
|
||||
# The selection as pool slots and as request-local compressed positions.
|
||||
out_page_indices: torch.Tensor
|
||||
out_raw_indices: torch.Tensor
|
||||
|
||||
|
||||
Metadata = TypeVar("Metadata", bound=CandidateMetadata)
|
||||
@@ -100,30 +108,16 @@ Metadata = TypeVar("Metadata", bound=CandidateMetadata)
|
||||
class PrefillCandidates(Protocol[Metadata]):
|
||||
"""The attention backend keeps a publish alive and hands it back unread."""
|
||||
|
||||
def publish_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
out: Selection,
|
||||
) -> Optional[Metadata]: ...
|
||||
def publish_prefill(self, inputs: PrefillInputs) -> Optional[Metadata]: ...
|
||||
|
||||
def consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[Metadata],
|
||||
out: Selection,
|
||||
self, inputs: PrefillInputs, published: Optional[Metadata]
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class DecodeCandidates(Protocol[Metadata]):
|
||||
def publish_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
out: Selection,
|
||||
) -> Optional[Metadata]: ...
|
||||
def publish_decode(self, inputs: DecodeInputs) -> Optional[Metadata]: ...
|
||||
|
||||
def consume_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[Metadata],
|
||||
out: Selection,
|
||||
self, inputs: DecodeInputs, published: Optional[Metadata]
|
||||
) -> None: ...
|
||||
|
||||
@@ -313,7 +313,13 @@ class EngramHasher(nn.Module):
|
||||
)
|
||||
lens = forward_batch.extend_seq_lens.to(torch.int64)
|
||||
starts = forward_batch.extend_start_loc.to(torch.int64)
|
||||
row = torch.repeat_interleave(torch.arange(bs, device=device), lens)
|
||||
lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
row = torch.repeat_interleave(
|
||||
torch.arange(bs, device=device),
|
||||
lens,
|
||||
# The host total skips the device sum's sync.
|
||||
output_size=sum(lens_cpu) if lens_cpu is not None else None,
|
||||
)
|
||||
num_real = row.shape[0]
|
||||
kmode = MODE_EXTEND
|
||||
if forward_batch.engram_history is not None:
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.managers.schedule_batch import ForwardMode
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.runtime_context import get_schedule
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.engram import EngramHasher
|
||||
@@ -200,7 +201,8 @@ class NgramEmbeddingManager:
|
||||
lo = max(0, start - n1)
|
||||
ids = req.full_untruncated_fill_ids[lo:start]
|
||||
rows.append([0] * (n1 - len(ids)) + list(ids))
|
||||
batch.engram_history = torch.tensor(
|
||||
# The forward stream waits on the schedule stream, which orders this copy.
|
||||
batch.engram_history = async_h2d(
|
||||
rows, dtype=torch.int32, device=self.engram_hasher.history.device
|
||||
).view(len(rows), n1)
|
||||
|
||||
|
||||
@@ -577,6 +577,13 @@ def is_pin_memory_available(device=None) -> bool:
|
||||
return current_platform.is_pin_memory_available(device)
|
||||
|
||||
|
||||
def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
|
||||
"""A host list on ``device``; staged in pinned memory so the copy is enqueued
|
||||
on the current stream instead of synchronizing like a pageable copy."""
|
||||
pin = is_pin_memory_available(device)
|
||||
return torch.tensor(args, dtype=dtype, pin_memory=pin).to(device, non_blocking=pin)
|
||||
|
||||
|
||||
def async_d2h(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Enqueue a CUDA-to-pinned-host copy on the current stream."""
|
||||
if not tensor.is_cuda:
|
||||
|
||||
Reference in New Issue
Block a user