Fix DSV4.1 Torch prefill indexer request-pool rows under CP
Materialize actual request-pool IDs from per-rank real row counts. Preserve zero-row requests, bounded-replay tails and the non-CP fast path. Add CPU regression coverage.
This commit is contained in:
@@ -28,7 +28,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
from .types import DecodeInputs, PrefillInputs
|
||||
from .types import DecodeInputs, PrefillInputs, materialize_prefill_request_rows
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
|
||||
@@ -295,8 +295,7 @@ def prefill_requests(
|
||||
pool = token_to_kv_pool
|
||||
ratio = inputs.compress_ratio
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
assert req is not None, "the torch indexer needs the batch's rows"
|
||||
req, pos = materialize_prefill_request_rows(inputs), inputs.positions
|
||||
inputs.reset_outputs()
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
|
||||
@@ -68,6 +68,26 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
self.out_page_indices.fill_(-1)
|
||||
|
||||
|
||||
def materialize_prefill_request_rows(inputs: PrefillInputs) -> torch.Tensor:
|
||||
"""Map real local query rows to request-pool IDs, including interleaved CP.
|
||||
|
||||
CP keeps request order but changes each request's local row count. Pool IDs
|
||||
are not batch ordinals, and zero-row requests must remain in the count map.
|
||||
The CPU counts also exclude padding and account for a bounded-replay tail.
|
||||
"""
|
||||
if inputs.req_rows is not None:
|
||||
return inputs.req_rows
|
||||
counts = inputs.rows_per_request
|
||||
assert counts is not None and len(counts) == inputs.req_pool_indices.numel()
|
||||
assert sum(counts) == inputs.positions.shape[0]
|
||||
repeats = inputs.rows_per_request_device
|
||||
if repeats is None:
|
||||
repeats = async_h2d(counts, dtype=torch.int32, device=inputs.positions.device)
|
||||
return torch.repeat_interleave(
|
||||
inputs.req_pool_indices, repeats, output_size=inputs.positions.shape[0]
|
||||
)
|
||||
|
||||
|
||||
class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Decode (one row per request) or verify (one row per draft token)."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user