Fix DSV4.1 Torch prefill indexer request-pool rows under CP

Materialize actual request-pool IDs from per-rank real row counts. Preserve zero-row requests, bounded-replay tails and the non-CP fast path. Add CPU regression coverage.
This commit is contained in:
冯骐
2026-10-20 10:16:54 +08:00
committed by minke.yu
parent e6f6420275
commit c82aed9fef
3 changed files with 92 additions and 3 deletions
@@ -28,7 +28,7 @@ from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils.common import async_h2d
from .types import DecodeInputs, PrefillInputs
from .types import DecodeInputs, PrefillInputs, materialize_prefill_request_rows
if TYPE_CHECKING:
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
@@ -295,8 +295,7 @@ def prefill_requests(
pool = token_to_kv_pool
ratio = inputs.compress_ratio
indexer = inputs.indexer
req, pos = inputs.req_rows, inputs.positions
assert req is not None, "the torch indexer needs the batch's rows"
req, pos = materialize_prefill_request_rows(inputs), inputs.positions
inputs.reset_outputs()
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
weights = indexer.head_weights(inputs.x)
@@ -68,6 +68,26 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
self.out_page_indices.fill_(-1)
def materialize_prefill_request_rows(inputs: PrefillInputs) -> torch.Tensor:
"""Map real local query rows to request-pool IDs, including interleaved CP.
CP keeps request order but changes each request's local row count. Pool IDs
are not batch ordinals, and zero-row requests must remain in the count map.
The CPU counts also exclude padding and account for a bounded-replay tail.
"""
if inputs.req_rows is not None:
return inputs.req_rows
counts = inputs.rows_per_request
assert counts is not None and len(counts) == inputs.req_pool_indices.numel()
assert sum(counts) == inputs.positions.shape[0]
repeats = inputs.rows_per_request_device
if repeats is None:
repeats = async_h2d(counts, dtype=torch.int32, device=inputs.positions.device)
return torch.repeat_interleave(
inputs.req_pool_indices, repeats, output_size=inputs.positions.shape[0]
)
class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
"""Decode (one row per request) or verify (one row per draft token)."""
@@ -0,0 +1,70 @@
"""Hopper prefill must index the same request pools after interleaved CP."""
from types import SimpleNamespace
import pytest
import torch
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
from sglang.srt.layers.attention.dsv4.v41_indexer.types import (
materialize_prefill_request_rows,
)
from sglang.srt.layers.cp.interleave import interleave_rows_per_request
@pytest.mark.parametrize("cp_size", [1, 2, 4, 8])
@pytest.mark.parametrize(
"lengths", [[1, 0, 2, 13], [3, 5, 7, 19], [0, 0, 0, 0], [1, 1, 1, 1]]
)
def test_pool_ids_follow_global_sharding(cp_size, lengths):
pools = torch.tensor([31, 4, 67, 9], dtype=torch.int32)
global_rows = pools.repeat_interleave(torch.tensor(lengths))
for rank in range(cp_size):
expected = global_rows[rank::cp_size]
counts = interleave_rows_per_request(lengths, rank, cp_size)
inputs = SimpleNamespace(
req_rows=None,
req_pool_indices=pools,
positions=torch.empty(expected.numel(), dtype=torch.int64),
rows_per_request=counts,
rows_per_request_device=torch.tensor(counts, dtype=torch.int32),
)
actual = materialize_prefill_request_rows(inputs)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
def test_local_tail_counts_exclude_padding_and_zero_row_requests():
inputs = SimpleNamespace(
req_rows=None,
req_pool_indices=torch.tensor([8, 41, 3, 27]),
positions=torch.arange(5),
rows_per_request=[0, 2, 0, 3],
rows_per_request_device=torch.tensor([0, 2, 0, 3]),
)
torch.testing.assert_close(
materialize_prefill_request_rows(inputs), torch.tensor([41, 41, 27, 27, 27])
)
def test_regular_prefill_reuses_its_existing_rows():
rows = torch.tensor([5, 5, 17])
assert materialize_prefill_request_rows(SimpleNamespace(req_rows=rows)) is rows
def test_rejects_padded_rows_as_real_queries():
inputs = SimpleNamespace(
req_rows=None,
req_pool_indices=torch.tensor([8, 41]),
positions=torch.arange(6),
rows_per_request=[2, 3],
rows_per_request_device=torch.tensor([2, 3]),
)
with pytest.raises(AssertionError):
materialize_prefill_request_rows(inputs)
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-v"]))