From c82aed9fef99dc888de7ff1feb9220290e8ce752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=AF=E9=AA=90?= Date: Wed, 7 Oct 2026 11:10:43 +0800 Subject: [PATCH] Fix DSV4.1 Torch prefill indexer request-pool rows under CP Materialize actual request-pool IDs from per-rank real row counts. Preserve zero-row requests, bounded-replay tails and the non-CP fast path. Add CPU regression coverage. --- .../attention/dsv4/v41_indexer/scoring.py | 5 +- .../attention/dsv4/v41_indexer/types.py | 20 ++++++ .../unit/layers/test_dsv41_cp_request_rows.py | 70 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/layers/test_dsv41_cp_request_rows.py diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index 62009cd5d9..d9022d318c 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -28,7 +28,7 @@ from sglang.srt.environ import envs from sglang.srt.runtime_context import get_platform from sglang.srt.utils.common import async_h2d -from .types import DecodeInputs, PrefillInputs +from .types import DecodeInputs, PrefillInputs, materialize_prefill_request_rows if TYPE_CHECKING: from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer @@ -295,8 +295,7 @@ def prefill_requests( pool = token_to_kv_pool ratio = inputs.compress_ratio indexer = inputs.indexer - req, pos = inputs.req_rows, inputs.positions - assert req is not None, "the torch indexer needs the batch's rows" + req, pos = materialize_prefill_request_rows(inputs), inputs.positions inputs.reset_outputs() q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos]) weights = indexer.head_weights(inputs.x) diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py index f02795e0b3..7b9410f334 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/types.py @@ -68,6 +68,26 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True): self.out_page_indices.fill_(-1) +def materialize_prefill_request_rows(inputs: PrefillInputs) -> torch.Tensor: + """Map real local query rows to request-pool IDs, including interleaved CP. + + CP keeps request order but changes each request's local row count. Pool IDs + are not batch ordinals, and zero-row requests must remain in the count map. + The CPU counts also exclude padding and account for a bounded-replay tail. + """ + if inputs.req_rows is not None: + return inputs.req_rows + counts = inputs.rows_per_request + assert counts is not None and len(counts) == inputs.req_pool_indices.numel() + assert sum(counts) == inputs.positions.shape[0] + repeats = inputs.rows_per_request_device + if repeats is None: + repeats = async_h2d(counts, dtype=torch.int32, device=inputs.positions.device) + return torch.repeat_interleave( + inputs.req_pool_indices, repeats, output_size=inputs.positions.shape[0] + ) + + class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True): """Decode (one row per request) or verify (one row per draft token).""" diff --git a/test/registered/unit/layers/test_dsv41_cp_request_rows.py b/test/registered/unit/layers/test_dsv41_cp_request_rows.py new file mode 100644 index 0000000000..fbe7deff21 --- /dev/null +++ b/test/registered/unit/layers/test_dsv41_cp_request_rows.py @@ -0,0 +1,70 @@ +"""Hopper prefill must index the same request pools after interleaved CP.""" + +from types import SimpleNamespace + +import pytest +import torch + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=20, suite="base-a-test-cpu") + +from sglang.srt.layers.attention.dsv4.v41_indexer.types import ( + materialize_prefill_request_rows, +) +from sglang.srt.layers.cp.interleave import interleave_rows_per_request + + +@pytest.mark.parametrize("cp_size", [1, 2, 4, 8]) +@pytest.mark.parametrize( + "lengths", [[1, 0, 2, 13], [3, 5, 7, 19], [0, 0, 0, 0], [1, 1, 1, 1]] +) +def test_pool_ids_follow_global_sharding(cp_size, lengths): + pools = torch.tensor([31, 4, 67, 9], dtype=torch.int32) + global_rows = pools.repeat_interleave(torch.tensor(lengths)) + for rank in range(cp_size): + expected = global_rows[rank::cp_size] + counts = interleave_rows_per_request(lengths, rank, cp_size) + inputs = SimpleNamespace( + req_rows=None, + req_pool_indices=pools, + positions=torch.empty(expected.numel(), dtype=torch.int64), + rows_per_request=counts, + rows_per_request_device=torch.tensor(counts, dtype=torch.int32), + ) + actual = materialize_prefill_request_rows(inputs) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_local_tail_counts_exclude_padding_and_zero_row_requests(): + inputs = SimpleNamespace( + req_rows=None, + req_pool_indices=torch.tensor([8, 41, 3, 27]), + positions=torch.arange(5), + rows_per_request=[0, 2, 0, 3], + rows_per_request_device=torch.tensor([0, 2, 0, 3]), + ) + torch.testing.assert_close( + materialize_prefill_request_rows(inputs), torch.tensor([41, 41, 27, 27, 27]) + ) + + +def test_regular_prefill_reuses_its_existing_rows(): + rows = torch.tensor([5, 5, 17]) + assert materialize_prefill_request_rows(SimpleNamespace(req_rows=rows)) is rows + + +def test_rejects_padded_rows_as_real_queries(): + inputs = SimpleNamespace( + req_rows=None, + req_pool_indices=torch.tensor([8, 41]), + positions=torch.arange(6), + rows_per_request=[2, 3], + rows_per_request_device=torch.tensor([2, 3]), + ) + with pytest.raises(AssertionError): + materialize_prefill_request_rows(inputs) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"]))