Fix DSV4.1 Torch prefill indexer request-pool rows under CP
Materialize actual request-pool IDs from per-rank real row counts. Preserve zero-row requests, bounded-replay tails and the non-CP fast path. Add CPU regression coverage.
This commit is contained in:
@@ -28,7 +28,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
from .types import DecodeInputs, PrefillInputs
|
||||
from .types import DecodeInputs, PrefillInputs, materialize_prefill_request_rows
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
|
||||
@@ -295,8 +295,7 @@ def prefill_requests(
|
||||
pool = token_to_kv_pool
|
||||
ratio = inputs.compress_ratio
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
assert req is not None, "the torch indexer needs the batch's rows"
|
||||
req, pos = materialize_prefill_request_rows(inputs), inputs.positions
|
||||
inputs.reset_outputs()
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
|
||||
@@ -68,6 +68,26 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
self.out_page_indices.fill_(-1)
|
||||
|
||||
|
||||
def materialize_prefill_request_rows(inputs: PrefillInputs) -> torch.Tensor:
|
||||
"""Map real local query rows to request-pool IDs, including interleaved CP.
|
||||
|
||||
CP keeps request order but changes each request's local row count. Pool IDs
|
||||
are not batch ordinals, and zero-row requests must remain in the count map.
|
||||
The CPU counts also exclude padding and account for a bounded-replay tail.
|
||||
"""
|
||||
if inputs.req_rows is not None:
|
||||
return inputs.req_rows
|
||||
counts = inputs.rows_per_request
|
||||
assert counts is not None and len(counts) == inputs.req_pool_indices.numel()
|
||||
assert sum(counts) == inputs.positions.shape[0]
|
||||
repeats = inputs.rows_per_request_device
|
||||
if repeats is None:
|
||||
repeats = async_h2d(counts, dtype=torch.int32, device=inputs.positions.device)
|
||||
return torch.repeat_interleave(
|
||||
inputs.req_pool_indices, repeats, output_size=inputs.positions.shape[0]
|
||||
)
|
||||
|
||||
|
||||
class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Decode (one row per request) or verify (one row per draft token)."""
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Hopper prefill must index the same request pools after interleaved CP."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
|
||||
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer.types import (
|
||||
materialize_prefill_request_rows,
|
||||
)
|
||||
from sglang.srt.layers.cp.interleave import interleave_rows_per_request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cp_size", [1, 2, 4, 8])
|
||||
@pytest.mark.parametrize(
|
||||
"lengths", [[1, 0, 2, 13], [3, 5, 7, 19], [0, 0, 0, 0], [1, 1, 1, 1]]
|
||||
)
|
||||
def test_pool_ids_follow_global_sharding(cp_size, lengths):
|
||||
pools = torch.tensor([31, 4, 67, 9], dtype=torch.int32)
|
||||
global_rows = pools.repeat_interleave(torch.tensor(lengths))
|
||||
for rank in range(cp_size):
|
||||
expected = global_rows[rank::cp_size]
|
||||
counts = interleave_rows_per_request(lengths, rank, cp_size)
|
||||
inputs = SimpleNamespace(
|
||||
req_rows=None,
|
||||
req_pool_indices=pools,
|
||||
positions=torch.empty(expected.numel(), dtype=torch.int64),
|
||||
rows_per_request=counts,
|
||||
rows_per_request_device=torch.tensor(counts, dtype=torch.int32),
|
||||
)
|
||||
actual = materialize_prefill_request_rows(inputs)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_local_tail_counts_exclude_padding_and_zero_row_requests():
|
||||
inputs = SimpleNamespace(
|
||||
req_rows=None,
|
||||
req_pool_indices=torch.tensor([8, 41, 3, 27]),
|
||||
positions=torch.arange(5),
|
||||
rows_per_request=[0, 2, 0, 3],
|
||||
rows_per_request_device=torch.tensor([0, 2, 0, 3]),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
materialize_prefill_request_rows(inputs), torch.tensor([41, 41, 27, 27, 27])
|
||||
)
|
||||
|
||||
|
||||
def test_regular_prefill_reuses_its_existing_rows():
|
||||
rows = torch.tensor([5, 5, 17])
|
||||
assert materialize_prefill_request_rows(SimpleNamespace(req_rows=rows)) is rows
|
||||
|
||||
|
||||
def test_rejects_padded_rows_as_real_queries():
|
||||
inputs = SimpleNamespace(
|
||||
req_rows=None,
|
||||
req_pool_indices=torch.tensor([8, 41]),
|
||||
positions=torch.arange(6),
|
||||
rows_per_request=[2, 3],
|
||||
rows_per_request_device=torch.tensor([2, 3]),
|
||||
)
|
||||
with pytest.raises(AssertionError):
|
||||
materialize_prefill_request_rows(inputs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user