From 23150f7ae005665d64561a128ec75337f4a1d365 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Mon, 12 Oct 2026 18:51:36 -0700 Subject: [PATCH] [DSv4.1] Reduce sparse-prefill workspace padding (#43921) --- .../layers/attention/deepseek_v4_backend.py | 1 + .../deepseek_v4_backend_hip_radix.py | 1 + .../attention/dsv4/sparse_prefill_utils.py | 44 ++-- test/manual/test_dsv41_ragged_prefill.py | 215 ++++++++++++++++++ .../unittests/dsv4/test_deepseek_v4.py | 28 ++- 5 files changed, 274 insertions(+), 15 deletions(-) create mode 100644 test/manual/test_dsv41_ragged_prefill.py diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 8acf85216c..e0c400c280 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -2483,6 +2483,7 @@ class DeepseekV4AttnBackend( swa_page_size=self.token_to_kv_pool.get_swa_key_page_size(), num_qo_tokens=num_qo_tokens, max_seq_len=max(seq_lens_cpu_list), + seq_lens_cpu=seq_lens_cpu_list, total_swa=total_swa, request_window_layout=request_layout, ) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index aaeb7cabcd..efcb29e500 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -2916,6 +2916,7 @@ class DeepseekV4HipRadixBackend( swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size, num_qo_tokens=num_tokens, max_seq_len=max(seq_lens_cpu), + seq_lens_cpu=seq_lens_cpu, total_swa=total_swa, ) diff --git a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py index ba4130dc62..4266493ad0 100644 --- a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py +++ b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py @@ -287,7 +287,7 @@ def build_swa_token_ids( class CompressedGather: """Positional layout of one compressed cache inside the workspace.""" - flat_token_ids: torch.Tensor # (num_reqs * c_max,) int32 + flat_token_ids: torch.Tensor # (total gathered rows,) int32 compressed_base: torch.Tensor # (num_reqs,) int32 swa_base: torch.Tensor # (num_reqs,) int32 # Tail stays at the -1 sentinel because the valid prefix length is @@ -338,6 +338,8 @@ class SparsePrefillChunkCache: compressed: Dict[int, CompressedGather] = field(default_factory=dict) swa_indices: Optional[torch.Tensor] = None swa_lengths: Optional[torch.Tensor] = None + # Host lengths avoid a device synchronization when sizing ragged gathers. + seq_lens_cpu: Optional[list[int]] = None @classmethod def build( @@ -355,6 +357,7 @@ class SparsePrefillChunkCache: max_seq_len: int, total_swa: int, request_window_layout=None, + seq_lens_cpu: Optional[list[int]] = None, ) -> "SparsePrefillChunkCache": """``query_lens`` / ``query_pos``: the rows this forward runs (the extend, or a CP rank's interleaved share of it); the SWA gather spans the whole extend.""" @@ -390,6 +393,7 @@ class SparsePrefillChunkCache: cache = cls( num_reqs=num_reqs, + seq_lens_cpu=seq_lens_cpu, num_qo_tokens=num_qo_tokens, max_seq_len=max_seq_len, swa_window_size=swa_window_size, @@ -551,20 +555,32 @@ class SparsePrefillChunkCache: first_q_per_req = ( self.query_start_loc[:-1].clamp_max(self.num_qo_tokens - 1).long() ) - num_blocks = (c_max + c_page_size - 1) // c_page_size - assert num_blocks <= page_table.shape[1] - per_req_page_table = page_table.narrow(1, 0, num_blocks).index_select( - 0, first_q_per_req - ) + # Each request needs only its own live compressed history. Keeping a + # rectangular [num_reqs, c_max] gather duplicates the longest extent + # for every short request in a mixed batch. + seq_lens_cpu = self.seq_lens_cpu + if seq_lens_cpu is None: + seq_lens_cpu = self.seq_lens.tolist() + lengths_cpu = [int(length) // compress_ratio for length in seq_lens_cpu] + total_compressed = sum(lengths_cpu) + lengths = torch.tensor(lengths_cpu, dtype=torch.int32, device=device) + ends = lengths.cumsum(0, dtype=torch.int32) + compressed_base = ends - lengths + swa_base = total_compressed + self.swa_offsets[:-1] - k_arange = torch.arange(c_max, dtype=torch.int32, device=device) - block_idx = (k_arange // c_page_size).long() - in_page = (k_arange % c_page_size).to(torch.int32) - token_ids_2d = ( - per_req_page_table.index_select(1, block_idx) * c_page_size + in_page - ).to(torch.int32) - flat_ids = token_ids_2d.reshape(-1).clamp_min(0) - compressed_base, swa_base = self._workspace_bases(c_max) + # output_size is known on the host, so repeat_interleave does not need + # to synchronize the device to determine its output allocation. + request_ids = torch.repeat_interleave( + torch.arange(self.num_reqs, device=device), + lengths, + output_size=total_compressed, + ) + positions = ( + torch.arange(total_compressed, device=device) - compressed_base[request_ids] + ) + pages = page_table[first_q_per_req[request_ids], positions // c_page_size] + flat_ids = (pages * c_page_size + positions % c_page_size).to(torch.int32) + flat_ids.clamp_min_(0) gather = CompressedGather( flat_token_ids=flat_ids, diff --git a/test/manual/test_dsv41_ragged_prefill.py b/test/manual/test_dsv41_ragged_prefill.py new file mode 100644 index 0000000000..c0e734b3b7 --- /dev/null +++ b/test/manual/test_dsv41_ragged_prefill.py @@ -0,0 +1,215 @@ +"""Standalone CUDA regression for ragged sparse-prefill history gathering.""" + +import unittest + +import torch + +from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import ( + CompressedGather, + SparsePrefillChunkCache, + SparsePrefillWorkspace, +) + + +def make_cache(lengths, query_lengths, device): + tensor = lambda values: torch.tensor(values, dtype=torch.int32, device=device) + starts = [0] + positions = [] + for length, queries in zip(lengths, query_lengths): + starts.append(starts[-1] + queries) + positions.extend(range(length - queries, length)) + return SparsePrefillChunkCache( + num_reqs=len(lengths), + num_qo_tokens=sum(query_lengths), + max_seq_len=max(lengths), + swa_window_size=128, + swa_page_size=256, + seq_lens=tensor(lengths), + seq_lens_cpu=lengths, + query_start_loc=tensor(starts), + query_pos=tensor(positions), + swa_token_ids=tensor([]), + swa_first_pos=tensor(lengths), + swa_gather_lens=tensor([0] * len(lengths)), + swa_offsets=tensor([0] * (len(lengths) + 1)), + ) + + +def rectangular_gather(cache, page_table, ratio, page_size): + width = max(cache.max_seq_len // ratio, 1) + first = cache.query_start_loc[:-1].clamp_max(cache.num_qo_tokens - 1).long() + positions = torch.arange(width, device=page_table.device) + slots = ( + ( + page_table[first][:, positions // page_size] * page_size + + positions % page_size + ) + .reshape(-1) + .clamp_min(0) + .int() + ) + bases, swa = cache._workspace_bases(width) + return CompressedGather(slots, bases, swa) + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required") +class TestRaggedPrefill(unittest.TestCase): + def test_gather_and_attention(self): + from sgl_kernel.flash_mla import flash_mla_sparse_fwd + + torch.manual_seed(42) + device = "cuda" + for ratio in (1, 2, 4): + for lengths, queries in ( + ([513, 99, 1, 2049], [3, 2, 1, 2]), + ([513, 99, 1, 2049], [3, 0, 1, 2]), + ([513, 513], [2, 2]), + ([1, 1], [1, 1]), + ): + with self.subTest(ratio=ratio, lengths=lengths, queries=queries): + cache = make_cache(lengths, queries, device) + old_cache = make_cache(lengths, queries, device) + page_size = 256 + blocks = (max(lengths) + page_size - 1) // page_size + # Non-contiguous page-table rows, with shuffled physical pages. + table = torch.randint( + 1, + 40, + (sum(queries), blocks * 2), + device=device, + dtype=torch.int32, + )[:, ::2] + for start, end in zip( + cache.query_start_loc.tolist()[:-1], + cache.query_start_loc.tolist()[1:], + ): + if end > start: + table[start:end] = table[start].clone() + old = rectangular_gather(old_cache, table, ratio, page_size) + old_cache.compressed[ratio] = old + new = cache.ensure_compressed(ratio, table, page_size) + self.assertEqual( + new.flat_token_ids.numel(), sum(x // ratio for x in lengths) + ) + for i, length in enumerate(lengths): + n = length // ratio + a, b = int(new.compressed_base[i]), int(old.compressed_base[i]) + torch.testing.assert_close( + new.flat_token_ids[a : a + n], old.flat_token_ids[b : b + n] + ) + # Include SWA rows as well as top-k and -1 sentinels. + swa_lens = [ + min(length, query + 127) + for length, query in zip(lengths, queries) + ] + swa_offsets = ( + torch.tensor([0] + swa_lens, device=device, dtype=torch.int32) + .cumsum(0) + .int() + ) + for c, g in ((cache, new), (old_cache, old)): + c.swa_gather_lens = torch.tensor( + swa_lens, device=device, dtype=torch.int32 + ) + c.swa_offsets = swa_offsets + g.swa_base = g.flat_token_ids.numel() + swa_offsets[:-1] + raw = torch.full( + (sum(queries), 128), -1, device=device, dtype=torch.int32 + ) + for row, pos in enumerate(cache.query_pos.tolist()): + n = min((pos + 1) // ratio, 128) + if n: + raw[row, :n] = torch.randperm( + (pos + 1) // ratio, device=device + )[:n].int() + ni, nl = cache.combine_compressed(ratio, raw) + oi, ol = old_cache.combine_compressed(ratio, raw) + torch.testing.assert_close(nl, ol) + values = ( + torch.randn( + 41 * page_size, 1, 512, device=device, dtype=torch.bfloat16 + ) + * 0.1 + ) + swa = ( + torch.randn( + sum(swa_lens), 1, 512, device=device, dtype=torch.bfloat16 + ) + * 0.1 + ) + nkv = torch.cat((values[new.flat_token_ids.long()], swa)) + okv = torch.cat((values[old.flat_token_ids.long()], swa)) + valid = ni >= 0 + torch.testing.assert_close( + nkv[ni[valid].long()], okv[oi[valid].long()] + ) + q = ( + torch.randn( + sum(queries), 64, 512, device=device, dtype=torch.bfloat16 + ) + * 0.1 + ) + + def attn(kv, indices, lens): + return flash_mla_sparse_fwd( + q=q, + kv=kv, + indices=indices.unsqueeze(1), + sm_scale=512**-0.5, + d_v=512, + attn_sink=torch.zeros(64, device=device), + topk_length=lens, + )[0] + + torch.testing.assert_close( + attn(nkv, ni, nl), attn(okv, oi, ol), rtol=0, atol=0 + ) + + def test_00_mixed_batch_under_memory_budget(self): + # Run this method alone in a fresh process. The allocator limit applies + # only to this test process, never to a resident serving process. + budget = 2 << 30 + total = torch.cuda.get_device_properties(0).total_memory + torch.cuda.set_per_process_memory_fraction(budget / total) + try: + cache = make_cache([44198, 577152, 95723, 51427, 6144], [1] * 5, "cuda") + table = torch.arange(2255, device="cuda", dtype=torch.int32).expand(5, -1) + with self.assertRaises(torch.OutOfMemoryError): + SparsePrefillWorkspace(torch.device("cuda")).get(2902511) + gather = cache.ensure_compressed(1, table, 256) + workspace = SparsePrefillWorkspace(torch.device("cuda")).get( + gather.flat_token_ids.numel() + 16751 + ) + self.assertEqual(workspace.shape[0], 791395) + print("2 GiB process budget: rectangular OOM, ragged PASS", flush=True) + finally: + torch.cuda.set_per_process_memory_fraction(1.0) + + def test_reported_workspace_sizes(self): + for lengths, swa, expected_old, expected_new in ( + ([44198, 577152, 95723, 51427, 6144], 16751, 2902511, 791395), + ([820552] * 14, 2786, 11490514, 11490514), + ): + cache = make_cache(lengths, [1] * len(lengths), "cuda") + blocks = (max(lengths) + 255) // 256 + table = torch.arange(blocks, device="cuda", dtype=torch.int32).expand( + len(lengths), -1 + ) + gather = cache.ensure_compressed(1, table, 256) + old_rows = len(lengths) * max(lengths) + swa + new_rows = gather.flat_token_ids.numel() + swa + self.assertEqual(old_rows, expected_old) + self.assertEqual(new_rows, expected_new) + ws = SparsePrefillWorkspace(torch.device("cuda")) + before = torch.cuda.memory_allocated() + buf = ws.get(new_rows) + allocated = torch.cuda.memory_allocated() - before + print( + f"WORKSPACE old={old_rows * 1024 / 2**30:.6f} GiB new={buf.numel() * buf.element_size() / 2**30:.6f} GiB allocated={allocated / 2**30:.6f} GiB", + flush=True, + ) + del ws, buf + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py index 82d4142b9f..3a7dd27354 100644 --- a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py +++ b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py @@ -772,10 +772,36 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): with self.subTest(max_seq_len=max_seq_len): cache = self._make_sparse_prefill_cache(max_seq_len) gather = cache.ensure_compressed(4, page_table, c_page_size=64) - expected_extent = max(max_seq_len // 4, 1) + expected_extent = max_seq_len // 4 self.assertEqual(gather.flat_token_ids.numel(), 2 * expected_extent) self.assertEqual(gather.compressed_base.tolist(), [0, expected_extent]) + def test_sparse_prefill_compressed_gather_packs_request_lengths(self): + for ratio in (1, 2, 4): + with self.subTest(ratio=ratio): + cache = self._make_sparse_prefill_cache(257) + cache.seq_lens = torch.tensor([257, 9], dtype=torch.int32) + cache.seq_lens_cpu = [257, 9] + cache.swa_offsets = torch.tensor([0, 8, 12], dtype=torch.int32) + # Deliberately non-contiguous, non-monotonic physical pages. + table = torch.tensor( + [[7, 0, 2, 0, 11, 0, 4, 0, 13, 0], [9, 0, 3, 0, 8, 0, 12, 0, 5, 0]], + dtype=torch.int32, + )[:, ::2] + gather = cache.ensure_compressed(ratio, table, c_page_size=64) + lengths = [257 // ratio, 9 // ratio] + expected = [ + int(table[r, pos // 64]) * 64 + pos % 64 + for r, length in enumerate(lengths) + for pos in range(length) + ] + self.assertEqual(gather.flat_token_ids.tolist(), expected) + self.assertEqual(gather.compressed_base.tolist(), [0, lengths[0]]) + self.assertEqual( + gather.swa_base.tolist(), [sum(lengths), sum(lengths) + 8] + ) + self.assertIs(cache.ensure_compressed(ratio, table, 64), gather) + def test_sparse_prefill_c128_uses_live_extent(self): from sglang.srt.layers.attention.dsv4 import sparse_prefill_utils