From 42f2e4fcb742318057e69be357f1aac9cf98df18 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:16:28 +0000 Subject: [PATCH] [DSA] Stop forcing the per-step CPU seq_lens sync for the k-pool indexer (#41987) Co-authored-by: Mohammad Angkad --- python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py | 5 +++-- python/sglang/srt/layers/attention/dsa_backend.py | 4 ++-- test/registered/unit/layers/test_kpool_stream_scheduling.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index 91f279ea13..ded594ee4e 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -1693,9 +1693,10 @@ class IndexerKPool(MultiPlatformOp): if metadata is None: return None - assert forward_batch.seq_lens_cpu is not None + # Spec-v2 decode/verify batches may carry no CPU seq_lens mirror (the + # backend opts out of the D2H); test emptiness on the device shape. mode = forward_batch.forward_mode - if mode.is_idle() or len(forward_batch.seq_lens_cpu) == 0: + if mode.is_idle() or forward_batch.seq_lens.shape[0] == 0: return torch.full( (x.shape[0], self.index_topk + self.index_kpool - 1), -1, diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 265563c5a9..307ad3c7d9 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -349,7 +349,8 @@ class DeepseekSparseAttnBackend( extend_dummy_seqs_capped_by_req_pool: bool = True # Decode/verify/draft graph replay rebuilds metadata from static buffers # (page-table width) and never reads seq_lens_cpu / seq_lens_sum; opt out of - # the D2H sync. The eager fallback derives lengths from GPU seq_lens. + # the D2H sync. The eager fallback derives lengths from GPU seq_lens, and the + # KPool indexer reads the host mirror only for extend, which always has it. needs_cpu_seq_lens: bool = False # init_cuda_graph_state sizes this for every backend, but only the TRT-LLM # branch of __init__ allocates one. @@ -380,7 +381,6 @@ class DeepseekSparseAttnBackend( ) self.dsa_index_topk = get_dsa_index_topk(hf_config) self.dsa_index_kpool = get_dsa_index_kpool(hf_config) - self.needs_cpu_seq_lens = self.dsa_index_kpool > 1 self._init_kpool_metadata_fusion() self.max_context_len = model_runner.model_config.context_len self._memory_saver_adapter = TorchMemorySaverAdapter.create( diff --git a/test/registered/unit/layers/test_kpool_stream_scheduling.py b/test/registered/unit/layers/test_kpool_stream_scheduling.py index 0ae4e845ac..23c8ff023b 100644 --- a/test/registered/unit/layers/test_kpool_stream_scheduling.py +++ b/test/registered/unit/layers/test_kpool_stream_scheduling.py @@ -224,6 +224,7 @@ class TestKPoolStreamScheduling(unittest.TestCase): ) batch = SimpleNamespace( forward_mode=ForwardMode.EXTEND, + seq_lens=torch.tensor([8192 + num_tokens]), seq_lens_cpu=torch.tensor([8192 + num_tokens]), ) prepare_qk = Mock(return_value=(x, x, None, None))