[DSA] Stop forcing the per-step CPU seq_lens sync for the k-pool indexer (#41987)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
(cherry picked from commit 42f2e4fcb7)
This commit is contained in:
@@ -1472,9 +1472,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
assert forward_batch.seq_lens_cpu is not None
|
||||
# Spec-v2 decode/verify batches may carry no CPU seq_lens mirror (the
|
||||
# backend opts out of the D2H); test emptiness on the device shape.
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_idle() or len(forward_batch.seq_lens_cpu) == 0:
|
||||
if mode.is_idle() or forward_batch.seq_lens.shape[0] == 0:
|
||||
return torch.full(
|
||||
(x.shape[0], self.index_topk + self.index_kpool - 1),
|
||||
-1,
|
||||
|
||||
@@ -306,7 +306,8 @@ class DeepseekSparseAttnBackend(
|
||||
extend_dummy_seqs_capped_by_req_pool: bool = True
|
||||
# Decode/verify/draft graph replay rebuilds metadata from static buffers
|
||||
# (page-table width) and never reads seq_lens_cpu / seq_lens_sum; opt out of
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens.
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens, and the
|
||||
# KPool indexer reads the host mirror only for extend, which always has it.
|
||||
needs_cpu_seq_lens: bool = False
|
||||
# init_cuda_graph_state sizes this for every backend, but only the TRT-LLM
|
||||
# branch of __init__ allocates one.
|
||||
@@ -337,7 +338,6 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
self.dsa_index_topk = get_dsa_index_topk(hf_config)
|
||||
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
|
||||
self.needs_cpu_seq_lens = self.dsa_index_kpool > 1
|
||||
self._init_kpool_metadata_fusion()
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
self.num_q_heads = (
|
||||
|
||||
@@ -219,6 +219,7 @@ class TestKPoolStreamScheduling(unittest.TestCase):
|
||||
)
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
seq_lens=torch.tensor([8192 + num_tokens]),
|
||||
seq_lens_cpu=torch.tensor([8192 + num_tokens]),
|
||||
)
|
||||
prepare_qk = Mock(return_value=(x, x, None, None))
|
||||
|
||||
Reference in New Issue
Block a user