Fix EAGLE draft graph seq_lens_sum padding (#26880)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Terry-UV
2026-06-22 18:00:25 -07:00
committed by GitHub
co-authored by hnyls2002
parent de3ec2c437
commit a17753e449
2 changed files with 208 additions and 0 deletions
@@ -564,11 +564,20 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
buffers.global_num_tokens_gpu.fill_(bs * self.num_tokens_per_bs)
buffers.global_num_tokens_for_logprob_gpu.fill_(bs * self.num_tokens_per_bs)
# Save the raw seq_lens_sum; it is restored after replay. While the graph
# runs it must reflect the padded fake rows (set below), since draft decode
# backends read seq_lens_sum to size/slice kv_indices.
raw_seq_lens_sum = forward_batch.seq_lens_sum
if bs != raw_bs:
forward_batch.batch_size = bs
forward_batch.seq_lens = buffers.seq_lens[:bs]
forward_batch.req_pool_indices = buffers.req_pool_indices[:bs]
forward_batch.positions = buffers.positions[:num_tokens]
if raw_seq_lens_sum is not None:
forward_batch.seq_lens_sum = (
raw_seq_lens_sum + (bs - raw_bs) * self.seq_len_fill_value
)
if buffers.rids_int is not None and forward_batch.rids_int is not None:
forward_batch.rids_int = buffers.rids_int[:bs]
if (
@@ -617,5 +626,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
]
if forward_batch.seq_lens_cpu is not None:
forward_batch.seq_lens_cpu = buffers.seq_lens_cpu[:raw_bs]
forward_batch.seq_lens_sum = raw_seq_lens_sum
return out