From 3e67398a96b341402eea7845de9a6221b6af5ca7 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Mon, 25 May 2026 03:29:05 -0700 Subject: [PATCH] Zero `req_pool_indices` padding in cuda-graph populate (#26292) --- python/sglang/srt/model_executor/cpu_graph_runner.py | 3 +++ python/sglang/srt/model_executor/cuda_graph_runner.py | 5 +++++ .../srt/speculative/eagle_draft_extend_cuda_graph_runner.py | 3 +++ .../srt/speculative/frozen_kv_mtp_cuda_graph_runner.py | 3 +++ 4 files changed, 14 insertions(+) diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index b80043e70..4b7c177a1 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -786,6 +786,9 @@ class CPUGraphRunner: assert captured_forward_batch is not None captured_forward_batch.seq_lens.fill_(self.seq_len_fill_value) captured_forward_batch.out_cache_loc.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init). + captured_forward_batch.req_pool_indices.zero_() captured_forward_batch.input_ids[:raw_num_token].copy_(forward_batch.input_ids) captured_forward_batch.req_pool_indices[:raw_bs].copy_( forward_batch.req_pool_indices diff --git a/python/sglang/srt/model_executor/cuda_graph_runner.py b/python/sglang/srt/model_executor/cuda_graph_runner.py index 63aff21f8..6501b6fe1 100644 --- a/python/sglang/srt/model_executor/cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/cuda_graph_runner.py @@ -286,6 +286,11 @@ class DecodeInputBuffers(ForwardInputBuffers): if bs != raw_bs: self.seq_lens.fill_(seq_len_fill_value) self.out_cache_loc.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init), + # so dummy attention reads land on slot 0 instead of a stale + # req_to_token row left by an earlier replay. + self.req_pool_indices.zero_() if self.mamba_track_indices is not None: self.mamba_track_indices.zero_() if self.mamba_track_mask is not None: diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index e58842278..8798086d1 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -460,6 +460,9 @@ class EAGLEDraftExtendCudaGraphRunner: buffers.seq_lens.fill_(self.seq_len_fill_value) buffers.out_cache_loc.zero_() buffers.positions.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init). + buffers.req_pool_indices.zero_() buffers.num_correct_drafts.fill_(self.num_tokens_per_bs) buffers.num_accept_tokens.fill_(self.num_tokens_per_bs) buffers.extend_seq_lens.fill_(self.num_tokens_per_bs) diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py index 8b1ac37f8..fd331dfde 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py @@ -352,6 +352,9 @@ class FrozenKVMTPCudaGraphRunner: if bs != raw_bs: buffers.seq_lens.fill_(self.seq_len_fill_value) buffers.positions.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init). + buffers.req_pool_indices.zero_() num_tokens = expanded_bs buffers.seq_lens[:raw_expanded_bs].copy_(forward_batch.seq_lens)