diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index d58f3f661..006911a48 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -714,12 +714,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): ) # Query-side max length, sourced from the host-resident extend lengths # (sync-free); for plain prefill these equal the full seq lens. - # NOTE: in piecewise CUDA graph warmup, extend_seq_lens_cpu is a torch.Tensor; - # Python's max() returns a 0-d tensor, but flashinfer expects an int. - max_q = max(forward_batch.extend_seq_lens_cpu) - metadata.max_seq_len_q = ( - int(max_q.item()) if isinstance(max_q, torch.Tensor) else int(max_q) - ) + metadata.max_seq_len_q = int(max(forward_batch.extend_seq_lens_cpu)) if ( forward_batch.extend_prefix_lens_cpu is not None and any(forward_batch.extend_prefix_lens_cpu) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 8a3a7b02c..1df55a02f 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1267,8 +1267,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self.extend_start_loc = torch.arange( bs, dtype=torch.int32, device=self.seq_lens.device ) - self.extend_prefix_lens_cpu = self.extend_prefix_lens.cpu() - self.extend_seq_lens_cpu = self.extend_seq_lens.cpu() + self.extend_prefix_lens_cpu = self.extend_prefix_lens.cpu().tolist() + self.extend_seq_lens_cpu = self.extend_seq_lens.cpu().tolist() self.extend_logprob_start_lens_cpu = self.extend_prefix_lens_cpu else: if self.spec_info is not None: diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index c6834a951..94970e151 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -728,11 +728,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): extend_seq_lens=shape_inputs["extend_seq_lens"], extend_prefix_lens=shape_inputs["extend_prefix_lens"], extend_start_loc=shape_inputs["extend_start_loc"], - extend_prefix_lens_cpu=torch.zeros( - (bs,), dtype=torch.int64, device="cpu" - ), - extend_seq_lens_cpu=torch.tensor(lens_cpu, device="cpu"), - extend_logprob_start_lens_cpu=torch.tensor(lens_cpu, device="cpu"), + extend_prefix_lens_cpu=[0] * bs, + extend_seq_lens_cpu=list(lens_cpu), + extend_logprob_start_lens_cpu=list(lens_cpu), positions=_slot("positions"), global_num_tokens_gpu=global_num_tokens_gpu, global_num_tokens_for_logprob_gpu=global_num_tokens_for_logprob_gpu, diff --git a/test/manual/attention/test_flashattn_backend.py b/test/manual/attention/test_flashattn_backend.py index 5b99abceb..51c04968d 100644 --- a/test/manual/attention/test_flashattn_backend.py +++ b/test/manual/attention/test_flashattn_backend.py @@ -222,15 +222,11 @@ class TestFlashAttentionBackend(CustomTestCase): extend_prefix_lens=torch.tensor( [prefix_len] * self.batch_size, device=self.device ), - extend_prefix_lens_cpu=torch.tensor( - [prefix_len] * self.batch_size, device="cpu" - ), + extend_prefix_lens_cpu=[prefix_len] * self.batch_size, extend_seq_lens=torch.tensor( [q_len] * self.batch_size, device=self.device ), - extend_seq_lens_cpu=torch.tensor( - [q_len] * self.batch_size, device="cpu" - ), + extend_seq_lens_cpu=[q_len] * self.batch_size, ) if attn_cp_size > 1: forward_batch.attn_cp_metadata = type( diff --git a/test/manual/attention/test_flashattn_mla_backend.py b/test/manual/attention/test_flashattn_mla_backend.py index fb44bf0f3..d19f992b6 100644 --- a/test/manual/attention/test_flashattn_mla_backend.py +++ b/test/manual/attention/test_flashattn_mla_backend.py @@ -190,15 +190,11 @@ class TestFlashAttentionMLABackend(CustomTestCase): extend_prefix_lens=torch.tensor( [prefix_len] * self.batch_size, device=self.device ), - extend_prefix_lens_cpu=torch.tensor( - [prefix_len] * self.batch_size, device="cpu" - ), + extend_prefix_lens_cpu=[prefix_len] * self.batch_size, extend_seq_lens=torch.tensor( [q_len] * self.batch_size, device=self.device ), - extend_seq_lens_cpu=torch.tensor( - [q_len] * self.batch_size, device="cpu" - ), + extend_seq_lens_cpu=[q_len] * self.batch_size, ) else: # ForwardMode.DECODE diff --git a/test/registered/kernels/test_dsa_indexer.py b/test/registered/kernels/test_dsa_indexer.py index 90d4bd9c9..3102f2126 100644 --- a/test/registered/kernels/test_dsa_indexer.py +++ b/test/registered/kernels/test_dsa_indexer.py @@ -367,11 +367,9 @@ class TestDSAIndexer(CustomTestCase): extend_prefix_lens=torch.tensor( [total_len - q_len] * batch_size, device=self.device ), - extend_prefix_lens_cpu=torch.tensor( - [total_len - q_len] * batch_size, device="cpu" - ), + extend_prefix_lens_cpu=[total_len - q_len] * batch_size, extend_seq_lens=torch.tensor([q_len] * batch_size, device=self.device), - extend_seq_lens_cpu=torch.tensor([q_len] * batch_size, device="cpu"), + extend_seq_lens_cpu=[q_len] * batch_size, ) else: # ForwardMode.DECODE decode_len = 1