From 52d5faed5bfaa9aedb80a2a4c7c64e526cbe557a Mon Sep 17 00:00:00 2001 From: Martin Hua Date: Thu, 1 Oct 2026 14:42:22 -0700 Subject: [PATCH] [Bugfix] Llama4 local attention: read page ids from the graph tables in CUDA-graph capture/replay (page_size > 1) (#38187) --- .../attention/flashattention_backend.py | 481 +++--------------- .../srt/layers/attention/local_attention.py | 378 ++++++++++++++ .../srt/layers/attention/xpu_backend.py | 7 +- .../layers/attention/test_local_attention.py | 221 ++++++++ 4 files changed, 674 insertions(+), 413 deletions(-) create mode 100644 python/sglang/srt/layers/attention/local_attention.py create mode 100644 test/registered/unit/layers/attention/test_local_attention.py diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index a8fceedfcf..5559e319ed 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, Optional -import numpy as np import torch from sglang.kernels.ops.attention.metadata import ( @@ -26,6 +25,10 @@ from sglang.srt.layers.attention.kv_shard_hooks import ( get_kv_shard_pool, prepare_kv_shard_forward, ) +from sglang.srt.layers.attention.local_attention import ( + LocalAttentionMetadata, + LocalAttentionMetadataBuilder, +) from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy from sglang.srt.layers.cp.utils import is_cp_active @@ -116,14 +119,6 @@ class FlashAttentionMetadata: # Page table for the encoder encoder_page_table: torch.Tensor = None - @dataclass - class LocalAttentionMetadata: - local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention - local_seqused_k: torch.Tensor = None # sequence lengths for local attention - local_block_table: torch.Tensor = None # block table for local attention - local_max_query_len: int = 0 # max query length for local attention - local_max_seq_len: int = 0 # max sequence length for local attention - local_attn_metadata: Optional[LocalAttentionMetadata] = None # For sliding window attention topk>1 spec decoding @@ -239,23 +234,14 @@ class FlashAttentionBackend(AttentionBackend): ) self.speculative_step_id = speculative_step_id - # Local attention settings - self.has_local_attention = model_runner.model_config.is_local_attention_model - # Local (chunked) attention derives its page table by re-translating - # metadata.page_table through the static full->swa map -- meaningless - # on the unified pool's kernel-facing tables, and no unified-eligible - # model uses it. Fail loud rather than silently double-translate. - assert not ( - self.kv_index_translator.is_translating and self.has_local_attention - ), ( - "--enable-unified-memory does not support local-attention models " - "on the fa3/fa4 backend." + self.local_attn_builder = self._maybe_init_local_attn_builder(model_runner) + # Read directly by MusaFlashAttentionBackend's copied forward paths. + self.has_local_attention = self.local_attn_builder is not None + self.attention_chunk_size = ( + self.local_attn_builder.attention_chunk_size + if self.local_attn_builder is not None + else None ) - if self.has_local_attention: - assert model_runner.attention_chunk_size is not None, ( - "Attention chunk size is required for local attention" - ) - self.attention_chunk_size = model_runner.attention_chunk_size # For each layer, the sliding_window_size can be different. This is only used for preparing SWA metadata. # We use `layer.sliding_window_size` to decide whether to use SWA for each layer. @@ -390,6 +376,32 @@ class FlashAttentionBackend(AttentionBackend): _should_disable_scheduler_metadata_precompute() ) + def _maybe_init_local_attn_builder( + self, model_runner: ModelRunner + ) -> Optional[LocalAttentionMetadataBuilder]: + if not model_runner.model_config.is_local_attention_model: + return None + # Local attention re-translates metadata.page_table through the static + # full->swa map, which is meaningless on the unified pool's kernel tables. + assert not self.kv_index_translator.is_translating, ( + "--enable-unified-memory does not support local-attention models " + "on the fa3/fa4 backend." + ) + assert model_runner.attention_chunk_size is not None, ( + "Attention chunk size is required for local attention" + ) + return LocalAttentionMetadataBuilder( + attention_chunk_size=model_runner.attention_chunk_size, + page_size=self.page_size, + max_context_len=self.max_context_len, + device=self.device, + swa_translate=( + self.token_to_kv_pool.translate_loc_from_full_to_swa + if self.use_sliding_window_kv_pool + else None + ), + ) + def _compute_scheduler_metadata( self, batch_size, max_seq_len_k, cache_seqlens, cu_seqlens_q ): @@ -624,7 +636,14 @@ class FlashAttentionBackend(AttentionBackend): # padded seq-len fill value (1), or a later long-context # replay would leave K/V tiles uncovered. metadata.max_seq_len_k = self.max_context_len - self._maybe_update_local_attn_metadata_for_capture(metadata, bs) + if self.local_attn_builder is not None: + metadata.local_attn_metadata = ( + self.local_attn_builder.build_for_capture( + metadata, + bs, + buffers=self.decode_cuda_graph_local_attn_metadata, + ) + ) if self._sched_meta_buf is not None: sched = self._compute_scheduler_metadata( bs, @@ -937,7 +956,7 @@ class FlashAttentionBackend(AttentionBackend): metadata.cu_seqlens_q, ) # TODO: we need to test this part for llama 4 eagle case - self._maybe_init_local_attn_metadata(forward_batch, metadata, device) + self._maybe_init_local_attn_metadata(metadata, device) elif forward_batch.forward_mode.is_target_verify(): if self.topk <= 1: ragged_layout = getattr( @@ -985,7 +1004,7 @@ class FlashAttentionBackend(AttentionBackend): forward_batch.req_pool_indices, : metadata.max_seq_len_k ] - self._maybe_init_local_attn_metadata(forward_batch, metadata, device) + self._maybe_init_local_attn_metadata(metadata, device) else: metadata.cache_seqlens_int32 = forward_batch.seq_lens.to(torch.int32) metadata.max_seq_len_q = self.speculative_num_draft_tokens @@ -1124,7 +1143,7 @@ class FlashAttentionBackend(AttentionBackend): # Setup local attention if enabled if forward_batch.forward_mode == ForwardMode.EXTEND: - self._maybe_init_local_attn_metadata(forward_batch, metadata, device) + self._maybe_init_local_attn_metadata(metadata, device) if self.is_prefill_aware_swa: self._pa_swa_prefill_lens[ @@ -1440,12 +1459,9 @@ class FlashAttentionBackend(AttentionBackend): kv_head_num=layer.tp_k_head_num, is_prefill=True, ) - # Check if we should use local attention use_local_attn = ( - self.has_local_attention - and self.attention_chunk_size is not None - and metadata.local_attn_metadata is not None - and (hasattr(layer, "use_irope") and layer.use_irope) + self.local_attn_builder is not None + and self.local_attn_builder.applies(layer, metadata) ) # We do cascade attention for Target Verify with topk > 1 @@ -1965,12 +1981,10 @@ class FlashAttentionBackend(AttentionBackend): # Use precomputed metadata across all layers metadata = self.forward_metadata - local_attn_metadata = getattr(metadata, "local_attn_metadata", None) + local_attn_metadata = metadata.local_attn_metadata use_local_attn = ( - self.has_local_attention - and self.attention_chunk_size is not None - and local_attn_metadata is not None - and (hasattr(layer, "use_irope") and layer.use_irope) + self.local_attn_builder is not None + and self.local_attn_builder.applies(layer, metadata) ) # When Spec Decode enabled, forward_decode would be called with two mode: @@ -2312,32 +2326,10 @@ class FlashAttentionBackend(AttentionBackend): else: self._sched_meta_buf = None - # Only allocate local attention buffers if local attention is enabled - # This prevents OOM errors when local attention is not being used - if self.has_local_attention: - # Estimate maximum sizes for local attention metadata - max_seq_len = self.max_context_len - page_size = self.page_size or 1 - attn_chunk_size = self.attention_chunk_size - max_virtual_batches = max_bs * ( - (max_seq_len + attn_chunk_size - 1) // attn_chunk_size + if self.local_attn_builder is not None: + self.decode_cuda_graph_local_attn_metadata = ( + self.local_attn_builder.alloc_cuda_graph_buffers(max_bs) ) - max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size - - self.decode_cuda_graph_local_attn_metadata = { - "local_query_start_loc": torch.zeros( - max_virtual_batches + 1, dtype=torch.int32, device=self.device - ), - "local_seqused_k": torch.zeros( - max_virtual_batches, dtype=torch.int32, device=self.device - ), - "local_block_table": torch.zeros( - max_virtual_batches, - max_pages_per_block, - dtype=torch.int32, - device=self.device, - ), - } if self.use_sliding_window_kv_pool: self.decode_cuda_graph_metadata["swa_page_table"] = torch.zeros( @@ -3008,10 +3000,12 @@ class FlashAttentionBackend(AttentionBackend): metadata, req_pool_indices, seq_lens, 0 ) - self._maybe_update_local_attn_metadata_for_replay( - metadata, - bs, - ) + if self.local_attn_builder is not None: + self.local_attn_builder.update_for_replay( + metadata, + bs, + buffers=self.decode_cuda_graph_local_attn_metadata, + ) # Recompute scheduler_metadata into pre-allocated buffer if ( @@ -3247,197 +3241,18 @@ class FlashAttentionBackend(AttentionBackend): return 1 def _maybe_init_local_attn_metadata( - self, - forwardbatch: ForwardBatch, - metadata: FlashAttentionMetadata, - device, - ): - """Centralized utility to initialize local_attn_metadata if chunked attention is enabled.""" - if not self.has_local_attention: + self, metadata: FlashAttentionMetadata, device + ) -> None: + if self.local_attn_builder is None: metadata.local_attn_metadata = None return - - cu_seqlens_q = metadata.cu_seqlens_q - cache_seqlens_int32 = metadata.cache_seqlens_int32 - if self.use_sliding_window_kv_pool: - page_table = self.token_to_kv_pool.translate_loc_from_full_to_swa( - metadata.page_table - ).to(torch.int32) - else: - page_table = metadata.page_table - if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None: - metadata.local_attn_metadata = None - return - if self.page_size > 1: - # Convert the eager token table to physical page indices. - page_table = page_table[:, :: self.page_size] // self.page_size - - cu_seqlens_q_np = cu_seqlens_q.cpu().numpy() - seq_lens_np = cache_seqlens_int32.cpu().numpy() - ( - seqlens_q_local_np, - cu_seqlens_q_local_np, - seqlens_k_local_np, - block_table_local, - ) = make_local_attention_virtual_batches( - self.attention_chunk_size, - cu_seqlens_q_np, - seq_lens_np, - page_table, - self.page_size, - preserve_attn_chunk_size=True, + metadata.local_attn_metadata = self.local_attn_builder.build( + cu_seqlens_q=metadata.cu_seqlens_q, + cache_seqlens_int32=metadata.cache_seqlens_int32, + page_table=metadata.page_table, + device=device, ) - local_metadata = FlashAttentionMetadata.LocalAttentionMetadata( - local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device), - local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device), - local_block_table=block_table_local.to(device), - local_max_query_len=int(seqlens_q_local_np.max()), - local_max_seq_len=int(seqlens_k_local_np.max()), - ) - metadata.local_attn_metadata = local_metadata - - def _maybe_update_local_attn_metadata_for_capture( - self, metadata: FlashAttentionMetadata, bs: int - ): - """Update local attention metadata during CUDA graph capture phase. - - This method calculates the exact buffer sizes needed for local attention metadata - during the CUDA graph capture phase, optimizing memory usage by creating views of - pre-allocated buffers with exactly the sizes needed. - """ - if not self.has_local_attention: - return - - seq_lens_capture = metadata.cache_seqlens_int32 - max_seq_len = int(seq_lens_capture.max().item()) - page_table_capture = metadata.page_table - - cu_seqlens_q_np = metadata.cu_seqlens_q.cpu().numpy() - seqlens_np = seq_lens_capture.cpu().numpy() - ( - seqlens_q_local_np, - cu_seqlens_q_local_np, - seqlens_k_local_np, - block_table_local_np, - ) = make_local_attention_virtual_batches( - self.attention_chunk_size, - cu_seqlens_q_np, - seqlens_np, - page_table_capture, - self.page_size, - preserve_attn_chunk_size=True, - ) - - # Get exact dimensions from the calculation - q_len = len(cu_seqlens_q_local_np) - k_len = len(seqlens_k_local_np) - b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs - b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1 - - # Create views of the pre-allocated buffers with exactly these sizes - # This is the key optimization - we only use the memory we actually need - local_query_start_loc = self.decode_cuda_graph_local_attn_metadata[ - "local_query_start_loc" - ][:q_len] - - local_seqused_k = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"][ - :k_len - ] - - local_block_table = self.decode_cuda_graph_local_attn_metadata[ - "local_block_table" - ][:b0, :b1] - - metadata.local_attn_metadata = FlashAttentionMetadata.LocalAttentionMetadata( - local_query_start_loc=local_query_start_loc, - local_seqused_k=local_seqused_k, - local_block_table=local_block_table, - local_max_query_len=1, - local_max_seq_len=max_seq_len, - ) - - def _maybe_update_local_attn_metadata_for_replay( - self, - metadata: FlashAttentionMetadata, - bs: int, - ): - """Update preallocated local attention metadata in-place before CUDA graph replay.""" - if not self.has_local_attention: - return - - # Access preallocated buffers - local_q_buf = self.decode_cuda_graph_local_attn_metadata[ - "local_query_start_loc" - ] - local_k_buf = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"] - local_block_buf = self.decode_cuda_graph_local_attn_metadata[ - "local_block_table" - ] - cu_seqlens_q = self.decode_cuda_graph_metadata["cu_seqlens_q"] - - # Create a modified version for local attention that only processes the last token - # This mimics the normal decode pattern - cu_seqlens_q = torch.arange( - bs + 1, device=cu_seqlens_q.device, dtype=cu_seqlens_q.dtype - ) - seqlens = metadata.cache_seqlens_int32[:bs] - # Slice the page_table to match the batch size and actual sequence length - # This serves three important purposes: - # 1. Ensures we only process the actual batch size (bs) and not the maximum batch size - # 2. Limits the sequence length to prevent processing padding tokens or garbage values - # 3. Prevents zeros in the block table which can cause garbage output during replay - # - # Without this slicing, the pre-allocated page_table may contain zeros or invalid indices - # beyond the actual sequence length, leading to incorrect attention calculations - max_seq_len = int(seqlens.max().item()) - if self.use_sliding_window_kv_pool: - sliced_page_table = self.token_to_kv_pool.translate_loc_from_full_to_swa( - metadata.page_table[:bs, :max_seq_len] - ).to(torch.int32) - else: - sliced_page_table = metadata.page_table[:bs, :max_seq_len] - - cu_seqlens_q_np = cu_seqlens_q.cpu().numpy() - seqlens_np = seqlens.cpu().numpy() - ( - seqlens_q_local_np, - cu_seqlens_q_local_np, - seqlens_k_local_np, - block_table_local, - ) = make_local_attention_virtual_batches( - self.attention_chunk_size, - cu_seqlens_q_np, - seqlens_np, - sliced_page_table, - self.page_size, - preserve_attn_chunk_size=True, - ) - - # Convert back to tensors - device = local_q_buf.device - cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device) - seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device) - block_table_local = block_table_local.to(device) - # Get sizes - q_len = cu_seqlens_q_local.shape[0] - k_len = seqlens_k_local.shape[0] - b0, b1 = block_table_local.shape - - # In-place updates into preallocated tensors and zero out the unused space - local_q_buf[:q_len].copy_(cu_seqlens_q_local) - local_q_buf[q_len:].fill_(0) - local_k_buf[:k_len].copy_(seqlens_k_local) - local_k_buf[k_len:].fill_(0) - local_block_buf[:b0, :b1].copy_(block_table_local) - local_block_buf[b0:, :].fill_(0) - local_block_buf[:b0, b1:].fill_(0) - - if metadata.local_attn_metadata is not None: - lam = metadata.local_attn_metadata - lam.local_max_query_len = int(seqlens_q_local_np.max()) - lam.local_max_seq_len = int(seqlens_k_local_np.max()) - def _init_sliding_window_attn_spec_metadata( self, metadata: FlashAttentionMetadata, @@ -3649,162 +3464,6 @@ def draft_decode_set_expand_metadata( # cu_seqlens_q_local = [0, 4, 6, 10, 14, 18, 19, 23, 24] # seqlens_k_local = [ 4, 2, 4, 4, 4, 1, 4, 1] # block_table_local : shape[local_virtual_batches, pages_per_local_batch] -def make_local_attention_virtual_batches( - attn_chunk_size: int, - query_start_loc_np: np.ndarray, - seq_lens_np: np.ndarray, - block_table: torch.Tensor, - page_size: int = 0, - preserve_attn_chunk_size: bool = False, -) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]: - """ - Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into - local attention blocks, where each block is passed to the attention kernel - as an independent local ("virtual") batch item. - - Args: - attn_chunk_size: Size of local attention chunks - query_start_loc_np: Cumulative sum of query lengths (numpy array) - seq_lens_np: Sequence lengths (numpy array) - block_table: Block table for KV cache - page_size: Size of each page in the KV cache - preserve_attn_chunk_size: Skip sequence-length-based chunk normalization. - - Returns: - seqlens_q_local: Query sequence lengths for local attention - cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention - seqlens_k_local: Key sequence lengths for local attention - block_table_local: Block table for local attention - """ - if not preserve_attn_chunk_size: - max_seq_len = seq_lens_np.max() - effective_chunk_size = min(attn_chunk_size, max_seq_len) - effective_chunk_size = (effective_chunk_size // page_size) * page_size - if effective_chunk_size < page_size: - effective_chunk_size = page_size - attn_chunk_size = effective_chunk_size - - q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1] - actual_batch_size = seq_lens_np.shape[0] - - # Handle if we are starting in the middle of a local attention block, - # we assume q_seqlens > 0 (for all elements), for each batch idx we compute - # the number of tokens that are not in the first local attention block and - # then we can simply use a cdiv for the rest. - # For example if we have: - # attn_chunk_size = 4 - # q_seqlens = [4, 10, 5] - # k_seqlens = [6, 17, 9] - # Then we would get: - # new_tokens_in_first_block = [2, 1, 4] - # local_blocks = [2, 4, 2] - q_tokens_in_first_block = np.minimum( - attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens - ).astype(np.int32) - tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size) - local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size) - - # Once we know the number of local blocks we can compute the request spans - # for each batch idx, we can figure out the number of "virtual" requests we - # have to make, - # For the above example we would get: - # seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1] - # - # First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1]) - # (TODO: max a utility to share this code with _prepare_inputs) - # arange step 1. [2, 4, 2] -> [2, 6, 8] - cu_num_blocks = np.cumsum(local_blocks) - virtual_batches = cu_num_blocks[-1] - # arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6] - block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks) - # arange step 3. [0, 1, 0, 1, 2, 3, 0, 1] - arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets - # also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0]) - rarange = np.repeat(local_blocks, local_blocks) - arange - 1 - # Then we can compute the seqlens_q_local, handling the fact that the - # first and last blocks could be partial - seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks) - # set the first block since this may be a partial block - seqlens_q_local[arange == 0] = q_tokens_in_first_block - # set the remaining blocks - seqlens_q_local[arange > 0] = np.minimum( - seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size - )[arange > 0] - - # convert from q_seqlens to cu_seqlens_q - cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32) - - # compute the seqlens_k_local, - # basically a full local attention block for all but the last block in each - # batch - # For our example this will be: - # seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1] - seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32) - seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block - - k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - ( - rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks) - ) - # For the example the local attention blocks start at: - # _b0_ _____b1_____ _b2_ - # k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8] - block_starts = k_seqstarts_absolute // page_size - - assert attn_chunk_size % page_size == 0, ( - f"attn_chunk_size {attn_chunk_size} is not divisible by page_size {page_size}" - ) - pages_per_local_batch = attn_chunk_size // page_size - - # Create a block_table for the local attention blocks - # For out example if we have a block-table like (assuming page_size=2): - # block_table = [ - # [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0 - # [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1 - # [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2 - # ] - # Then for the local batches we would want a block-table like - # block_table_local = [ - # [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0]) - # [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4]) - # [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4]) - # [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8]) - # [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12]) - # [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16]) - # [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4]) - # [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8]) - # ] - block_indices = np.broadcast_to( - np.arange(pages_per_local_batch, dtype=np.int32), - (virtual_batches, pages_per_local_batch), - ) + np.expand_dims(block_starts, axis=1) - # Ensure block_indices doesn't exceed block_table dimensions - # This is a critical safety check that prevents index out of bounds errors - # when dealing with large sequences (>8192 tokens) or when the block_table - # dimensions are smaller than what would be needed for the full attention chunk size. - block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1) - batch_indices = np.repeat( - np.arange(actual_batch_size, dtype=np.int32), - local_blocks * pages_per_local_batch, - ) - - # NOTE: https://github.com/pytorch/pytorch/pull/160256 causes performance - # regression when using numpy arrays (batch and block indices) to index into - # torch tensor (block_table). As a workaround, convert numpy arrays to torch - # tensor first, which recovers perf. - batch_indices_torch = torch.from_numpy(batch_indices) - block_indices_torch = torch.from_numpy(block_indices) - block_table_local = block_table[batch_indices_torch, block_indices_torch].view( - virtual_batches, -1 - ) - - return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local - - -def cdiv(a: int, b: int) -> int: - """Ceiling division.""" - return -(a // -b) - - # TODO(hebiao064): remove this once we have a better way to handle the merge_state_v2 torch.compile issue @torch._dynamo.disable() def merge_state_v2_wrapper(o, s_a, o_exp, s_b): diff --git a/python/sglang/srt/layers/attention/local_attention.py b/python/sglang/srt/layers/attention/local_attention.py new file mode 100644 index 0000000000..cdf684e1f1 --- /dev/null +++ b/python/sglang/srt/layers/attention/local_attention.py @@ -0,0 +1,378 @@ +"""Chunked (local) attention for Llama4 iRoPE layers on the FA3/FA4 backend. + +Each sequence is split into ``attention_chunk_size`` blocks that the kernel sees +as independent "virtual" batch items. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Dict, Optional + +import numpy as np +import torch + +if TYPE_CHECKING: + from sglang.srt.layers.attention.flashattention_backend import ( + FlashAttentionMetadata, + ) + from sglang.srt.layers.radix_attention import RadixAttention + + +@dataclass +class LocalAttentionMetadata: + local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention + local_seqused_k: torch.Tensor = None # sequence lengths for local attention + local_block_table: torch.Tensor = None # block table for local attention + local_max_query_len: int = 0 # max query length for local attention + local_max_seq_len: int = 0 # max sequence length for local attention + + +class LocalAttentionMetadataBuilder: + """The backend owns the CUDA-graph buffers; ``alloc_cuda_graph_buffers`` + returns them and the capture/replay methods take them back in.""" + + def __init__( + self, + *, + attention_chunk_size: int, + page_size: int, + max_context_len: int, + device: torch.device, + swa_translate: Optional[Callable[[torch.Tensor], torch.Tensor]], + ): + self.attention_chunk_size = attention_chunk_size + self.page_size = page_size + self.max_context_len = max_context_len + self.device = device + self.swa_translate = swa_translate + + def applies(self, layer: RadixAttention, metadata: FlashAttentionMetadata) -> bool: + return metadata.local_attn_metadata is not None and layer.use_irope + + def alloc_cuda_graph_buffers(self, max_bs: int) -> Dict[str, torch.Tensor]: + page_size = self.page_size or 1 + attn_chunk_size = self.attention_chunk_size + max_virtual_batches = max_bs * ( + (self.max_context_len + attn_chunk_size - 1) // attn_chunk_size + ) + max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size + return { + "local_query_start_loc": torch.zeros( + max_virtual_batches + 1, dtype=torch.int32, device=self.device + ), + "local_seqused_k": torch.zeros( + max_virtual_batches, dtype=torch.int32, device=self.device + ), + "local_block_table": torch.zeros( + max_virtual_batches, + max_pages_per_block, + dtype=torch.int32, + device=self.device, + ), + } + + def _kernel_page_table(self, page_table: torch.Tensor) -> torch.Tensor: + # Eager path only: the extend metadata holds token slots, which the + # full->swa mapping is indexed by. + if self.swa_translate is not None: + return self.swa_translate(page_table).to(torch.int32) + return page_table + + def _graph_page_table( + self, metadata: FlashAttentionMetadata, bs: int, max_seq_len: int + ) -> torch.Tensor: + # CUDA-graph metadata already holds page ids, and swa_page_table is the + # same table translated into the swa pool; the token-slot mapping must + # not be applied to either. Columns are pages, so slice by pages. + table = ( + metadata.swa_page_table + if self.swa_translate is not None + else metadata.page_table + ) + num_pages = (max_seq_len + self.page_size - 1) // self.page_size + return table[:bs, :num_pages] + + def build( + self, + *, + cu_seqlens_q: Optional[torch.Tensor], + cache_seqlens_int32: Optional[torch.Tensor], + page_table: Optional[torch.Tensor], + device: torch.device, + ) -> Optional[LocalAttentionMetadata]: + if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None: + return None + page_table = self._kernel_page_table(page_table) + if self.page_size > 1: + # Convert the eager token table to physical page indices. + page_table = page_table[:, :: self.page_size] // self.page_size + + ( + seqlens_q_local_np, + cu_seqlens_q_local_np, + seqlens_k_local_np, + block_table_local, + ) = make_local_attention_virtual_batches( + self.attention_chunk_size, + cu_seqlens_q.cpu().numpy(), + cache_seqlens_int32.cpu().numpy(), + page_table, + self.page_size, + preserve_attn_chunk_size=True, + ) + return LocalAttentionMetadata( + local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device), + local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device), + local_block_table=block_table_local.to(device), + local_max_query_len=int(seqlens_q_local_np.max()), + local_max_seq_len=int(seqlens_k_local_np.max()), + ) + + def build_for_capture( + self, + metadata: FlashAttentionMetadata, + bs: int, + *, + buffers: Dict[str, torch.Tensor], + ) -> LocalAttentionMetadata: + """Views of the preallocated buffers sized to what this capture needs.""" + seq_lens_capture = metadata.cache_seqlens_int32 + max_seq_len = int(seq_lens_capture.max().item()) + + ( + seqlens_q_local_np, + cu_seqlens_q_local_np, + seqlens_k_local_np, + block_table_local_np, + ) = make_local_attention_virtual_batches( + self.attention_chunk_size, + metadata.cu_seqlens_q.cpu().numpy(), + seq_lens_capture.cpu().numpy(), + self._graph_page_table(metadata, bs, max_seq_len), + self.page_size, + preserve_attn_chunk_size=True, + ) + q_len = len(cu_seqlens_q_local_np) + k_len = len(seqlens_k_local_np) + b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs + b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1 + return LocalAttentionMetadata( + local_query_start_loc=buffers["local_query_start_loc"][:q_len], + local_seqused_k=buffers["local_seqused_k"][:k_len], + local_block_table=buffers["local_block_table"][:b0, :b1], + local_max_query_len=1, + local_max_seq_len=max_seq_len, + ) + + def update_for_replay( + self, + metadata: FlashAttentionMetadata, + bs: int, + *, + buffers: Dict[str, torch.Tensor], + ) -> None: + """Refill the preallocated buffers in place before a CUDA-graph replay.""" + local_q_buf = buffers["local_query_start_loc"] + local_k_buf = buffers["local_seqused_k"] + local_block_buf = buffers["local_block_table"] + + # Decode: one query token per request. + cu_seqlens_q = torch.arange( + bs + 1, device=local_q_buf.device, dtype=local_q_buf.dtype + ) + seqlens = metadata.cache_seqlens_int32[:bs] + # Rows past bs and columns past the live pages hold stale ids. + max_seq_len = int(seqlens.max().item()) + sliced_page_table = self._graph_page_table(metadata, bs, max_seq_len) + + ( + seqlens_q_local_np, + cu_seqlens_q_local_np, + seqlens_k_local_np, + block_table_local, + ) = make_local_attention_virtual_batches( + self.attention_chunk_size, + cu_seqlens_q.cpu().numpy(), + seqlens.cpu().numpy(), + sliced_page_table, + self.page_size, + preserve_attn_chunk_size=True, + ) + + device = local_q_buf.device + cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device) + seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device) + block_table_local = block_table_local.to(device) + q_len = cu_seqlens_q_local.shape[0] + k_len = seqlens_k_local.shape[0] + b0, b1 = block_table_local.shape + + local_q_buf[:q_len].copy_(cu_seqlens_q_local) + local_q_buf[q_len:].fill_(0) + local_k_buf[:k_len].copy_(seqlens_k_local) + local_k_buf[k_len:].fill_(0) + local_block_buf[:b0, :b1].copy_(block_table_local) + local_block_buf[b0:, :].fill_(0) + local_block_buf[:b0, b1:].fill_(0) + + if metadata.local_attn_metadata is not None: + lam = metadata.local_attn_metadata + lam.local_max_query_len = int(seqlens_q_local_np.max()) + lam.local_max_seq_len = int(seqlens_k_local_np.max()) + + +def make_local_attention_virtual_batches( + attn_chunk_size: int, + query_start_loc_np: np.ndarray, + seq_lens_np: np.ndarray, + block_table: torch.Tensor, + page_size: int = 0, + preserve_attn_chunk_size: bool = False, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]: + """ + Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into + local attention blocks, where each block is passed to the attention kernel + as an independent local ("virtual") batch item. + + Args: + attn_chunk_size: Size of local attention chunks + query_start_loc_np: Cumulative sum of query lengths (numpy array) + seq_lens_np: Sequence lengths (numpy array) + block_table: Block table for KV cache + page_size: Size of each page in the KV cache + preserve_attn_chunk_size: Skip sequence-length-based chunk normalization. + + Returns: + seqlens_q_local: Query sequence lengths for local attention + cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention + seqlens_k_local: Key sequence lengths for local attention + block_table_local: Block table for local attention + """ + if not preserve_attn_chunk_size: + max_seq_len = seq_lens_np.max() + effective_chunk_size = min(attn_chunk_size, max_seq_len) + effective_chunk_size = (effective_chunk_size // page_size) * page_size + if effective_chunk_size < page_size: + effective_chunk_size = page_size + attn_chunk_size = effective_chunk_size + + q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1] + actual_batch_size = seq_lens_np.shape[0] + + # Handle if we are starting in the middle of a local attention block, + # we assume q_seqlens > 0 (for all elements), for each batch idx we compute + # the number of tokens that are not in the first local attention block and + # then we can simply use a cdiv for the rest. + # For example if we have: + # attn_chunk_size = 4 + # q_seqlens = [4, 10, 5] + # k_seqlens = [6, 17, 9] + # Then we would get: + # new_tokens_in_first_block = [2, 1, 4] + # local_blocks = [2, 4, 2] + q_tokens_in_first_block = np.minimum( + attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens + ).astype(np.int32) + tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size) + local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size) + + # Once we know the number of local blocks we can compute the request spans + # for each batch idx, we can figure out the number of "virtual" requests we + # have to make, + # For the above example we would get: + # seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1] + # + # First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1]) + # (TODO: max a utility to share this code with _prepare_inputs) + # arange step 1. [2, 4, 2] -> [2, 6, 8] + cu_num_blocks = np.cumsum(local_blocks) + virtual_batches = cu_num_blocks[-1] + # arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6] + block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks) + # arange step 3. [0, 1, 0, 1, 2, 3, 0, 1] + arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets + # also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0]) + rarange = np.repeat(local_blocks, local_blocks) - arange - 1 + # Then we can compute the seqlens_q_local, handling the fact that the + # first and last blocks could be partial + seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks) + # set the first block since this may be a partial block + seqlens_q_local[arange == 0] = q_tokens_in_first_block + # set the remaining blocks + seqlens_q_local[arange > 0] = np.minimum( + seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size + )[arange > 0] + + # convert from q_seqlens to cu_seqlens_q + cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32) + + # compute the seqlens_k_local, + # basically a full local attention block for all but the last block in each + # batch + # For our example this will be: + # seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1] + seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32) + seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block + + k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - ( + rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks) + ) + # For the example the local attention blocks start at: + # _b0_ _____b1_____ _b2_ + # k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8] + block_starts = k_seqstarts_absolute // page_size + + assert attn_chunk_size % page_size == 0, ( + f"attn_chunk_size {attn_chunk_size} is not divisible by page_size {page_size}" + ) + pages_per_local_batch = attn_chunk_size // page_size + + # Create a block_table for the local attention blocks + # For out example if we have a block-table like (assuming page_size=2): + # block_table = [ + # [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0 + # [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1 + # [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2 + # ] + # Then for the local batches we would want a block-table like + # block_table_local = [ + # [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0]) + # [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4]) + # [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4]) + # [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8]) + # [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12]) + # [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16]) + # [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4]) + # [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8]) + # ] + block_indices = np.broadcast_to( + np.arange(pages_per_local_batch, dtype=np.int32), + (virtual_batches, pages_per_local_batch), + ) + np.expand_dims(block_starts, axis=1) + # Ensure block_indices doesn't exceed block_table dimensions + # This is a critical safety check that prevents index out of bounds errors + # when dealing with large sequences (>8192 tokens) or when the block_table + # dimensions are smaller than what would be needed for the full attention chunk size. + block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1) + batch_indices = np.repeat( + np.arange(actual_batch_size, dtype=np.int32), + local_blocks * pages_per_local_batch, + ) + + # NOTE: https://github.com/pytorch/pytorch/pull/160256 causes performance + # regression when using numpy arrays (batch and block indices) to index into + # torch tensor (block_table). As a workaround, convert numpy arrays to torch + # tensor first, which recovers perf. + batch_indices_torch = torch.from_numpy(batch_indices) + block_indices_torch = torch.from_numpy(block_indices) + block_table_local = block_table[batch_indices_torch, block_indices_torch].view( + virtual_batches, -1 + ) + + return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local + + +def cdiv(a: int, b: int) -> int: + """Ceiling division.""" + return -(a // -b) diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index 83e456c674..c2df104293 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -8,10 +8,13 @@ from sglang.srt.configs.model_config import AttentionArch from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.flashattention_backend import ( FlashAttentionMetadata, - make_local_attention_virtual_batches, merge_state_v2_wrapper, prepare_swa_spec_page_table_triton, ) +from sglang.srt.layers.attention.local_attention import ( + LocalAttentionMetadata, + make_local_attention_virtual_batches, +) from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -1480,7 +1483,7 @@ class XPUAttentionBackend(AttentionBackend): self.page_size, ) - local_metadata = FlashAttentionMetadata.LocalAttentionMetadata( + local_metadata = LocalAttentionMetadata( local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device), local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device), local_block_table=block_table_local.to(device), diff --git a/test/registered/unit/layers/attention/test_local_attention.py b/test/registered/unit/layers/attention/test_local_attention.py new file mode 100644 index 0000000000..2dafbc4704 --- /dev/null +++ b/test/registered/unit/layers/attention/test_local_attention.py @@ -0,0 +1,221 @@ +"""CPU tests for srt/layers/attention/local_attention: the virtual-batch +decomposition and the metadata builder.""" + +import unittest +from types import SimpleNamespace + +import numpy as np +import torch + +from sglang.srt.layers.attention.local_attention import ( + LocalAttentionMetadata, + LocalAttentionMetadataBuilder, + make_local_attention_virtual_batches, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +CPU = torch.device("cpu") + + +def _page_table(bs: int, width: int = 32) -> torch.Tensor: + return torch.arange(bs * width, dtype=torch.int32).view(bs, width) + + +class TestVirtualBatches(CustomTestCase): + def test_worked_example_from_docstring(self): + """chunk=4, q=[4,10,5], k=[6,17,9] must split into local_blocks=[2,4,2].""" + q_start = np.array([0, 4, 14, 19], dtype=np.int32) + seq_lens = np.array([6, 17, 9], dtype=np.int32) + q_local, cu_q_local, k_local, block_table_local = ( + make_local_attention_virtual_batches( + 4, q_start, seq_lens, _page_table(3), 1 + ) + ) + self.assertEqual(q_local.tolist(), [2, 2, 1, 4, 4, 1, 4, 1]) + self.assertEqual(cu_q_local.tolist(), [0, 2, 4, 5, 9, 13, 14, 18, 19]) + # Every block sees a full chunk of keys except the last block of each request. + self.assertEqual(k_local.tolist(), [4, 2, 4, 4, 4, 1, 4, 1]) + self.assertEqual(tuple(block_table_local.shape), (8, 4)) + + def test_decode_is_one_block_per_request(self): + q_start = np.array([0, 1, 2], dtype=np.int32) + seq_lens = np.array([6, 9], dtype=np.int32) + q_local, cu_q_local, k_local, _ = make_local_attention_virtual_batches( + 4, q_start, seq_lens, _page_table(2), 1 + ) + self.assertEqual(q_local.tolist(), [1, 1]) + self.assertEqual(cu_q_local.tolist(), [0, 1, 2]) + # Only the tail of the current chunk is attended: 6 % 4 == 2, 9 % 4 == 1. + self.assertEqual(k_local.tolist(), [2, 1]) + + def test_chunk_is_clamped_to_the_longest_sequence(self): + q_start = np.array([0, 6], dtype=np.int32) + seq_lens = np.array([6], dtype=np.int32) + _, _, k_local, block_table_local = make_local_attention_virtual_batches( + 8, q_start, seq_lens, _page_table(1), 1 + ) + self.assertEqual(k_local.tolist(), [6]) + self.assertEqual(tuple(block_table_local.shape), (1, 6)) + + def test_page_size_divides_the_block_table_width(self): + q_start = np.array([0, 6], dtype=np.int32) + seq_lens = np.array([6], dtype=np.int32) + q_local, _, k_local, block_table_local = make_local_attention_virtual_batches( + 4, q_start, seq_lens, _page_table(1), 2 + ) + self.assertEqual(q_local.tolist(), [4, 2]) + self.assertEqual(k_local.tolist(), [4, 2]) + self.assertEqual(tuple(block_table_local.shape), (2, 2)) # chunk 4 / page 2 + + +class TestLocalAttentionMetadataBuilder(CustomTestCase): + def _builder(self, swa_translate=None, max_context_len=16): + return LocalAttentionMetadataBuilder( + attention_chunk_size=4, + page_size=1, + max_context_len=max_context_len, + device=CPU, + swa_translate=swa_translate, + ) + + def test_build_returns_none_without_a_complete_batch(self): + b = self._builder() + self.assertIsNone( + b.build( + cu_seqlens_q=None, + cache_seqlens_int32=torch.tensor([6]), + page_table=_page_table(1), + device=CPU, + ) + ) + + def test_build_applies_the_swa_translation_to_the_block_table(self): + args = dict( + cu_seqlens_q=torch.tensor([0, 6], dtype=torch.int32), + cache_seqlens_int32=torch.tensor([6], dtype=torch.int32), + page_table=_page_table(1), + device=CPU, + ) + plain = self._builder().build(**args) + swa = self._builder(swa_translate=lambda t: t + 100).build(**args) + self.assertEqual(plain.local_max_query_len, 4) + self.assertEqual(plain.local_max_seq_len, 4) + torch.testing.assert_close(swa.local_block_table, plain.local_block_table + 100) + + def test_cuda_graph_buffers_fit_the_worst_case_decomposition(self): + """Buffer sizing must match the most virtual batches max_bs x max_context_len can produce.""" + b = self._builder(max_context_len=16) + bufs = b.alloc_cuda_graph_buffers(max_bs=2) + md = b.build( + cu_seqlens_q=torch.tensor([0, 16, 32], dtype=torch.int32), + cache_seqlens_int32=torch.tensor([16, 16], dtype=torch.int32), + page_table=_page_table(2), + device=CPU, + ) + self.assertEqual( + md.local_query_start_loc.numel(), bufs["local_query_start_loc"].numel() + ) + self.assertEqual(md.local_seqused_k.numel(), bufs["local_seqused_k"].numel()) + self.assertEqual( + tuple(md.local_block_table.shape), tuple(bufs["local_block_table"].shape) + ) + + def test_capture_returns_views_of_the_preallocated_buffers(self): + b = self._builder() + bufs = b.alloc_cuda_graph_buffers(max_bs=2) + metadata = SimpleNamespace( + cu_seqlens_q=torch.tensor([0, 1, 2], dtype=torch.int32), + cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32), + page_table=_page_table(2), + ) + md = b.build_for_capture(metadata, 2, buffers=bufs) + self.assertEqual(md.local_query_start_loc.numel(), 3) + self.assertEqual(md.local_seqused_k.numel(), 2) + self.assertEqual(tuple(md.local_block_table.shape), (2, 4)) + self.assertEqual(md.local_max_query_len, 1) + self.assertEqual(md.local_max_seq_len, 9) + self.assertEqual( + md.local_query_start_loc.data_ptr(), + bufs["local_query_start_loc"].data_ptr(), + ) + self.assertEqual( + md.local_block_table.data_ptr(), bufs["local_block_table"].data_ptr() + ) + + def _paged_builder(self, swa_translate): + # chunk 32 / page 16: two pages per local block. + return LocalAttentionMetadataBuilder( + attention_chunk_size=32, + page_size=16, + max_context_len=64, + device=CPU, + swa_translate=swa_translate, + ) + + def test_replay_reads_page_ids_from_swa_page_table(self): + """Regression: with page_size > 1 the CUDA-graph replay must not push the + page-granular table through the token-slot full->swa mapping.""" + # Any translation applied to page ids would land far outside {7, 8}. + b = self._paged_builder(swa_translate=lambda t: t * 1000) + bufs = b.alloc_cuda_graph_buffers(max_bs=2) + metadata = SimpleNamespace( + cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32), + page_table=torch.tensor([[3], [5]], dtype=torch.int32), + swa_page_table=torch.tensor([[7], [8]], dtype=torch.int32), + local_attn_metadata=LocalAttentionMetadata(), + ) + b.update_for_replay(metadata, 2, buffers=bufs) + table = bufs["local_block_table"][:2] + self.assertEqual(table[:, 0].tolist(), [7, 8]) + self.assertTrue(set(table.flatten().tolist()) <= {0, 7, 8}) + + def test_replay_without_swa_pool_reads_page_table(self): + b = self._paged_builder(swa_translate=None) + bufs = b.alloc_cuda_graph_buffers(max_bs=2) + metadata = SimpleNamespace( + cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32), + page_table=torch.tensor([[3], [5]], dtype=torch.int32), + local_attn_metadata=LocalAttentionMetadata(), + ) + b.update_for_replay(metadata, 2, buffers=bufs) + self.assertEqual(bufs["local_block_table"][:2, 0].tolist(), [3, 5]) + + def test_capture_and_replay_share_the_table_source(self): + b = self._paged_builder(swa_translate=lambda t: t * 1000) + bufs = b.alloc_cuda_graph_buffers(max_bs=2) + metadata = SimpleNamespace( + cu_seqlens_q=torch.tensor([0, 1, 2], dtype=torch.int32), + cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32), + page_table=torch.tensor([[3], [5]], dtype=torch.int32), + swa_page_table=torch.tensor([[7], [8]], dtype=torch.int32), + ) + md = b.build_for_capture(metadata, 2, buffers=bufs) + self.assertEqual( + tuple(md.local_block_table.shape), (2, 2) + ) # 2 reqs x 2 pages per block + + def test_replay_refills_buffers_in_place_and_zeroes_the_tail(self): + b = self._builder() + bufs = b.alloc_cuda_graph_buffers(max_bs=2) + for t in bufs.values(): + t.fill_(7) # stale contents from an earlier, larger batch + lam = LocalAttentionMetadata() + metadata = SimpleNamespace( + cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32), + page_table=_page_table(2), + local_attn_metadata=lam, + ) + b.update_for_replay(metadata, 2, buffers=bufs) + self.assertEqual(bufs["local_query_start_loc"][:3].tolist(), [0, 1, 2]) + self.assertEqual(bufs["local_query_start_loc"][3:].abs().sum().item(), 0) + self.assertEqual(bufs["local_seqused_k"][:2].tolist(), [2, 1]) + self.assertEqual(bufs["local_seqused_k"][2:].abs().sum().item(), 0) + self.assertEqual(bufs["local_block_table"][2:].abs().sum().item(), 0) + self.assertEqual((lam.local_max_query_len, lam.local_max_seq_len), (1, 2)) + + +if __name__ == "__main__": + unittest.main()