[Bugfix] Llama4 local attention: read page ids from the graph tables in CUDA-graph capture/replay (page_size > 1) (#38187)

This commit is contained in:
Martin Hua
2026-10-01 14:42:22 -07:00
committed by GitHub
parent bf2d686a82
commit 52d5faed5b
4 changed files with 674 additions and 413 deletions
@@ -3,7 +3,6 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
import numpy as np
import torch
from sglang.kernels.ops.attention.metadata import (
@@ -26,6 +25,10 @@ from sglang.srt.layers.attention.kv_shard_hooks import (
get_kv_shard_pool,
prepare_kv_shard_forward,
)
from sglang.srt.layers.attention.local_attention import (
LocalAttentionMetadata,
LocalAttentionMetadataBuilder,
)
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
from sglang.srt.layers.cp.utils import is_cp_active
@@ -116,14 +119,6 @@ class FlashAttentionMetadata:
# Page table for the encoder
encoder_page_table: torch.Tensor = None
@dataclass
class LocalAttentionMetadata:
local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention
local_seqused_k: torch.Tensor = None # sequence lengths for local attention
local_block_table: torch.Tensor = None # block table for local attention
local_max_query_len: int = 0 # max query length for local attention
local_max_seq_len: int = 0 # max sequence length for local attention
local_attn_metadata: Optional[LocalAttentionMetadata] = None
# For sliding window attention topk>1 spec decoding
@@ -239,23 +234,14 @@ class FlashAttentionBackend(AttentionBackend):
)
self.speculative_step_id = speculative_step_id
# Local attention settings
self.has_local_attention = model_runner.model_config.is_local_attention_model
# Local (chunked) attention derives its page table by re-translating
# metadata.page_table through the static full->swa map -- meaningless
# on the unified pool's kernel-facing tables, and no unified-eligible
# model uses it. Fail loud rather than silently double-translate.
assert not (
self.kv_index_translator.is_translating and self.has_local_attention
), (
"--enable-unified-memory does not support local-attention models "
"on the fa3/fa4 backend."
self.local_attn_builder = self._maybe_init_local_attn_builder(model_runner)
# Read directly by MusaFlashAttentionBackend's copied forward paths.
self.has_local_attention = self.local_attn_builder is not None
self.attention_chunk_size = (
self.local_attn_builder.attention_chunk_size
if self.local_attn_builder is not None
else None
)
if self.has_local_attention:
assert model_runner.attention_chunk_size is not None, (
"Attention chunk size is required for local attention"
)
self.attention_chunk_size = model_runner.attention_chunk_size
# For each layer, the sliding_window_size can be different. This is only used for preparing SWA metadata.
# We use `layer.sliding_window_size` to decide whether to use SWA for each layer.
@@ -390,6 +376,32 @@ class FlashAttentionBackend(AttentionBackend):
_should_disable_scheduler_metadata_precompute()
)
def _maybe_init_local_attn_builder(
self, model_runner: ModelRunner
) -> Optional[LocalAttentionMetadataBuilder]:
if not model_runner.model_config.is_local_attention_model:
return None
# Local attention re-translates metadata.page_table through the static
# full->swa map, which is meaningless on the unified pool's kernel tables.
assert not self.kv_index_translator.is_translating, (
"--enable-unified-memory does not support local-attention models "
"on the fa3/fa4 backend."
)
assert model_runner.attention_chunk_size is not None, (
"Attention chunk size is required for local attention"
)
return LocalAttentionMetadataBuilder(
attention_chunk_size=model_runner.attention_chunk_size,
page_size=self.page_size,
max_context_len=self.max_context_len,
device=self.device,
swa_translate=(
self.token_to_kv_pool.translate_loc_from_full_to_swa
if self.use_sliding_window_kv_pool
else None
),
)
def _compute_scheduler_metadata(
self, batch_size, max_seq_len_k, cache_seqlens, cu_seqlens_q
):
@@ -624,7 +636,14 @@ class FlashAttentionBackend(AttentionBackend):
# padded seq-len fill value (1), or a later long-context
# replay would leave K/V tiles uncovered.
metadata.max_seq_len_k = self.max_context_len
self._maybe_update_local_attn_metadata_for_capture(metadata, bs)
if self.local_attn_builder is not None:
metadata.local_attn_metadata = (
self.local_attn_builder.build_for_capture(
metadata,
bs,
buffers=self.decode_cuda_graph_local_attn_metadata,
)
)
if self._sched_meta_buf is not None:
sched = self._compute_scheduler_metadata(
bs,
@@ -937,7 +956,7 @@ class FlashAttentionBackend(AttentionBackend):
metadata.cu_seqlens_q,
)
# TODO: we need to test this part for llama 4 eagle case
self._maybe_init_local_attn_metadata(forward_batch, metadata, device)
self._maybe_init_local_attn_metadata(metadata, device)
elif forward_batch.forward_mode.is_target_verify():
if self.topk <= 1:
ragged_layout = getattr(
@@ -985,7 +1004,7 @@ class FlashAttentionBackend(AttentionBackend):
forward_batch.req_pool_indices, : metadata.max_seq_len_k
]
self._maybe_init_local_attn_metadata(forward_batch, metadata, device)
self._maybe_init_local_attn_metadata(metadata, device)
else:
metadata.cache_seqlens_int32 = forward_batch.seq_lens.to(torch.int32)
metadata.max_seq_len_q = self.speculative_num_draft_tokens
@@ -1124,7 +1143,7 @@ class FlashAttentionBackend(AttentionBackend):
# Setup local attention if enabled
if forward_batch.forward_mode == ForwardMode.EXTEND:
self._maybe_init_local_attn_metadata(forward_batch, metadata, device)
self._maybe_init_local_attn_metadata(metadata, device)
if self.is_prefill_aware_swa:
self._pa_swa_prefill_lens[
@@ -1440,12 +1459,9 @@ class FlashAttentionBackend(AttentionBackend):
kv_head_num=layer.tp_k_head_num,
is_prefill=True,
)
# Check if we should use local attention
use_local_attn = (
self.has_local_attention
and self.attention_chunk_size is not None
and metadata.local_attn_metadata is not None
and (hasattr(layer, "use_irope") and layer.use_irope)
self.local_attn_builder is not None
and self.local_attn_builder.applies(layer, metadata)
)
# We do cascade attention for Target Verify with topk > 1
@@ -1965,12 +1981,10 @@ class FlashAttentionBackend(AttentionBackend):
# Use precomputed metadata across all layers
metadata = self.forward_metadata
local_attn_metadata = getattr(metadata, "local_attn_metadata", None)
local_attn_metadata = metadata.local_attn_metadata
use_local_attn = (
self.has_local_attention
and self.attention_chunk_size is not None
and local_attn_metadata is not None
and (hasattr(layer, "use_irope") and layer.use_irope)
self.local_attn_builder is not None
and self.local_attn_builder.applies(layer, metadata)
)
# When Spec Decode enabled, forward_decode would be called with two mode:
@@ -2312,32 +2326,10 @@ class FlashAttentionBackend(AttentionBackend):
else:
self._sched_meta_buf = None
# Only allocate local attention buffers if local attention is enabled
# This prevents OOM errors when local attention is not being used
if self.has_local_attention:
# Estimate maximum sizes for local attention metadata
max_seq_len = self.max_context_len
page_size = self.page_size or 1
attn_chunk_size = self.attention_chunk_size
max_virtual_batches = max_bs * (
(max_seq_len + attn_chunk_size - 1) // attn_chunk_size
if self.local_attn_builder is not None:
self.decode_cuda_graph_local_attn_metadata = (
self.local_attn_builder.alloc_cuda_graph_buffers(max_bs)
)
max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size
self.decode_cuda_graph_local_attn_metadata = {
"local_query_start_loc": torch.zeros(
max_virtual_batches + 1, dtype=torch.int32, device=self.device
),
"local_seqused_k": torch.zeros(
max_virtual_batches, dtype=torch.int32, device=self.device
),
"local_block_table": torch.zeros(
max_virtual_batches,
max_pages_per_block,
dtype=torch.int32,
device=self.device,
),
}
if self.use_sliding_window_kv_pool:
self.decode_cuda_graph_metadata["swa_page_table"] = torch.zeros(
@@ -3008,10 +3000,12 @@ class FlashAttentionBackend(AttentionBackend):
metadata, req_pool_indices, seq_lens, 0
)
self._maybe_update_local_attn_metadata_for_replay(
metadata,
bs,
)
if self.local_attn_builder is not None:
self.local_attn_builder.update_for_replay(
metadata,
bs,
buffers=self.decode_cuda_graph_local_attn_metadata,
)
# Recompute scheduler_metadata into pre-allocated buffer
if (
@@ -3247,197 +3241,18 @@ class FlashAttentionBackend(AttentionBackend):
return 1
def _maybe_init_local_attn_metadata(
self,
forwardbatch: ForwardBatch,
metadata: FlashAttentionMetadata,
device,
):
"""Centralized utility to initialize local_attn_metadata if chunked attention is enabled."""
if not self.has_local_attention:
self, metadata: FlashAttentionMetadata, device
) -> None:
if self.local_attn_builder is None:
metadata.local_attn_metadata = None
return
cu_seqlens_q = metadata.cu_seqlens_q
cache_seqlens_int32 = metadata.cache_seqlens_int32
if self.use_sliding_window_kv_pool:
page_table = self.token_to_kv_pool.translate_loc_from_full_to_swa(
metadata.page_table
).to(torch.int32)
else:
page_table = metadata.page_table
if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None:
metadata.local_attn_metadata = None
return
if self.page_size > 1:
# Convert the eager token table to physical page indices.
page_table = page_table[:, :: self.page_size] // self.page_size
cu_seqlens_q_np = cu_seqlens_q.cpu().numpy()
seq_lens_np = cache_seqlens_int32.cpu().numpy()
(
seqlens_q_local_np,
cu_seqlens_q_local_np,
seqlens_k_local_np,
block_table_local,
) = make_local_attention_virtual_batches(
self.attention_chunk_size,
cu_seqlens_q_np,
seq_lens_np,
page_table,
self.page_size,
preserve_attn_chunk_size=True,
metadata.local_attn_metadata = self.local_attn_builder.build(
cu_seqlens_q=metadata.cu_seqlens_q,
cache_seqlens_int32=metadata.cache_seqlens_int32,
page_table=metadata.page_table,
device=device,
)
local_metadata = FlashAttentionMetadata.LocalAttentionMetadata(
local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device),
local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device),
local_block_table=block_table_local.to(device),
local_max_query_len=int(seqlens_q_local_np.max()),
local_max_seq_len=int(seqlens_k_local_np.max()),
)
metadata.local_attn_metadata = local_metadata
def _maybe_update_local_attn_metadata_for_capture(
self, metadata: FlashAttentionMetadata, bs: int
):
"""Update local attention metadata during CUDA graph capture phase.
This method calculates the exact buffer sizes needed for local attention metadata
during the CUDA graph capture phase, optimizing memory usage by creating views of
pre-allocated buffers with exactly the sizes needed.
"""
if not self.has_local_attention:
return
seq_lens_capture = metadata.cache_seqlens_int32
max_seq_len = int(seq_lens_capture.max().item())
page_table_capture = metadata.page_table
cu_seqlens_q_np = metadata.cu_seqlens_q.cpu().numpy()
seqlens_np = seq_lens_capture.cpu().numpy()
(
seqlens_q_local_np,
cu_seqlens_q_local_np,
seqlens_k_local_np,
block_table_local_np,
) = make_local_attention_virtual_batches(
self.attention_chunk_size,
cu_seqlens_q_np,
seqlens_np,
page_table_capture,
self.page_size,
preserve_attn_chunk_size=True,
)
# Get exact dimensions from the calculation
q_len = len(cu_seqlens_q_local_np)
k_len = len(seqlens_k_local_np)
b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs
b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1
# Create views of the pre-allocated buffers with exactly these sizes
# This is the key optimization - we only use the memory we actually need
local_query_start_loc = self.decode_cuda_graph_local_attn_metadata[
"local_query_start_loc"
][:q_len]
local_seqused_k = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"][
:k_len
]
local_block_table = self.decode_cuda_graph_local_attn_metadata[
"local_block_table"
][:b0, :b1]
metadata.local_attn_metadata = FlashAttentionMetadata.LocalAttentionMetadata(
local_query_start_loc=local_query_start_loc,
local_seqused_k=local_seqused_k,
local_block_table=local_block_table,
local_max_query_len=1,
local_max_seq_len=max_seq_len,
)
def _maybe_update_local_attn_metadata_for_replay(
self,
metadata: FlashAttentionMetadata,
bs: int,
):
"""Update preallocated local attention metadata in-place before CUDA graph replay."""
if not self.has_local_attention:
return
# Access preallocated buffers
local_q_buf = self.decode_cuda_graph_local_attn_metadata[
"local_query_start_loc"
]
local_k_buf = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"]
local_block_buf = self.decode_cuda_graph_local_attn_metadata[
"local_block_table"
]
cu_seqlens_q = self.decode_cuda_graph_metadata["cu_seqlens_q"]
# Create a modified version for local attention that only processes the last token
# This mimics the normal decode pattern
cu_seqlens_q = torch.arange(
bs + 1, device=cu_seqlens_q.device, dtype=cu_seqlens_q.dtype
)
seqlens = metadata.cache_seqlens_int32[:bs]
# Slice the page_table to match the batch size and actual sequence length
# This serves three important purposes:
# 1. Ensures we only process the actual batch size (bs) and not the maximum batch size
# 2. Limits the sequence length to prevent processing padding tokens or garbage values
# 3. Prevents zeros in the block table which can cause garbage output during replay
#
# Without this slicing, the pre-allocated page_table may contain zeros or invalid indices
# beyond the actual sequence length, leading to incorrect attention calculations
max_seq_len = int(seqlens.max().item())
if self.use_sliding_window_kv_pool:
sliced_page_table = self.token_to_kv_pool.translate_loc_from_full_to_swa(
metadata.page_table[:bs, :max_seq_len]
).to(torch.int32)
else:
sliced_page_table = metadata.page_table[:bs, :max_seq_len]
cu_seqlens_q_np = cu_seqlens_q.cpu().numpy()
seqlens_np = seqlens.cpu().numpy()
(
seqlens_q_local_np,
cu_seqlens_q_local_np,
seqlens_k_local_np,
block_table_local,
) = make_local_attention_virtual_batches(
self.attention_chunk_size,
cu_seqlens_q_np,
seqlens_np,
sliced_page_table,
self.page_size,
preserve_attn_chunk_size=True,
)
# Convert back to tensors
device = local_q_buf.device
cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device)
seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device)
block_table_local = block_table_local.to(device)
# Get sizes
q_len = cu_seqlens_q_local.shape[0]
k_len = seqlens_k_local.shape[0]
b0, b1 = block_table_local.shape
# In-place updates into preallocated tensors and zero out the unused space
local_q_buf[:q_len].copy_(cu_seqlens_q_local)
local_q_buf[q_len:].fill_(0)
local_k_buf[:k_len].copy_(seqlens_k_local)
local_k_buf[k_len:].fill_(0)
local_block_buf[:b0, :b1].copy_(block_table_local)
local_block_buf[b0:, :].fill_(0)
local_block_buf[:b0, b1:].fill_(0)
if metadata.local_attn_metadata is not None:
lam = metadata.local_attn_metadata
lam.local_max_query_len = int(seqlens_q_local_np.max())
lam.local_max_seq_len = int(seqlens_k_local_np.max())
def _init_sliding_window_attn_spec_metadata(
self,
metadata: FlashAttentionMetadata,
@@ -3649,162 +3464,6 @@ def draft_decode_set_expand_metadata(
# cu_seqlens_q_local = [0, 4, 6, 10, 14, 18, 19, 23, 24]
# seqlens_k_local = [ 4, 2, 4, 4, 4, 1, 4, 1]
# block_table_local : shape[local_virtual_batches, pages_per_local_batch]
def make_local_attention_virtual_batches(
attn_chunk_size: int,
query_start_loc_np: np.ndarray,
seq_lens_np: np.ndarray,
block_table: torch.Tensor,
page_size: int = 0,
preserve_attn_chunk_size: bool = False,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]:
"""
Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into
local attention blocks, where each block is passed to the attention kernel
as an independent local ("virtual") batch item.
Args:
attn_chunk_size: Size of local attention chunks
query_start_loc_np: Cumulative sum of query lengths (numpy array)
seq_lens_np: Sequence lengths (numpy array)
block_table: Block table for KV cache
page_size: Size of each page in the KV cache
preserve_attn_chunk_size: Skip sequence-length-based chunk normalization.
Returns:
seqlens_q_local: Query sequence lengths for local attention
cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention
seqlens_k_local: Key sequence lengths for local attention
block_table_local: Block table for local attention
"""
if not preserve_attn_chunk_size:
max_seq_len = seq_lens_np.max()
effective_chunk_size = min(attn_chunk_size, max_seq_len)
effective_chunk_size = (effective_chunk_size // page_size) * page_size
if effective_chunk_size < page_size:
effective_chunk_size = page_size
attn_chunk_size = effective_chunk_size
q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1]
actual_batch_size = seq_lens_np.shape[0]
# Handle if we are starting in the middle of a local attention block,
# we assume q_seqlens > 0 (for all elements), for each batch idx we compute
# the number of tokens that are not in the first local attention block and
# then we can simply use a cdiv for the rest.
# For example if we have:
# attn_chunk_size = 4
# q_seqlens = [4, 10, 5]
# k_seqlens = [6, 17, 9]
# Then we would get:
# new_tokens_in_first_block = [2, 1, 4]
# local_blocks = [2, 4, 2]
q_tokens_in_first_block = np.minimum(
attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens
).astype(np.int32)
tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size)
local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size)
# Once we know the number of local blocks we can compute the request spans
# for each batch idx, we can figure out the number of "virtual" requests we
# have to make,
# For the above example we would get:
# seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1]
#
# First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1])
# (TODO: max a utility to share this code with _prepare_inputs)
# arange step 1. [2, 4, 2] -> [2, 6, 8]
cu_num_blocks = np.cumsum(local_blocks)
virtual_batches = cu_num_blocks[-1]
# arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6]
block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks)
# arange step 3. [0, 1, 0, 1, 2, 3, 0, 1]
arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets
# also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0])
rarange = np.repeat(local_blocks, local_blocks) - arange - 1
# Then we can compute the seqlens_q_local, handling the fact that the
# first and last blocks could be partial
seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks)
# set the first block since this may be a partial block
seqlens_q_local[arange == 0] = q_tokens_in_first_block
# set the remaining blocks
seqlens_q_local[arange > 0] = np.minimum(
seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size
)[arange > 0]
# convert from q_seqlens to cu_seqlens_q
cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32)
# compute the seqlens_k_local,
# basically a full local attention block for all but the last block in each
# batch
# For our example this will be:
# seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1]
seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32)
seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block
k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - (
rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks)
)
# For the example the local attention blocks start at:
# _b0_ _____b1_____ _b2_
# k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8]
block_starts = k_seqstarts_absolute // page_size
assert attn_chunk_size % page_size == 0, (
f"attn_chunk_size {attn_chunk_size} is not divisible by page_size {page_size}"
)
pages_per_local_batch = attn_chunk_size // page_size
# Create a block_table for the local attention blocks
# For out example if we have a block-table like (assuming page_size=2):
# block_table = [
# [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0
# [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1
# [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2
# ]
# Then for the local batches we would want a block-table like
# block_table_local = [
# [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0])
# [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4])
# [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4])
# [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8])
# [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12])
# [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16])
# [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4])
# [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8])
# ]
block_indices = np.broadcast_to(
np.arange(pages_per_local_batch, dtype=np.int32),
(virtual_batches, pages_per_local_batch),
) + np.expand_dims(block_starts, axis=1)
# Ensure block_indices doesn't exceed block_table dimensions
# This is a critical safety check that prevents index out of bounds errors
# when dealing with large sequences (>8192 tokens) or when the block_table
# dimensions are smaller than what would be needed for the full attention chunk size.
block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1)
batch_indices = np.repeat(
np.arange(actual_batch_size, dtype=np.int32),
local_blocks * pages_per_local_batch,
)
# NOTE: https://github.com/pytorch/pytorch/pull/160256 causes performance
# regression when using numpy arrays (batch and block indices) to index into
# torch tensor (block_table). As a workaround, convert numpy arrays to torch
# tensor first, which recovers perf.
batch_indices_torch = torch.from_numpy(batch_indices)
block_indices_torch = torch.from_numpy(block_indices)
block_table_local = block_table[batch_indices_torch, block_indices_torch].view(
virtual_batches, -1
)
return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local
def cdiv(a: int, b: int) -> int:
"""Ceiling division."""
return -(a // -b)
# TODO(hebiao064): remove this once we have a better way to handle the merge_state_v2 torch.compile issue
@torch._dynamo.disable()
def merge_state_v2_wrapper(o, s_a, o_exp, s_b):
@@ -0,0 +1,378 @@
"""Chunked (local) attention for Llama4 iRoPE layers on the FA3/FA4 backend.
Each sequence is split into ``attention_chunk_size`` blocks that the kernel sees
as independent "virtual" batch items.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Callable, Dict, Optional
import numpy as np
import torch
if TYPE_CHECKING:
from sglang.srt.layers.attention.flashattention_backend import (
FlashAttentionMetadata,
)
from sglang.srt.layers.radix_attention import RadixAttention
@dataclass
class LocalAttentionMetadata:
local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention
local_seqused_k: torch.Tensor = None # sequence lengths for local attention
local_block_table: torch.Tensor = None # block table for local attention
local_max_query_len: int = 0 # max query length for local attention
local_max_seq_len: int = 0 # max sequence length for local attention
class LocalAttentionMetadataBuilder:
"""The backend owns the CUDA-graph buffers; ``alloc_cuda_graph_buffers``
returns them and the capture/replay methods take them back in."""
def __init__(
self,
*,
attention_chunk_size: int,
page_size: int,
max_context_len: int,
device: torch.device,
swa_translate: Optional[Callable[[torch.Tensor], torch.Tensor]],
):
self.attention_chunk_size = attention_chunk_size
self.page_size = page_size
self.max_context_len = max_context_len
self.device = device
self.swa_translate = swa_translate
def applies(self, layer: RadixAttention, metadata: FlashAttentionMetadata) -> bool:
return metadata.local_attn_metadata is not None and layer.use_irope
def alloc_cuda_graph_buffers(self, max_bs: int) -> Dict[str, torch.Tensor]:
page_size = self.page_size or 1
attn_chunk_size = self.attention_chunk_size
max_virtual_batches = max_bs * (
(self.max_context_len + attn_chunk_size - 1) // attn_chunk_size
)
max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size
return {
"local_query_start_loc": torch.zeros(
max_virtual_batches + 1, dtype=torch.int32, device=self.device
),
"local_seqused_k": torch.zeros(
max_virtual_batches, dtype=torch.int32, device=self.device
),
"local_block_table": torch.zeros(
max_virtual_batches,
max_pages_per_block,
dtype=torch.int32,
device=self.device,
),
}
def _kernel_page_table(self, page_table: torch.Tensor) -> torch.Tensor:
# Eager path only: the extend metadata holds token slots, which the
# full->swa mapping is indexed by.
if self.swa_translate is not None:
return self.swa_translate(page_table).to(torch.int32)
return page_table
def _graph_page_table(
self, metadata: FlashAttentionMetadata, bs: int, max_seq_len: int
) -> torch.Tensor:
# CUDA-graph metadata already holds page ids, and swa_page_table is the
# same table translated into the swa pool; the token-slot mapping must
# not be applied to either. Columns are pages, so slice by pages.
table = (
metadata.swa_page_table
if self.swa_translate is not None
else metadata.page_table
)
num_pages = (max_seq_len + self.page_size - 1) // self.page_size
return table[:bs, :num_pages]
def build(
self,
*,
cu_seqlens_q: Optional[torch.Tensor],
cache_seqlens_int32: Optional[torch.Tensor],
page_table: Optional[torch.Tensor],
device: torch.device,
) -> Optional[LocalAttentionMetadata]:
if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None:
return None
page_table = self._kernel_page_table(page_table)
if self.page_size > 1:
# Convert the eager token table to physical page indices.
page_table = page_table[:, :: self.page_size] // self.page_size
(
seqlens_q_local_np,
cu_seqlens_q_local_np,
seqlens_k_local_np,
block_table_local,
) = make_local_attention_virtual_batches(
self.attention_chunk_size,
cu_seqlens_q.cpu().numpy(),
cache_seqlens_int32.cpu().numpy(),
page_table,
self.page_size,
preserve_attn_chunk_size=True,
)
return LocalAttentionMetadata(
local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device),
local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device),
local_block_table=block_table_local.to(device),
local_max_query_len=int(seqlens_q_local_np.max()),
local_max_seq_len=int(seqlens_k_local_np.max()),
)
def build_for_capture(
self,
metadata: FlashAttentionMetadata,
bs: int,
*,
buffers: Dict[str, torch.Tensor],
) -> LocalAttentionMetadata:
"""Views of the preallocated buffers sized to what this capture needs."""
seq_lens_capture = metadata.cache_seqlens_int32
max_seq_len = int(seq_lens_capture.max().item())
(
seqlens_q_local_np,
cu_seqlens_q_local_np,
seqlens_k_local_np,
block_table_local_np,
) = make_local_attention_virtual_batches(
self.attention_chunk_size,
metadata.cu_seqlens_q.cpu().numpy(),
seq_lens_capture.cpu().numpy(),
self._graph_page_table(metadata, bs, max_seq_len),
self.page_size,
preserve_attn_chunk_size=True,
)
q_len = len(cu_seqlens_q_local_np)
k_len = len(seqlens_k_local_np)
b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs
b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1
return LocalAttentionMetadata(
local_query_start_loc=buffers["local_query_start_loc"][:q_len],
local_seqused_k=buffers["local_seqused_k"][:k_len],
local_block_table=buffers["local_block_table"][:b0, :b1],
local_max_query_len=1,
local_max_seq_len=max_seq_len,
)
def update_for_replay(
self,
metadata: FlashAttentionMetadata,
bs: int,
*,
buffers: Dict[str, torch.Tensor],
) -> None:
"""Refill the preallocated buffers in place before a CUDA-graph replay."""
local_q_buf = buffers["local_query_start_loc"]
local_k_buf = buffers["local_seqused_k"]
local_block_buf = buffers["local_block_table"]
# Decode: one query token per request.
cu_seqlens_q = torch.arange(
bs + 1, device=local_q_buf.device, dtype=local_q_buf.dtype
)
seqlens = metadata.cache_seqlens_int32[:bs]
# Rows past bs and columns past the live pages hold stale ids.
max_seq_len = int(seqlens.max().item())
sliced_page_table = self._graph_page_table(metadata, bs, max_seq_len)
(
seqlens_q_local_np,
cu_seqlens_q_local_np,
seqlens_k_local_np,
block_table_local,
) = make_local_attention_virtual_batches(
self.attention_chunk_size,
cu_seqlens_q.cpu().numpy(),
seqlens.cpu().numpy(),
sliced_page_table,
self.page_size,
preserve_attn_chunk_size=True,
)
device = local_q_buf.device
cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device)
seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device)
block_table_local = block_table_local.to(device)
q_len = cu_seqlens_q_local.shape[0]
k_len = seqlens_k_local.shape[0]
b0, b1 = block_table_local.shape
local_q_buf[:q_len].copy_(cu_seqlens_q_local)
local_q_buf[q_len:].fill_(0)
local_k_buf[:k_len].copy_(seqlens_k_local)
local_k_buf[k_len:].fill_(0)
local_block_buf[:b0, :b1].copy_(block_table_local)
local_block_buf[b0:, :].fill_(0)
local_block_buf[:b0, b1:].fill_(0)
if metadata.local_attn_metadata is not None:
lam = metadata.local_attn_metadata
lam.local_max_query_len = int(seqlens_q_local_np.max())
lam.local_max_seq_len = int(seqlens_k_local_np.max())
def make_local_attention_virtual_batches(
attn_chunk_size: int,
query_start_loc_np: np.ndarray,
seq_lens_np: np.ndarray,
block_table: torch.Tensor,
page_size: int = 0,
preserve_attn_chunk_size: bool = False,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]:
"""
Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into
local attention blocks, where each block is passed to the attention kernel
as an independent local ("virtual") batch item.
Args:
attn_chunk_size: Size of local attention chunks
query_start_loc_np: Cumulative sum of query lengths (numpy array)
seq_lens_np: Sequence lengths (numpy array)
block_table: Block table for KV cache
page_size: Size of each page in the KV cache
preserve_attn_chunk_size: Skip sequence-length-based chunk normalization.
Returns:
seqlens_q_local: Query sequence lengths for local attention
cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention
seqlens_k_local: Key sequence lengths for local attention
block_table_local: Block table for local attention
"""
if not preserve_attn_chunk_size:
max_seq_len = seq_lens_np.max()
effective_chunk_size = min(attn_chunk_size, max_seq_len)
effective_chunk_size = (effective_chunk_size // page_size) * page_size
if effective_chunk_size < page_size:
effective_chunk_size = page_size
attn_chunk_size = effective_chunk_size
q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1]
actual_batch_size = seq_lens_np.shape[0]
# Handle if we are starting in the middle of a local attention block,
# we assume q_seqlens > 0 (for all elements), for each batch idx we compute
# the number of tokens that are not in the first local attention block and
# then we can simply use a cdiv for the rest.
# For example if we have:
# attn_chunk_size = 4
# q_seqlens = [4, 10, 5]
# k_seqlens = [6, 17, 9]
# Then we would get:
# new_tokens_in_first_block = [2, 1, 4]
# local_blocks = [2, 4, 2]
q_tokens_in_first_block = np.minimum(
attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens
).astype(np.int32)
tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size)
local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size)
# Once we know the number of local blocks we can compute the request spans
# for each batch idx, we can figure out the number of "virtual" requests we
# have to make,
# For the above example we would get:
# seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1]
#
# First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1])
# (TODO: max a utility to share this code with _prepare_inputs)
# arange step 1. [2, 4, 2] -> [2, 6, 8]
cu_num_blocks = np.cumsum(local_blocks)
virtual_batches = cu_num_blocks[-1]
# arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6]
block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks)
# arange step 3. [0, 1, 0, 1, 2, 3, 0, 1]
arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets
# also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0])
rarange = np.repeat(local_blocks, local_blocks) - arange - 1
# Then we can compute the seqlens_q_local, handling the fact that the
# first and last blocks could be partial
seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks)
# set the first block since this may be a partial block
seqlens_q_local[arange == 0] = q_tokens_in_first_block
# set the remaining blocks
seqlens_q_local[arange > 0] = np.minimum(
seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size
)[arange > 0]
# convert from q_seqlens to cu_seqlens_q
cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32)
# compute the seqlens_k_local,
# basically a full local attention block for all but the last block in each
# batch
# For our example this will be:
# seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1]
seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32)
seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block
k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - (
rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks)
)
# For the example the local attention blocks start at:
# _b0_ _____b1_____ _b2_
# k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8]
block_starts = k_seqstarts_absolute // page_size
assert attn_chunk_size % page_size == 0, (
f"attn_chunk_size {attn_chunk_size} is not divisible by page_size {page_size}"
)
pages_per_local_batch = attn_chunk_size // page_size
# Create a block_table for the local attention blocks
# For out example if we have a block-table like (assuming page_size=2):
# block_table = [
# [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0
# [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1
# [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2
# ]
# Then for the local batches we would want a block-table like
# block_table_local = [
# [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0])
# [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4])
# [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4])
# [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8])
# [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12])
# [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16])
# [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4])
# [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8])
# ]
block_indices = np.broadcast_to(
np.arange(pages_per_local_batch, dtype=np.int32),
(virtual_batches, pages_per_local_batch),
) + np.expand_dims(block_starts, axis=1)
# Ensure block_indices doesn't exceed block_table dimensions
# This is a critical safety check that prevents index out of bounds errors
# when dealing with large sequences (>8192 tokens) or when the block_table
# dimensions are smaller than what would be needed for the full attention chunk size.
block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1)
batch_indices = np.repeat(
np.arange(actual_batch_size, dtype=np.int32),
local_blocks * pages_per_local_batch,
)
# NOTE: https://github.com/pytorch/pytorch/pull/160256 causes performance
# regression when using numpy arrays (batch and block indices) to index into
# torch tensor (block_table). As a workaround, convert numpy arrays to torch
# tensor first, which recovers perf.
batch_indices_torch = torch.from_numpy(batch_indices)
block_indices_torch = torch.from_numpy(block_indices)
block_table_local = block_table[batch_indices_torch, block_indices_torch].view(
virtual_batches, -1
)
return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local
def cdiv(a: int, b: int) -> int:
"""Ceiling division."""
return -(a // -b)
@@ -8,10 +8,13 @@ from sglang.srt.configs.model_config import AttentionArch
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.layers.attention.flashattention_backend import (
FlashAttentionMetadata,
make_local_attention_virtual_batches,
merge_state_v2_wrapper,
prepare_swa_spec_page_table_triton,
)
from sglang.srt.layers.attention.local_attention import (
LocalAttentionMetadata,
make_local_attention_virtual_batches,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -1480,7 +1483,7 @@ class XPUAttentionBackend(AttentionBackend):
self.page_size,
)
local_metadata = FlashAttentionMetadata.LocalAttentionMetadata(
local_metadata = LocalAttentionMetadata(
local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device),
local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device),
local_block_table=block_table_local.to(device),
@@ -0,0 +1,221 @@
"""CPU tests for srt/layers/attention/local_attention: the virtual-batch
decomposition and the metadata builder."""
import unittest
from types import SimpleNamespace
import numpy as np
import torch
from sglang.srt.layers.attention.local_attention import (
LocalAttentionMetadata,
LocalAttentionMetadataBuilder,
make_local_attention_virtual_batches,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
CPU = torch.device("cpu")
def _page_table(bs: int, width: int = 32) -> torch.Tensor:
return torch.arange(bs * width, dtype=torch.int32).view(bs, width)
class TestVirtualBatches(CustomTestCase):
def test_worked_example_from_docstring(self):
"""chunk=4, q=[4,10,5], k=[6,17,9] must split into local_blocks=[2,4,2]."""
q_start = np.array([0, 4, 14, 19], dtype=np.int32)
seq_lens = np.array([6, 17, 9], dtype=np.int32)
q_local, cu_q_local, k_local, block_table_local = (
make_local_attention_virtual_batches(
4, q_start, seq_lens, _page_table(3), 1
)
)
self.assertEqual(q_local.tolist(), [2, 2, 1, 4, 4, 1, 4, 1])
self.assertEqual(cu_q_local.tolist(), [0, 2, 4, 5, 9, 13, 14, 18, 19])
# Every block sees a full chunk of keys except the last block of each request.
self.assertEqual(k_local.tolist(), [4, 2, 4, 4, 4, 1, 4, 1])
self.assertEqual(tuple(block_table_local.shape), (8, 4))
def test_decode_is_one_block_per_request(self):
q_start = np.array([0, 1, 2], dtype=np.int32)
seq_lens = np.array([6, 9], dtype=np.int32)
q_local, cu_q_local, k_local, _ = make_local_attention_virtual_batches(
4, q_start, seq_lens, _page_table(2), 1
)
self.assertEqual(q_local.tolist(), [1, 1])
self.assertEqual(cu_q_local.tolist(), [0, 1, 2])
# Only the tail of the current chunk is attended: 6 % 4 == 2, 9 % 4 == 1.
self.assertEqual(k_local.tolist(), [2, 1])
def test_chunk_is_clamped_to_the_longest_sequence(self):
q_start = np.array([0, 6], dtype=np.int32)
seq_lens = np.array([6], dtype=np.int32)
_, _, k_local, block_table_local = make_local_attention_virtual_batches(
8, q_start, seq_lens, _page_table(1), 1
)
self.assertEqual(k_local.tolist(), [6])
self.assertEqual(tuple(block_table_local.shape), (1, 6))
def test_page_size_divides_the_block_table_width(self):
q_start = np.array([0, 6], dtype=np.int32)
seq_lens = np.array([6], dtype=np.int32)
q_local, _, k_local, block_table_local = make_local_attention_virtual_batches(
4, q_start, seq_lens, _page_table(1), 2
)
self.assertEqual(q_local.tolist(), [4, 2])
self.assertEqual(k_local.tolist(), [4, 2])
self.assertEqual(tuple(block_table_local.shape), (2, 2)) # chunk 4 / page 2
class TestLocalAttentionMetadataBuilder(CustomTestCase):
def _builder(self, swa_translate=None, max_context_len=16):
return LocalAttentionMetadataBuilder(
attention_chunk_size=4,
page_size=1,
max_context_len=max_context_len,
device=CPU,
swa_translate=swa_translate,
)
def test_build_returns_none_without_a_complete_batch(self):
b = self._builder()
self.assertIsNone(
b.build(
cu_seqlens_q=None,
cache_seqlens_int32=torch.tensor([6]),
page_table=_page_table(1),
device=CPU,
)
)
def test_build_applies_the_swa_translation_to_the_block_table(self):
args = dict(
cu_seqlens_q=torch.tensor([0, 6], dtype=torch.int32),
cache_seqlens_int32=torch.tensor([6], dtype=torch.int32),
page_table=_page_table(1),
device=CPU,
)
plain = self._builder().build(**args)
swa = self._builder(swa_translate=lambda t: t + 100).build(**args)
self.assertEqual(plain.local_max_query_len, 4)
self.assertEqual(plain.local_max_seq_len, 4)
torch.testing.assert_close(swa.local_block_table, plain.local_block_table + 100)
def test_cuda_graph_buffers_fit_the_worst_case_decomposition(self):
"""Buffer sizing must match the most virtual batches max_bs x max_context_len can produce."""
b = self._builder(max_context_len=16)
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
md = b.build(
cu_seqlens_q=torch.tensor([0, 16, 32], dtype=torch.int32),
cache_seqlens_int32=torch.tensor([16, 16], dtype=torch.int32),
page_table=_page_table(2),
device=CPU,
)
self.assertEqual(
md.local_query_start_loc.numel(), bufs["local_query_start_loc"].numel()
)
self.assertEqual(md.local_seqused_k.numel(), bufs["local_seqused_k"].numel())
self.assertEqual(
tuple(md.local_block_table.shape), tuple(bufs["local_block_table"].shape)
)
def test_capture_returns_views_of_the_preallocated_buffers(self):
b = self._builder()
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
metadata = SimpleNamespace(
cu_seqlens_q=torch.tensor([0, 1, 2], dtype=torch.int32),
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
page_table=_page_table(2),
)
md = b.build_for_capture(metadata, 2, buffers=bufs)
self.assertEqual(md.local_query_start_loc.numel(), 3)
self.assertEqual(md.local_seqused_k.numel(), 2)
self.assertEqual(tuple(md.local_block_table.shape), (2, 4))
self.assertEqual(md.local_max_query_len, 1)
self.assertEqual(md.local_max_seq_len, 9)
self.assertEqual(
md.local_query_start_loc.data_ptr(),
bufs["local_query_start_loc"].data_ptr(),
)
self.assertEqual(
md.local_block_table.data_ptr(), bufs["local_block_table"].data_ptr()
)
def _paged_builder(self, swa_translate):
# chunk 32 / page 16: two pages per local block.
return LocalAttentionMetadataBuilder(
attention_chunk_size=32,
page_size=16,
max_context_len=64,
device=CPU,
swa_translate=swa_translate,
)
def test_replay_reads_page_ids_from_swa_page_table(self):
"""Regression: with page_size > 1 the CUDA-graph replay must not push the
page-granular table through the token-slot full->swa mapping."""
# Any translation applied to page ids would land far outside {7, 8}.
b = self._paged_builder(swa_translate=lambda t: t * 1000)
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
metadata = SimpleNamespace(
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
page_table=torch.tensor([[3], [5]], dtype=torch.int32),
swa_page_table=torch.tensor([[7], [8]], dtype=torch.int32),
local_attn_metadata=LocalAttentionMetadata(),
)
b.update_for_replay(metadata, 2, buffers=bufs)
table = bufs["local_block_table"][:2]
self.assertEqual(table[:, 0].tolist(), [7, 8])
self.assertTrue(set(table.flatten().tolist()) <= {0, 7, 8})
def test_replay_without_swa_pool_reads_page_table(self):
b = self._paged_builder(swa_translate=None)
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
metadata = SimpleNamespace(
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
page_table=torch.tensor([[3], [5]], dtype=torch.int32),
local_attn_metadata=LocalAttentionMetadata(),
)
b.update_for_replay(metadata, 2, buffers=bufs)
self.assertEqual(bufs["local_block_table"][:2, 0].tolist(), [3, 5])
def test_capture_and_replay_share_the_table_source(self):
b = self._paged_builder(swa_translate=lambda t: t * 1000)
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
metadata = SimpleNamespace(
cu_seqlens_q=torch.tensor([0, 1, 2], dtype=torch.int32),
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
page_table=torch.tensor([[3], [5]], dtype=torch.int32),
swa_page_table=torch.tensor([[7], [8]], dtype=torch.int32),
)
md = b.build_for_capture(metadata, 2, buffers=bufs)
self.assertEqual(
tuple(md.local_block_table.shape), (2, 2)
) # 2 reqs x 2 pages per block
def test_replay_refills_buffers_in_place_and_zeroes_the_tail(self):
b = self._builder()
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
for t in bufs.values():
t.fill_(7) # stale contents from an earlier, larger batch
lam = LocalAttentionMetadata()
metadata = SimpleNamespace(
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
page_table=_page_table(2),
local_attn_metadata=lam,
)
b.update_for_replay(metadata, 2, buffers=bufs)
self.assertEqual(bufs["local_query_start_loc"][:3].tolist(), [0, 1, 2])
self.assertEqual(bufs["local_query_start_loc"][3:].abs().sum().item(), 0)
self.assertEqual(bufs["local_seqused_k"][:2].tolist(), [2, 1])
self.assertEqual(bufs["local_seqused_k"][2:].abs().sum().item(), 0)
self.assertEqual(bufs["local_block_table"][2:].abs().sum().item(), 0)
self.assertEqual((lam.local_max_query_len, lam.local_max_seq_len), (1, 2))
if __name__ == "__main__":
unittest.main()