[Bugfix] Llama4 local attention: read page ids from the graph tables in CUDA-graph capture/replay (page_size > 1) (#38187)
This commit is contained in:
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.metadata import (
|
||||
@@ -26,6 +25,10 @@ from sglang.srt.layers.attention.kv_shard_hooks import (
|
||||
get_kv_shard_pool,
|
||||
prepare_kv_shard_forward,
|
||||
)
|
||||
from sglang.srt.layers.attention.local_attention import (
|
||||
LocalAttentionMetadata,
|
||||
LocalAttentionMetadataBuilder,
|
||||
)
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
|
||||
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
@@ -116,14 +119,6 @@ class FlashAttentionMetadata:
|
||||
# Page table for the encoder
|
||||
encoder_page_table: torch.Tensor = None
|
||||
|
||||
@dataclass
|
||||
class LocalAttentionMetadata:
|
||||
local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention
|
||||
local_seqused_k: torch.Tensor = None # sequence lengths for local attention
|
||||
local_block_table: torch.Tensor = None # block table for local attention
|
||||
local_max_query_len: int = 0 # max query length for local attention
|
||||
local_max_seq_len: int = 0 # max sequence length for local attention
|
||||
|
||||
local_attn_metadata: Optional[LocalAttentionMetadata] = None
|
||||
|
||||
# For sliding window attention topk>1 spec decoding
|
||||
@@ -239,23 +234,14 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
)
|
||||
self.speculative_step_id = speculative_step_id
|
||||
|
||||
# Local attention settings
|
||||
self.has_local_attention = model_runner.model_config.is_local_attention_model
|
||||
# Local (chunked) attention derives its page table by re-translating
|
||||
# metadata.page_table through the static full->swa map -- meaningless
|
||||
# on the unified pool's kernel-facing tables, and no unified-eligible
|
||||
# model uses it. Fail loud rather than silently double-translate.
|
||||
assert not (
|
||||
self.kv_index_translator.is_translating and self.has_local_attention
|
||||
), (
|
||||
"--enable-unified-memory does not support local-attention models "
|
||||
"on the fa3/fa4 backend."
|
||||
self.local_attn_builder = self._maybe_init_local_attn_builder(model_runner)
|
||||
# Read directly by MusaFlashAttentionBackend's copied forward paths.
|
||||
self.has_local_attention = self.local_attn_builder is not None
|
||||
self.attention_chunk_size = (
|
||||
self.local_attn_builder.attention_chunk_size
|
||||
if self.local_attn_builder is not None
|
||||
else None
|
||||
)
|
||||
if self.has_local_attention:
|
||||
assert model_runner.attention_chunk_size is not None, (
|
||||
"Attention chunk size is required for local attention"
|
||||
)
|
||||
self.attention_chunk_size = model_runner.attention_chunk_size
|
||||
|
||||
# For each layer, the sliding_window_size can be different. This is only used for preparing SWA metadata.
|
||||
# We use `layer.sliding_window_size` to decide whether to use SWA for each layer.
|
||||
@@ -390,6 +376,32 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
_should_disable_scheduler_metadata_precompute()
|
||||
)
|
||||
|
||||
def _maybe_init_local_attn_builder(
|
||||
self, model_runner: ModelRunner
|
||||
) -> Optional[LocalAttentionMetadataBuilder]:
|
||||
if not model_runner.model_config.is_local_attention_model:
|
||||
return None
|
||||
# Local attention re-translates metadata.page_table through the static
|
||||
# full->swa map, which is meaningless on the unified pool's kernel tables.
|
||||
assert not self.kv_index_translator.is_translating, (
|
||||
"--enable-unified-memory does not support local-attention models "
|
||||
"on the fa3/fa4 backend."
|
||||
)
|
||||
assert model_runner.attention_chunk_size is not None, (
|
||||
"Attention chunk size is required for local attention"
|
||||
)
|
||||
return LocalAttentionMetadataBuilder(
|
||||
attention_chunk_size=model_runner.attention_chunk_size,
|
||||
page_size=self.page_size,
|
||||
max_context_len=self.max_context_len,
|
||||
device=self.device,
|
||||
swa_translate=(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa
|
||||
if self.use_sliding_window_kv_pool
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _compute_scheduler_metadata(
|
||||
self, batch_size, max_seq_len_k, cache_seqlens, cu_seqlens_q
|
||||
):
|
||||
@@ -624,7 +636,14 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# padded seq-len fill value (1), or a later long-context
|
||||
# replay would leave K/V tiles uncovered.
|
||||
metadata.max_seq_len_k = self.max_context_len
|
||||
self._maybe_update_local_attn_metadata_for_capture(metadata, bs)
|
||||
if self.local_attn_builder is not None:
|
||||
metadata.local_attn_metadata = (
|
||||
self.local_attn_builder.build_for_capture(
|
||||
metadata,
|
||||
bs,
|
||||
buffers=self.decode_cuda_graph_local_attn_metadata,
|
||||
)
|
||||
)
|
||||
if self._sched_meta_buf is not None:
|
||||
sched = self._compute_scheduler_metadata(
|
||||
bs,
|
||||
@@ -937,7 +956,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata.cu_seqlens_q,
|
||||
)
|
||||
# TODO: we need to test this part for llama 4 eagle case
|
||||
self._maybe_init_local_attn_metadata(forward_batch, metadata, device)
|
||||
self._maybe_init_local_attn_metadata(metadata, device)
|
||||
elif forward_batch.forward_mode.is_target_verify():
|
||||
if self.topk <= 1:
|
||||
ragged_layout = getattr(
|
||||
@@ -985,7 +1004,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
forward_batch.req_pool_indices, : metadata.max_seq_len_k
|
||||
]
|
||||
|
||||
self._maybe_init_local_attn_metadata(forward_batch, metadata, device)
|
||||
self._maybe_init_local_attn_metadata(metadata, device)
|
||||
else:
|
||||
metadata.cache_seqlens_int32 = forward_batch.seq_lens.to(torch.int32)
|
||||
metadata.max_seq_len_q = self.speculative_num_draft_tokens
|
||||
@@ -1124,7 +1143,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
|
||||
# Setup local attention if enabled
|
||||
if forward_batch.forward_mode == ForwardMode.EXTEND:
|
||||
self._maybe_init_local_attn_metadata(forward_batch, metadata, device)
|
||||
self._maybe_init_local_attn_metadata(metadata, device)
|
||||
|
||||
if self.is_prefill_aware_swa:
|
||||
self._pa_swa_prefill_lens[
|
||||
@@ -1440,12 +1459,9 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
kv_head_num=layer.tp_k_head_num,
|
||||
is_prefill=True,
|
||||
)
|
||||
# Check if we should use local attention
|
||||
use_local_attn = (
|
||||
self.has_local_attention
|
||||
and self.attention_chunk_size is not None
|
||||
and metadata.local_attn_metadata is not None
|
||||
and (hasattr(layer, "use_irope") and layer.use_irope)
|
||||
self.local_attn_builder is not None
|
||||
and self.local_attn_builder.applies(layer, metadata)
|
||||
)
|
||||
|
||||
# We do cascade attention for Target Verify with topk > 1
|
||||
@@ -1965,12 +1981,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
|
||||
# Use precomputed metadata across all layers
|
||||
metadata = self.forward_metadata
|
||||
local_attn_metadata = getattr(metadata, "local_attn_metadata", None)
|
||||
local_attn_metadata = metadata.local_attn_metadata
|
||||
use_local_attn = (
|
||||
self.has_local_attention
|
||||
and self.attention_chunk_size is not None
|
||||
and local_attn_metadata is not None
|
||||
and (hasattr(layer, "use_irope") and layer.use_irope)
|
||||
self.local_attn_builder is not None
|
||||
and self.local_attn_builder.applies(layer, metadata)
|
||||
)
|
||||
|
||||
# When Spec Decode enabled, forward_decode would be called with two mode:
|
||||
@@ -2312,32 +2326,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else:
|
||||
self._sched_meta_buf = None
|
||||
|
||||
# Only allocate local attention buffers if local attention is enabled
|
||||
# This prevents OOM errors when local attention is not being used
|
||||
if self.has_local_attention:
|
||||
# Estimate maximum sizes for local attention metadata
|
||||
max_seq_len = self.max_context_len
|
||||
page_size = self.page_size or 1
|
||||
attn_chunk_size = self.attention_chunk_size
|
||||
max_virtual_batches = max_bs * (
|
||||
(max_seq_len + attn_chunk_size - 1) // attn_chunk_size
|
||||
if self.local_attn_builder is not None:
|
||||
self.decode_cuda_graph_local_attn_metadata = (
|
||||
self.local_attn_builder.alloc_cuda_graph_buffers(max_bs)
|
||||
)
|
||||
max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size
|
||||
|
||||
self.decode_cuda_graph_local_attn_metadata = {
|
||||
"local_query_start_loc": torch.zeros(
|
||||
max_virtual_batches + 1, dtype=torch.int32, device=self.device
|
||||
),
|
||||
"local_seqused_k": torch.zeros(
|
||||
max_virtual_batches, dtype=torch.int32, device=self.device
|
||||
),
|
||||
"local_block_table": torch.zeros(
|
||||
max_virtual_batches,
|
||||
max_pages_per_block,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
),
|
||||
}
|
||||
|
||||
if self.use_sliding_window_kv_pool:
|
||||
self.decode_cuda_graph_metadata["swa_page_table"] = torch.zeros(
|
||||
@@ -3008,10 +3000,12 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata, req_pool_indices, seq_lens, 0
|
||||
)
|
||||
|
||||
self._maybe_update_local_attn_metadata_for_replay(
|
||||
metadata,
|
||||
bs,
|
||||
)
|
||||
if self.local_attn_builder is not None:
|
||||
self.local_attn_builder.update_for_replay(
|
||||
metadata,
|
||||
bs,
|
||||
buffers=self.decode_cuda_graph_local_attn_metadata,
|
||||
)
|
||||
|
||||
# Recompute scheduler_metadata into pre-allocated buffer
|
||||
if (
|
||||
@@ -3247,197 +3241,18 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
return 1
|
||||
|
||||
def _maybe_init_local_attn_metadata(
|
||||
self,
|
||||
forwardbatch: ForwardBatch,
|
||||
metadata: FlashAttentionMetadata,
|
||||
device,
|
||||
):
|
||||
"""Centralized utility to initialize local_attn_metadata if chunked attention is enabled."""
|
||||
if not self.has_local_attention:
|
||||
self, metadata: FlashAttentionMetadata, device
|
||||
) -> None:
|
||||
if self.local_attn_builder is None:
|
||||
metadata.local_attn_metadata = None
|
||||
return
|
||||
|
||||
cu_seqlens_q = metadata.cu_seqlens_q
|
||||
cache_seqlens_int32 = metadata.cache_seqlens_int32
|
||||
if self.use_sliding_window_kv_pool:
|
||||
page_table = self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
metadata.page_table
|
||||
).to(torch.int32)
|
||||
else:
|
||||
page_table = metadata.page_table
|
||||
if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None:
|
||||
metadata.local_attn_metadata = None
|
||||
return
|
||||
if self.page_size > 1:
|
||||
# Convert the eager token table to physical page indices.
|
||||
page_table = page_table[:, :: self.page_size] // self.page_size
|
||||
|
||||
cu_seqlens_q_np = cu_seqlens_q.cpu().numpy()
|
||||
seq_lens_np = cache_seqlens_int32.cpu().numpy()
|
||||
(
|
||||
seqlens_q_local_np,
|
||||
cu_seqlens_q_local_np,
|
||||
seqlens_k_local_np,
|
||||
block_table_local,
|
||||
) = make_local_attention_virtual_batches(
|
||||
self.attention_chunk_size,
|
||||
cu_seqlens_q_np,
|
||||
seq_lens_np,
|
||||
page_table,
|
||||
self.page_size,
|
||||
preserve_attn_chunk_size=True,
|
||||
metadata.local_attn_metadata = self.local_attn_builder.build(
|
||||
cu_seqlens_q=metadata.cu_seqlens_q,
|
||||
cache_seqlens_int32=metadata.cache_seqlens_int32,
|
||||
page_table=metadata.page_table,
|
||||
device=device,
|
||||
)
|
||||
|
||||
local_metadata = FlashAttentionMetadata.LocalAttentionMetadata(
|
||||
local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device),
|
||||
local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device),
|
||||
local_block_table=block_table_local.to(device),
|
||||
local_max_query_len=int(seqlens_q_local_np.max()),
|
||||
local_max_seq_len=int(seqlens_k_local_np.max()),
|
||||
)
|
||||
metadata.local_attn_metadata = local_metadata
|
||||
|
||||
def _maybe_update_local_attn_metadata_for_capture(
|
||||
self, metadata: FlashAttentionMetadata, bs: int
|
||||
):
|
||||
"""Update local attention metadata during CUDA graph capture phase.
|
||||
|
||||
This method calculates the exact buffer sizes needed for local attention metadata
|
||||
during the CUDA graph capture phase, optimizing memory usage by creating views of
|
||||
pre-allocated buffers with exactly the sizes needed.
|
||||
"""
|
||||
if not self.has_local_attention:
|
||||
return
|
||||
|
||||
seq_lens_capture = metadata.cache_seqlens_int32
|
||||
max_seq_len = int(seq_lens_capture.max().item())
|
||||
page_table_capture = metadata.page_table
|
||||
|
||||
cu_seqlens_q_np = metadata.cu_seqlens_q.cpu().numpy()
|
||||
seqlens_np = seq_lens_capture.cpu().numpy()
|
||||
(
|
||||
seqlens_q_local_np,
|
||||
cu_seqlens_q_local_np,
|
||||
seqlens_k_local_np,
|
||||
block_table_local_np,
|
||||
) = make_local_attention_virtual_batches(
|
||||
self.attention_chunk_size,
|
||||
cu_seqlens_q_np,
|
||||
seqlens_np,
|
||||
page_table_capture,
|
||||
self.page_size,
|
||||
preserve_attn_chunk_size=True,
|
||||
)
|
||||
|
||||
# Get exact dimensions from the calculation
|
||||
q_len = len(cu_seqlens_q_local_np)
|
||||
k_len = len(seqlens_k_local_np)
|
||||
b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs
|
||||
b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1
|
||||
|
||||
# Create views of the pre-allocated buffers with exactly these sizes
|
||||
# This is the key optimization - we only use the memory we actually need
|
||||
local_query_start_loc = self.decode_cuda_graph_local_attn_metadata[
|
||||
"local_query_start_loc"
|
||||
][:q_len]
|
||||
|
||||
local_seqused_k = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"][
|
||||
:k_len
|
||||
]
|
||||
|
||||
local_block_table = self.decode_cuda_graph_local_attn_metadata[
|
||||
"local_block_table"
|
||||
][:b0, :b1]
|
||||
|
||||
metadata.local_attn_metadata = FlashAttentionMetadata.LocalAttentionMetadata(
|
||||
local_query_start_loc=local_query_start_loc,
|
||||
local_seqused_k=local_seqused_k,
|
||||
local_block_table=local_block_table,
|
||||
local_max_query_len=1,
|
||||
local_max_seq_len=max_seq_len,
|
||||
)
|
||||
|
||||
def _maybe_update_local_attn_metadata_for_replay(
|
||||
self,
|
||||
metadata: FlashAttentionMetadata,
|
||||
bs: int,
|
||||
):
|
||||
"""Update preallocated local attention metadata in-place before CUDA graph replay."""
|
||||
if not self.has_local_attention:
|
||||
return
|
||||
|
||||
# Access preallocated buffers
|
||||
local_q_buf = self.decode_cuda_graph_local_attn_metadata[
|
||||
"local_query_start_loc"
|
||||
]
|
||||
local_k_buf = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"]
|
||||
local_block_buf = self.decode_cuda_graph_local_attn_metadata[
|
||||
"local_block_table"
|
||||
]
|
||||
cu_seqlens_q = self.decode_cuda_graph_metadata["cu_seqlens_q"]
|
||||
|
||||
# Create a modified version for local attention that only processes the last token
|
||||
# This mimics the normal decode pattern
|
||||
cu_seqlens_q = torch.arange(
|
||||
bs + 1, device=cu_seqlens_q.device, dtype=cu_seqlens_q.dtype
|
||||
)
|
||||
seqlens = metadata.cache_seqlens_int32[:bs]
|
||||
# Slice the page_table to match the batch size and actual sequence length
|
||||
# This serves three important purposes:
|
||||
# 1. Ensures we only process the actual batch size (bs) and not the maximum batch size
|
||||
# 2. Limits the sequence length to prevent processing padding tokens or garbage values
|
||||
# 3. Prevents zeros in the block table which can cause garbage output during replay
|
||||
#
|
||||
# Without this slicing, the pre-allocated page_table may contain zeros or invalid indices
|
||||
# beyond the actual sequence length, leading to incorrect attention calculations
|
||||
max_seq_len = int(seqlens.max().item())
|
||||
if self.use_sliding_window_kv_pool:
|
||||
sliced_page_table = self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
metadata.page_table[:bs, :max_seq_len]
|
||||
).to(torch.int32)
|
||||
else:
|
||||
sliced_page_table = metadata.page_table[:bs, :max_seq_len]
|
||||
|
||||
cu_seqlens_q_np = cu_seqlens_q.cpu().numpy()
|
||||
seqlens_np = seqlens.cpu().numpy()
|
||||
(
|
||||
seqlens_q_local_np,
|
||||
cu_seqlens_q_local_np,
|
||||
seqlens_k_local_np,
|
||||
block_table_local,
|
||||
) = make_local_attention_virtual_batches(
|
||||
self.attention_chunk_size,
|
||||
cu_seqlens_q_np,
|
||||
seqlens_np,
|
||||
sliced_page_table,
|
||||
self.page_size,
|
||||
preserve_attn_chunk_size=True,
|
||||
)
|
||||
|
||||
# Convert back to tensors
|
||||
device = local_q_buf.device
|
||||
cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device)
|
||||
seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device)
|
||||
block_table_local = block_table_local.to(device)
|
||||
# Get sizes
|
||||
q_len = cu_seqlens_q_local.shape[0]
|
||||
k_len = seqlens_k_local.shape[0]
|
||||
b0, b1 = block_table_local.shape
|
||||
|
||||
# In-place updates into preallocated tensors and zero out the unused space
|
||||
local_q_buf[:q_len].copy_(cu_seqlens_q_local)
|
||||
local_q_buf[q_len:].fill_(0)
|
||||
local_k_buf[:k_len].copy_(seqlens_k_local)
|
||||
local_k_buf[k_len:].fill_(0)
|
||||
local_block_buf[:b0, :b1].copy_(block_table_local)
|
||||
local_block_buf[b0:, :].fill_(0)
|
||||
local_block_buf[:b0, b1:].fill_(0)
|
||||
|
||||
if metadata.local_attn_metadata is not None:
|
||||
lam = metadata.local_attn_metadata
|
||||
lam.local_max_query_len = int(seqlens_q_local_np.max())
|
||||
lam.local_max_seq_len = int(seqlens_k_local_np.max())
|
||||
|
||||
def _init_sliding_window_attn_spec_metadata(
|
||||
self,
|
||||
metadata: FlashAttentionMetadata,
|
||||
@@ -3649,162 +3464,6 @@ def draft_decode_set_expand_metadata(
|
||||
# cu_seqlens_q_local = [0, 4, 6, 10, 14, 18, 19, 23, 24]
|
||||
# seqlens_k_local = [ 4, 2, 4, 4, 4, 1, 4, 1]
|
||||
# block_table_local : shape[local_virtual_batches, pages_per_local_batch]
|
||||
def make_local_attention_virtual_batches(
|
||||
attn_chunk_size: int,
|
||||
query_start_loc_np: np.ndarray,
|
||||
seq_lens_np: np.ndarray,
|
||||
block_table: torch.Tensor,
|
||||
page_size: int = 0,
|
||||
preserve_attn_chunk_size: bool = False,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]:
|
||||
"""
|
||||
Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into
|
||||
local attention blocks, where each block is passed to the attention kernel
|
||||
as an independent local ("virtual") batch item.
|
||||
|
||||
Args:
|
||||
attn_chunk_size: Size of local attention chunks
|
||||
query_start_loc_np: Cumulative sum of query lengths (numpy array)
|
||||
seq_lens_np: Sequence lengths (numpy array)
|
||||
block_table: Block table for KV cache
|
||||
page_size: Size of each page in the KV cache
|
||||
preserve_attn_chunk_size: Skip sequence-length-based chunk normalization.
|
||||
|
||||
Returns:
|
||||
seqlens_q_local: Query sequence lengths for local attention
|
||||
cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention
|
||||
seqlens_k_local: Key sequence lengths for local attention
|
||||
block_table_local: Block table for local attention
|
||||
"""
|
||||
if not preserve_attn_chunk_size:
|
||||
max_seq_len = seq_lens_np.max()
|
||||
effective_chunk_size = min(attn_chunk_size, max_seq_len)
|
||||
effective_chunk_size = (effective_chunk_size // page_size) * page_size
|
||||
if effective_chunk_size < page_size:
|
||||
effective_chunk_size = page_size
|
||||
attn_chunk_size = effective_chunk_size
|
||||
|
||||
q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1]
|
||||
actual_batch_size = seq_lens_np.shape[0]
|
||||
|
||||
# Handle if we are starting in the middle of a local attention block,
|
||||
# we assume q_seqlens > 0 (for all elements), for each batch idx we compute
|
||||
# the number of tokens that are not in the first local attention block and
|
||||
# then we can simply use a cdiv for the rest.
|
||||
# For example if we have:
|
||||
# attn_chunk_size = 4
|
||||
# q_seqlens = [4, 10, 5]
|
||||
# k_seqlens = [6, 17, 9]
|
||||
# Then we would get:
|
||||
# new_tokens_in_first_block = [2, 1, 4]
|
||||
# local_blocks = [2, 4, 2]
|
||||
q_tokens_in_first_block = np.minimum(
|
||||
attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens
|
||||
).astype(np.int32)
|
||||
tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size)
|
||||
local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size)
|
||||
|
||||
# Once we know the number of local blocks we can compute the request spans
|
||||
# for each batch idx, we can figure out the number of "virtual" requests we
|
||||
# have to make,
|
||||
# For the above example we would get:
|
||||
# seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1]
|
||||
#
|
||||
# First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1])
|
||||
# (TODO: max a utility to share this code with _prepare_inputs)
|
||||
# arange step 1. [2, 4, 2] -> [2, 6, 8]
|
||||
cu_num_blocks = np.cumsum(local_blocks)
|
||||
virtual_batches = cu_num_blocks[-1]
|
||||
# arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6]
|
||||
block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks)
|
||||
# arange step 3. [0, 1, 0, 1, 2, 3, 0, 1]
|
||||
arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets
|
||||
# also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0])
|
||||
rarange = np.repeat(local_blocks, local_blocks) - arange - 1
|
||||
# Then we can compute the seqlens_q_local, handling the fact that the
|
||||
# first and last blocks could be partial
|
||||
seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks)
|
||||
# set the first block since this may be a partial block
|
||||
seqlens_q_local[arange == 0] = q_tokens_in_first_block
|
||||
# set the remaining blocks
|
||||
seqlens_q_local[arange > 0] = np.minimum(
|
||||
seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size
|
||||
)[arange > 0]
|
||||
|
||||
# convert from q_seqlens to cu_seqlens_q
|
||||
cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32)
|
||||
|
||||
# compute the seqlens_k_local,
|
||||
# basically a full local attention block for all but the last block in each
|
||||
# batch
|
||||
# For our example this will be:
|
||||
# seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1]
|
||||
seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32)
|
||||
seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block
|
||||
|
||||
k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - (
|
||||
rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks)
|
||||
)
|
||||
# For the example the local attention blocks start at:
|
||||
# _b0_ _____b1_____ _b2_
|
||||
# k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8]
|
||||
block_starts = k_seqstarts_absolute // page_size
|
||||
|
||||
assert attn_chunk_size % page_size == 0, (
|
||||
f"attn_chunk_size {attn_chunk_size} is not divisible by page_size {page_size}"
|
||||
)
|
||||
pages_per_local_batch = attn_chunk_size // page_size
|
||||
|
||||
# Create a block_table for the local attention blocks
|
||||
# For out example if we have a block-table like (assuming page_size=2):
|
||||
# block_table = [
|
||||
# [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0
|
||||
# [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1
|
||||
# [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2
|
||||
# ]
|
||||
# Then for the local batches we would want a block-table like
|
||||
# block_table_local = [
|
||||
# [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0])
|
||||
# [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4])
|
||||
# [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4])
|
||||
# [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8])
|
||||
# [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12])
|
||||
# [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16])
|
||||
# [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4])
|
||||
# [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8])
|
||||
# ]
|
||||
block_indices = np.broadcast_to(
|
||||
np.arange(pages_per_local_batch, dtype=np.int32),
|
||||
(virtual_batches, pages_per_local_batch),
|
||||
) + np.expand_dims(block_starts, axis=1)
|
||||
# Ensure block_indices doesn't exceed block_table dimensions
|
||||
# This is a critical safety check that prevents index out of bounds errors
|
||||
# when dealing with large sequences (>8192 tokens) or when the block_table
|
||||
# dimensions are smaller than what would be needed for the full attention chunk size.
|
||||
block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1)
|
||||
batch_indices = np.repeat(
|
||||
np.arange(actual_batch_size, dtype=np.int32),
|
||||
local_blocks * pages_per_local_batch,
|
||||
)
|
||||
|
||||
# NOTE: https://github.com/pytorch/pytorch/pull/160256 causes performance
|
||||
# regression when using numpy arrays (batch and block indices) to index into
|
||||
# torch tensor (block_table). As a workaround, convert numpy arrays to torch
|
||||
# tensor first, which recovers perf.
|
||||
batch_indices_torch = torch.from_numpy(batch_indices)
|
||||
block_indices_torch = torch.from_numpy(block_indices)
|
||||
block_table_local = block_table[batch_indices_torch, block_indices_torch].view(
|
||||
virtual_batches, -1
|
||||
)
|
||||
|
||||
return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local
|
||||
|
||||
|
||||
def cdiv(a: int, b: int) -> int:
|
||||
"""Ceiling division."""
|
||||
return -(a // -b)
|
||||
|
||||
|
||||
# TODO(hebiao064): remove this once we have a better way to handle the merge_state_v2 torch.compile issue
|
||||
@torch._dynamo.disable()
|
||||
def merge_state_v2_wrapper(o, s_a, o_exp, s_b):
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""Chunked (local) attention for Llama4 iRoPE layers on the FA3/FA4 backend.
|
||||
|
||||
Each sequence is split into ``attention_chunk_size`` blocks that the kernel sees
|
||||
as independent "virtual" batch items.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.flashattention_backend import (
|
||||
FlashAttentionMetadata,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
|
||||
|
||||
@dataclass
|
||||
class LocalAttentionMetadata:
|
||||
local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention
|
||||
local_seqused_k: torch.Tensor = None # sequence lengths for local attention
|
||||
local_block_table: torch.Tensor = None # block table for local attention
|
||||
local_max_query_len: int = 0 # max query length for local attention
|
||||
local_max_seq_len: int = 0 # max sequence length for local attention
|
||||
|
||||
|
||||
class LocalAttentionMetadataBuilder:
|
||||
"""The backend owns the CUDA-graph buffers; ``alloc_cuda_graph_buffers``
|
||||
returns them and the capture/replay methods take them back in."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
attention_chunk_size: int,
|
||||
page_size: int,
|
||||
max_context_len: int,
|
||||
device: torch.device,
|
||||
swa_translate: Optional[Callable[[torch.Tensor], torch.Tensor]],
|
||||
):
|
||||
self.attention_chunk_size = attention_chunk_size
|
||||
self.page_size = page_size
|
||||
self.max_context_len = max_context_len
|
||||
self.device = device
|
||||
self.swa_translate = swa_translate
|
||||
|
||||
def applies(self, layer: RadixAttention, metadata: FlashAttentionMetadata) -> bool:
|
||||
return metadata.local_attn_metadata is not None and layer.use_irope
|
||||
|
||||
def alloc_cuda_graph_buffers(self, max_bs: int) -> Dict[str, torch.Tensor]:
|
||||
page_size = self.page_size or 1
|
||||
attn_chunk_size = self.attention_chunk_size
|
||||
max_virtual_batches = max_bs * (
|
||||
(self.max_context_len + attn_chunk_size - 1) // attn_chunk_size
|
||||
)
|
||||
max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size
|
||||
return {
|
||||
"local_query_start_loc": torch.zeros(
|
||||
max_virtual_batches + 1, dtype=torch.int32, device=self.device
|
||||
),
|
||||
"local_seqused_k": torch.zeros(
|
||||
max_virtual_batches, dtype=torch.int32, device=self.device
|
||||
),
|
||||
"local_block_table": torch.zeros(
|
||||
max_virtual_batches,
|
||||
max_pages_per_block,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
),
|
||||
}
|
||||
|
||||
def _kernel_page_table(self, page_table: torch.Tensor) -> torch.Tensor:
|
||||
# Eager path only: the extend metadata holds token slots, which the
|
||||
# full->swa mapping is indexed by.
|
||||
if self.swa_translate is not None:
|
||||
return self.swa_translate(page_table).to(torch.int32)
|
||||
return page_table
|
||||
|
||||
def _graph_page_table(
|
||||
self, metadata: FlashAttentionMetadata, bs: int, max_seq_len: int
|
||||
) -> torch.Tensor:
|
||||
# CUDA-graph metadata already holds page ids, and swa_page_table is the
|
||||
# same table translated into the swa pool; the token-slot mapping must
|
||||
# not be applied to either. Columns are pages, so slice by pages.
|
||||
table = (
|
||||
metadata.swa_page_table
|
||||
if self.swa_translate is not None
|
||||
else metadata.page_table
|
||||
)
|
||||
num_pages = (max_seq_len + self.page_size - 1) // self.page_size
|
||||
return table[:bs, :num_pages]
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
cu_seqlens_q: Optional[torch.Tensor],
|
||||
cache_seqlens_int32: Optional[torch.Tensor],
|
||||
page_table: Optional[torch.Tensor],
|
||||
device: torch.device,
|
||||
) -> Optional[LocalAttentionMetadata]:
|
||||
if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None:
|
||||
return None
|
||||
page_table = self._kernel_page_table(page_table)
|
||||
if self.page_size > 1:
|
||||
# Convert the eager token table to physical page indices.
|
||||
page_table = page_table[:, :: self.page_size] // self.page_size
|
||||
|
||||
(
|
||||
seqlens_q_local_np,
|
||||
cu_seqlens_q_local_np,
|
||||
seqlens_k_local_np,
|
||||
block_table_local,
|
||||
) = make_local_attention_virtual_batches(
|
||||
self.attention_chunk_size,
|
||||
cu_seqlens_q.cpu().numpy(),
|
||||
cache_seqlens_int32.cpu().numpy(),
|
||||
page_table,
|
||||
self.page_size,
|
||||
preserve_attn_chunk_size=True,
|
||||
)
|
||||
return LocalAttentionMetadata(
|
||||
local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device),
|
||||
local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device),
|
||||
local_block_table=block_table_local.to(device),
|
||||
local_max_query_len=int(seqlens_q_local_np.max()),
|
||||
local_max_seq_len=int(seqlens_k_local_np.max()),
|
||||
)
|
||||
|
||||
def build_for_capture(
|
||||
self,
|
||||
metadata: FlashAttentionMetadata,
|
||||
bs: int,
|
||||
*,
|
||||
buffers: Dict[str, torch.Tensor],
|
||||
) -> LocalAttentionMetadata:
|
||||
"""Views of the preallocated buffers sized to what this capture needs."""
|
||||
seq_lens_capture = metadata.cache_seqlens_int32
|
||||
max_seq_len = int(seq_lens_capture.max().item())
|
||||
|
||||
(
|
||||
seqlens_q_local_np,
|
||||
cu_seqlens_q_local_np,
|
||||
seqlens_k_local_np,
|
||||
block_table_local_np,
|
||||
) = make_local_attention_virtual_batches(
|
||||
self.attention_chunk_size,
|
||||
metadata.cu_seqlens_q.cpu().numpy(),
|
||||
seq_lens_capture.cpu().numpy(),
|
||||
self._graph_page_table(metadata, bs, max_seq_len),
|
||||
self.page_size,
|
||||
preserve_attn_chunk_size=True,
|
||||
)
|
||||
q_len = len(cu_seqlens_q_local_np)
|
||||
k_len = len(seqlens_k_local_np)
|
||||
b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs
|
||||
b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1
|
||||
return LocalAttentionMetadata(
|
||||
local_query_start_loc=buffers["local_query_start_loc"][:q_len],
|
||||
local_seqused_k=buffers["local_seqused_k"][:k_len],
|
||||
local_block_table=buffers["local_block_table"][:b0, :b1],
|
||||
local_max_query_len=1,
|
||||
local_max_seq_len=max_seq_len,
|
||||
)
|
||||
|
||||
def update_for_replay(
|
||||
self,
|
||||
metadata: FlashAttentionMetadata,
|
||||
bs: int,
|
||||
*,
|
||||
buffers: Dict[str, torch.Tensor],
|
||||
) -> None:
|
||||
"""Refill the preallocated buffers in place before a CUDA-graph replay."""
|
||||
local_q_buf = buffers["local_query_start_loc"]
|
||||
local_k_buf = buffers["local_seqused_k"]
|
||||
local_block_buf = buffers["local_block_table"]
|
||||
|
||||
# Decode: one query token per request.
|
||||
cu_seqlens_q = torch.arange(
|
||||
bs + 1, device=local_q_buf.device, dtype=local_q_buf.dtype
|
||||
)
|
||||
seqlens = metadata.cache_seqlens_int32[:bs]
|
||||
# Rows past bs and columns past the live pages hold stale ids.
|
||||
max_seq_len = int(seqlens.max().item())
|
||||
sliced_page_table = self._graph_page_table(metadata, bs, max_seq_len)
|
||||
|
||||
(
|
||||
seqlens_q_local_np,
|
||||
cu_seqlens_q_local_np,
|
||||
seqlens_k_local_np,
|
||||
block_table_local,
|
||||
) = make_local_attention_virtual_batches(
|
||||
self.attention_chunk_size,
|
||||
cu_seqlens_q.cpu().numpy(),
|
||||
seqlens.cpu().numpy(),
|
||||
sliced_page_table,
|
||||
self.page_size,
|
||||
preserve_attn_chunk_size=True,
|
||||
)
|
||||
|
||||
device = local_q_buf.device
|
||||
cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device)
|
||||
seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device)
|
||||
block_table_local = block_table_local.to(device)
|
||||
q_len = cu_seqlens_q_local.shape[0]
|
||||
k_len = seqlens_k_local.shape[0]
|
||||
b0, b1 = block_table_local.shape
|
||||
|
||||
local_q_buf[:q_len].copy_(cu_seqlens_q_local)
|
||||
local_q_buf[q_len:].fill_(0)
|
||||
local_k_buf[:k_len].copy_(seqlens_k_local)
|
||||
local_k_buf[k_len:].fill_(0)
|
||||
local_block_buf[:b0, :b1].copy_(block_table_local)
|
||||
local_block_buf[b0:, :].fill_(0)
|
||||
local_block_buf[:b0, b1:].fill_(0)
|
||||
|
||||
if metadata.local_attn_metadata is not None:
|
||||
lam = metadata.local_attn_metadata
|
||||
lam.local_max_query_len = int(seqlens_q_local_np.max())
|
||||
lam.local_max_seq_len = int(seqlens_k_local_np.max())
|
||||
|
||||
|
||||
def make_local_attention_virtual_batches(
|
||||
attn_chunk_size: int,
|
||||
query_start_loc_np: np.ndarray,
|
||||
seq_lens_np: np.ndarray,
|
||||
block_table: torch.Tensor,
|
||||
page_size: int = 0,
|
||||
preserve_attn_chunk_size: bool = False,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]:
|
||||
"""
|
||||
Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into
|
||||
local attention blocks, where each block is passed to the attention kernel
|
||||
as an independent local ("virtual") batch item.
|
||||
|
||||
Args:
|
||||
attn_chunk_size: Size of local attention chunks
|
||||
query_start_loc_np: Cumulative sum of query lengths (numpy array)
|
||||
seq_lens_np: Sequence lengths (numpy array)
|
||||
block_table: Block table for KV cache
|
||||
page_size: Size of each page in the KV cache
|
||||
preserve_attn_chunk_size: Skip sequence-length-based chunk normalization.
|
||||
|
||||
Returns:
|
||||
seqlens_q_local: Query sequence lengths for local attention
|
||||
cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention
|
||||
seqlens_k_local: Key sequence lengths for local attention
|
||||
block_table_local: Block table for local attention
|
||||
"""
|
||||
if not preserve_attn_chunk_size:
|
||||
max_seq_len = seq_lens_np.max()
|
||||
effective_chunk_size = min(attn_chunk_size, max_seq_len)
|
||||
effective_chunk_size = (effective_chunk_size // page_size) * page_size
|
||||
if effective_chunk_size < page_size:
|
||||
effective_chunk_size = page_size
|
||||
attn_chunk_size = effective_chunk_size
|
||||
|
||||
q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1]
|
||||
actual_batch_size = seq_lens_np.shape[0]
|
||||
|
||||
# Handle if we are starting in the middle of a local attention block,
|
||||
# we assume q_seqlens > 0 (for all elements), for each batch idx we compute
|
||||
# the number of tokens that are not in the first local attention block and
|
||||
# then we can simply use a cdiv for the rest.
|
||||
# For example if we have:
|
||||
# attn_chunk_size = 4
|
||||
# q_seqlens = [4, 10, 5]
|
||||
# k_seqlens = [6, 17, 9]
|
||||
# Then we would get:
|
||||
# new_tokens_in_first_block = [2, 1, 4]
|
||||
# local_blocks = [2, 4, 2]
|
||||
q_tokens_in_first_block = np.minimum(
|
||||
attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens
|
||||
).astype(np.int32)
|
||||
tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size)
|
||||
local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size)
|
||||
|
||||
# Once we know the number of local blocks we can compute the request spans
|
||||
# for each batch idx, we can figure out the number of "virtual" requests we
|
||||
# have to make,
|
||||
# For the above example we would get:
|
||||
# seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1]
|
||||
#
|
||||
# First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1])
|
||||
# (TODO: max a utility to share this code with _prepare_inputs)
|
||||
# arange step 1. [2, 4, 2] -> [2, 6, 8]
|
||||
cu_num_blocks = np.cumsum(local_blocks)
|
||||
virtual_batches = cu_num_blocks[-1]
|
||||
# arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6]
|
||||
block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks)
|
||||
# arange step 3. [0, 1, 0, 1, 2, 3, 0, 1]
|
||||
arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets
|
||||
# also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0])
|
||||
rarange = np.repeat(local_blocks, local_blocks) - arange - 1
|
||||
# Then we can compute the seqlens_q_local, handling the fact that the
|
||||
# first and last blocks could be partial
|
||||
seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks)
|
||||
# set the first block since this may be a partial block
|
||||
seqlens_q_local[arange == 0] = q_tokens_in_first_block
|
||||
# set the remaining blocks
|
||||
seqlens_q_local[arange > 0] = np.minimum(
|
||||
seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size
|
||||
)[arange > 0]
|
||||
|
||||
# convert from q_seqlens to cu_seqlens_q
|
||||
cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32)
|
||||
|
||||
# compute the seqlens_k_local,
|
||||
# basically a full local attention block for all but the last block in each
|
||||
# batch
|
||||
# For our example this will be:
|
||||
# seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1]
|
||||
seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32)
|
||||
seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block
|
||||
|
||||
k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - (
|
||||
rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks)
|
||||
)
|
||||
# For the example the local attention blocks start at:
|
||||
# _b0_ _____b1_____ _b2_
|
||||
# k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8]
|
||||
block_starts = k_seqstarts_absolute // page_size
|
||||
|
||||
assert attn_chunk_size % page_size == 0, (
|
||||
f"attn_chunk_size {attn_chunk_size} is not divisible by page_size {page_size}"
|
||||
)
|
||||
pages_per_local_batch = attn_chunk_size // page_size
|
||||
|
||||
# Create a block_table for the local attention blocks
|
||||
# For out example if we have a block-table like (assuming page_size=2):
|
||||
# block_table = [
|
||||
# [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0
|
||||
# [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1
|
||||
# [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2
|
||||
# ]
|
||||
# Then for the local batches we would want a block-table like
|
||||
# block_table_local = [
|
||||
# [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0])
|
||||
# [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4])
|
||||
# [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4])
|
||||
# [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8])
|
||||
# [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12])
|
||||
# [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16])
|
||||
# [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4])
|
||||
# [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8])
|
||||
# ]
|
||||
block_indices = np.broadcast_to(
|
||||
np.arange(pages_per_local_batch, dtype=np.int32),
|
||||
(virtual_batches, pages_per_local_batch),
|
||||
) + np.expand_dims(block_starts, axis=1)
|
||||
# Ensure block_indices doesn't exceed block_table dimensions
|
||||
# This is a critical safety check that prevents index out of bounds errors
|
||||
# when dealing with large sequences (>8192 tokens) or when the block_table
|
||||
# dimensions are smaller than what would be needed for the full attention chunk size.
|
||||
block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1)
|
||||
batch_indices = np.repeat(
|
||||
np.arange(actual_batch_size, dtype=np.int32),
|
||||
local_blocks * pages_per_local_batch,
|
||||
)
|
||||
|
||||
# NOTE: https://github.com/pytorch/pytorch/pull/160256 causes performance
|
||||
# regression when using numpy arrays (batch and block indices) to index into
|
||||
# torch tensor (block_table). As a workaround, convert numpy arrays to torch
|
||||
# tensor first, which recovers perf.
|
||||
batch_indices_torch = torch.from_numpy(batch_indices)
|
||||
block_indices_torch = torch.from_numpy(block_indices)
|
||||
block_table_local = block_table[batch_indices_torch, block_indices_torch].view(
|
||||
virtual_batches, -1
|
||||
)
|
||||
|
||||
return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local
|
||||
|
||||
|
||||
def cdiv(a: int, b: int) -> int:
|
||||
"""Ceiling division."""
|
||||
return -(a // -b)
|
||||
@@ -8,10 +8,13 @@ from sglang.srt.configs.model_config import AttentionArch
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.layers.attention.flashattention_backend import (
|
||||
FlashAttentionMetadata,
|
||||
make_local_attention_virtual_batches,
|
||||
merge_state_v2_wrapper,
|
||||
prepare_swa_spec_page_table_triton,
|
||||
)
|
||||
from sglang.srt.layers.attention.local_attention import (
|
||||
LocalAttentionMetadata,
|
||||
make_local_attention_virtual_batches,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
@@ -1480,7 +1483,7 @@ class XPUAttentionBackend(AttentionBackend):
|
||||
self.page_size,
|
||||
)
|
||||
|
||||
local_metadata = FlashAttentionMetadata.LocalAttentionMetadata(
|
||||
local_metadata = LocalAttentionMetadata(
|
||||
local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device),
|
||||
local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device),
|
||||
local_block_table=block_table_local.to(device),
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""CPU tests for srt/layers/attention/local_attention: the virtual-batch
|
||||
decomposition and the metadata builder."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.local_attention import (
|
||||
LocalAttentionMetadata,
|
||||
LocalAttentionMetadataBuilder,
|
||||
make_local_attention_virtual_batches,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
CPU = torch.device("cpu")
|
||||
|
||||
|
||||
def _page_table(bs: int, width: int = 32) -> torch.Tensor:
|
||||
return torch.arange(bs * width, dtype=torch.int32).view(bs, width)
|
||||
|
||||
|
||||
class TestVirtualBatches(CustomTestCase):
|
||||
def test_worked_example_from_docstring(self):
|
||||
"""chunk=4, q=[4,10,5], k=[6,17,9] must split into local_blocks=[2,4,2]."""
|
||||
q_start = np.array([0, 4, 14, 19], dtype=np.int32)
|
||||
seq_lens = np.array([6, 17, 9], dtype=np.int32)
|
||||
q_local, cu_q_local, k_local, block_table_local = (
|
||||
make_local_attention_virtual_batches(
|
||||
4, q_start, seq_lens, _page_table(3), 1
|
||||
)
|
||||
)
|
||||
self.assertEqual(q_local.tolist(), [2, 2, 1, 4, 4, 1, 4, 1])
|
||||
self.assertEqual(cu_q_local.tolist(), [0, 2, 4, 5, 9, 13, 14, 18, 19])
|
||||
# Every block sees a full chunk of keys except the last block of each request.
|
||||
self.assertEqual(k_local.tolist(), [4, 2, 4, 4, 4, 1, 4, 1])
|
||||
self.assertEqual(tuple(block_table_local.shape), (8, 4))
|
||||
|
||||
def test_decode_is_one_block_per_request(self):
|
||||
q_start = np.array([0, 1, 2], dtype=np.int32)
|
||||
seq_lens = np.array([6, 9], dtype=np.int32)
|
||||
q_local, cu_q_local, k_local, _ = make_local_attention_virtual_batches(
|
||||
4, q_start, seq_lens, _page_table(2), 1
|
||||
)
|
||||
self.assertEqual(q_local.tolist(), [1, 1])
|
||||
self.assertEqual(cu_q_local.tolist(), [0, 1, 2])
|
||||
# Only the tail of the current chunk is attended: 6 % 4 == 2, 9 % 4 == 1.
|
||||
self.assertEqual(k_local.tolist(), [2, 1])
|
||||
|
||||
def test_chunk_is_clamped_to_the_longest_sequence(self):
|
||||
q_start = np.array([0, 6], dtype=np.int32)
|
||||
seq_lens = np.array([6], dtype=np.int32)
|
||||
_, _, k_local, block_table_local = make_local_attention_virtual_batches(
|
||||
8, q_start, seq_lens, _page_table(1), 1
|
||||
)
|
||||
self.assertEqual(k_local.tolist(), [6])
|
||||
self.assertEqual(tuple(block_table_local.shape), (1, 6))
|
||||
|
||||
def test_page_size_divides_the_block_table_width(self):
|
||||
q_start = np.array([0, 6], dtype=np.int32)
|
||||
seq_lens = np.array([6], dtype=np.int32)
|
||||
q_local, _, k_local, block_table_local = make_local_attention_virtual_batches(
|
||||
4, q_start, seq_lens, _page_table(1), 2
|
||||
)
|
||||
self.assertEqual(q_local.tolist(), [4, 2])
|
||||
self.assertEqual(k_local.tolist(), [4, 2])
|
||||
self.assertEqual(tuple(block_table_local.shape), (2, 2)) # chunk 4 / page 2
|
||||
|
||||
|
||||
class TestLocalAttentionMetadataBuilder(CustomTestCase):
|
||||
def _builder(self, swa_translate=None, max_context_len=16):
|
||||
return LocalAttentionMetadataBuilder(
|
||||
attention_chunk_size=4,
|
||||
page_size=1,
|
||||
max_context_len=max_context_len,
|
||||
device=CPU,
|
||||
swa_translate=swa_translate,
|
||||
)
|
||||
|
||||
def test_build_returns_none_without_a_complete_batch(self):
|
||||
b = self._builder()
|
||||
self.assertIsNone(
|
||||
b.build(
|
||||
cu_seqlens_q=None,
|
||||
cache_seqlens_int32=torch.tensor([6]),
|
||||
page_table=_page_table(1),
|
||||
device=CPU,
|
||||
)
|
||||
)
|
||||
|
||||
def test_build_applies_the_swa_translation_to_the_block_table(self):
|
||||
args = dict(
|
||||
cu_seqlens_q=torch.tensor([0, 6], dtype=torch.int32),
|
||||
cache_seqlens_int32=torch.tensor([6], dtype=torch.int32),
|
||||
page_table=_page_table(1),
|
||||
device=CPU,
|
||||
)
|
||||
plain = self._builder().build(**args)
|
||||
swa = self._builder(swa_translate=lambda t: t + 100).build(**args)
|
||||
self.assertEqual(plain.local_max_query_len, 4)
|
||||
self.assertEqual(plain.local_max_seq_len, 4)
|
||||
torch.testing.assert_close(swa.local_block_table, plain.local_block_table + 100)
|
||||
|
||||
def test_cuda_graph_buffers_fit_the_worst_case_decomposition(self):
|
||||
"""Buffer sizing must match the most virtual batches max_bs x max_context_len can produce."""
|
||||
b = self._builder(max_context_len=16)
|
||||
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
|
||||
md = b.build(
|
||||
cu_seqlens_q=torch.tensor([0, 16, 32], dtype=torch.int32),
|
||||
cache_seqlens_int32=torch.tensor([16, 16], dtype=torch.int32),
|
||||
page_table=_page_table(2),
|
||||
device=CPU,
|
||||
)
|
||||
self.assertEqual(
|
||||
md.local_query_start_loc.numel(), bufs["local_query_start_loc"].numel()
|
||||
)
|
||||
self.assertEqual(md.local_seqused_k.numel(), bufs["local_seqused_k"].numel())
|
||||
self.assertEqual(
|
||||
tuple(md.local_block_table.shape), tuple(bufs["local_block_table"].shape)
|
||||
)
|
||||
|
||||
def test_capture_returns_views_of_the_preallocated_buffers(self):
|
||||
b = self._builder()
|
||||
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
|
||||
metadata = SimpleNamespace(
|
||||
cu_seqlens_q=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
|
||||
page_table=_page_table(2),
|
||||
)
|
||||
md = b.build_for_capture(metadata, 2, buffers=bufs)
|
||||
self.assertEqual(md.local_query_start_loc.numel(), 3)
|
||||
self.assertEqual(md.local_seqused_k.numel(), 2)
|
||||
self.assertEqual(tuple(md.local_block_table.shape), (2, 4))
|
||||
self.assertEqual(md.local_max_query_len, 1)
|
||||
self.assertEqual(md.local_max_seq_len, 9)
|
||||
self.assertEqual(
|
||||
md.local_query_start_loc.data_ptr(),
|
||||
bufs["local_query_start_loc"].data_ptr(),
|
||||
)
|
||||
self.assertEqual(
|
||||
md.local_block_table.data_ptr(), bufs["local_block_table"].data_ptr()
|
||||
)
|
||||
|
||||
def _paged_builder(self, swa_translate):
|
||||
# chunk 32 / page 16: two pages per local block.
|
||||
return LocalAttentionMetadataBuilder(
|
||||
attention_chunk_size=32,
|
||||
page_size=16,
|
||||
max_context_len=64,
|
||||
device=CPU,
|
||||
swa_translate=swa_translate,
|
||||
)
|
||||
|
||||
def test_replay_reads_page_ids_from_swa_page_table(self):
|
||||
"""Regression: with page_size > 1 the CUDA-graph replay must not push the
|
||||
page-granular table through the token-slot full->swa mapping."""
|
||||
# Any translation applied to page ids would land far outside {7, 8}.
|
||||
b = self._paged_builder(swa_translate=lambda t: t * 1000)
|
||||
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
|
||||
metadata = SimpleNamespace(
|
||||
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
|
||||
page_table=torch.tensor([[3], [5]], dtype=torch.int32),
|
||||
swa_page_table=torch.tensor([[7], [8]], dtype=torch.int32),
|
||||
local_attn_metadata=LocalAttentionMetadata(),
|
||||
)
|
||||
b.update_for_replay(metadata, 2, buffers=bufs)
|
||||
table = bufs["local_block_table"][:2]
|
||||
self.assertEqual(table[:, 0].tolist(), [7, 8])
|
||||
self.assertTrue(set(table.flatten().tolist()) <= {0, 7, 8})
|
||||
|
||||
def test_replay_without_swa_pool_reads_page_table(self):
|
||||
b = self._paged_builder(swa_translate=None)
|
||||
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
|
||||
metadata = SimpleNamespace(
|
||||
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
|
||||
page_table=torch.tensor([[3], [5]], dtype=torch.int32),
|
||||
local_attn_metadata=LocalAttentionMetadata(),
|
||||
)
|
||||
b.update_for_replay(metadata, 2, buffers=bufs)
|
||||
self.assertEqual(bufs["local_block_table"][:2, 0].tolist(), [3, 5])
|
||||
|
||||
def test_capture_and_replay_share_the_table_source(self):
|
||||
b = self._paged_builder(swa_translate=lambda t: t * 1000)
|
||||
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
|
||||
metadata = SimpleNamespace(
|
||||
cu_seqlens_q=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
|
||||
page_table=torch.tensor([[3], [5]], dtype=torch.int32),
|
||||
swa_page_table=torch.tensor([[7], [8]], dtype=torch.int32),
|
||||
)
|
||||
md = b.build_for_capture(metadata, 2, buffers=bufs)
|
||||
self.assertEqual(
|
||||
tuple(md.local_block_table.shape), (2, 2)
|
||||
) # 2 reqs x 2 pages per block
|
||||
|
||||
def test_replay_refills_buffers_in_place_and_zeroes_the_tail(self):
|
||||
b = self._builder()
|
||||
bufs = b.alloc_cuda_graph_buffers(max_bs=2)
|
||||
for t in bufs.values():
|
||||
t.fill_(7) # stale contents from an earlier, larger batch
|
||||
lam = LocalAttentionMetadata()
|
||||
metadata = SimpleNamespace(
|
||||
cache_seqlens_int32=torch.tensor([6, 9], dtype=torch.int32),
|
||||
page_table=_page_table(2),
|
||||
local_attn_metadata=lam,
|
||||
)
|
||||
b.update_for_replay(metadata, 2, buffers=bufs)
|
||||
self.assertEqual(bufs["local_query_start_loc"][:3].tolist(), [0, 1, 2])
|
||||
self.assertEqual(bufs["local_query_start_loc"][3:].abs().sum().item(), 0)
|
||||
self.assertEqual(bufs["local_seqused_k"][:2].tolist(), [2, 1])
|
||||
self.assertEqual(bufs["local_seqused_k"][2:].abs().sum().item(), 0)
|
||||
self.assertEqual(bufs["local_block_table"][2:].abs().sum().item(), 0)
|
||||
self.assertEqual((lam.local_max_query_len, lam.local_max_seq_len), (1, 2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user