[Refactor] attention: Separate decode metadata from pool ownership (#44969)

This commit is contained in:
Cheng Wan
2026-10-18 05:39:44 -07:00
committed by GitHub
parent 3edb156dec
commit a5b21145e1
5 changed files with 251 additions and 356 deletions
+116 -313
View File
@@ -1,12 +1,9 @@
from typing import TYPE_CHECKING, Optional
from typing import Optional
import torch
import triton
import triton.language as tl
if TYPE_CHECKING:
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@triton.jit
def get_num_kv_splits_triton(
@@ -163,8 +160,24 @@ def prepare_swa_spec_page_table_triton(
@triton.jit
def _fused_metadata_kernel_general(
# Input tensors
def _resolve_metadata_page_ids(
token_ids,
mask,
PAGE_SIZE: tl.constexpr,
token_mapping=None,
token_mapping_stride=0,
):
# Static SWA maps token slots. An absent mapping compiles to an identity.
if token_mapping is not None:
token_ids = tl.load(
token_mapping + token_ids * token_mapping_stride, mask=mask, other=0
)
page_ids = token_ids // PAGE_SIZE
return page_ids
@triton.jit
def _fused_decode_metadata_kernel(
seq_lens,
seq_lens_stride_0,
req_to_token,
@@ -172,7 +185,6 @@ def _fused_metadata_kernel_general(
req_to_token_stride_1,
req_pool_indices,
req_pool_indices_stride_0,
# Output buffers
cache_seqlens_int32,
cache_seqlens_int32_stride_0,
cu_seqlens_k,
@@ -183,168 +195,76 @@ def _fused_metadata_kernel_general(
swa_page_table,
swa_page_table_stride_0,
swa_page_table_stride_1,
full_to_swa_mapping,
full_to_swa_index_mapping,
full_to_swa_mapping_stride_0,
# Scalar parameters
B,
max_seq_pages,
page_size: tl.constexpr,
seq_len_delta: tl.constexpr,
use_swa: tl.constexpr,
SHIFT: tl.constexpr,
PAGE_SIZE: tl.constexpr,
SEQ_LEN_DELTA: tl.constexpr,
BLOCK_COLS: tl.constexpr,
SKIP_PAGE_TABLE: tl.constexpr = 0,
SKIP_PAGE_TABLE: tl.constexpr,
):
pid_b = tl.program_id(0) # batch index
pid_c = tl.program_id(1) # column chunk index
pid_b = tl.program_id(0)
pid_c = tl.program_id(1)
# 1. Prefix sum (only one block does it)
# One block computes the sequence lengths and prefix sum.
if pid_b == 0 and pid_c == 0:
acc = 0
for idx in range(B):
seq = tl.load(seq_lens + idx * seq_lens_stride_0)
val = (seq + seq_len_delta).to(tl.int32)
val = (seq + SEQ_LEN_DELTA).to(tl.int32)
tl.store(cache_seqlens_int32 + idx * cache_seqlens_int32_stride_0, val)
tl.store(cu_seqlens_k + idx * cu_seqlens_k_stride_0, acc)
acc += val
tl.store(cu_seqlens_k + B * cu_seqlens_k_stride_0, acc)
# 2. Gather for this batch and column chunk
if SKIP_PAGE_TABLE:
return
if max_seq_pages == 0:
return
i = pid_b
# Self-guard on the device-side seq_len: skip column chunks past the
# request's live pages (tails keep stale values the attention kernels
# never read past cache_seqlens).
seq_len = tl.load(seq_lens + i * seq_lens_stride_0).to(tl.int32)
if page_size == 1:
num_live_pages = seq_len + seq_len_delta
else:
num_live_pages = (seq_len + seq_len_delta + (1 << SHIFT) - 1) >> SHIFT
num_live_pages = tl.minimum(num_live_pages, max_seq_pages)
seq_len = tl.load(seq_lens + pid_b * seq_lens_stride_0).to(tl.int32)
num_live_pages = tl.minimum(
tl.cdiv(seq_len + SEQ_LEN_DELTA, PAGE_SIZE), max_seq_pages
)
col_start = pid_c * BLOCK_COLS
if col_start >= num_live_pages:
return
# Load row index for this batch (all threads in block have same i)
row_idx = tl.load(req_pool_indices + i * req_pool_indices_stride_0)
row_offset = row_idx * req_to_token_stride_0
col_offsets = col_start + tl.arange(0, BLOCK_COLS)
mask = col_offsets < num_live_pages
# Compute column indices in the source tensor
if page_size == 1:
col_idx = col_offsets
else:
col_idx = col_offsets << SHIFT # faster than multiplication for power-of-two
# Load page indices from req_to_token
rt_offsets = row_offset + col_idx * req_to_token_stride_1
page_index = tl.load(
req_to_token + rt_offsets, mask=mask, other=0, cache_modifier=".cg"
row_idx = tl.load(req_pool_indices + pid_b * req_pool_indices_stride_0)
cols = col_start + tl.arange(0, BLOCK_COLS)
live_mask = cols < num_live_pages
token_ids = tl.load(
req_to_token
+ row_idx * req_to_token_stride_0
+ cols * PAGE_SIZE * req_to_token_stride_1,
mask=live_mask,
other=0,
cache_modifier=".cg",
)
# Compute page_table
if page_size == 1:
page_table_val = page_index
else:
page_table_val = page_index >> SHIFT
# Store to page_table
pt_offsets = i * page_table_stride_0 + col_offsets * page_table_stride_1
tl.store(page_table + pt_offsets, page_table_val, mask=mask, cache_modifier=".cg")
if use_swa:
swa_slot = tl.load(
full_to_swa_mapping + page_index * full_to_swa_mapping_stride_0,
mask=mask,
other=0,
full_pages = _resolve_metadata_page_ids(token_ids, live_mask, PAGE_SIZE)
tl.store(
page_table + pid_b * page_table_stride_0 + cols * page_table_stride_1,
full_pages,
mask=live_mask,
cache_modifier=".cg",
)
if swa_page_table is not None:
swa_pages = _resolve_metadata_page_ids(
token_ids,
live_mask,
PAGE_SIZE,
full_to_swa_index_mapping,
full_to_swa_mapping_stride_0,
)
tl.store(
swa_page_table
+ pid_b * swa_page_table_stride_0
+ cols * swa_page_table_stride_1,
swa_pages,
mask=live_mask,
cache_modifier=".cg",
)
if page_size == 1:
swa_val = swa_slot
else:
swa_val = swa_slot >> SHIFT
swa_offsets = (
i * swa_page_table_stride_0 + col_offsets * swa_page_table_stride_1
)
tl.store(swa_page_table + swa_offsets, swa_val, mask=mask, cache_modifier=".cg")
@triton.jit
def _fused_metadata_kernel_ps1_no_swa(
# Input tensors
seq_lens,
seq_lens_stride_0,
req_to_token,
req_to_token_stride_0,
req_to_token_stride_1,
req_pool_indices,
req_pool_indices_stride_0,
# Output buffers
cache_seqlens_int32,
cache_seqlens_int32_stride_0,
cu_seqlens_k,
cu_seqlens_k_stride_0,
page_table,
page_table_stride_0,
page_table_stride_1,
# Scalar parameters
B,
max_seq_pages,
seq_len_delta: tl.constexpr,
BLOCK_COLS: tl.constexpr,
SKIP_PAGE_TABLE: tl.constexpr = 0,
):
pid_b = tl.program_id(0) # batch index
pid_c = tl.program_id(1) # column chunk index
# 1. Prefix sum (only one block does it)
if pid_b == 0 and pid_c == 0:
acc = 0
for idx in range(B):
seq = tl.load(seq_lens + idx * seq_lens_stride_0)
val = (seq + seq_len_delta).to(tl.int32)
tl.store(cache_seqlens_int32 + idx * cache_seqlens_int32_stride_0, val)
tl.store(cu_seqlens_k + idx * cu_seqlens_k_stride_0, acc)
acc += val
tl.store(cu_seqlens_k + B * cu_seqlens_k_stride_0, acc)
# 2. Gather for this batch and column chunk
if SKIP_PAGE_TABLE:
return
if max_seq_pages == 0:
return
i = pid_b
# Self-guard on the device-side seq_len: skip column chunks past the
# request's live pages (tails keep stale values the attention kernels
# never read past cache_seqlens).
seq_len = tl.load(seq_lens + i * seq_lens_stride_0).to(tl.int32)
num_live_pages = tl.minimum(seq_len + seq_len_delta, max_seq_pages)
col_start = pid_c * BLOCK_COLS
if col_start >= num_live_pages:
return
# Load row index for this batch (all threads in block have same i)
row_idx = tl.load(req_pool_indices + i * req_pool_indices_stride_0)
row_offset = row_idx * req_to_token_stride_0
col_offsets = col_start + tl.arange(0, BLOCK_COLS)
mask = col_offsets < num_live_pages
# page_size = 1: col_idx = col_offsets
rt_offsets = row_offset + col_offsets * req_to_token_stride_1
page_index = tl.load(
req_to_token + rt_offsets, mask=mask, other=0, cache_modifier=".cg"
)
pt_offsets = i * page_table_stride_0 + col_offsets * page_table_stride_1
tl.store(page_table + pt_offsets, page_index, mask=mask, cache_modifier=".cg")
@triton.jit
@@ -563,188 +483,71 @@ def normal_decode_set_metadata(
page_table: torch.Tensor,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
max_seq_pages: torch.Tensor,
max_seq_pages: int,
seq_lens: torch.Tensor,
seq_len_delta: int,
page_size: int,
swa_page_table: Optional[torch.Tensor] = None,
token_to_kv_pool: Optional["SWAKVPool"] = None,
full_to_swa_index_mapping: Optional[torch.Tensor] = None,
skip_page_table: bool = False,
):
"""
Fused Triton implementation that replaces 4-5 sequential CUDA kernels with 1-2 kernels:
1. cache_seqlens = seq_lens + seq_len_delta (int64->int32 cast)
2. cu_seqlens_k = cumsum(cache_seqlens) (prefix-sum)
3. page_indices = req_to_token[pool_idx, stride_idx] (2-D gather)
4. page_table = page_indices // page_size (floor-divide)
5. (optional) swa_page_table for sliding window attention
"""Fill sequence lengths, their prefix sum and page tables in one launch.
Unified pool (``skip_page_table=True``): the translator has already filled
the page tables in place, so only steps 1-2 run.
``req_to_token`` contains token slots. Static SWA uses
``full_to_swa_index_mapping`` to map FULL token slots to SWA token slots.
Runtime callers resolve this tensor from their pool before calling.
Achieves ~5.2x speedup on H200 hardware for typical decode workloads.
Contract: only the live prefix (cdiv(cache_seqlens, page_size) pages) of each
page_table / swa_page_table row is (re)written; the tail keeps stale values
across CUDA-graph replays, so consumers must bound reads by cache_seqlens.
Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten;
consumers must bound reads by ``cache_seqlens_int32``. For pretranslated
tables, ``skip_page_table=True`` leaves both page tables untouched and
computes only lengths and the prefix sum.
"""
assert page_size > 0 and (page_size & (page_size - 1)) == 0, (
f"page_size must be a power of two, got {page_size}"
)
if swa_page_table is not None and not skip_page_table:
assert full_to_swa_index_mapping is not None, (
"SWA output requires a token-slot mapping"
)
batch_size = cache_seqlens_int32.shape[0]
device = seq_lens.device
# Ensure contiguous memory layout for efficient Triton access
seq_lens = seq_lens.contiguous()
req_to_token = req_to_token.contiguous()
req_pool_indices = req_pool_indices.contiguous()
# Prepare tensor strides
seq_lens_stride_0 = seq_lens.stride(0)
req_to_token_stride_0 = req_to_token.stride(0)
req_to_token_stride_1 = req_to_token.stride(1)
req_pool_indices_stride_0 = req_pool_indices.stride(0)
cache_seqlens_int32_stride_0 = cache_seqlens_int32.stride(0)
cu_seqlens_k_stride_0 = cu_seqlens_k.stride(0)
page_table_stride_0 = page_table.stride(0)
page_table_stride_1 = page_table.stride(1)
if skip_page_table:
# One block does the prefix sum, so one block is the whole grid.
_fused_metadata_kernel_ps1_no_swa[(1, 1)](
seq_lens,
seq_lens_stride_0,
page_table,
page_table_stride_0,
page_table_stride_1,
req_pool_indices,
req_pool_indices_stride_0,
cache_seqlens_int32,
cache_seqlens_int32_stride_0,
cu_seqlens_k,
cu_seqlens_k_stride_0,
page_table,
page_table_stride_0,
page_table_stride_1,
batch_size,
0,
seq_len_delta,
BLOCK_COLS=256,
SKIP_PAGE_TABLE=1,
num_warps=8,
num_stages=3,
)
return
use_swa = swa_page_table is not None and token_to_kv_pool is not None
# Unified SWA uses an independent SWA v2p table.
swa_v2p_page_table = None
if use_swa and token_to_kv_pool.full_to_swa_index_mapping is None:
from sglang.srt.mem_cache.unified_memory_pool import UnifiedSWAKVPool
assert isinstance(token_to_kv_pool, UnifiedSWAKVPool)
assert token_to_kv_pool._swa_allocator is not None
swa_v2p_page_table = token_to_kv_pool._swa_allocator.virtual_to_physical
# Check if we should use the specialized fast path for page_size=1, no SWA
swa_uses_v2p = swa_v2p_page_table is not None
if page_size == 1 and not use_swa:
# Specialized kernel for the common case (page_size=1, no SWA)
BLOCK_COLS = 256
if max_seq_pages == 0:
grid = (1, 1)
else:
num_blocks_j = triton.cdiv(max_seq_pages, BLOCK_COLS)
grid = (batch_size, num_blocks_j)
_fused_metadata_kernel_ps1_no_swa[grid](
seq_lens,
seq_lens_stride_0,
req_to_token,
req_to_token_stride_0,
req_to_token_stride_1,
req_pool_indices,
req_pool_indices_stride_0,
cache_seqlens_int32,
cache_seqlens_int32_stride_0,
cu_seqlens_k,
cu_seqlens_k_stride_0,
page_table,
page_table_stride_0,
page_table_stride_1,
batch_size,
max_seq_pages,
seq_len_delta,
BLOCK_COLS=BLOCK_COLS,
num_warps=8,
num_stages=3,
)
else:
# General kernel for page_size > 1 or SWA cases
# SWA parameters
if use_swa:
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
swa_page_table = swa_page_table.contiguous()
swa_page_table_stride_0 = swa_page_table.stride(0)
swa_page_table_stride_1 = swa_page_table.stride(1)
if swa_uses_v2p:
full_to_swa_mapping = swa_v2p_page_table.contiguous()
full_to_swa_mapping_stride_0 = 0
else:
assert isinstance(token_to_kv_pool, SWAKVPool)
full_to_swa_mapping = (
token_to_kv_pool.full_to_swa_index_mapping.contiguous()
)
full_to_swa_mapping_stride_0 = full_to_swa_mapping.stride(0)
else:
# Dummy tensors (not used)
swa_page_table = torch.empty(0, dtype=torch.int32, device=device)
swa_page_table_stride_0 = 0
swa_page_table_stride_1 = 0
full_to_swa_mapping = torch.empty(0, dtype=torch.int32, device=device)
full_to_swa_mapping_stride_0 = 0
# Kernel configuration
BLOCK_COLS = 128
shift = (page_size).bit_length() - 1 if page_size > 1 else 0
if max_seq_pages == 0:
grid = (1, 1)
else:
num_blocks_j = triton.cdiv(max_seq_pages, BLOCK_COLS)
grid = (batch_size, num_blocks_j)
_fused_metadata_kernel_general[grid](
seq_lens,
seq_lens_stride_0,
req_to_token,
req_to_token_stride_0,
req_to_token_stride_1,
req_pool_indices,
req_pool_indices_stride_0,
cache_seqlens_int32,
cache_seqlens_int32_stride_0,
cu_seqlens_k,
cu_seqlens_k_stride_0,
page_table,
page_table_stride_0,
page_table_stride_1,
swa_page_table,
swa_page_table_stride_0,
swa_page_table_stride_1,
full_to_swa_mapping,
full_to_swa_mapping_stride_0,
batch_size,
max_seq_pages,
page_size,
seq_len_delta,
use_swa,
shift,
BLOCK_COLS=BLOCK_COLS,
num_warps=4,
num_stages=3,
)
# Lengths-only/token gathers use 256 columns; page/SWA gathers use 128.
block_cols = (
256 if skip_page_table or (page_size == 1 and swa_page_table is None) else 128
)
grid = (
(1, 1)
if skip_page_table or max_seq_pages == 0
else (batch_size, triton.cdiv(max_seq_pages, block_cols))
)
_fused_decode_metadata_kernel[grid](
seq_lens,
seq_lens.stride(0),
req_to_token,
req_to_token.stride(0),
req_to_token.stride(1),
req_pool_indices,
req_pool_indices.stride(0),
cache_seqlens_int32,
cache_seqlens_int32.stride(0),
cu_seqlens_k,
cu_seqlens_k.stride(0),
page_table,
page_table.stride(0),
page_table.stride(1),
swa_page_table,
swa_page_table.stride(0) if swa_page_table is not None else 0,
swa_page_table.stride(1) if swa_page_table is not None else 0,
full_to_swa_index_mapping,
full_to_swa_index_mapping.stride(0)
if full_to_swa_index_mapping is not None
else 0,
batch_size,
max_seq_pages,
PAGE_SIZE=page_size,
SEQ_LEN_DELTA=seq_len_delta,
BLOCK_COLS=block_cols,
SKIP_PAGE_TABLE=skip_page_table,
num_warps=block_cols // 32,
num_stages=3,
)
@@ -2926,7 +2926,11 @@ class FlashAttentionBackend(AttentionBackend):
seq_len_delta,
self.page_size,
metadata.swa_page_table,
self.token_to_kv_pool if self.use_sliding_window_kv_pool else None,
full_to_swa_index_mapping=(
self.token_to_kv_pool.full_to_swa_index_mapping
if self.use_sliding_window_kv_pool
else None
),
skip_page_table=translated,
)
if translated:
@@ -1786,7 +1786,9 @@ class XPUAttentionBackend(AttentionBackend):
seq_len_delta=seq_len_delta,
page_size=self.page_size,
swa_page_table=metadata.swa_page_table if use_swa else None,
token_to_kv_pool=self.token_to_kv_pool if use_swa else None,
full_to_swa_index_mapping=(
self.token_to_kv_pool.full_to_swa_index_mapping if use_swa else None
),
)
def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch):