diff --git a/python/sglang/kernels/ops/attention/metadata.py b/python/sglang/kernels/ops/attention/metadata.py index 88230f5e31..f995f9ea3d 100644 --- a/python/sglang/kernels/ops/attention/metadata.py +++ b/python/sglang/kernels/ops/attention/metadata.py @@ -1,12 +1,9 @@ -from typing import TYPE_CHECKING, Optional +from typing import Optional import torch import triton import triton.language as tl -if TYPE_CHECKING: - from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool - @triton.jit def get_num_kv_splits_triton( @@ -163,8 +160,24 @@ def prepare_swa_spec_page_table_triton( @triton.jit -def _fused_metadata_kernel_general( - # Input tensors +def _resolve_metadata_page_ids( + token_ids, + mask, + PAGE_SIZE: tl.constexpr, + token_mapping=None, + token_mapping_stride=0, +): + # Static SWA maps token slots. An absent mapping compiles to an identity. + if token_mapping is not None: + token_ids = tl.load( + token_mapping + token_ids * token_mapping_stride, mask=mask, other=0 + ) + page_ids = token_ids // PAGE_SIZE + return page_ids + + +@triton.jit +def _fused_decode_metadata_kernel( seq_lens, seq_lens_stride_0, req_to_token, @@ -172,7 +185,6 @@ def _fused_metadata_kernel_general( req_to_token_stride_1, req_pool_indices, req_pool_indices_stride_0, - # Output buffers cache_seqlens_int32, cache_seqlens_int32_stride_0, cu_seqlens_k, @@ -183,168 +195,76 @@ def _fused_metadata_kernel_general( swa_page_table, swa_page_table_stride_0, swa_page_table_stride_1, - full_to_swa_mapping, + full_to_swa_index_mapping, full_to_swa_mapping_stride_0, - # Scalar parameters B, max_seq_pages, - page_size: tl.constexpr, - seq_len_delta: tl.constexpr, - use_swa: tl.constexpr, - SHIFT: tl.constexpr, + PAGE_SIZE: tl.constexpr, + SEQ_LEN_DELTA: tl.constexpr, BLOCK_COLS: tl.constexpr, - SKIP_PAGE_TABLE: tl.constexpr = 0, + SKIP_PAGE_TABLE: tl.constexpr, ): - pid_b = tl.program_id(0) # batch index - pid_c = tl.program_id(1) # column chunk index + pid_b = tl.program_id(0) + pid_c = tl.program_id(1) - # 1. Prefix sum (only one block does it) + # One block computes the sequence lengths and prefix sum. if pid_b == 0 and pid_c == 0: acc = 0 for idx in range(B): seq = tl.load(seq_lens + idx * seq_lens_stride_0) - val = (seq + seq_len_delta).to(tl.int32) + val = (seq + SEQ_LEN_DELTA).to(tl.int32) tl.store(cache_seqlens_int32 + idx * cache_seqlens_int32_stride_0, val) tl.store(cu_seqlens_k + idx * cu_seqlens_k_stride_0, acc) acc += val tl.store(cu_seqlens_k + B * cu_seqlens_k_stride_0, acc) - # 2. Gather for this batch and column chunk if SKIP_PAGE_TABLE: return if max_seq_pages == 0: return - i = pid_b - # Self-guard on the device-side seq_len: skip column chunks past the - # request's live pages (tails keep stale values the attention kernels - # never read past cache_seqlens). - seq_len = tl.load(seq_lens + i * seq_lens_stride_0).to(tl.int32) - if page_size == 1: - num_live_pages = seq_len + seq_len_delta - else: - num_live_pages = (seq_len + seq_len_delta + (1 << SHIFT) - 1) >> SHIFT - num_live_pages = tl.minimum(num_live_pages, max_seq_pages) + seq_len = tl.load(seq_lens + pid_b * seq_lens_stride_0).to(tl.int32) + num_live_pages = tl.minimum( + tl.cdiv(seq_len + SEQ_LEN_DELTA, PAGE_SIZE), max_seq_pages + ) col_start = pid_c * BLOCK_COLS if col_start >= num_live_pages: return - # Load row index for this batch (all threads in block have same i) - row_idx = tl.load(req_pool_indices + i * req_pool_indices_stride_0) - row_offset = row_idx * req_to_token_stride_0 - - col_offsets = col_start + tl.arange(0, BLOCK_COLS) - mask = col_offsets < num_live_pages - - # Compute column indices in the source tensor - if page_size == 1: - col_idx = col_offsets - else: - col_idx = col_offsets << SHIFT # faster than multiplication for power-of-two - - # Load page indices from req_to_token - rt_offsets = row_offset + col_idx * req_to_token_stride_1 - page_index = tl.load( - req_to_token + rt_offsets, mask=mask, other=0, cache_modifier=".cg" + row_idx = tl.load(req_pool_indices + pid_b * req_pool_indices_stride_0) + cols = col_start + tl.arange(0, BLOCK_COLS) + live_mask = cols < num_live_pages + token_ids = tl.load( + req_to_token + + row_idx * req_to_token_stride_0 + + cols * PAGE_SIZE * req_to_token_stride_1, + mask=live_mask, + other=0, + cache_modifier=".cg", ) - - # Compute page_table - if page_size == 1: - page_table_val = page_index - else: - page_table_val = page_index >> SHIFT - - # Store to page_table - pt_offsets = i * page_table_stride_0 + col_offsets * page_table_stride_1 - tl.store(page_table + pt_offsets, page_table_val, mask=mask, cache_modifier=".cg") - - if use_swa: - swa_slot = tl.load( - full_to_swa_mapping + page_index * full_to_swa_mapping_stride_0, - mask=mask, - other=0, + full_pages = _resolve_metadata_page_ids(token_ids, live_mask, PAGE_SIZE) + tl.store( + page_table + pid_b * page_table_stride_0 + cols * page_table_stride_1, + full_pages, + mask=live_mask, + cache_modifier=".cg", + ) + if swa_page_table is not None: + swa_pages = _resolve_metadata_page_ids( + token_ids, + live_mask, + PAGE_SIZE, + full_to_swa_index_mapping, + full_to_swa_mapping_stride_0, + ) + tl.store( + swa_page_table + + pid_b * swa_page_table_stride_0 + + cols * swa_page_table_stride_1, + swa_pages, + mask=live_mask, cache_modifier=".cg", ) - if page_size == 1: - swa_val = swa_slot - else: - swa_val = swa_slot >> SHIFT - swa_offsets = ( - i * swa_page_table_stride_0 + col_offsets * swa_page_table_stride_1 - ) - tl.store(swa_page_table + swa_offsets, swa_val, mask=mask, cache_modifier=".cg") - - -@triton.jit -def _fused_metadata_kernel_ps1_no_swa( - # Input tensors - seq_lens, - seq_lens_stride_0, - req_to_token, - req_to_token_stride_0, - req_to_token_stride_1, - req_pool_indices, - req_pool_indices_stride_0, - # Output buffers - cache_seqlens_int32, - cache_seqlens_int32_stride_0, - cu_seqlens_k, - cu_seqlens_k_stride_0, - page_table, - page_table_stride_0, - page_table_stride_1, - # Scalar parameters - B, - max_seq_pages, - seq_len_delta: tl.constexpr, - BLOCK_COLS: tl.constexpr, - SKIP_PAGE_TABLE: tl.constexpr = 0, -): - pid_b = tl.program_id(0) # batch index - pid_c = tl.program_id(1) # column chunk index - - # 1. Prefix sum (only one block does it) - if pid_b == 0 and pid_c == 0: - acc = 0 - for idx in range(B): - seq = tl.load(seq_lens + idx * seq_lens_stride_0) - val = (seq + seq_len_delta).to(tl.int32) - tl.store(cache_seqlens_int32 + idx * cache_seqlens_int32_stride_0, val) - tl.store(cu_seqlens_k + idx * cu_seqlens_k_stride_0, acc) - acc += val - tl.store(cu_seqlens_k + B * cu_seqlens_k_stride_0, acc) - - # 2. Gather for this batch and column chunk - if SKIP_PAGE_TABLE: - return - if max_seq_pages == 0: - return - - i = pid_b - # Self-guard on the device-side seq_len: skip column chunks past the - # request's live pages (tails keep stale values the attention kernels - # never read past cache_seqlens). - seq_len = tl.load(seq_lens + i * seq_lens_stride_0).to(tl.int32) - num_live_pages = tl.minimum(seq_len + seq_len_delta, max_seq_pages) - col_start = pid_c * BLOCK_COLS - if col_start >= num_live_pages: - return - - # Load row index for this batch (all threads in block have same i) - row_idx = tl.load(req_pool_indices + i * req_pool_indices_stride_0) - row_offset = row_idx * req_to_token_stride_0 - - col_offsets = col_start + tl.arange(0, BLOCK_COLS) - mask = col_offsets < num_live_pages - - # page_size = 1: col_idx = col_offsets - rt_offsets = row_offset + col_offsets * req_to_token_stride_1 - page_index = tl.load( - req_to_token + rt_offsets, mask=mask, other=0, cache_modifier=".cg" - ) - - pt_offsets = i * page_table_stride_0 + col_offsets * page_table_stride_1 - tl.store(page_table + pt_offsets, page_index, mask=mask, cache_modifier=".cg") @triton.jit @@ -563,188 +483,71 @@ def normal_decode_set_metadata( page_table: torch.Tensor, req_to_token: torch.Tensor, req_pool_indices: torch.Tensor, - max_seq_pages: torch.Tensor, + max_seq_pages: int, seq_lens: torch.Tensor, seq_len_delta: int, page_size: int, swa_page_table: Optional[torch.Tensor] = None, - token_to_kv_pool: Optional["SWAKVPool"] = None, + full_to_swa_index_mapping: Optional[torch.Tensor] = None, skip_page_table: bool = False, ): - """ - Fused Triton implementation that replaces 4-5 sequential CUDA kernels with 1-2 kernels: - 1. cache_seqlens = seq_lens + seq_len_delta (int64->int32 cast) - 2. cu_seqlens_k = cumsum(cache_seqlens) (prefix-sum) - 3. page_indices = req_to_token[pool_idx, stride_idx] (2-D gather) - 4. page_table = page_indices // page_size (floor-divide) - 5. (optional) swa_page_table for sliding window attention + """Fill sequence lengths, their prefix sum and page tables in one launch. - Unified pool (``skip_page_table=True``): the translator has already filled - the page tables in place, so only steps 1-2 run. + ``req_to_token`` contains token slots. Static SWA uses + ``full_to_swa_index_mapping`` to map FULL token slots to SWA token slots. + Runtime callers resolve this tensor from their pool before calling. - Achieves ~5.2x speedup on H200 hardware for typical decode workloads. - - Contract: only the live prefix (cdiv(cache_seqlens, page_size) pages) of each - page_table / swa_page_table row is (re)written; the tail keeps stale values - across CUDA-graph replays, so consumers must bound reads by cache_seqlens. + Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten; + consumers must bound reads by ``cache_seqlens_int32``. For pretranslated + tables, ``skip_page_table=True`` leaves both page tables untouched and + computes only lengths and the prefix sum. """ assert page_size > 0 and (page_size & (page_size - 1)) == 0, ( f"page_size must be a power of two, got {page_size}" ) + if swa_page_table is not None and not skip_page_table: + assert full_to_swa_index_mapping is not None, ( + "SWA output requires a token-slot mapping" + ) batch_size = cache_seqlens_int32.shape[0] - device = seq_lens.device - - # Ensure contiguous memory layout for efficient Triton access - seq_lens = seq_lens.contiguous() - req_to_token = req_to_token.contiguous() - req_pool_indices = req_pool_indices.contiguous() - - # Prepare tensor strides - seq_lens_stride_0 = seq_lens.stride(0) - req_to_token_stride_0 = req_to_token.stride(0) - req_to_token_stride_1 = req_to_token.stride(1) - req_pool_indices_stride_0 = req_pool_indices.stride(0) - cache_seqlens_int32_stride_0 = cache_seqlens_int32.stride(0) - cu_seqlens_k_stride_0 = cu_seqlens_k.stride(0) - page_table_stride_0 = page_table.stride(0) - page_table_stride_1 = page_table.stride(1) - - if skip_page_table: - # One block does the prefix sum, so one block is the whole grid. - _fused_metadata_kernel_ps1_no_swa[(1, 1)]( - seq_lens, - seq_lens_stride_0, - page_table, - page_table_stride_0, - page_table_stride_1, - req_pool_indices, - req_pool_indices_stride_0, - cache_seqlens_int32, - cache_seqlens_int32_stride_0, - cu_seqlens_k, - cu_seqlens_k_stride_0, - page_table, - page_table_stride_0, - page_table_stride_1, - batch_size, - 0, - seq_len_delta, - BLOCK_COLS=256, - SKIP_PAGE_TABLE=1, - num_warps=8, - num_stages=3, - ) - return - - use_swa = swa_page_table is not None and token_to_kv_pool is not None - - # Unified SWA uses an independent SWA v2p table. - swa_v2p_page_table = None - if use_swa and token_to_kv_pool.full_to_swa_index_mapping is None: - from sglang.srt.mem_cache.unified_memory_pool import UnifiedSWAKVPool - - assert isinstance(token_to_kv_pool, UnifiedSWAKVPool) - assert token_to_kv_pool._swa_allocator is not None - swa_v2p_page_table = token_to_kv_pool._swa_allocator.virtual_to_physical - - # Check if we should use the specialized fast path for page_size=1, no SWA - swa_uses_v2p = swa_v2p_page_table is not None - - if page_size == 1 and not use_swa: - # Specialized kernel for the common case (page_size=1, no SWA) - BLOCK_COLS = 256 - if max_seq_pages == 0: - grid = (1, 1) - else: - num_blocks_j = triton.cdiv(max_seq_pages, BLOCK_COLS) - grid = (batch_size, num_blocks_j) - - _fused_metadata_kernel_ps1_no_swa[grid]( - seq_lens, - seq_lens_stride_0, - req_to_token, - req_to_token_stride_0, - req_to_token_stride_1, - req_pool_indices, - req_pool_indices_stride_0, - cache_seqlens_int32, - cache_seqlens_int32_stride_0, - cu_seqlens_k, - cu_seqlens_k_stride_0, - page_table, - page_table_stride_0, - page_table_stride_1, - batch_size, - max_seq_pages, - seq_len_delta, - BLOCK_COLS=BLOCK_COLS, - num_warps=8, - num_stages=3, - ) - else: - # General kernel for page_size > 1 or SWA cases - # SWA parameters - if use_swa: - from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool - - swa_page_table = swa_page_table.contiguous() - swa_page_table_stride_0 = swa_page_table.stride(0) - swa_page_table_stride_1 = swa_page_table.stride(1) - if swa_uses_v2p: - full_to_swa_mapping = swa_v2p_page_table.contiguous() - full_to_swa_mapping_stride_0 = 0 - else: - assert isinstance(token_to_kv_pool, SWAKVPool) - full_to_swa_mapping = ( - token_to_kv_pool.full_to_swa_index_mapping.contiguous() - ) - full_to_swa_mapping_stride_0 = full_to_swa_mapping.stride(0) - else: - # Dummy tensors (not used) - swa_page_table = torch.empty(0, dtype=torch.int32, device=device) - swa_page_table_stride_0 = 0 - swa_page_table_stride_1 = 0 - full_to_swa_mapping = torch.empty(0, dtype=torch.int32, device=device) - full_to_swa_mapping_stride_0 = 0 - - # Kernel configuration - BLOCK_COLS = 128 - shift = (page_size).bit_length() - 1 if page_size > 1 else 0 - - if max_seq_pages == 0: - grid = (1, 1) - else: - num_blocks_j = triton.cdiv(max_seq_pages, BLOCK_COLS) - grid = (batch_size, num_blocks_j) - - _fused_metadata_kernel_general[grid]( - seq_lens, - seq_lens_stride_0, - req_to_token, - req_to_token_stride_0, - req_to_token_stride_1, - req_pool_indices, - req_pool_indices_stride_0, - cache_seqlens_int32, - cache_seqlens_int32_stride_0, - cu_seqlens_k, - cu_seqlens_k_stride_0, - page_table, - page_table_stride_0, - page_table_stride_1, - swa_page_table, - swa_page_table_stride_0, - swa_page_table_stride_1, - full_to_swa_mapping, - full_to_swa_mapping_stride_0, - batch_size, - max_seq_pages, - page_size, - seq_len_delta, - use_swa, - shift, - BLOCK_COLS=BLOCK_COLS, - num_warps=4, - num_stages=3, - ) + # Lengths-only/token gathers use 256 columns; page/SWA gathers use 128. + block_cols = ( + 256 if skip_page_table or (page_size == 1 and swa_page_table is None) else 128 + ) + grid = ( + (1, 1) + if skip_page_table or max_seq_pages == 0 + else (batch_size, triton.cdiv(max_seq_pages, block_cols)) + ) + _fused_decode_metadata_kernel[grid]( + seq_lens, + seq_lens.stride(0), + req_to_token, + req_to_token.stride(0), + req_to_token.stride(1), + req_pool_indices, + req_pool_indices.stride(0), + cache_seqlens_int32, + cache_seqlens_int32.stride(0), + cu_seqlens_k, + cu_seqlens_k.stride(0), + page_table, + page_table.stride(0), + page_table.stride(1), + swa_page_table, + swa_page_table.stride(0) if swa_page_table is not None else 0, + swa_page_table.stride(1) if swa_page_table is not None else 0, + full_to_swa_index_mapping, + full_to_swa_index_mapping.stride(0) + if full_to_swa_index_mapping is not None + else 0, + batch_size, + max_seq_pages, + PAGE_SIZE=page_size, + SEQ_LEN_DELTA=seq_len_delta, + BLOCK_COLS=block_cols, + SKIP_PAGE_TABLE=skip_page_table, + num_warps=block_cols // 32, + num_stages=3, + ) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index abb0a8cf4a..4f6ed47abf 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -2926,7 +2926,11 @@ class FlashAttentionBackend(AttentionBackend): seq_len_delta, self.page_size, metadata.swa_page_table, - self.token_to_kv_pool if self.use_sliding_window_kv_pool else None, + full_to_swa_index_mapping=( + self.token_to_kv_pool.full_to_swa_index_mapping + if self.use_sliding_window_kv_pool + else None + ), skip_page_table=translated, ) if translated: diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index c4c99ff554..bcf5a1d7a4 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -1786,7 +1786,9 @@ class XPUAttentionBackend(AttentionBackend): seq_len_delta=seq_len_delta, page_size=self.page_size, swa_page_table=metadata.swa_page_table if use_swa else None, - token_to_kv_pool=self.token_to_kv_pool if use_swa else None, + full_to_swa_index_mapping=( + self.token_to_kv_pool.full_to_swa_index_mapping if use_swa else None + ), ) def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch): diff --git a/test/registered/attention/test_normal_decode_set_metadata.py b/test/registered/attention/test_normal_decode_set_metadata.py index 8cbf512ee6..3e73660aa8 100644 --- a/test/registered/attention/test_normal_decode_set_metadata.py +++ b/test/registered/attention/test_normal_decode_set_metadata.py @@ -13,10 +13,7 @@ import unittest import torch -from sglang.srt.layers.attention.flashattention_backend import ( - normal_decode_set_metadata, -) -from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool +from sglang.kernels.ops.attention.metadata import normal_decode_set_metadata from sglang.srt.utils import get_device from sglang.test.ci.ci_register import ( register_amd_ci, @@ -26,7 +23,7 @@ from sglang.test.ci.ci_register import ( from sglang.test.test_utils import CustomTestCase # Register this test for CUDA CI in base-b (fast attention/kernel tests) -register_cuda_ci(est_time=11, stage="base-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=15, stage="base-b", runner_config="1-gpu-large") register_amd_ci(est_time=17, suite="stage-b-test-1-gpu-large-amd") # The XPU attention backend calls the same fused kernel (Triton XPU backend). register_xpu_ci(est_time=15, suite="stage-b-test-1-gpu-xpu") @@ -43,7 +40,7 @@ def reference_normal_decode_set_metadata( seq_len_delta: int, page_size: int, swa_page_table: torch.Tensor = None, - token_to_kv_pool=None, + full_to_swa_index_mapping=None, ): """ Reference implementation using original PyTorch operations. @@ -62,8 +59,8 @@ def reference_normal_decode_set_metadata( ] page_table[:, :max_seq_pages].copy_(page_indices // page_size) - if swa_page_table is not None and token_to_kv_pool is not None: - swa_page_indices = token_to_kv_pool.translate_loc_from_full_to_swa(page_indices) + if swa_page_table is not None and full_to_swa_index_mapping is not None: + swa_page_indices = full_to_swa_index_mapping[page_indices] swa_page_table[:, :max_seq_pages].copy_(swa_page_indices // page_size) @@ -151,13 +148,12 @@ class TestNormalDecodeSetMetadata(CustomTestCase): # SWA setup if needed swa_page_table = None - token_to_kv_pool = None + full_to_swa_index_mapping = None if has_swa: swa_page_table = torch.zeros( (batch_size, max_seq_pages + 10), dtype=torch.int32, device=self.device ) - # Create a simple SWA KV pool for testing - token_to_kv_pool = self._create_swa_kv_pool(10000, page_size) + full_to_swa_index_mapping = torch.randperm(10000, device=self.device).int() return { "cache_seqlens_int32": cache_seqlens_int32, @@ -171,33 +167,9 @@ class TestNormalDecodeSetMetadata(CustomTestCase): "seq_len_delta": seq_len_delta, "page_size": page_size, "swa_page_table": swa_page_table, - "token_to_kv_pool": token_to_kv_pool, + "full_to_swa_index_mapping": full_to_swa_index_mapping, } - def _create_swa_kv_pool(self, size: int, page_size: int): - """Create a mock SWA KV pool for testing that inherits from SWAKVPool.""" - - # Create a minimal mock that inherits from SWAKVPool to pass isinstance check - class MinimalSWAKVPool(SWAKVPool): - def __init__(self, size, device): - # Don't call super().__init__() to avoid complex initialization - # Just set the minimal attributes needed for the test - self.full_to_swa_index_mapping = torch.arange( - size, dtype=torch.int32, device=device - ) - # Add some randomness to simulate real SWA mapping - self.full_to_swa_index_mapping = ( - self.full_to_swa_index_mapping - + torch.randint(0, 100, (size,), device=device) - ) % size - self.device = device - - def translate_loc_from_full_to_swa(self, page_indices): - """Mock translation method.""" - return self.full_to_swa_index_mapping[page_indices] - - return MinimalSWAKVPool(size, self.device) - def _run_test( self, batch_size: int, @@ -232,7 +204,7 @@ class TestNormalDecodeSetMetadata(CustomTestCase): test_data["seq_len_delta"], test_data["page_size"], ref_data["swa_page_table"], - test_data["token_to_kv_pool"], + test_data["full_to_swa_index_mapping"], ) # Run fused Triton implementation @@ -247,7 +219,7 @@ class TestNormalDecodeSetMetadata(CustomTestCase): test_data["seq_len_delta"], test_data["page_size"], test_data["swa_page_table"], - test_data["token_to_kv_pool"], + test_data["full_to_swa_index_mapping"], ) # Compare results @@ -286,7 +258,7 @@ class TestNormalDecodeSetMetadata(CustomTestCase): f"swa_page_table mismatch at bs={batch_size}, page_size={page_size}", ) - # Test cases for page_size=1 (uses specialized kernel _fused_metadata_kernel_ps1_no_swa) + # Test cases for page_size=1 def test_page_size_1_small_batch(self): """Test with page_size=1, small batch.""" self._run_test(batch_size=2, max_seq_len=128, page_size=1, has_swa=False) @@ -305,7 +277,7 @@ class TestNormalDecodeSetMetadata(CustomTestCase): batch_size=8, max_seq_len=200, page_size=1, has_swa=False, seq_len_delta=5 ) - # Test cases for page_size > 1 (uses general kernel _fused_metadata_kernel_general) + # Test cases for page_size > 1 def test_page_size_16_small_batch(self): """Test with page_size=16, small batch.""" self._run_test(batch_size=4, max_seq_len=256, page_size=16, has_swa=False) @@ -368,7 +340,7 @@ class TestNormalDecodeSetMetadata(CustomTestCase): test_data["seq_len_delta"], test_data["page_size"], test_data["swa_page_table"], - test_data["token_to_kv_pool"], + test_data["full_to_swa_index_mapping"], ) # Verify no crashes and basic properties diff --git a/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py b/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py new file mode 100644 index 0000000000..bb358dbe2e --- /dev/null +++ b/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py @@ -0,0 +1,114 @@ +"""Decode metadata must preserve strided storage and untouched row tails.""" + +import itertools +import unittest + +import torch + +from sglang.kernels.ops.attention.metadata import normal_decode_set_metadata +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestDecodeMetadataMapping(CustomTestCase): + def test_static_mapping_and_strides(self): + for ps, use_swa in itertools.product((1, 128), (False, True)): + with self.subTest(ps=ps, use_swa=use_swa): + self._check_tables(ps, use_swa) + + def _check_tables(self, ps, use_swa): + device = "cuda" + bs, width, delta, sentinel = 3, 300, 3, -17 + # Use strided inputs/outputs and mappings: graph replay must read the + # original storage, without materializing a contiguous mapping copy. + virtual_pages = torch.arange(bs * width, device=device).view(bs, width) % 31 + tokens = ( + virtual_pages[:, :, None] * ps + torch.arange(ps, device=device) + ).reshape(bs, width * ps) + req_to_token = torch.stack((tokens, tokens), dim=-1)[:, :, 0] + rows = torch.tensor([1, 9, 2, 9, 0, 9], device=device)[::2] + lengths = torch.tensor([2 * ps + 1, 1, 0], device=device, dtype=torch.int32) + seq_lens = torch.stack((lengths - delta, lengths), dim=1)[:, 0] + slots = torch.arange(32 * ps, device=device) + token_map = ((slots // ps * 3 + 5) % 31) * ps + slots % ps + token_map = torch.stack((token_map, token_map), dim=1)[:, 0] + token_map = token_map if use_swa else None + full_storage = torch.full( + (bs, 2 * (width + 2)), sentinel, device=device, dtype=torch.int32 + ) + swa_storage = torch.full_like(full_storage, sentinel) + page_table = full_storage[:, ::2] + swa_table = swa_storage[:, ::2] if use_swa else None + cache_lens = torch.empty_like(lengths) + cu_lens = torch.empty(bs + 1, device=device, dtype=torch.int32) + + normal_decode_set_metadata( + cache_lens, + cu_lens, + page_table, + req_to_token, + rows, + width, + seq_lens, + delta, + ps, + swa_page_table=swa_table, + full_to_swa_index_mapping=token_map, + ) + + torch.testing.assert_close(cache_lens, lengths) + torch.testing.assert_close(cu_lens[1:], lengths.cumsum(0).int()) + self.assertEqual(cu_lens[0].item(), 0) + gathered = tokens[rows, ::ps] + live = ( + torch.arange(width, device=device)[None, :] + < ((lengths + ps - 1) // ps)[:, None] + ) + expected_full = gathered // ps + expected_swa = token_map[gathered] // ps if use_swa else None + for storage, expected in ( + (full_storage, expected_full), + (swa_storage, expected_swa), + ): + if expected is None: + continue + want = torch.full_like(storage, sentinel) + want[:, : 2 * width : 2] = torch.where(live, expected, sentinel) + torch.testing.assert_close(storage, want) + + def test_lengths_only_leaves_tables_untouched(self): + for ps, skip in itertools.product((1, 128), (False, True)): + with self.subTest(ps=ps, skip=skip): + seq_lens = torch.tensor([0, 9, 130], device="cuda") + cache_lens = torch.empty(3, device="cuda", dtype=torch.int32) + cu_lens = torch.empty(4, device="cuda", dtype=torch.int32) + table = torch.full((3, 4), 23, device="cuda", dtype=torch.int32) + swa_table = table.clone() + normal_decode_set_metadata( + cache_lens, + cu_lens, + table, + torch.zeros((3, 4 * ps), device="cuda", dtype=torch.int64), + torch.arange(3, device="cuda"), + 4 if skip else 0, + seq_lens, + 3, + ps, + swa_page_table=swa_table, + full_to_swa_index_mapping=torch.arange(4 * ps, device="cuda"), + skip_page_table=skip, + ) + torch.testing.assert_close(cache_lens, (seq_lens + 3).int()) + torch.testing.assert_close( + cu_lens, + torch.tensor([0, 3, 15, 148], device="cuda", dtype=torch.int32), + ) + torch.testing.assert_close(table, torch.full_like(table, 23)) + torch.testing.assert_close(swa_table, table) + + +if __name__ == "__main__": + unittest.main()