diff --git a/python/sglang/kernels/ops/kvcache/zero_pages.py b/python/sglang/kernels/ops/kvcache/zero_pages.py index afa3c00710..2c456b2302 100644 --- a/python/sglang/kernels/ops/kvcache/zero_pages.py +++ b/python/sglang/kernels/ops/kvcache/zero_pages.py @@ -1,8 +1,8 @@ """Zero whole page envelopes of the unified pool by physical page id. -The pool is viewed as int64 words (the MLA page envelope is always -8-byte-aligned: entry bytes per layer = kv_cache_dim * itemsize, a multiple -of 8), one wide element per lane; grid = (num_pages, page word blocks). +The pool is viewed as int64 words (MHA and MLA page envelopes are +8-byte-aligned), one wide element per lane; grid = (num_pages, page word +blocks). """ from __future__ import annotations @@ -41,6 +41,9 @@ def zero_pages( if m == 0: return assert raw.dtype == torch.uint8, f"expected uint8 pool, got {raw.dtype}" + if not raw.is_cuda: + raw[: num_pages * page_bytes].view(num_pages, page_bytes)[pages] = 0 + return assert page_bytes % 8 == 0, f"page_bytes {page_bytes} not int64-aligned" page_words = page_bytes // 8 words = raw[: num_pages * page_bytes].view(torch.int64) diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index e20817a4e3..85ab055ee4 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -491,7 +491,7 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset( # The MHA backends that read through the KV-index translator on every path a # draft forward takes. A fused draft region holds dense K/V rows, so a fused # draft runs only on these. -TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3"}) +TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"}) def _assert_spec_verify_backends( @@ -707,18 +707,6 @@ def handle_unified_memory_pool(server_args: Any) -> None: sorted(full_cg_backends), sorted(backends), ) - # trtllm_mha refills its graph page table before replay from - # `cache_seqlens_int32`, which only the in-graph metadata kernel writes, so - # it uses the previous replay's lengths. Refuse until it uses the batch's. - _, decode_backend = attention_backends_of(resolved_view(server_args)) - if decode_backend == "trtllm_mha": - assert _cg_cfg is None or _cg_cfg.decode.backend == Backend.DISABLED, ( - "--enable-unified-memory does not yet support decode cuda graphs " - "with the trtllm_mha attention backend: its replay refills the page " - "table from the previous replay's sequence lengths. Pass " - "--disable-cuda-graph, or pick another decode attention backend " - "(fa3 / fa4 / flashinfer / triton)." - ) def _validate_unified_memory_dcp(server_args: Any) -> None: diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index 630cab92f6..f5b37798ad 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -78,6 +78,9 @@ if TYPE_CHECKING: # Default workspace size in MB for TRTLLM MHA # Can be configured via SGLANG_FLASHINFER_WORKSPACE_SIZE environment variable DEFAULT_WORKSPACE_SIZE_MB = 512 +# flashinfer's XQA decode keeps its multi-block semaphores in the first 8 MiB +# of the workspace and needs them zero at every launch. +_XQA_SEMAPHORE_BYTES = 8 * 1024 * 1024 # Reuse this workspace buffer across all TRTLLM MHA wrappers @@ -161,6 +164,11 @@ class TRTLLMMHAMetadata: # Prefix before trailing one-token requests in an extend-shaped batch. mixed_prefill_reqs: int = 0 mixed_prefill_tokens: int = 0 + # fmha_v2 prefill: the (row, column) of each page-table entry the batch + # reads, in copy order, and the block table over that copy. + fmha_v2_copy_rows: torch.Tensor = None + fmha_v2_copy_cols: torch.Tensor = None + fmha_v2_block_table: torch.Tensor = None class TRTLLMHAAttnBackend(FlashInferAttnBackend): @@ -295,15 +303,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): # Workspace allocation self.workspace_size = workspace_size_bytes + semaphore_bytes = _XQA_SEMAPHORE_BYTES if self.is_xqa_impl else 0 # Allocate buffers self.workspace_buffer = get_buffer( "trtllm_mha_zero_workspace", lambda: torch.zeros( - self.workspace_size, + semaphore_bytes + self.workspace_size, dtype=torch.uint8, device=model_runner.device, ), ) + self.fmha_v2_workspace_buffer = self.workspace_buffer[semaphore_bytes:] # CUDA graph state self.decode_cuda_graph_metadata = {} @@ -1282,13 +1292,14 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): ) else: metadata.cu_seqlens_q = metadata.cu_seqlens_k + if self.use_fmha_v2 and not forward_batch.forward_mode.is_draft_extend_v2(): + self._init_fmha_v2_batch_pages(metadata, forward_batch) kv_view = self.kv_index_translator.read_table( forward_batch.kv_loc_plan, rows=forward_batch.batch_size ) if kv_view.is_translated: - # No fill kernel: the kernels take the tensor's own width/stride - # and bound their reads by cache_seqlens. + # No fill kernel: the kernels bound their reads by cache_seqlens. metadata.page_table = kv_view.ids metadata.swa_page_table = ( self.kv_index_translator.read_table( @@ -1300,6 +1311,15 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): is not None else None ) + # XQA and fmha_v2 take a page table's row length from its width and + # ignore its strides, so a view into a wider table would be misread. + assert metadata.page_table.is_contiguous() and ( + metadata.swa_page_table is None + or metadata.swa_page_table.is_contiguous() + ), ( + "trtllm_mha reads its page tables as packed rows, but the plan's " + "table is a strided view of a wider buffer" + ) else: has_swa = self._swa_kv_pool is not None metadata.page_table = torch.empty( @@ -1334,6 +1354,63 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): self.forward_metadata = metadata + def _init_fmha_v2_batch_pages( + self, metadata: TRTLLMMHAMetadata, forward_batch: ForwardBatch + ) -> None: + """Index the pages this batch reads, for the fmha_v2 prefill copy. + + Sized by the host seq_lens, which are live only alongside seq_lens_sum + (a GPU-only batch keeps a stale seq_lens_cpu); without them the prefill + copies the whole pool. + """ + seq_lens_cpu = forward_batch.seq_lens_cpu + if ( + forward_batch.seq_lens_sum is None + or seq_lens_cpu is None + or seq_lens_cpu.numel() == 0 + ): + return + host_pages = (seq_lens_cpu + self.page_size - 1) // self.page_size + total_pages = int(host_pages.sum()) + if total_pages == 0: + return + num_pages = ( + (metadata.cache_seqlens_int32 + self.page_size - 1) // self.page_size + ).to(torch.int64) + device = num_pages.device + # Row r's pages land in copy rows [first[r], first[r] + num_pages[r]). + first = torch.cumsum(num_pages, 0) - num_pages + rows = torch.repeat_interleave(num_pages, output_size=total_pages) + metadata.fmha_v2_copy_rows = rows + metadata.fmha_v2_copy_cols = ( + torch.arange(total_pages, device=device) - first[rows] + ) + # Past a row's last page, entries only need to stay in range. + metadata.fmha_v2_block_table = ( + (first[:, None] + torch.arange(int(host_pages.max()), device=device)) + .clamp_(max=total_pages - 1) + .to(torch.int32) + ) + + def _fmha_v2_paged_kv( + self, k_cache: torch.Tensor, v_cache: torch.Tensor, page_table: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """K and V as the [pages, 2, page_size, heads, head_dim] tensor fmha_v2 + takes, and the block table that addresses it. + + fmha_v2 derives strides from shapes, so this is a copy: of the batch's + pages only, unless they are no fewer than the pool's. + """ + metadata = self.forward_metadata + rows = metadata.fmha_v2_copy_rows + if rows is None or rows.numel() >= k_cache.shape[0]: + return torch.stack([k_cache, v_cache], dim=1), page_table + pages = page_table[rows, metadata.fmha_v2_copy_cols] + paged_kv = k_cache.new_empty((rows.numel(), 2, *k_cache.shape[1:])) + torch.index_select(k_cache, 0, pages, out=paged_kv[:, 0]) + torch.index_select(v_cache, 0, pages, out=paged_kv[:, 1]) + return paged_kv, metadata.fmha_v2_block_table + def _reshape_paged_kv_cache( self, k_cache: torch.Tensor, @@ -1760,12 +1837,14 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): elif self.use_fmha_v2 and not cp_active: # CP must go through cp_strategy.run_attention (per-shard # masking); the plain-causal fmha_v2 call below would be wrong. - paged_kv = torch.stack([k_cache, v_cache], dim=1) + paged_kv, block_tables = self._fmha_v2_paged_kv( + k_cache, v_cache, page_table + ) out = forward_batch._attn_output o = flashinfer.prefill.trtllm_fmha_v2_prefill( (q, paged_kv), input_layout="Q_PAGED_KV_NHD", - workspace_buffer=self.workspace_buffer, + workspace_buffer=self.fmha_v2_workspace_buffer, seq_lens=self.forward_metadata.cache_seqlens_int32, max_q_len=self.forward_metadata.max_seq_len_q, max_kv_len=self.max_context_len, @@ -1774,7 +1853,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): batch_size=forward_batch.batch_size, cum_seq_lens_q=self.forward_metadata.cu_seqlens_q, cum_seq_lens_kv=self.forward_metadata.cu_seqlens_k, - block_tables=page_table, + block_tables=block_tables, out=None if out is None else out.view_as(q), out_dtype=self.q_data_type, mask_mode=( diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 334965d7f7..7e564964c2 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -55,6 +55,7 @@ from sglang.srt.mem_cache.allocator.paged import ( ) from sglang.srt.mem_cache.unified_memory_pool import ( UnifiedKVPool, + UnifiedMHATokenToKVPool, UnifiedMLATokenToKVPool, ) from sglang.srt.runtime_context import get_parallel @@ -327,7 +328,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self.min_slot_index = unified_buffer.min_slot_index(sub_pool_name) self.is_id_owner = is_id_owner # Zero page envelopes on hand-out -- see _maybe_zero_pages. - self._zero_pages_on_alloc = isinstance(kvcache, UnifiedMLATokenToKVPool) + self._zero_pages_on_alloc = isinstance( + kvcache, (UnifiedMHATokenToKVPool, UnifiedMLATokenToKVPool) + ) # Overlap mode: `free` drops a wait_stream(forward_stream) barrier so its # v2p writes + move kernel serialize after the in-flight forward. self.forward_stream = forward_stream @@ -1003,9 +1006,11 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): return phys_pages def _maybe_zero_pages(self, phys_pages: torch.Tensor) -> None: - """Zero the page ENVELOPES on hand-out (MLA full pool only): the MLA - kernels arithmetically mask the rows beyond seq_len, so never-written page - bytes must read as finite values. + """Zero the page ENVELOPES on hand-out (MHA and MLA pools): the MLA + kernels and trtllm_mha's fmha_v2 prefill (and fp8 XQA decode) load whole + pages and give the rows past seq_len a zero weight, so a NaN there still + reaches the output. Never-written bytes must read as finite values; a + mamba neighbour's fp32 state, read as bf16, may not. """ if not self._zero_pages_on_alloc or phys_pages.numel() == 0: return @@ -2351,6 +2356,7 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): if phys_pages is None: return None self.bind(v_pages, phys_pages) + self._maybe_zero_pages(phys_pages) return phys_pages # -- free: hole-marking, boundary absorption, park-on-empty -- diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 874137136e..8859cbbbdb 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -1065,9 +1065,7 @@ class KVCacheConfigurator: {draft_backend} if draft_backend else set(attention_backends()) - {None} ) # A fused draft reads its rows through the KV-index translator, which - # other backends miss on some draft path: trtllm_mha's graph replay - # refills its page table from stale lengths, and its eager build does - # not widen the table by the draft block. + # other backends miss on some draft path. if not draft_backends <= TRANSLATED_MHA_RAILS: return FusedDraftDecision( declined=( @@ -1203,7 +1201,7 @@ class KVCacheConfigurator: would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that does not fuse is refused. A DSPARK draft that does not fuse keeps its private pool, unpriced, so DSPARK still boots where its draft declines - fusion (as the Kimi-Linear default, a trtllm_mha draft, does).""" + fusion.""" decision = self._fused_draft_decision() placement = self._fused_draft_for_pool_factory(decision) if ( diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 4039c4b341..ab34887220 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -456,6 +456,7 @@ class UnifiedKVPool: # 0 (up to page_size * entry_bytes), not one slot entry -- reserve the # max of both. reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size) + self._reserved_floor = reserved_floor with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE): if post_capture_active: @@ -566,6 +567,7 @@ class UnifiedKVPool: # Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes # fail deterministically. raw.view(torch.int16).fill_(0x7FC1) + raw[: self._reserved_floor].zero_() logger.warning( "[unified-memory-pool] POISONED: backed pool bytes filled with " "bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)" @@ -840,6 +842,17 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool): self._page_bytes, ) + def zero_physical_pages(self, phys_pages: torch.Tensor) -> None: + """Zero whole page envelopes (PHYSICAL page ids) on allocator + hand-out.""" + assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0 + zero_pages( + self._unified_buffer._raw, + phys_pages, + self._num_pages, + self._page_bytes, + ) + def get_contiguous_buf_infos(self): """Register the raw buffer as physical page envelopes for PD transfer. diff --git a/test/registered/attention/test_trtllm_mha_graph_metadata.py b/test/registered/attention/test_trtllm_mha_graph_metadata.py index c4bd867935..579b999166 100644 --- a/test/registered/attention/test_trtllm_mha_graph_metadata.py +++ b/test/registered/attention/test_trtllm_mha_graph_metadata.py @@ -6,10 +6,14 @@ cu_seqlens_q (all 3 q-modes) / page_table / swa_page_table / swa_out_cache_loc, with the SWA -1 sentinel guard. """ +import contextlib +import weakref from types import SimpleNamespace import pytest import torch +from torch.utils import _pytree as pytree +from torch.utils._python_dispatch import TorchDispatchMode import sglang.srt.layers.attention.trtllm_mha_backend as trtllm_mha_backend from sglang.kernels.ops.kvcache.trtllm_mha_graph_metadata import ( @@ -18,6 +22,7 @@ from sglang.kernels.ops.kvcache.trtllm_mha_graph_metadata import ( Q_MODE_STRIDED, update_trtllm_mha_graph_metadata, ) +from sglang.srt.environ import envs from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cuda_ci @@ -202,6 +207,249 @@ def test_decode_mode_query_dtype_with_fp8_kv( assert query_dtypes == [expected_q_dtype] +@pytest.mark.parametrize( + "explicit_bytes", [None, 8 * 1024 * 1024], ids=["default", "explicit_8MiB"] +) +def test_fmha_v2_prefill_workspace_skips_xqa_semaphores( + make_xqa_backend, monkeypatch, explicit_bytes +): + # XQA decode keeps its multi-block semaphores in the first 8 MiB of the + # workspace and needs them zero at launch; fmha_v2 prefill, sharing the + # buffer, leaves its tile counters at the base of what it is handed. + monkeypatch.setattr(trtllm_mha_backend, "DEFAULT_WORKSPACE_SIZE_MB", 16) + configured = explicit_bytes or 16 * 1024 * 1024 + with ( + envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.override(explicit_bytes) + if explicit_bytes is not None + else contextlib.nullcontext() + ): + backend = make_xqa_backend(None) + heads, head_dim, bs = 2, 4, 2 + seq_lens = torch.tensor([30, 10], dtype=torch.int32) + cu_seqlens = torch.tensor([0, 30, 40], dtype=torch.int32) + backend.forward_metadata = trtllm_mha_backend.TRTLLMMHAMetadata( + cache_seqlens_int32=seq_lens, + max_seq_len_q=30, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + page_table=torch.arange(bs, dtype=torch.int32)[:, None], + ) + kv = torch.zeros(bs * backend.page_size, heads, head_dim, dtype=torch.bfloat16) + backend.token_to_kv_pool = SimpleNamespace(get_kv_buffer=lambda _: (kv, kv)) + monkeypatch.setattr(trtllm_mha_backend, "is_cp_active", lambda _: False) + workspaces = {} + + def decode(**kwargs): + workspaces["xqa"] = kwargs["workspace_buffer"] + return torch.zeros_like(kwargs["query"]) + + def prefill(qkv, **kwargs): + workspaces["fmha_v2"] = kwargs["workspace_buffer"] + return torch.zeros_like(qkv[0]) + + monkeypatch.setattr( + trtllm_mha_backend, + "flashinfer", + SimpleNamespace( + decode=SimpleNamespace(trtllm_batch_decode_with_kv_cache=decode), + prefill=SimpleNamespace(trtllm_fmha_v2_prefill=prefill), + ), + raising=False, + ) + layer = SimpleNamespace( + layer_id=0, + tp_q_head_num=heads, + tp_k_head_num=heads, + tp_v_head_num=heads, + head_dim=head_dim, + scaling=1.0, + sliding_window_size=-1, + attn_type=trtllm_mha_backend.AttentionType.DECODER, + ) + q = torch.zeros(40, heads * head_dim, dtype=torch.bfloat16) + extend = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, batch_size=bs, _attn_output=None + ) + backend.forward_extend(q, None, None, layer, extend, save_kv_cache=False) + decode_batch = SimpleNamespace(forward_mode=ForwardMode.DECODE, batch_size=bs) + backend.forward_decode(q[:bs], None, None, layer, decode_batch, save_kv_cache=False) + + semaphores = workspaces["xqa"].data_ptr() + fmha_v2 = workspaces["fmha_v2"] + start = fmha_v2.data_ptr() + end = start + fmha_v2.numel() * fmha_v2.element_size() + # The semaphores come on top of the configured size, so fmha_v2 keeps all of it. + assert end - start == configured + assert end <= semaphores or start >= semaphores + 8 * 1024 * 1024, ( + f"fmha_v2 workspace [{start - semaphores}, {end - semaphores}) overlaps " + "XQA's semaphores [0, 8 MiB)" + ) + + +@pytest.mark.parametrize( + "host_lens_live,pool_pages,copied_pages", + [ + # The rows' own 3 + 1 pages, not the pool. + (True, 64, 4), + # No live host lens (seq_lens_sum is None): the whole pool. + (False, 64, 64), + # The batch's pages are no fewer than the pool's: the whole pool. + (True, 4, 4), + ], +) +def test_fmha_v2_prefill_copies_only_the_batch_pages( + make_xqa_backend, monkeypatch, host_lens_live, pool_pages, copied_pages +): + backend = make_xqa_backend(None) + page_size, heads, head_dim = backend.page_size, 2, 4 + seq_lens = torch.tensor([70, 20], dtype=torch.int32) + row_pages = [[3, 1, 0], [2]] + # Past each row's last page the table is unwritten. + page_table = torch.full((2, 4), 1 << 20, dtype=torch.int32) + for row, pages in enumerate(row_pages): + page_table[row, : len(pages)] = torch.tensor(pages) + k = torch.randn(pool_pages * page_size, heads, head_dim).to(torch.bfloat16) + v = torch.randn_like(k) + backend.token_to_kv_pool = SimpleNamespace(get_kv_buffer=lambda _: (k, v)) + + backend.kv_index_translator = SimpleNamespace( + read_table=lambda plan, rows: SimpleNamespace( + ids=page_table, is_translated=True + ), + space=lambda kind: None, + ) + backend.use_sliding_window_kv_pool = False + monkeypatch.setattr(trtllm_mha_backend, "is_cp_active", lambda _: False) + calls = [] + + def prefill(qkv, **kwargs): + calls.append((qkv[1], kwargs["block_tables"])) + return torch.zeros_like(qkv[0]) + + monkeypatch.setattr( + trtllm_mha_backend, + "flashinfer", + SimpleNamespace(prefill=SimpleNamespace(trtllm_fmha_v2_prefill=prefill)), + raising=False, + ) + fb = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + batch_size=2, + spec_info=None, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens.long(), + seq_lens_sum=int(seq_lens.sum()) if host_lens_live else None, + extend_seq_lens_cpu=seq_lens.tolist(), + extend_prefix_lens_cpu=[0, 0], + req_pool_indices=torch.arange(2), + kv_loc_plan=None, + _attn_output=None, + ) + layer = SimpleNamespace( + layer_id=0, + tp_q_head_num=heads, + tp_k_head_num=heads, + tp_v_head_num=heads, + head_dim=head_dim, + scaling=1.0, + sliding_window_size=-1, + attn_type=trtllm_mha_backend.AttentionType.DECODER, + ) + backend.init_forward_metadata(fb) + q = torch.zeros(int(seq_lens.sum()), heads * head_dim, dtype=torch.bfloat16) + backend.forward_extend(q, None, None, layer, fb, save_kv_cache=False) + + assert len(calls) == 1 + paged_kv, block_tables = calls[0] + assert paged_kv.shape == (copied_pages, 2, page_size, heads, head_dim) + if copied_pages < pool_pages: + # A compact copy's table addresses the copy, past a row's last page too. + assert int(block_tables.max()) < copied_pages + k_pages = k.view(pool_pages, page_size, heads, head_dim) + v_pages = v.view(pool_pages, page_size, heads, head_dim) + for row, pages in enumerate(row_pages): + for col, page in enumerate(pages): + block = block_tables[row, col] + assert torch.equal(paged_kv[block, 0], k_pages[page]) + assert torch.equal(paged_kv[block, 1], v_pages[page]) + + +class _LiveBytes(TorchDispatchMode): + """Peak bytes held by the tensors the dispatched ops allocate.""" + + def __init__(self): + super().__init__() + self.live = self.peak = 0 + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + given = { + t.untyped_storage().data_ptr() + for t in pytree.tree_leaves((args, kwargs)) + if isinstance(t, torch.Tensor) + } + out = func(*args, **kwargs) + for t in pytree.tree_leaves(out): + if ( + isinstance(t, torch.Tensor) + and t.untyped_storage().data_ptr() not in given + ): + nbytes = t.untyped_storage().nbytes() + self.live += nbytes + self.peak = max(self.peak, self.live) + weakref.finalize(t, self._release, nbytes) + return out + + def _release(self, nbytes): + self.live -= nbytes + + +@pytest.mark.parametrize("strided", [False, True], ids=["packed_pool", "strided_pool"]) +@pytest.mark.parametrize("pool_fraction", [0.5, 0.75, 0.97]) +def test_fmha_v2_prefill_copy_peaks_at_its_own_size( + make_xqa_backend, strided, pool_fraction +): + """Bug regression: gathering K and V apart and then stacking them held the + batch's pages twice, more than the whole-pool copy once they pass half of it.""" + backend = make_xqa_backend(None) + page_size, heads, head_dim, pool_pages = backend.page_size, 2, 16, 64 + batch_pages = int(pool_pages * pool_fraction) + row_lens = [batch_pages // 2, batch_pages - batch_pages // 2] + perm = torch.randperm(pool_pages) + row_pages = [perm[: row_lens[0]], perm[row_lens[0] : batch_pages]] + page_table = torch.full((2, max(row_lens)), 1 << 20, dtype=torch.int32) + for row, pages in enumerate(row_pages): + page_table[row, : len(pages)] = pages.to(torch.int32) + shape = (pool_pages, page_size, heads, head_dim) + if strided: + kv = torch.randn(pool_pages, 2, *shape[1:]).to(torch.bfloat16) + k, v = kv[:, 0], kv[:, 1] + else: + k = torch.randn(shape).to(torch.bfloat16) + v = torch.randn_like(k) + # A partial last page still counts whole. + seq_lens = torch.tensor([n * page_size - 3 for n in row_lens], dtype=torch.int32) + metadata = trtllm_mha_backend.TRTLLMMHAMetadata(cache_seqlens_int32=seq_lens) + backend._init_fmha_v2_batch_pages( + metadata, + SimpleNamespace(seq_lens_cpu=seq_lens.long(), seq_lens_sum=int(seq_lens.sum())), + ) + backend.forward_metadata = metadata + + with _LiveBytes() as live: + paged_kv, block_table = backend._fmha_v2_paged_kv(k, v, page_table) + + copy_bytes = paged_kv.numel() * paged_kv.element_size() + page_bytes = copy_bytes // (2 * batch_pages) + assert paged_kv.shape[0] == batch_pages + assert live.peak < copy_bytes + page_bytes + assert live.peak < 2 * k.numel() * k.element_size() + for row, pages in enumerate(row_pages): + blocks = block_table[row, : len(pages)].long() + assert torch.equal(paged_kv[blocks, 0], k[pages]) + assert torch.equal(paged_kv[blocks, 1], v[pages]) + + @pytest.mark.parametrize( "max_running_requests,max_draft_tokens,max_cuda_graph_bs,expected", [ @@ -251,6 +499,7 @@ def _make_backend_for_hook_test(speculative_num_draft_tokens=None): backend.max_num_pages = 8 backend.req_to_token = torch.zeros(4, 1024, dtype=torch.int32) backend.use_sliding_window_kv_pool = False + backend.use_fmha_v2 = False backend._swa_kv_pool = None backend._swa_full_to_swa_mapping = None backend.speculative_step_id = 0 @@ -367,6 +616,252 @@ def test_draft_extend_in_graph_uses_captured_static_q_stride(monkeypatch): assert calls[0]["q_stride"] == 4 +def _translate_through_a_moving_page_map(backend): + """Give the backend a translating pool whose virtual->physical page map the + test rewrites before every replay, so a page-table column the refill missed + still holds an earlier replay's page.""" + from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator + from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind + + rows, width = backend.req_to_token.shape + # Row r holds virtual pages r * width / PAGE_SIZE + 1, ...; page 0 is the sink. + backend.req_to_token.copy_( + torch.arange(rows * width, dtype=torch.int32).view(rows, width) + PAGE_SIZE + ) + v2p = torch.zeros(rows * width // PAGE_SIZE + 1, dtype=torch.int64) + translator = KVIndexTranslator.__new__(KVIndexTranslator) + translator.req_to_token = backend.req_to_token + translator.page_size = PAGE_SIZE + translator.device = "cpu" + translator.is_translating = True + translator.defer_read_translate = False + translator._capture_page_size = PAGE_SIZE + translator._full_v2p_table = v2p + translator._spaces = { + IdSpaceKind.FULL: IdSpace( + key=(IdSpaceKind.FULL, "test"), write=lambda ids: ids, read_v2p=v2p + ) + } + translator._rows = torch.arange(rows, dtype=torch.int64) + backend.kv_index_translator = translator + return v2p + + +def _move_every_page(v2p, replay): + v2p[1:] = torch.arange(1, v2p.numel()) + 1000 * replay + + +def _assert_reads_only_current_pages(backend, v2p, bs=2): + metadata = backend.forward_metadata + for row, length in enumerate(metadata.cache_seqlens_int32[:bs].tolist()): + pages = -(-length // PAGE_SIZE) + first_tokens = backend.req_to_token[row, : pages * PAGE_SIZE : PAGE_SIZE] + expected = v2p[first_tokens.long() // PAGE_SIZE].to(torch.int32) + stale = (metadata.page_table[row, :pages] != expected).nonzero().flatten() + assert stale.numel() == 0, ( + f"row {row} reads {length} tokens ({pages} pages), but page-table " + f"columns {stale.tolist()} hold another replay's pages" + ) + + +def _graph_batch(seq_lens, forward_mode, spec_info, tokens_per_req): + seq_lens = torch.tensor(seq_lens, dtype=torch.int32) + bs = seq_lens.numel() + return SimpleNamespace( + batch_size=bs, + req_pool_indices=torch.arange(bs, dtype=torch.int64), + seq_lens=seq_lens, + seq_lens_cpu=seq_lens.clone(), + forward_mode=forward_mode, + spec_info=spec_info, + positions=torch.arange(bs * tokens_per_req, dtype=torch.int64), + # The window it writes; the reads under test do not depend on its ids. + out_cache_loc=torch.zeros(bs * tokens_per_req, dtype=torch.int64), + ) + + +def _replay_and_check_every_read( + monkeypatch, + backend, + batch, + bind_plan, + replays=([5, 9], [300, 3], [129, 700]), +): + # The recorded kernel writes cache_seqlens = seq_lens + offset only when the + # graph runs, which is after replay-prep has refilled the page table. + def recorded_kernel(**kwargs): + bs = kwargs["bs"] + kwargs["cache_seqlens"][:bs].copy_( + kwargs["seq_lens"][:bs] + kwargs["seqlen_offset"] + ) + + monkeypatch.setattr( + trtllm_mha_backend, "update_trtllm_mha_graph_metadata", recorded_kernel + ) + v2p = _translate_through_a_moving_page_map(backend) + capture = batch([1, 1]) + bind_plan(backend.kv_index_translator, capture) + backend.init_forward_metadata_out_graph(capture, in_capture=True) + backend.init_forward_metadata_in_graph(capture) + # Rows grow across page boundaries and trade lengths between replays. + for n, seq_lens in enumerate(replays, start=1): + _move_every_page(v2p, n) + replay = batch(seq_lens) + bind_plan(backend.kv_index_translator, replay) + backend.init_forward_metadata_out_graph(replay) + backend.init_forward_metadata_in_graph(replay) + _assert_reads_only_current_pages(backend, v2p) + + +def test_unified_decode_replay_reads_only_this_replays_pages(monkeypatch): + """A decode replay's page table is the plan of its own batch: the replay + reads no page an earlier replay refilled.""" + + def batch(seq_lens): + return _graph_batch(seq_lens, ForwardMode.DECODE, None, 1) + + _replay_and_check_every_read( + monkeypatch, + _make_backend_for_hook_test(), + batch, + lambda translator, fb: translator.bind_own_plan(fb), + ) + + +def _eager_batch(seq_lens, forward_mode, spec_info=None, tokens_per_req=1): + seq_lens = torch.tensor(seq_lens, dtype=torch.int32) + bs = seq_lens.numel() + return SimpleNamespace( + batch_size=bs, + req_pool_indices=torch.arange(bs, dtype=torch.int64), + seq_lens=seq_lens, + seq_lens_cpu=seq_lens.clone(), + forward_mode=forward_mode, + spec_info=spec_info, + input_ids=torch.zeros(bs * tokens_per_req, dtype=torch.int64), + out_cache_loc=torch.zeros(bs * tokens_per_req, dtype=torch.int64), + extend_seq_lens=seq_lens, + extend_seq_lens_cpu=seq_lens.tolist(), + extend_prefix_lens_cpu=[0] * bs, + ) + + +def test_unified_eager_table_must_be_packed(): + """XQA and fmha_v2 index a page table as packed rows (row length = width, + strides ignored). A plan whose table an earlier captured reader had built in + its own wider buffer hands a graphs-off reader a strided view, which they + would misread for every row after the first.""" + backend = _make_backend_for_hook_test() + v2p = _translate_through_a_moving_page_map(backend) + _move_every_page(v2p, 1) + translator = backend.kv_index_translator + + forward_batch = _eager_batch([127, 300], ForwardMode.DECODE) + translator.bind_own_plan(forward_batch) + backend.init_forward_metadata(forward_batch) + assert backend.forward_metadata.page_table.is_contiguous() + _assert_reads_only_current_pages(backend, v2p) + + forward_batch = _eager_batch([127, 300], ForwardMode.DECODE) + translator.bind_own_plan(forward_batch) + captured = torch.zeros((8, backend.max_num_pages), dtype=torch.int32) + translator.copy_page_table(forward_batch.kv_loc_plan, out=captured[:2]) + with pytest.raises(AssertionError, match="packed rows"): + backend.init_forward_metadata(forward_batch) + + +def _bind_iteration_plan(translator, forward_batch, window=4): + """The plan a speculative worker builds once per iteration: its reads reach + the whole draft window past the committed lengths.""" + translator.plan( + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=forward_batch.seq_lens, + seq_lens_cpu=forward_batch.seq_lens_cpu, + write_virtual=forward_batch.out_cache_loc, + read_extent=window, + ).bind(forward_batch, translator) + + +@pytest.mark.parametrize( + "forward_mode,spec_info,step_id,tokens_per_req", + [ + (ForwardMode.DECODE, SimpleNamespace(), 1, 1), + (ForwardMode.TARGET_VERIFY, SimpleNamespace(ragged_verify_layout=None), 0, 4), + (ForwardMode.DRAFT_EXTEND_V2, SimpleNamespace(num_tokens_per_req=4), 0, 4), + ], + ids=["draft_decode", "target_verify", "draft_extend"], +) +def test_unified_spec_replay_reads_only_this_replays_pages( + monkeypatch, forward_mode, spec_info, step_id, tokens_per_req +): + """A speculative replay reads past seq_lens: a draft step reads back its + earlier steps, a verify the drafts it scores.""" + backend = _make_backend_for_hook_test(speculative_num_draft_tokens=4) + backend.speculative_step_id = step_id + + def batch(seq_lens): + return _graph_batch(seq_lens, forward_mode, spec_info, tokens_per_req) + + # 127 committed tokens plus the drafts reach a page the prefix does not. + _replay_and_check_every_read( + monkeypatch, + backend, + batch, + _bind_iteration_plan, + replays=([5, 9], [127, 300], [129, 700]), + ) + + +def test_unified_ragged_verify_reads_only_this_replays_pages(monkeypatch): + # Ragged verify writes per-row lengths before the refill; its geometry needs + # GPU kernels, so this stand-in writes the same lengths the real one does. + verify_lens = torch.tensor([1, 7], dtype=torch.int32) + + def write_ragged(metadata, forward_batch, ragged_layout, bs, in_capture=False): + metadata.cache_seqlens_int32[:bs].copy_( + forward_batch.seq_lens[:bs] + verify_lens + ) + + backend = _make_backend_for_hook_test(speculative_num_draft_tokens=4) + backend.is_xqa_impl = False + monkeypatch.setattr(backend, "_write_ragged_verify_graph_metadata", write_ragged) + spec_info = SimpleNamespace(ragged_verify_layout=object()) + + def batch(seq_lens): + return _graph_batch(seq_lens, ForwardMode.TARGET_VERIFY, spec_info, 8) + + # 123 + 7 crosses a page boundary that 123 plus the uniform draft width does not. + _replay_and_check_every_read( + monkeypatch, + backend, + batch, + lambda translator, fb: _bind_iteration_plan(translator, fb, window=8), + replays=([5, 123], [300, 3]), + ) + + +@pytest.mark.parametrize( + "forward_mode,spec_info,step_id,tokens_per_req", + [ + (ForwardMode.DECODE, SimpleNamespace(), 1, 1), + (ForwardMode.TARGET_VERIFY, SimpleNamespace(ragged_verify_layout=None), 0, 4), + ], + ids=["draft_decode", "target_verify"], +) +def test_unified_eager_spec_table_covers_every_page_read( + forward_mode, spec_info, step_id, tokens_per_req +): + backend = _make_backend_for_hook_test(speculative_num_draft_tokens=4) + backend.speculative_step_id = step_id + v2p = _translate_through_a_moving_page_map(backend) + _move_every_page(v2p, 1) + # 127 committed tokens plus the drafts reach a page the prefix does not. + forward_batch = _eager_batch([127, 3], forward_mode, spec_info, tokens_per_req) + _bind_iteration_plan(backend.kv_index_translator, forward_batch) + backend.init_forward_metadata(forward_batch) + _assert_reads_only_current_pages(backend, v2p) + + def test_hybrid_wrappers_forward_in_graph_hook(): # The hybrid backend reads the mode from the published configuration. from sglang.srt.runtime_context import get_context diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py index a8f089038e..cc1abf68a0 100644 --- a/test/registered/unit/mem_cache/test_fused_draft_placement.py +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -224,13 +224,13 @@ class TestFusedDraftDecision(CustomTestCase): def test_a_draft_off_the_translated_rails_keeps_the_private_pool(self): """A fused draft reads its rows through the KV-index translator, which - only triton, flashinfer and fa3 carry on every draft path. The draft's - backend is the published one, a model hook's declaration included; - otherwise it is what the draft runner inherits from the target.""" - # Kimi-Linear + DSPARK on SM100, where a model hook declares trtllm_mha. - declined = self._decide(algorithm="DSPARK", draft_backend="trtllm_mha") + only triton, flashinfer, fa3 and trtllm_mha carry on every draft path. + The draft's backend is the published one, a model hook's declaration + included; otherwise it is what the draft runner inherits from the + target.""" + declined = self._decide(algorithm="DSPARK", draft_backend="fa4") self.assertIsNone(declined.placement) - self.assertIn("trtllm_mha", declined.declined) + self.assertIn("fa4", declined.declined) # A DFLASH-family draft runs the target's prefill backend, or the # platform default when that is not a draft backend. for target_backends, fuses in ( @@ -243,6 +243,17 @@ class TestFusedDraftDecision(CustomTestCase): # An EAGLE draft with no backend of its own runs the target's pair. self.assertIsNone(self._decide(target_backends=("fa3", "flashmla")).placement) + def test_a_trtllm_mha_draft_fuses(self): + """trtllm_mha refills its graph page table to what each replay reads + and widens its eager table by the draft block, so its draft fuses.""" + # Kimi-Linear + DSPARK on SM100, where a model hook declares trtllm_mha. + for algorithm in ("EAGLE", "DFLASH", "DSPARK"): + decision = self._decide(algorithm=algorithm, draft_backend="trtllm_mha") + self.assertIsNotNone(decision.placement, algorithm) + # A draft with no backend of its own inherits the target's trtllm_mha. + decision = self._decide(target_backends=("trtllm_mha", "trtllm_mha")) + self.assertIsNotNone(decision.placement) + def test_dcp_keeps_the_private_pool(self): """Under --dcp-size > 1 each rank's host rows hold only its share of the widened id space, while the draft, outside the DCP group, reads diff --git a/test/registered/unit/mem_cache/test_unified_handout_zeroing.py b/test/registered/unit/mem_cache/test_unified_handout_zeroing.py index 3691a577c3..088521d59d 100644 --- a/test/registered/unit/mem_cache/test_unified_handout_zeroing.py +++ b/test/registered/unit/mem_cache/test_unified_handout_zeroing.py @@ -6,16 +6,27 @@ register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-small") register_amd_ci(est_time=9, suite="stage-b-test-1-gpu-small-amd") import unittest +from types import SimpleNamespace import torch +from sglang.kernels.ops.kvcache.kv_read_table import build_kv_read_table +from sglang.srt.environ import envs +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedMambaSWATokenToKVPoolAllocator, +) from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator from sglang.srt.mem_cache.unified_memory_pool import ( MambaSubPoolSpec, + MHASubPoolSpec, MLASubPoolSpec, UnifiedKVPool, + UnifiedMHATokenToKVPool, UnifiedMLATokenToKVPool, + init_unified_swa_pools, ) +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs BF16_NAN = 0x7FC1 # LE bf16 NaN bit pattern, as SGLANG_DEBUG_POISON_POOL fills @@ -68,6 +79,130 @@ def _build(device, page_size=1): return buf, kvcache, allocator +def _build_mha(device, page_size=8): + """The same pool with an MHA full side, as a GDN hybrid (Qwen3.5) builds it: + its mamba neighbour's fp32 state shares the bytes the KV pages reuse.""" + full_spec = MHASubPoolSpec( + name="full", + layer_num=2, + grow_direction="up", + head_num=2, + head_dim=8, + store_dtype=torch.bfloat16, + ) + mamba_spec = MambaSubPoolSpec( + name="mamba", + layer_num=1, + grow_direction="down", + conv_state_shapes=((8, 3),), + conv_dtype=torch.bfloat16, + temporal_state_shape=(2, 4, 4), + temporal_dtype=torch.float32, + ) + buf = UnifiedKVPool( + total_bytes=4096 * full_spec.entry_bytes(), + sub_pool_specs=[full_spec, mamba_spec], + device=device, + enable_memory_saver=False, + page_size=page_size, + ) + kvcache = UnifiedMHATokenToKVPool( + unified_buffer=buf, sub_pool_name="full", page_size=page_size + ) + allocator = MultiEndedAllocator( + kvcache=kvcache, + unified_buffer=buf, + sub_pool_name="full", + device=device, + is_id_owner=True, + page_size=page_size, + ) + return buf, kvcache, allocator + + +def _build_tri(device, page_size=4): + """A tri-pool, [mamba (up) | swa (float) | full (down)], with MHA pools on + both KV sides, as a mamba + sliding-window hybrid builds it.""" + full_spec = MHASubPoolSpec( + name="full", + layer_num=2, + grow_direction="down", + head_num=2, + head_dim=8, + store_dtype=torch.bfloat16, + ) + swa_spec = MHASubPoolSpec( + name="swa", + layer_num=2, + grow_direction="float", + head_num=2, + head_dim=8, + store_dtype=torch.bfloat16, + ) + mamba_spec = MambaSubPoolSpec( + name="mamba", + layer_num=1, + grow_direction="up", + conv_state_shapes=((3, 8),), + conv_dtype=torch.bfloat16, + temporal_state_shape=(0, 0, 0), + temporal_dtype=torch.float32, + ) + n_full, n_swa = 256, 128 + buf = UnifiedKVPool( + total_bytes=n_full * full_spec.entry_bytes() + + n_swa * swa_spec.entry_bytes() + + 16 * mamba_spec.entry_bytes(), + sub_pool_specs=[full_spec, swa_spec, mamba_spec], + device=device, + enable_memory_saver=False, + page_size=page_size, + ) + kvcache = SimpleNamespace( + full_kv_pool=UnifiedMHATokenToKVPool( + unified_buffer=buf, sub_pool_name="full", page_size=page_size + ), + swa_kv_pool=UnifiedMHATokenToKVPool( + unified_buffer=buf, sub_pool_name="swa", page_size=page_size + ), + attach_allocators=lambda **_: None, + ) + allocator = UnifiedMambaSWATokenToKVPoolAllocator( + unified_buffer=buf, + kvcache=kvcache, + mamba_kvcache=SimpleNamespace(move_kv_cache=lambda dst, src: None), + device=device, + full_max_total_num_tokens=n_full, + swa_max_total_num_tokens=n_swa, + page_size=page_size, + ) + return buf, kvcache.swa_kv_pool, allocator + + +def _build_swa_poisoned(device, page_size=4): + """A full + SWA unified pool built under SGLANG_DEBUG_POISON_POOL.""" + with envs.SGLANG_DEBUG_POISON_POOL.override(True): + return init_unified_swa_pools( + device=device, + kv_cache_dtype=torch.bfloat16, + head_num=2, + head_dim=8, + v_head_dim=8, + swa_head_num=2, + swa_head_dim=8, + swa_v_head_dim=8, + page_size=page_size, + start_layer=0, + end_layer=2, + swa_attention_layer_ids=[1], + full_attention_layer_ids=[0], + full_max_total_num_tokens=64, + swa_max_total_num_tokens=32, + enable_memory_saver=False, + need_sort=False, + ) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA required (fused alloc kernel)") class TestUnifiedHandoutZeroing(unittest.TestCase): """Root-cause guard: pages must leave the allocator ZEROED. @@ -77,6 +212,8 @@ class TestUnifiedHandoutZeroing(unittest.TestCase): bytes must never carry NaN bit patterns. Static pools get this from torch.zeros; the unified pool must re-establish it at every hand-out.""" + build = staticmethod(_build) + def _poison(self, buf): buf._raw.view(torch.int16).fill_(BF16_NAN) @@ -89,7 +226,7 @@ class TestUnifiedHandoutZeroing(unittest.TestCase): return (allocator.translate_kv_loc(virt_tokens) // allocator.page_size).unique() def test_fresh_and_recycled_pages_zeroed(self): - buf, kvcache, allocator = _build("cuda") + buf, kvcache, allocator = self.build("cuda") env = self._env(buf, kvcache) # Fresh hand-out over a poisoned pool (the deterministic form of @@ -117,7 +254,7 @@ class TestUnifiedHandoutZeroing(unittest.TestCase): # UnifiedMLATokenToKVPool has NaN-unsafe partial-page reads whatever # its geometry; zeroing must key on the pool type, not on any # id-space scale. - buf, kvcache, allocator = _build("cuda") + buf, kvcache, allocator = self.build("cuda") self.assertTrue(allocator._zero_pages_on_alloc) self._poison(buf) out = allocator.alloc(8) @@ -127,5 +264,70 @@ class TestUnifiedHandoutZeroing(unittest.TestCase): self.assertTrue((env[pages] == 0).all().item()) +class TestUnifiedMHAHandoutZeroing(TestUnifiedHandoutZeroing): + """The same contract for an MHA pool: trtllm_mha's fmha_v2 prefill loads a + request's last partial page whole and multiplies its unwritten V rows by a + zero probability, so one NaN there turns the request's output into NaN.""" + + build = staticmethod(_build_mha) + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required (fused alloc kernel)") +class TestTriPoolSWAHandoutZeroing(unittest.TestCase): + """A tri-pool hands its sliding-window pages out through the float middle's + own allocator; they must leave it zeroed like every other MHA page.""" + + def test_fresh_and_recycled_swa_pages_zeroed(self): + buf, swa, allocator = _build_tri("cuda") + sa = allocator.swa_attn_allocator + self.assertTrue(sa._zero_pages_on_alloc) + env = buf._raw[: swa._num_pages * swa._page_bytes].view( + swa._num_pages, swa._page_bytes + ) + for _ in range(2): # fresh, then recycled + buf._raw.view(torch.int16).fill_(BF16_NAN) + out = allocator.alloc(16) + self.assertIsNotNone(out) + pages = (sa.translate_kv_loc(out) // sa.page_size).unique() + self.assertTrue((env[pages] == 0).all().item()) + allocator.free(out) + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required (fused alloc kernel)") +class TestPoisonSparesTheSlotZeroSink(unittest.TestCase): + """SGLANG_DEBUG_POISON_POOL marks never-written bytes. The slot-0 sink is + not one: a freed window page's read-table entry resolves to it, kernels that + load whole tiles read its rows (at zero weight), and in serving it holds + zeros. So the poison must leave it zero, or every window model fails it.""" + + def setUp(self): + publish(ServerArgs(model_path="dummy"), role="tokenizer") + self.addCleanup(reset_context) + + def test_freed_window_page_reads_zeros(self): + ps = 4 + bundle = _build_swa_poisoned("cuda", page_size=ps) + alloc = bundle.token_to_kv_pool_allocator + ids = alloc.alloc(4 * ps) + self.assertIsNotNone(ids) + # The window moved past the request's first two pages. + alloc.free_swa_segment(ids[: 2 * ps], start_pos=0) + + dev = ids.device + table = torch.full((1, 4), -1, dtype=torch.int32, device=dev) + build_kv_read_table( + req_to_token=ids.to(torch.int32).view(1, -1), + req_pool_indices=torch.tensor([0], device=dev), + seq_lens=torch.tensor([4 * ps], device=dev), + v2p=alloc.swa_attn_allocator.virtual_to_physical, + page_size=ps, + max_pages=4, + out=table, + ) + self.assertEqual(table[0, :2].tolist(), [0, 0]) # freed: the sink + v = bundle.token_to_kv_pool.swa_kv_pool.v_buffer[0] + self.assertFalse(torch.isnan(v[:ps].float()).any().item()) + + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/test/registered/unit/server_args/test_unified_memory_spec_gate.py b/test/registered/unit/server_args/test_unified_memory_spec_gate.py index bb9db7c723..bc3aa24674 100644 --- a/test/registered/unit/server_args/test_unified_memory_spec_gate.py +++ b/test/registered/unit/server_args/test_unified_memory_spec_gate.py @@ -97,19 +97,20 @@ _VERIFY_BACKENDS = ( "flashinfer", "fa3", ) -_MHA_RAILS = ("triton", "flashinfer", "fa3") +_MHA_RAILS = ("triton", "flashinfer", "fa3", "trtllm_mha") class TestUnifiedMemorySpecGate(unittest.TestCase): VERIFY_BACKENDS = _VERIFY_BACKENDS MHA_RAILS = _MHA_RAILS MLA_ONLY = tuple(b for b in _VERIFY_BACKENDS if b not in _MHA_RAILS) - UNAUDITED = ("fa4", "trtllm_mha") + UNAUDITED = ("fa4",) def test_dspark_verify_backends(self): for backend in self.VERIFY_BACKENDS: self.assertTrue(_accepts("DSPARK", backend=backend), backend) - for backend in self.UNAUDITED: + # trtllm_mha is audited for the EAGLE and DFLASH arms only. + for backend in self.UNAUDITED + ("trtllm_mha",): self.assertFalse(_accepts("DSPARK", backend=backend), backend) def test_dcp_keeps_flashinfer_off_the_verify_list(self): diff --git a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py index 3c86460f20..aa0d732045 100644 --- a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py +++ b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py @@ -30,10 +30,6 @@ other backend (Inkling declares FULL as a MODEL default, so refusing to boot would fail on a flag the user never typed), and decode capture is never touched. -Decode capture has one refusal of its own: trtllm_mha refills its graph page -table before the replay from lengths only the recorded kernel writes, so a -replay translates the previous step's lengths. That combination does not boot. - python -m pytest test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py -v """ @@ -114,25 +110,12 @@ class TestUnifiedPrefillCudaGraphGate(unittest.TestCase): class TestUnifiedTrtllmMhaDecodeGraphGate(unittest.TestCase): - def test_trtllm_mha_decode_capture_is_refused(self): - with self.assertRaisesRegex(AssertionError, "trtllm_mha"): - _run_handler( - prefill_backend=Backend.BREAKABLE, - attention_backends=("trtllm_mha", "trtllm_mha"), - ) - - def test_trtllm_mha_without_decode_capture_boots(self): + def test_trtllm_mha_decode_capture_boots(self): + """Its replay refills the page table from this batch's lengths, so the + gate leaves trtllm_mha decode capture enabled.""" cg = _run_handler( prefill_backend=Backend.BREAKABLE, attention_backends=("trtllm_mha", "trtllm_mha"), - decode_backend=Backend.DISABLED, - ) - self.assertEqual(cg.decode.backend, Backend.DISABLED) - - def test_trtllm_mha_prefill_with_another_decode_backend_boots(self): - cg = _run_handler( - prefill_backend=Backend.BREAKABLE, - attention_backends=("trtllm_mha", "fa3"), ) self.assertEqual(cg.decode.backend, Backend.FULL)