[Fix] unified-memory: enabling trtllm_mha with decode graphs and speculative decoding (#42930)

This commit is contained in:
caihuali95
2026-10-16 02:24:06 -07:00
committed by GitHub
parent 6efc96abf9
commit 8d229d96e0
11 changed files with 840 additions and 61 deletions
@@ -1,8 +1,8 @@
"""Zero whole page envelopes of the unified pool by physical page id.
The pool is viewed as int64 words (the MLA page envelope is always
8-byte-aligned: entry bytes per layer = kv_cache_dim * itemsize, a multiple
of 8), one wide element per lane; grid = (num_pages, page word blocks).
The pool is viewed as int64 words (MHA and MLA page envelopes are
8-byte-aligned), one wide element per lane; grid = (num_pages, page word
blocks).
"""
from __future__ import annotations
@@ -41,6 +41,9 @@ def zero_pages(
if m == 0:
return
assert raw.dtype == torch.uint8, f"expected uint8 pool, got {raw.dtype}"
if not raw.is_cuda:
raw[: num_pages * page_bytes].view(num_pages, page_bytes)[pages] = 0
return
assert page_bytes % 8 == 0, f"page_bytes {page_bytes} not int64-aligned"
page_words = page_bytes // 8
words = raw[: num_pages * page_bytes].view(torch.int64)
+1 -13
View File
@@ -491,7 +491,7 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
# The MHA backends that read through the KV-index translator on every path a
# draft forward takes. A fused draft region holds dense K/V rows, so a fused
# draft runs only on these.
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3"})
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"})
def _assert_spec_verify_backends(
@@ -707,18 +707,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
sorted(full_cg_backends),
sorted(backends),
)
# trtllm_mha refills its graph page table before replay from
# `cache_seqlens_int32`, which only the in-graph metadata kernel writes, so
# it uses the previous replay's lengths. Refuse until it uses the batch's.
_, decode_backend = attention_backends_of(resolved_view(server_args))
if decode_backend == "trtllm_mha":
assert _cg_cfg is None or _cg_cfg.decode.backend == Backend.DISABLED, (
"--enable-unified-memory does not yet support decode cuda graphs "
"with the trtllm_mha attention backend: its replay refills the page "
"table from the previous replay's sequence lengths. Pass "
"--disable-cuda-graph, or pick another decode attention backend "
"(fa3 / fa4 / flashinfer / triton)."
)
def _validate_unified_memory_dcp(server_args: Any) -> None:
@@ -78,6 +78,9 @@ if TYPE_CHECKING:
# Default workspace size in MB for TRTLLM MHA
# Can be configured via SGLANG_FLASHINFER_WORKSPACE_SIZE environment variable
DEFAULT_WORKSPACE_SIZE_MB = 512
# flashinfer's XQA decode keeps its multi-block semaphores in the first 8 MiB
# of the workspace and needs them zero at every launch.
_XQA_SEMAPHORE_BYTES = 8 * 1024 * 1024
# Reuse this workspace buffer across all TRTLLM MHA wrappers
@@ -161,6 +164,11 @@ class TRTLLMMHAMetadata:
# Prefix before trailing one-token requests in an extend-shaped batch.
mixed_prefill_reqs: int = 0
mixed_prefill_tokens: int = 0
# fmha_v2 prefill: the (row, column) of each page-table entry the batch
# reads, in copy order, and the block table over that copy.
fmha_v2_copy_rows: torch.Tensor = None
fmha_v2_copy_cols: torch.Tensor = None
fmha_v2_block_table: torch.Tensor = None
class TRTLLMHAAttnBackend(FlashInferAttnBackend):
@@ -295,15 +303,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
# Workspace allocation
self.workspace_size = workspace_size_bytes
semaphore_bytes = _XQA_SEMAPHORE_BYTES if self.is_xqa_impl else 0
# Allocate buffers
self.workspace_buffer = get_buffer(
"trtllm_mha_zero_workspace",
lambda: torch.zeros(
self.workspace_size,
semaphore_bytes + self.workspace_size,
dtype=torch.uint8,
device=model_runner.device,
),
)
self.fmha_v2_workspace_buffer = self.workspace_buffer[semaphore_bytes:]
# CUDA graph state
self.decode_cuda_graph_metadata = {}
@@ -1282,13 +1292,14 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
)
else:
metadata.cu_seqlens_q = metadata.cu_seqlens_k
if self.use_fmha_v2 and not forward_batch.forward_mode.is_draft_extend_v2():
self._init_fmha_v2_batch_pages(metadata, forward_batch)
kv_view = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
)
if kv_view.is_translated:
# No fill kernel: the kernels take the tensor's own width/stride
# and bound their reads by cache_seqlens.
# No fill kernel: the kernels bound their reads by cache_seqlens.
metadata.page_table = kv_view.ids
metadata.swa_page_table = (
self.kv_index_translator.read_table(
@@ -1300,6 +1311,15 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
is not None
else None
)
# XQA and fmha_v2 take a page table's row length from its width and
# ignore its strides, so a view into a wider table would be misread.
assert metadata.page_table.is_contiguous() and (
metadata.swa_page_table is None
or metadata.swa_page_table.is_contiguous()
), (
"trtllm_mha reads its page tables as packed rows, but the plan's "
"table is a strided view of a wider buffer"
)
else:
has_swa = self._swa_kv_pool is not None
metadata.page_table = torch.empty(
@@ -1334,6 +1354,63 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
self.forward_metadata = metadata
def _init_fmha_v2_batch_pages(
self, metadata: TRTLLMMHAMetadata, forward_batch: ForwardBatch
) -> None:
"""Index the pages this batch reads, for the fmha_v2 prefill copy.
Sized by the host seq_lens, which are live only alongside seq_lens_sum
(a GPU-only batch keeps a stale seq_lens_cpu); without them the prefill
copies the whole pool.
"""
seq_lens_cpu = forward_batch.seq_lens_cpu
if (
forward_batch.seq_lens_sum is None
or seq_lens_cpu is None
or seq_lens_cpu.numel() == 0
):
return
host_pages = (seq_lens_cpu + self.page_size - 1) // self.page_size
total_pages = int(host_pages.sum())
if total_pages == 0:
return
num_pages = (
(metadata.cache_seqlens_int32 + self.page_size - 1) // self.page_size
).to(torch.int64)
device = num_pages.device
# Row r's pages land in copy rows [first[r], first[r] + num_pages[r]).
first = torch.cumsum(num_pages, 0) - num_pages
rows = torch.repeat_interleave(num_pages, output_size=total_pages)
metadata.fmha_v2_copy_rows = rows
metadata.fmha_v2_copy_cols = (
torch.arange(total_pages, device=device) - first[rows]
)
# Past a row's last page, entries only need to stay in range.
metadata.fmha_v2_block_table = (
(first[:, None] + torch.arange(int(host_pages.max()), device=device))
.clamp_(max=total_pages - 1)
.to(torch.int32)
)
def _fmha_v2_paged_kv(
self, k_cache: torch.Tensor, v_cache: torch.Tensor, page_table: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""K and V as the [pages, 2, page_size, heads, head_dim] tensor fmha_v2
takes, and the block table that addresses it.
fmha_v2 derives strides from shapes, so this is a copy: of the batch's
pages only, unless they are no fewer than the pool's.
"""
metadata = self.forward_metadata
rows = metadata.fmha_v2_copy_rows
if rows is None or rows.numel() >= k_cache.shape[0]:
return torch.stack([k_cache, v_cache], dim=1), page_table
pages = page_table[rows, metadata.fmha_v2_copy_cols]
paged_kv = k_cache.new_empty((rows.numel(), 2, *k_cache.shape[1:]))
torch.index_select(k_cache, 0, pages, out=paged_kv[:, 0])
torch.index_select(v_cache, 0, pages, out=paged_kv[:, 1])
return paged_kv, metadata.fmha_v2_block_table
def _reshape_paged_kv_cache(
self,
k_cache: torch.Tensor,
@@ -1760,12 +1837,14 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
elif self.use_fmha_v2 and not cp_active:
# CP must go through cp_strategy.run_attention (per-shard
# masking); the plain-causal fmha_v2 call below would be wrong.
paged_kv = torch.stack([k_cache, v_cache], dim=1)
paged_kv, block_tables = self._fmha_v2_paged_kv(
k_cache, v_cache, page_table
)
out = forward_batch._attn_output
o = flashinfer.prefill.trtllm_fmha_v2_prefill(
(q, paged_kv),
input_layout="Q_PAGED_KV_NHD",
workspace_buffer=self.workspace_buffer,
workspace_buffer=self.fmha_v2_workspace_buffer,
seq_lens=self.forward_metadata.cache_seqlens_int32,
max_q_len=self.forward_metadata.max_seq_len_q,
max_kv_len=self.max_context_len,
@@ -1774,7 +1853,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
batch_size=forward_batch.batch_size,
cum_seq_lens_q=self.forward_metadata.cu_seqlens_q,
cum_seq_lens_kv=self.forward_metadata.cu_seqlens_k,
block_tables=page_table,
block_tables=block_tables,
out=None if out is None else out.view_as(q),
out_dtype=self.q_data_type,
mask_mode=(
@@ -55,6 +55,7 @@ from sglang.srt.mem_cache.allocator.paged import (
)
from sglang.srt.mem_cache.unified_memory_pool import (
UnifiedKVPool,
UnifiedMHATokenToKVPool,
UnifiedMLATokenToKVPool,
)
from sglang.srt.runtime_context import get_parallel
@@ -327,7 +328,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.min_slot_index = unified_buffer.min_slot_index(sub_pool_name)
self.is_id_owner = is_id_owner
# Zero page envelopes on hand-out -- see _maybe_zero_pages.
self._zero_pages_on_alloc = isinstance(kvcache, UnifiedMLATokenToKVPool)
self._zero_pages_on_alloc = isinstance(
kvcache, (UnifiedMHATokenToKVPool, UnifiedMLATokenToKVPool)
)
# Overlap mode: `free` drops a wait_stream(forward_stream) barrier so its
# v2p writes + move kernel serialize after the in-flight forward.
self.forward_stream = forward_stream
@@ -1003,9 +1006,11 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
return phys_pages
def _maybe_zero_pages(self, phys_pages: torch.Tensor) -> None:
"""Zero the page ENVELOPES on hand-out (MLA full pool only): the MLA
kernels arithmetically mask the rows beyond seq_len, so never-written page
bytes must read as finite values.
"""Zero the page ENVELOPES on hand-out (MHA and MLA pools): the MLA
kernels and trtllm_mha's fmha_v2 prefill (and fp8 XQA decode) load whole
pages and give the rows past seq_len a zero weight, so a NaN there still
reaches the output. Never-written bytes must read as finite values; a
mamba neighbour's fp32 state, read as bf16, may not.
"""
if not self._zero_pages_on_alloc or phys_pages.numel() == 0:
return
@@ -2351,6 +2356,7 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
if phys_pages is None:
return None
self.bind(v_pages, phys_pages)
self._maybe_zero_pages(phys_pages)
return phys_pages
# -- free: hole-marking, boundary absorption, park-on-empty --
@@ -1065,9 +1065,7 @@ class KVCacheConfigurator:
{draft_backend} if draft_backend else set(attention_backends()) - {None}
)
# A fused draft reads its rows through the KV-index translator, which
# other backends miss on some draft path: trtllm_mha's graph replay
# refills its page table from stale lengths, and its eager build does
# not widen the table by the draft block.
# other backends miss on some draft path.
if not draft_backends <= TRANSLATED_MHA_RAILS:
return FusedDraftDecision(
declined=(
@@ -1203,7 +1201,7 @@ class KVCacheConfigurator:
would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that
does not fuse is refused. A DSPARK draft that does not fuse keeps its
private pool, unpriced, so DSPARK still boots where its draft declines
fusion (as the Kimi-Linear default, a trtllm_mha draft, does)."""
fusion."""
decision = self._fused_draft_decision()
placement = self._fused_draft_for_pool_factory(decision)
if (
@@ -456,6 +456,7 @@ class UnifiedKVPool:
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
# max of both.
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
self._reserved_floor = reserved_floor
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
if post_capture_active:
@@ -566,6 +567,7 @@ class UnifiedKVPool:
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
# fail deterministically.
raw.view(torch.int16).fill_(0x7FC1)
raw[: self._reserved_floor].zero_()
logger.warning(
"[unified-memory-pool] POISONED: backed pool bytes filled with "
"bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)"
@@ -840,6 +842,17 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
self._page_bytes,
)
def zero_physical_pages(self, phys_pages: torch.Tensor) -> None:
"""Zero whole page envelopes (PHYSICAL page ids) on allocator
hand-out."""
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
zero_pages(
self._unified_buffer._raw,
phys_pages,
self._num_pages,
self._page_bytes,
)
def get_contiguous_buf_infos(self):
"""Register the raw buffer as physical page envelopes for PD transfer.