[Fix] unified-memory: enabling trtllm_mha with decode graphs and speculative decoding (#42930)
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
"""Zero whole page envelopes of the unified pool by physical page id.
|
||||
|
||||
The pool is viewed as int64 words (the MLA page envelope is always
|
||||
8-byte-aligned: entry bytes per layer = kv_cache_dim * itemsize, a multiple
|
||||
of 8), one wide element per lane; grid = (num_pages, page word blocks).
|
||||
The pool is viewed as int64 words (MHA and MLA page envelopes are
|
||||
8-byte-aligned), one wide element per lane; grid = (num_pages, page word
|
||||
blocks).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -41,6 +41,9 @@ def zero_pages(
|
||||
if m == 0:
|
||||
return
|
||||
assert raw.dtype == torch.uint8, f"expected uint8 pool, got {raw.dtype}"
|
||||
if not raw.is_cuda:
|
||||
raw[: num_pages * page_bytes].view(num_pages, page_bytes)[pages] = 0
|
||||
return
|
||||
assert page_bytes % 8 == 0, f"page_bytes {page_bytes} not int64-aligned"
|
||||
page_words = page_bytes // 8
|
||||
words = raw[: num_pages * page_bytes].view(torch.int64)
|
||||
|
||||
@@ -491,7 +491,7 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
|
||||
# The MHA backends that read through the KV-index translator on every path a
|
||||
# draft forward takes. A fused draft region holds dense K/V rows, so a fused
|
||||
# draft runs only on these.
|
||||
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3"})
|
||||
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"})
|
||||
|
||||
|
||||
def _assert_spec_verify_backends(
|
||||
@@ -707,18 +707,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
sorted(full_cg_backends),
|
||||
sorted(backends),
|
||||
)
|
||||
# trtllm_mha refills its graph page table before replay from
|
||||
# `cache_seqlens_int32`, which only the in-graph metadata kernel writes, so
|
||||
# it uses the previous replay's lengths. Refuse until it uses the batch's.
|
||||
_, decode_backend = attention_backends_of(resolved_view(server_args))
|
||||
if decode_backend == "trtllm_mha":
|
||||
assert _cg_cfg is None or _cg_cfg.decode.backend == Backend.DISABLED, (
|
||||
"--enable-unified-memory does not yet support decode cuda graphs "
|
||||
"with the trtllm_mha attention backend: its replay refills the page "
|
||||
"table from the previous replay's sequence lengths. Pass "
|
||||
"--disable-cuda-graph, or pick another decode attention backend "
|
||||
"(fa3 / fa4 / flashinfer / triton)."
|
||||
)
|
||||
|
||||
|
||||
def _validate_unified_memory_dcp(server_args: Any) -> None:
|
||||
|
||||
@@ -78,6 +78,9 @@ if TYPE_CHECKING:
|
||||
# Default workspace size in MB for TRTLLM MHA
|
||||
# Can be configured via SGLANG_FLASHINFER_WORKSPACE_SIZE environment variable
|
||||
DEFAULT_WORKSPACE_SIZE_MB = 512
|
||||
# flashinfer's XQA decode keeps its multi-block semaphores in the first 8 MiB
|
||||
# of the workspace and needs them zero at every launch.
|
||||
_XQA_SEMAPHORE_BYTES = 8 * 1024 * 1024
|
||||
|
||||
# Reuse this workspace buffer across all TRTLLM MHA wrappers
|
||||
|
||||
@@ -161,6 +164,11 @@ class TRTLLMMHAMetadata:
|
||||
# Prefix before trailing one-token requests in an extend-shaped batch.
|
||||
mixed_prefill_reqs: int = 0
|
||||
mixed_prefill_tokens: int = 0
|
||||
# fmha_v2 prefill: the (row, column) of each page-table entry the batch
|
||||
# reads, in copy order, and the block table over that copy.
|
||||
fmha_v2_copy_rows: torch.Tensor = None
|
||||
fmha_v2_copy_cols: torch.Tensor = None
|
||||
fmha_v2_block_table: torch.Tensor = None
|
||||
|
||||
|
||||
class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
@@ -295,15 +303,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
|
||||
# Workspace allocation
|
||||
self.workspace_size = workspace_size_bytes
|
||||
semaphore_bytes = _XQA_SEMAPHORE_BYTES if self.is_xqa_impl else 0
|
||||
# Allocate buffers
|
||||
self.workspace_buffer = get_buffer(
|
||||
"trtllm_mha_zero_workspace",
|
||||
lambda: torch.zeros(
|
||||
self.workspace_size,
|
||||
semaphore_bytes + self.workspace_size,
|
||||
dtype=torch.uint8,
|
||||
device=model_runner.device,
|
||||
),
|
||||
)
|
||||
self.fmha_v2_workspace_buffer = self.workspace_buffer[semaphore_bytes:]
|
||||
|
||||
# CUDA graph state
|
||||
self.decode_cuda_graph_metadata = {}
|
||||
@@ -1282,13 +1292,14 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
)
|
||||
else:
|
||||
metadata.cu_seqlens_q = metadata.cu_seqlens_k
|
||||
if self.use_fmha_v2 and not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
self._init_fmha_v2_batch_pages(metadata, forward_batch)
|
||||
|
||||
kv_view = self.kv_index_translator.read_table(
|
||||
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
|
||||
)
|
||||
if kv_view.is_translated:
|
||||
# No fill kernel: the kernels take the tensor's own width/stride
|
||||
# and bound their reads by cache_seqlens.
|
||||
# No fill kernel: the kernels bound their reads by cache_seqlens.
|
||||
metadata.page_table = kv_view.ids
|
||||
metadata.swa_page_table = (
|
||||
self.kv_index_translator.read_table(
|
||||
@@ -1300,6 +1311,15 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
is not None
|
||||
else None
|
||||
)
|
||||
# XQA and fmha_v2 take a page table's row length from its width and
|
||||
# ignore its strides, so a view into a wider table would be misread.
|
||||
assert metadata.page_table.is_contiguous() and (
|
||||
metadata.swa_page_table is None
|
||||
or metadata.swa_page_table.is_contiguous()
|
||||
), (
|
||||
"trtllm_mha reads its page tables as packed rows, but the plan's "
|
||||
"table is a strided view of a wider buffer"
|
||||
)
|
||||
else:
|
||||
has_swa = self._swa_kv_pool is not None
|
||||
metadata.page_table = torch.empty(
|
||||
@@ -1334,6 +1354,63 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
|
||||
self.forward_metadata = metadata
|
||||
|
||||
def _init_fmha_v2_batch_pages(
|
||||
self, metadata: TRTLLMMHAMetadata, forward_batch: ForwardBatch
|
||||
) -> None:
|
||||
"""Index the pages this batch reads, for the fmha_v2 prefill copy.
|
||||
|
||||
Sized by the host seq_lens, which are live only alongside seq_lens_sum
|
||||
(a GPU-only batch keeps a stale seq_lens_cpu); without them the prefill
|
||||
copies the whole pool.
|
||||
"""
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
if (
|
||||
forward_batch.seq_lens_sum is None
|
||||
or seq_lens_cpu is None
|
||||
or seq_lens_cpu.numel() == 0
|
||||
):
|
||||
return
|
||||
host_pages = (seq_lens_cpu + self.page_size - 1) // self.page_size
|
||||
total_pages = int(host_pages.sum())
|
||||
if total_pages == 0:
|
||||
return
|
||||
num_pages = (
|
||||
(metadata.cache_seqlens_int32 + self.page_size - 1) // self.page_size
|
||||
).to(torch.int64)
|
||||
device = num_pages.device
|
||||
# Row r's pages land in copy rows [first[r], first[r] + num_pages[r]).
|
||||
first = torch.cumsum(num_pages, 0) - num_pages
|
||||
rows = torch.repeat_interleave(num_pages, output_size=total_pages)
|
||||
metadata.fmha_v2_copy_rows = rows
|
||||
metadata.fmha_v2_copy_cols = (
|
||||
torch.arange(total_pages, device=device) - first[rows]
|
||||
)
|
||||
# Past a row's last page, entries only need to stay in range.
|
||||
metadata.fmha_v2_block_table = (
|
||||
(first[:, None] + torch.arange(int(host_pages.max()), device=device))
|
||||
.clamp_(max=total_pages - 1)
|
||||
.to(torch.int32)
|
||||
)
|
||||
|
||||
def _fmha_v2_paged_kv(
|
||||
self, k_cache: torch.Tensor, v_cache: torch.Tensor, page_table: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""K and V as the [pages, 2, page_size, heads, head_dim] tensor fmha_v2
|
||||
takes, and the block table that addresses it.
|
||||
|
||||
fmha_v2 derives strides from shapes, so this is a copy: of the batch's
|
||||
pages only, unless they are no fewer than the pool's.
|
||||
"""
|
||||
metadata = self.forward_metadata
|
||||
rows = metadata.fmha_v2_copy_rows
|
||||
if rows is None or rows.numel() >= k_cache.shape[0]:
|
||||
return torch.stack([k_cache, v_cache], dim=1), page_table
|
||||
pages = page_table[rows, metadata.fmha_v2_copy_cols]
|
||||
paged_kv = k_cache.new_empty((rows.numel(), 2, *k_cache.shape[1:]))
|
||||
torch.index_select(k_cache, 0, pages, out=paged_kv[:, 0])
|
||||
torch.index_select(v_cache, 0, pages, out=paged_kv[:, 1])
|
||||
return paged_kv, metadata.fmha_v2_block_table
|
||||
|
||||
def _reshape_paged_kv_cache(
|
||||
self,
|
||||
k_cache: torch.Tensor,
|
||||
@@ -1760,12 +1837,14 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
elif self.use_fmha_v2 and not cp_active:
|
||||
# CP must go through cp_strategy.run_attention (per-shard
|
||||
# masking); the plain-causal fmha_v2 call below would be wrong.
|
||||
paged_kv = torch.stack([k_cache, v_cache], dim=1)
|
||||
paged_kv, block_tables = self._fmha_v2_paged_kv(
|
||||
k_cache, v_cache, page_table
|
||||
)
|
||||
out = forward_batch._attn_output
|
||||
o = flashinfer.prefill.trtllm_fmha_v2_prefill(
|
||||
(q, paged_kv),
|
||||
input_layout="Q_PAGED_KV_NHD",
|
||||
workspace_buffer=self.workspace_buffer,
|
||||
workspace_buffer=self.fmha_v2_workspace_buffer,
|
||||
seq_lens=self.forward_metadata.cache_seqlens_int32,
|
||||
max_q_len=self.forward_metadata.max_seq_len_q,
|
||||
max_kv_len=self.max_context_len,
|
||||
@@ -1774,7 +1853,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
batch_size=forward_batch.batch_size,
|
||||
cum_seq_lens_q=self.forward_metadata.cu_seqlens_q,
|
||||
cum_seq_lens_kv=self.forward_metadata.cu_seqlens_k,
|
||||
block_tables=page_table,
|
||||
block_tables=block_tables,
|
||||
out=None if out is None else out.view_as(q),
|
||||
out_dtype=self.q_data_type,
|
||||
mask_mode=(
|
||||
|
||||
@@ -55,6 +55,7 @@ from sglang.srt.mem_cache.allocator.paged import (
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
UnifiedKVPool,
|
||||
UnifiedMHATokenToKVPool,
|
||||
UnifiedMLATokenToKVPool,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
@@ -327,7 +328,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.min_slot_index = unified_buffer.min_slot_index(sub_pool_name)
|
||||
self.is_id_owner = is_id_owner
|
||||
# Zero page envelopes on hand-out -- see _maybe_zero_pages.
|
||||
self._zero_pages_on_alloc = isinstance(kvcache, UnifiedMLATokenToKVPool)
|
||||
self._zero_pages_on_alloc = isinstance(
|
||||
kvcache, (UnifiedMHATokenToKVPool, UnifiedMLATokenToKVPool)
|
||||
)
|
||||
# Overlap mode: `free` drops a wait_stream(forward_stream) barrier so its
|
||||
# v2p writes + move kernel serialize after the in-flight forward.
|
||||
self.forward_stream = forward_stream
|
||||
@@ -1003,9 +1006,11 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
return phys_pages
|
||||
|
||||
def _maybe_zero_pages(self, phys_pages: torch.Tensor) -> None:
|
||||
"""Zero the page ENVELOPES on hand-out (MLA full pool only): the MLA
|
||||
kernels arithmetically mask the rows beyond seq_len, so never-written page
|
||||
bytes must read as finite values.
|
||||
"""Zero the page ENVELOPES on hand-out (MHA and MLA pools): the MLA
|
||||
kernels and trtllm_mha's fmha_v2 prefill (and fp8 XQA decode) load whole
|
||||
pages and give the rows past seq_len a zero weight, so a NaN there still
|
||||
reaches the output. Never-written bytes must read as finite values; a
|
||||
mamba neighbour's fp32 state, read as bf16, may not.
|
||||
"""
|
||||
if not self._zero_pages_on_alloc or phys_pages.numel() == 0:
|
||||
return
|
||||
@@ -2351,6 +2356,7 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
|
||||
if phys_pages is None:
|
||||
return None
|
||||
self.bind(v_pages, phys_pages)
|
||||
self._maybe_zero_pages(phys_pages)
|
||||
return phys_pages
|
||||
|
||||
# -- free: hole-marking, boundary absorption, park-on-empty --
|
||||
|
||||
@@ -1065,9 +1065,7 @@ class KVCacheConfigurator:
|
||||
{draft_backend} if draft_backend else set(attention_backends()) - {None}
|
||||
)
|
||||
# A fused draft reads its rows through the KV-index translator, which
|
||||
# other backends miss on some draft path: trtllm_mha's graph replay
|
||||
# refills its page table from stale lengths, and its eager build does
|
||||
# not widen the table by the draft block.
|
||||
# other backends miss on some draft path.
|
||||
if not draft_backends <= TRANSLATED_MHA_RAILS:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
@@ -1203,7 +1201,7 @@ class KVCacheConfigurator:
|
||||
would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that
|
||||
does not fuse is refused. A DSPARK draft that does not fuse keeps its
|
||||
private pool, unpriced, so DSPARK still boots where its draft declines
|
||||
fusion (as the Kimi-Linear default, a trtllm_mha draft, does)."""
|
||||
fusion."""
|
||||
decision = self._fused_draft_decision()
|
||||
placement = self._fused_draft_for_pool_factory(decision)
|
||||
if (
|
||||
|
||||
@@ -456,6 +456,7 @@ class UnifiedKVPool:
|
||||
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
|
||||
# max of both.
|
||||
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
|
||||
self._reserved_floor = reserved_floor
|
||||
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
if post_capture_active:
|
||||
@@ -566,6 +567,7 @@ class UnifiedKVPool:
|
||||
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
|
||||
# fail deterministically.
|
||||
raw.view(torch.int16).fill_(0x7FC1)
|
||||
raw[: self._reserved_floor].zero_()
|
||||
logger.warning(
|
||||
"[unified-memory-pool] POISONED: backed pool bytes filled with "
|
||||
"bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)"
|
||||
@@ -840,6 +842,17 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
self._page_bytes,
|
||||
)
|
||||
|
||||
def zero_physical_pages(self, phys_pages: torch.Tensor) -> None:
|
||||
"""Zero whole page envelopes (PHYSICAL page ids) on allocator
|
||||
hand-out."""
|
||||
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
||||
zero_pages(
|
||||
self._unified_buffer._raw,
|
||||
phys_pages,
|
||||
self._num_pages,
|
||||
self._page_bytes,
|
||||
)
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
"""Register the raw buffer as physical page envelopes for PD transfer.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user