From 8683f8ad4ade22b33eb15414f61602c91850360a Mon Sep 17 00:00:00 2001 From: kpjeeja Date: Tue, 20 Oct 2026 08:12:21 +0530 Subject: [PATCH] [Feat] [Intel] hicache: Enable L1<->L2 support on XPU (#32503) --- python/sglang/srt/mem_cache/l2_transfer.py | 4 +- .../sglang/srt/mem_cache/memory_pool_host.py | 56 +++- .../srt/mem_cache/pool_host/_kvcacheio.py | 130 ++++++++ .../sglang/srt/mem_cache/pool_host/common.py | 31 +- python/sglang/srt/mem_cache/pool_host/dsa.py | 25 ++ .../sglang/srt/mem_cache/pool_host/mamba.py | 12 +- python/sglang/srt/mem_cache/pool_host/mha.py | 87 +++++ python/sglang/srt/mem_cache/pool_host/mla.py | 25 +- python/sglang/test/hicache_xpu_common.py | 118 +++++++ .../hicache/test_hicache_xpu_mla_kernels.py | 249 ++++++++++++++ .../hicache/test_hicache_xpu_no_storage.py | 161 +++++++++ .../mem_cache/test_hicache_host_register.py | 30 ++ .../mem_cache/test_hicache_kvcacheio_gate.py | 312 ++++++++++++++++++ .../unit/mem_cache/test_hicache_mtp_state.py | 2 + ...test_hicache_staged_write_back_dispatch.py | 62 ++++ .../unit/mem_cache/test_mem_pool_host.py | 39 ++- .../test_hicache_transfer_round_trip_xpu.py | 169 ++++++++++ 17 files changed, 1498 insertions(+), 14 deletions(-) create mode 100644 python/sglang/srt/mem_cache/pool_host/_kvcacheio.py create mode 100644 python/sglang/test/hicache_xpu_common.py create mode 100644 test/registered/e2e/hicache/test_hicache_xpu_mla_kernels.py create mode 100644 test/registered/e2e/hicache/test_hicache_xpu_no_storage.py create mode 100644 test/registered/unit/mem_cache/test_hicache_kvcacheio_gate.py create mode 100644 test/registered/xpu/test_hicache_transfer_round_trip_xpu.py diff --git a/python/sglang/srt/mem_cache/l2_transfer.py b/python/sglang/srt/mem_cache/l2_transfer.py index 76d85ede6e..82f67bc86b 100644 --- a/python/sglang/srt/mem_cache/l2_transfer.py +++ b/python/sglang/srt/mem_cache/l2_transfer.py @@ -197,5 +197,7 @@ class L2TransferEngine: # Keep temporary translated indices alive until the transfer completes. tensors.extend(resolved) for indices in tensors: - if indices is not None and indices.is_cuda: + if indices is not None and ( + indices.is_cuda or indices.device.type == "xpu" + ): indices.record_stream(stream) diff --git a/python/sglang/srt/mem_cache/memory_pool_host.py b/python/sglang/srt/mem_cache/memory_pool_host.py index 34067a730a..8300d4d28e 100644 --- a/python/sglang/srt/mem_cache/memory_pool_host.py +++ b/python/sglang/srt/mem_cache/memory_pool_host.py @@ -13,6 +13,11 @@ from sglang.kernels.ops.kvcache.hicache import ( transfer_hicache_all_layer_mla_staged_lf_pf as jit_transfer_hicache_all_layer_mla_staged_lf_pf, ) from sglang.kernels.ops.kvcache.hisparse import transfer_cache_dsv4_mla +from sglang.srt.mem_cache.pool_host._kvcacheio import ( + MLA_KVCACHEIO_OPS, + require_kvcacheio_ops, + unavailable_stub, +) from sglang.srt.platforms import current_platform from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu @@ -21,11 +26,22 @@ _is_hip = is_hip() _is_npu = is_npu() _is_xpu = is_xpu() _is_mps = is_mps() +_dsv4_mla_unavailable_reason: Optional[str] = None if _is_xpu: - from sgl_kernel import transfer_cache_dsv4_mla + try: + from sgl_kernel import transfer_cache_dsv4_mla + except ImportError as e: + # Only DSV4 needs this op; a wheel without it must not break scheduler import. + _dsv4_mla_unavailable_reason = ( + f"sgl_kernel.transfer_cache_dsv4_mla failed to import ({e}); XPU " + "takes it from the out-of-tree sgl-kernel-xpu wheel" + ) + transfer_cache_dsv4_mla = unavailable_stub( + "transfer_cache_dsv4_mla", _dsv4_mla_unavailable_reason + ) else: from sglang.kernels.ops.kvcache.hisparse import transfer_cache_dsv4_mla -if _is_cuda or _is_hip or _is_xpu: +if _is_cuda or _is_hip: from sgl_kernel.kvcacheio import ( transfer_kv_all_layer_direct_lf_pf, transfer_kv_all_layer_mla, @@ -35,6 +51,16 @@ if _is_cuda or _is_hip or _is_xpu: transfer_kv_per_layer_mla, transfer_kv_per_layer_mla_pf_lf, ) +elif _is_xpu: + from sglang.srt.mem_cache.pool_host._kvcacheio import ( + transfer_kv_all_layer_direct_lf_pf, + transfer_kv_all_layer_mla, + transfer_kv_all_layer_mla_lf_pf, + transfer_kv_direct, + transfer_kv_per_layer_direct_pf_lf, + transfer_kv_per_layer_mla, + transfer_kv_per_layer_mla_pf_lf, + ) if _is_npu: from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange @@ -223,6 +249,18 @@ class DeepSeekV4PagedHostPool(HiSparseHostPoolMixin, HostKVCache): allocator_type: str = "default", page_aligned_only: bool = False, ): + if _is_xpu: + require_kvcacheio_ops( + pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops + ) + # Token-granular transfers go through transfer_cache_dsv4_mla whatever + # the io backend and layout, so it is checked here, not in the table. + if not page_aligned_only and _dsv4_mla_unavailable_reason is not None: + raise ValueError( + f"HiCache cannot build the {type(self).__name__} host pool here: " + f"{_dsv4_mla_unavailable_reason}. Launch without HiCache " + "(--enable-hierarchical-cache)." + ) self.pool_name = pool_name self.layer_num = len(device_buffers) self.item_bytes = item_bytes @@ -455,6 +493,11 @@ class DeepSeekV4PagedHostPool(HiSparseHostPoolMixin, HostKVCache): self.num_release_slots += len(indices_cpu) return len(indices) + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + _kvcacheio_ops = MLA_KVCACHEIO_OPS + def backup_from_device_all_layer( self, device_pool, host_indices, device_indices, io_backend ): @@ -734,6 +777,10 @@ class DeepSeekV4StateHostPool(HostKVCache): ): if any(pool is None for pool in state_pools): raise ValueError(f"{pool_name} state_pools must not contain None") + if _is_xpu: + require_kvcacheio_ops( + pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops + ) self.pool_name = pool_name self.state_pools = state_pools @@ -956,6 +1003,11 @@ class DeepSeekV4StateHostPool(HostKVCache): f"{self.pool_name} reuses SWA transfer indices and has no free list" ) + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + _kvcacheio_ops = MLA_KVCACHEIO_OPS + def backup_from_device_all_layer( self, device_pool, host_indices, device_indices, io_backend ): diff --git a/python/sglang/srt/mem_cache/pool_host/_kvcacheio.py b/python/sglang/srt/mem_cache/pool_host/_kvcacheio.py new file mode 100644 index 0000000000..ba21052d10 --- /dev/null +++ b/python/sglang/srt/mem_cache/pool_host/_kvcacheio.py @@ -0,0 +1,130 @@ +"""`sgl_kernel.kvcacheio` KV-transfer ops for XPU, from the sgl-kernel-xpu wheel. + +CUDA/ROCm keep importing `sgl_kernel.kvcacheio` directly in each pool module; +only XPU imports the ops from here. The XPU wheel can lag the CUDA op set, so +each op is bound on its own: a missing one becomes a stub naming it, is recorded +in `missing_ops`, and host pools refuse at construction via +`require_kvcacheio_ops`. `unavailable_reason` is set when the module itself +does not import. Off XPU nothing is bound and `missing_ops` stays empty. +`hisparse_memory_pool.py` still imports `transfer_kv_all_layer_mla` directly, +so on XPU a wheel without that op or without the module fails at import. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +from sglang.srt.utils import is_xpu + +_is_xpu = is_xpu() + + +def unavailable_stub(name: str, reason: str): + """A stand-in for one missing op that names it when called.""" + + def _raise(*args, **kwargs): + raise RuntimeError( + f"HiCache host<->device KV transfer needs {name}, which is not " + f"available here: {reason}" + ) + + return _raise + + +# The table shared by the pools that move one latent buffer per layer +# (MLA, DSA indexer, DeepSeek-V4). Read-only, since four classes share it. +MLA_KVCACHEIO_OPS = MappingProxyType( + { + "kernel": MappingProxyType( + { + "layer_first": ( + "transfer_kv_per_layer_mla", + "transfer_kv_all_layer_mla", + ), + "page_first": ( + "transfer_kv_per_layer_mla_pf_lf", + "transfer_kv_all_layer_mla_lf_pf", + ), + } + ), + "direct": MappingProxyType( + { + "layer_first": ("transfer_kv_direct",), + "page_first_direct": ( + "transfer_kv_per_layer_direct_pf_lf", + "transfer_kv_all_layer_direct_lf_pf", + ), + } + ), + } +) + + +def require_kvcacheio_ops( + *, pool: str, layout: str, ops: Mapping[str, Mapping[str, tuple[str, ...]]] +) -> None: + """Fail at host-pool construction if this build lacks an op the pool will call. + + `ops` is the pool's own `{io_backend: {layout: op names}}` table, declared + next to the transfer methods that dispatch to those ops. + """ + if not missing_ops: + return + # Only read when an op is missing, so pools stay constructible without + # published server args wherever the build is complete. + from sglang.srt.runtime_context import get_memory + + io_backend = get_memory().hicache_io_backend + missing = [ + name for name in ops.get(io_backend, {}).get(layout, ()) if name in missing_ops + ] + if missing: + if unavailable_reason is None: + fix = "Pick another io backend or layout, or launch without HiCache" + else: + fix = "Launch without HiCache" + raise ValueError( + f"HiCache cannot build the {pool} host pool for --hicache-io-backend " + f"{io_backend} --hicache-mem-layout {layout} here: " + f"{missing_ops[missing[0]]}. {fix} (--enable-hierarchical-cache)." + ) + + +missing_ops: dict[str, str] = {} +unavailable_reason: str | None = None +if _is_xpu: + try: + import sgl_kernel.kvcacheio as _kvcacheio + except ImportError as e: + _kvcacheio = None + unavailable_reason = ( + f"sgl_kernel.kvcacheio failed to import ({e}); XPU takes it from " + "the out-of-tree sgl-kernel-xpu wheel" + ) + + def _bind(name: str): + # getattr, not a from-import: the op set varies by sgl-kernel-xpu wheel, + # and an absent op must stub only itself. + op = None if _kvcacheio is None else getattr(_kvcacheio, name, None) + if op is not None: + return op + reason = unavailable_reason or ( + f"this sgl-kernel-xpu build has no sgl_kernel.kvcacheio.{name}" + ) + missing_ops[name] = reason + return unavailable_stub(name, reason) + + transfer_kv_all_layer = _bind("transfer_kv_all_layer") + transfer_kv_all_layer_direct_lf_pf = _bind("transfer_kv_all_layer_direct_lf_pf") + transfer_kv_all_layer_lf_pf = _bind("transfer_kv_all_layer_lf_pf") + transfer_kv_all_layer_lf_ph = _bind("transfer_kv_all_layer_lf_ph") + transfer_kv_all_layer_mla = _bind("transfer_kv_all_layer_mla") + transfer_kv_all_layer_mla_lf_pf = _bind("transfer_kv_all_layer_mla_lf_pf") + transfer_kv_direct = _bind("transfer_kv_direct") + transfer_kv_per_layer = _bind("transfer_kv_per_layer") + transfer_kv_per_layer_direct_pf_lf = _bind("transfer_kv_per_layer_direct_pf_lf") + transfer_kv_per_layer_mla = _bind("transfer_kv_per_layer_mla") + transfer_kv_per_layer_mla_pf_lf = _bind("transfer_kv_per_layer_mla_pf_lf") + transfer_kv_per_layer_pf_lf = _bind("transfer_kv_per_layer_pf_lf") + transfer_kv_per_layer_ph_lf = _bind("transfer_kv_per_layer_ph_lf") diff --git a/python/sglang/srt/mem_cache/pool_host/common.py b/python/sglang/srt/mem_cache/pool_host/common.py index decd4b368f..305a54fbd1 100644 --- a/python/sglang/srt/mem_cache/pool_host/common.py +++ b/python/sglang/srt/mem_cache/pool_host/common.py @@ -365,6 +365,35 @@ def alloc_with_pin_memory( return buffer +def alloc_with_pin_memory_xpu( + dims: tuple, + dtype: torch.dtype, + device: str, + pin_memory: bool, + allocator: None, + registration_granularity_bytes: int | None = None, +) -> torch.Tensor: + """`alloc_with_pin_memory`, failing on a null pinned buffer. + + torch-xpu returns `data_ptr() == 0` from a failed pinned allocation instead of + raising; an empty tensor has a null pointer on every backend and is fine. + """ + buffer = alloc_with_pin_memory( + dims=dims, + dtype=dtype, + device=device, + pin_memory=pin_memory, + allocator=allocator, + ) + if pin_memory and buffer.numel() > 0 and buffer.data_ptr() == 0: + n_bytes = buffer.numel() * buffer.element_size() + raise RuntimeError( + f"Pinned host memory allocation of {n_bytes / (1024**3):.1f} GiB " + f"({n_bytes} bytes) returned a null pointer" + ) + return buffer + + @lru_cache(maxsize=1) def _resolve_device_accessible_ptr_fn(): try: @@ -426,6 +455,6 @@ ALLOC_MEMORY_FUNCS = defaultdict( { "npu": alloc_with_pin_memory, "musa": alloc_with_pin_memory, - "xpu": alloc_with_pin_memory, + "xpu": alloc_with_pin_memory_xpu, }, ) diff --git a/python/sglang/srt/mem_cache/pool_host/dsa.py b/python/sglang/srt/mem_cache/pool_host/dsa.py index b27d7d1b7b..cce1676867 100644 --- a/python/sglang/srt/mem_cache/pool_host/dsa.py +++ b/python/sglang/srt/mem_cache/pool_host/dsa.py @@ -17,6 +17,10 @@ from sglang.kernels.ops.kvcache.hicache import ( ) from sglang.srt.mem_cache.hicache_storage import PoolName from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool +from sglang.srt.mem_cache.pool_host._kvcacheio import ( + MLA_KVCACHEIO_OPS, + require_kvcacheio_ops, +) from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, HostKVCache, @@ -50,6 +54,16 @@ if _is_cuda or _is_hip: transfer_kv_per_layer_mla, transfer_kv_per_layer_mla_pf_lf, ) +elif _is_xpu: + from sglang.srt.mem_cache.pool_host._kvcacheio import ( + transfer_kv_all_layer_direct_lf_pf, + transfer_kv_all_layer_mla, + transfer_kv_all_layer_mla_lf_pf, + transfer_kv_direct, + transfer_kv_per_layer_direct_pf_lf, + transfer_kv_per_layer_mla, + transfer_kv_per_layer_mla_pf_lf, + ) logger = logging.getLogger(__name__) @@ -142,6 +156,12 @@ class DSAIndexerPoolHost(HostKVCache): allocator_type: str = "default", is_dummy: bool = False, ): + if _is_xpu and not is_dummy: + require_kvcacheio_ops( + pool=type(self).__name__, + layout=anchor_host.layout, + ops=self._kvcacheio_ops, + ) self._is_dummy = is_dummy self._destroyed = False self.decl = decl @@ -348,6 +368,11 @@ class DSAIndexerPoolHost(HostKVCache): ) return host_page_indices, device_page_indices + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + _kvcacheio_ops = MLA_KVCACHEIO_OPS + def load_to_device_per_layer( self, device_pool, diff --git a/python/sglang/srt/mem_cache/pool_host/mamba.py b/python/sglang/srt/mem_cache/pool_host/mamba.py index ea36ee7779..2dd729f45b 100644 --- a/python/sglang/srt/mem_cache/pool_host/mamba.py +++ b/python/sglang/srt/mem_cache/pool_host/mamba.py @@ -19,11 +19,12 @@ from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, get_allocator_from_storage, ) -from sglang.srt.utils import is_cuda, is_hip, is_npu +from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu _is_cuda = is_cuda() _is_hip = is_hip() _is_npu = is_npu() +_is_xpu = is_xpu() transfer_state_per_layer_direct_pf_lf = None transfer_state_all_layer_direct_lf_pf = None transfer_mamba_state = None @@ -104,6 +105,15 @@ class MambaPoolHost(HostKVCache): "page_first_direct", ], f"Unsupported layout: {layout}" + if _is_xpu: + raise ValueError( + "HiCache does not support hybrid Mamba models on XPU: the Mamba KV " + "transfer kernels are JIT-built from CUDA source. " + "Launch without HiCache host offload " + "(--enable-hierarchical-cache, " + "--disaggregation-decode-retraction-backup=host_pool)." + ) + self.layout = layout self.pin_memory = pin_memory self.device = device diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index 2af000962d..efe3dadb41 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -35,6 +35,7 @@ from sglang.srt.mem_cache.device_pool_info import ( from sglang.srt.mem_cache.memory_pool import ( MHATokenToKOnlyPool, ) +from sglang.srt.mem_cache.pool_host._kvcacheio import require_kvcacheio_ops from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, HostKVCache, @@ -71,6 +72,21 @@ if _is_cuda or _is_hip: transfer_kv_per_layer_pf_lf, transfer_kv_per_layer_ph_lf, ) +elif _is_xpu: + from sglang.srt.mem_cache.pool_host._kvcacheio import ( + transfer_kv_all_layer, + transfer_kv_all_layer_direct_lf_pf, + transfer_kv_all_layer_lf_pf, + transfer_kv_all_layer_lf_ph, + transfer_kv_all_layer_mla_lf_pf, + transfer_kv_direct, + transfer_kv_per_layer, + transfer_kv_per_layer_direct_pf_lf, + transfer_kv_per_layer_mla, + transfer_kv_per_layer_mla_pf_lf, + transfer_kv_per_layer_pf_lf, + transfer_kv_per_layer_ph_lf, + ) if _is_npu: from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange @@ -136,6 +152,10 @@ class MHATokenToKVPoolHost(HostKVCache): pool_label: str = "kv", head_group_num: int = 1, ): + if _is_xpu: + require_kvcacheio_ops( + pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops + ) infos = (buffer_info, *draft_buffer_infos) for info in infos: if ( @@ -375,6 +395,30 @@ class MHATokenToKVPoolHost(HostKVCache): def v_buffer(self): return self.kv_buffer[1] + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + _kvcacheio_ops = { + "kernel": { + "layer_first": ("transfer_kv_per_layer", "transfer_kv_all_layer"), + "page_first": ( + "transfer_kv_per_layer_pf_lf", + "transfer_kv_all_layer_lf_pf", + ), + "page_head": ( + "transfer_kv_per_layer_ph_lf", + "transfer_kv_all_layer_lf_ph", + ), + }, + "direct": { + "layer_first": ("transfer_kv_direct",), + "page_first_direct": ( + "transfer_kv_per_layer_direct_pf_lf", + "transfer_kv_all_layer_direct_lf_pf", + ), + }, + } + def load_to_device_per_layer( self, device_pool, @@ -870,6 +914,10 @@ class MHATokenToKOnlyPoolHost(HostKVCache): allocator_type: str = "default", ): self._destroyed = False + if _is_xpu: + require_kvcacheio_ops( + pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops + ) self.device_pool = device_pool self.page_size = anchor_host.page_size self.layout = layout @@ -979,6 +1027,27 @@ class MHATokenToKOnlyPoolHost(HostKVCache): def get_hybrid_pool_buffer(self): return [self.k_buffer] + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + # layer_first backs up one layer at a time. + _kvcacheio_ops = { + "kernel": { + "layer_first": ("transfer_kv_per_layer_mla",), + "page_first": ( + "transfer_kv_per_layer_mla_pf_lf", + "transfer_kv_all_layer_mla_lf_pf", + ), + }, + "direct": { + "layer_first": ("transfer_kv_direct",), + "page_first_direct": ( + "transfer_kv_per_layer_direct_pf_lf", + "transfer_kv_all_layer_direct_lf_pf", + ), + }, + } + def load_to_device_per_layer( self, device_pool, @@ -1352,6 +1421,24 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): "Use a backend that does not use this interface (e.g. mooncake, simm)." ) + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + _kvcacheio_ops = { + "kernel": { + "page_first": ( + "transfer_kv_per_layer_mla_pf_lf", + "transfer_kv_all_layer_mla_lf_pf", + ), + }, + "direct": { + "page_first_direct": ( + "transfer_kv_per_layer_direct_pf_lf", + "transfer_kv_all_layer_direct_lf_pf", + ), + }, + } + def load_to_device_per_layer( self, device_pool, diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index 9480cddabd..0cced5fc97 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -21,6 +21,10 @@ from sglang.kernels.ops.kvcache.hicache import ( ) from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices from sglang.srt.mem_cache.device_pool_info import MLABufferInfo, MLASplitBuffers +from sglang.srt.mem_cache.pool_host._kvcacheio import ( + MLA_KVCACHEIO_OPS, + require_kvcacheio_ops, +) from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, HostKVCache, @@ -47,7 +51,7 @@ _is_hip = is_hip() _is_npu = is_npu() _is_xpu = is_xpu() _is_mps = is_mps() -if _is_cuda or _is_hip or _is_xpu: +if _is_cuda or _is_hip: from sgl_kernel.kvcacheio import ( transfer_kv_all_layer_direct_lf_pf, transfer_kv_all_layer_mla, @@ -57,6 +61,16 @@ if _is_cuda or _is_hip or _is_xpu: transfer_kv_per_layer_mla, transfer_kv_per_layer_mla_pf_lf, ) +elif _is_xpu: + from sglang.srt.mem_cache.pool_host._kvcacheio import ( + transfer_kv_all_layer_direct_lf_pf, + transfer_kv_all_layer_mla, + transfer_kv_all_layer_mla_lf_pf, + transfer_kv_direct, + transfer_kv_per_layer_direct_pf_lf, + transfer_kv_per_layer_mla, + transfer_kv_per_layer_mla_pf_lf, + ) if _is_npu: from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange @@ -106,6 +120,10 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ): self._is_dummy = isinstance(buffer_info, MLAHostAllocation) self.override_kv_cache_dim = override_kv_cache_dim + if _is_xpu and not self._is_dummy: + require_kvcacheio_ops( + pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops + ) self.dtype = buffer_info.dtype self.layout = layout self.kv_lora_rank = buffer_info.kv_lora_rank @@ -668,6 +686,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): if ret != 0: raise RuntimeError(f"offload.kv_exchange_copy failed with code {ret}") + if _is_xpu: + # {io_backend: {layout: kvcacheio ops}} the transfer methods below call, + # checked at construction; keep in step with their branches. + _kvcacheio_ops = MLA_KVCACHEIO_OPS + def load_to_device_per_layer( self, device_pool, diff --git a/python/sglang/test/hicache_xpu_common.py b/python/sglang/test/hicache_xpu_common.py new file mode 100644 index 0000000000..decfea6a12 --- /dev/null +++ b/python/sglang/test/hicache_xpu_common.py @@ -0,0 +1,118 @@ +"""Shared helpers for the Intel XPU HiCache E2E tests.""" + +import os +import time + +import requests +import torch + +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + find_available_port, + popen_launch_server, +) + +XPU_AVAILABLE = torch.xpu.is_available() if hasattr(torch, "xpu") else False + +# Device KV pool, in tokens. --mem-fraction-static cannot size it small enough +# to evict anything on a 22 GB Arc; 1024 tokens (64 pages at --page-size 16) is +# below every scenario's prompt set, so the reload has to come from the host. +EVICT_DEVICE_POOL_TOKENS = 1024 +# Keeps the host tier (~230 MB) well clear of every working set, so the prefix +# under test is not evicted from the host too and recomputed. +EVICT_HICACHE_RATIO = 8 +# Tokens compared golden-vs-restore. A restored prefix sits on different pages, +# so bf16 reduction order eventually flips a near-tie token even on an exact +# transfer. Measured on Arc (Qwen2.5-1.5B, page_size 16): an exact restore holds +# for ~30 tokens, so 16 leaves margin. Exact identity is checked in +# test_hicache_transfer_round_trip_xpu.py. +COMPARE_TOKENS = 16 + + +def resolve_base_url() -> str: + """A probed-free port, so back-to-back launches in one file cannot collide.""" + default_port = int(DEFAULT_URL_FOR_TEST.rsplit(":", 1)[1]) + return f"http://127.0.0.1:{find_available_port(default_port)}" + + +def launch_server(model: str, base_url: str, other_args: list, env_extra=None): + """Launch a server with env_extra layered over the current environment.""" + return popen_launch_server( + model=model, + base_url=base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={**os.environ, **(env_extra or {})}, + ) + + +def complete(base_url, model, prompt, max_tokens=32, timeout=90, want_cached=False): + resp = requests.post( + f"{base_url}/v1/completions", + json={ + "model": model, + "prompt": prompt, + "max_tokens": max_tokens, + "temperature": 0.0, + }, + timeout=timeout, + ) + resp.raise_for_status() + data = resp.json() + text = data["choices"][0]["text"] + if not want_cached: + return text + cached = 0 + usage = data.get("usage") or {} + details = usage.get("prompt_tokens_details") or {} + if details: + cached = details.get("cached_tokens", 0) or 0 + return text, cached + + +def prime_cache(base_url, model, prompt, max_tokens=COMPARE_TOKENS, timeout=90) -> None: + """Serve `prompt` once so the next identical request is a cache hit. + + The golden must be a cache hit too: cold and prefix-reuse prefill are + different numeric paths and diverge within COMPARE_TOKENS on Arc. + """ + complete(base_url, model, prompt, max_tokens=max_tokens, timeout=timeout) + + +def load_back_tokens(base_url) -> float: + """Tokens loaded host (L2) -> device (L1) since launch, all pools summed. + + Requires --enable-metrics. The only evidence the load kernel ran: output + agreement and cached_tokens > 0 are both satisfied by a device-tier hit. + """ + body = requests.get(f"{base_url}/metrics", timeout=30).text + return sum( + float(line.rsplit(" ", 1)[1]) + for line in body.splitlines() + if line.startswith("sglang:load_back_tokens_total") + ) + + +def wait_load_back_tokens(base_url, above: float, timeout_s: float = 20.0) -> float: + """Poll load_back_tokens_total until it passes `above`, then return it. + + The counter is incremented when the scheduler reaps the load ack + (hiradix_cache.py load_back bookkeeping), which can trail the HTTP response + by a loop iteration, so a bare read right after the completion races. + Returns the last value observed either way; the caller does the asserting. + """ + deadline = time.time() + timeout_s + seen = load_back_tokens(base_url) + while seen <= above and time.time() < deadline: + time.sleep(0.5) + seen = load_back_tokens(base_url) + return seen + + +def shared_prefix_len(a: str, b: str) -> int: + n = min(len(a), len(b)) + i = 0 + while i < n and a[i] == b[i]: + i += 1 + return i diff --git a/test/registered/e2e/hicache/test_hicache_xpu_mla_kernels.py b/test/registered/e2e/hicache/test_hicache_xpu_mla_kernels.py new file mode 100644 index 0000000000..9a3e13ba5e --- /dev/null +++ b/test/registered/e2e/hicache/test_hicache_xpu_mla_kernels.py @@ -0,0 +1,249 @@ +"""MLA (DeepSeek-family) *_mla_* KV-transfer kernels on Intel XPU. + +Kernel selection (see pool_host/mla.py load_to_device_per_layer / +backup_from_device_all_layer): + + io_backend mem_layout load kernel backup kernel + ---------- ------------ ------------------------------- ------------------------------- + kernel layer_first transfer_kv_per_layer_mla transfer_kv_all_layer_mla + kernel page_first transfer_kv_per_layer_mla_pf_lf transfer_kv_all_layer_mla_lf_pf + +The table holds on XPU because MLATokenToKVPoolHost gates can_use_jit on +(_is_cuda or _is_hip) in mla.py, so the jit_transfer_hicache_*_mla branch +preceding each transfer_kv_* call never fires here. +""" + +import shutil +import tempfile +import unittest + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.hicache_xpu_common import ( + COMPARE_TOKENS, + EVICT_DEVICE_POOL_TOKENS, + EVICT_HICACHE_RATIO, + XPU_AVAILABLE, + complete, + launch_server, + load_back_tokens, + prime_cache, + resolve_base_url, + shared_prefix_len, + wait_load_back_tokens, +) +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + CustomTestCase, + terminate_and_kill_process_tree, +) + +register_xpu_ci(est_time=120, suite="stage-b-test-1-gpu-xpu") + + +def _write_tiny_mla_model(model_dir): + """Save a random 2-layer DeepseekV3 with the real MLA dims to model_dir. + + Built locally so the test needs no private or third-party HF repo; + DEFAULT_MODEL_NAME_FOR_TEST_MLA is private. The tokenizer is borrowed from + the Qwen model the other XPU tests already download. Accuracy is + irrelevant: a restore reloads the same KV bytes it wrote. + """ + import torch + from transformers import AutoTokenizer, DeepseekV3Config, DeepseekV3ForCausalLM + + tokenizer = AutoTokenizer.from_pretrained(DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN) + tokenizer.save_pretrained(model_dir) + config = DeepseekV3Config( + vocab_size=-(-len(tokenizer) // 64) * 64, + hidden_size=256, + intermediate_size=512, + num_hidden_layers=2, + # Every layer dense, so no MoE weights are built. Past num_hidden_layers + 1 + # because sglang also probes layer_id + 1, which reads the moe_layer_freq + # that transformers 5 no longer saves. + first_k_dense_replace=3, + n_routed_experts=8, + num_experts_per_tok=2, + n_group=1, + topk_group=1, + moe_intermediate_size=128, + num_attention_heads=2, + num_key_value_heads=2, + # DeepSeek-V3's MLA dims: these set the KV-cache shape the kernels move. + q_lora_rank=1536, + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + max_position_embeddings=4096, + bos_token_id=tokenizer.bos_token_id, + eos_token_id=tokenizer.eos_token_id, + tie_word_embeddings=False, + ) + torch.manual_seed(0) + model = DeepseekV3ForCausalLM(config).to(torch.bfloat16) + model.save_pretrained(model_dir) + + +_model_dir = None + + +def setUpModule(): + """Build the checkpoint once; both server classes load the same weights.""" + global _model_dir + if not XPU_AVAILABLE: + return + _model_dir = tempfile.mkdtemp(prefix="hc_xpu_mla_model_") + try: + _write_tiny_mla_model(_model_dir) + except BaseException: + # tearDownModule does not run when setUpModule raises. + shutil.rmtree(_model_dir, ignore_errors=True) + _model_dir = None + raise + + +def tearDownModule(): + if _model_dir is not None: + shutil.rmtree(_model_dir, ignore_errors=True) + + +class _MlaKernelServer(CustomTestCase): + """One MLA HiCache server with a host tier only, plus the scenario it runs.""" + + model = None + io_backend = None + mem_layout = None + process = None + + @classmethod + def setUpClass(cls): + if cls is _MlaKernelServer: + raise unittest.SkipTest( + "abstract base; concrete subclasses set io_backend and mem_layout" + ) + cls.base_url = resolve_base_url() + cls.model = _model_dir + cls.process = launch_server( + cls.model, + cls.base_url, + [ + "--tp", + 1, + "--attention-backend", + "triton", + "--enable-hierarchical-cache", + # The device KV pool is (mem-fraction x device mem), so 0.1 + # keeps the host pin under ~3 GB on a 22 GB Arc and leaves the + # rest of the card for the fixture's weights. + "--mem-fraction-static", + 0.1, + "--max-total-tokens", + EVICT_DEVICE_POOL_TOKENS, + "--hicache-ratio", + EVICT_HICACHE_RATIO, + "--page-size", + 16, + "--hicache-io-backend", + cls.io_backend, + "--hicache-mem-layout", + cls.mem_layout, + "--enable-cache-report", + "--enable-metrics", + ], + ) + + @classmethod + def tearDownClass(cls): + if cls.process is not None: + terminate_and_kill_process_tree(cls.process) + + def test_offload_reload_round_trip(self): + """Evict a long prefix to the host tier, reload it, and compare greedily. + + One method rather than several: the cached-token assertions are only + meaningful after the eviction has run, and unittest orders methods + alphabetically, not causally. + """ + # 444 + 8 x 128 tokens overflow the 1024-token device pool, forcing a reload. + prefix = "The history of computing is a long and fascinating story. " * 40 + prime_cache( + self.base_url, + self.model, + prefix + " In summary,", + max_tokens=COMPARE_TOKENS, + timeout=120, + ) + out1, out1_cached = complete( + self.base_url, + self.model, + prefix + " In summary,", + max_tokens=COMPARE_TOKENS, + timeout=120, + want_cached=True, + ) + for i in range(8): + complete( + self.base_url, + self.model, + f"Unrelated filler request {i}: " + "lorem ipsum " * 60, + max_tokens=16, + timeout=120, + ) + loaded_before = load_back_tokens(self.base_url) + out2, cached = complete( + self.base_url, + self.model, + prefix + " In summary,", + max_tokens=COMPARE_TOKENS, + timeout=120, + want_cached=True, + ) + + tag = f"[{self.io_backend}/{self.mem_layout}]" + loaded_after = wait_load_back_tokens(self.base_url, loaded_before) + self.assertGreater( + loaded_after, + loaded_before, + f"{tag} no tokens loaded back from the host tier -> the reload was " + f"served without running the load kernel, so the assertions below " + f"cannot distinguish it from a device-tier hit", + ) + self.assertGreater(len(out1.strip()), 0, f"{tag} empty output") + self.assertGreater( + cached, 0, f"{tag} reload reported 0 cached tokens -> KV was recomputed" + ) + self.assertEqual( + cached, + out1_cached, + f"{tag} reload reused {cached} tokens vs the baseline's " + f"{out1_cached} -> the tiers disagree on the prefix boundary", + ) + spl = shared_prefix_len(out2, out1) + self.assertEqual( + out2, + out1, + f"{tag} reloaded continuation diverged after {spl} chars " + f"(first={out1!r} reloaded={out2!r}) -> the host round trip " + f"corrupted the KV", + ) + + +@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") +class TestMlaKernelLayerFirst(_MlaKernelServer): + """kernel/layer_first: transfer_kv_per_layer_mla / transfer_kv_all_layer_mla.""" + + io_backend = "kernel" + mem_layout = "layer_first" + + +@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") +class TestMlaKernelPageFirst(_MlaKernelServer): + """kernel/page_first: transfer_kv_per_layer_mla_pf_lf / *_all_layer_mla_lf_pf.""" + + io_backend = "kernel" + mem_layout = "page_first" + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/registered/e2e/hicache/test_hicache_xpu_no_storage.py b/test/registered/e2e/hicache/test_hicache_xpu_no_storage.py new file mode 100644 index 0000000000..c5def6c48b --- /dev/null +++ b/test/registered/e2e/hicache/test_hicache_xpu_no_storage.py @@ -0,0 +1,161 @@ +"""HiCache device<->host on Intel XPU with NO Tier-3 storage. + +Isolates the plain device<->host copy so a regression there is not attributed to +the storage tier. Both KV-transfer io-backends are covered. +""" + +import unittest + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.hicache_xpu_common import ( + COMPARE_TOKENS, + EVICT_DEVICE_POOL_TOKENS, + EVICT_HICACHE_RATIO, + XPU_AVAILABLE, + complete, + launch_server, + load_back_tokens, + prime_cache, + resolve_base_url, + shared_prefix_len, + wait_load_back_tokens, +) +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + CustomTestCase, + terminate_and_kill_process_tree, +) + +register_xpu_ci(est_time=400, suite="stage-b-test-1-gpu-xpu") + + +class _NoStorageServer(CustomTestCase): + """One HiCache server, no storage tier, plus the scenario it runs.""" + + model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN + io_backend = None + mem_layout = None + process = None + + @classmethod + def setUpClass(cls): + if cls is _NoStorageServer: + raise unittest.SkipTest( + "abstract base; concrete subclasses set io_backend and mem_layout" + ) + cls.base_url = resolve_base_url() + cls.process = launch_server( + cls.model, + cls.base_url, + [ + "--tp", + 1, + "--enable-hierarchical-cache", + # 1.5B weights plus a 0.4-fraction KV pool leave room for the + # host pin on a 22 GB Arc. + "--mem-fraction-static", + 0.4, + "--max-total-tokens", + EVICT_DEVICE_POOL_TOKENS, + "--hicache-ratio", + EVICT_HICACHE_RATIO, + "--page-size", + 16, + "--hicache-io-backend", + cls.io_backend, + "--hicache-mem-layout", + cls.mem_layout, + "--enable-cache-report", + "--enable-metrics", + ], + ) + + @classmethod + def tearDownClass(cls): + if cls.process is not None: + terminate_and_kill_process_tree(cls.process) + + def test_repeat_after_eviction_agrees(self): + """A repeat reloaded from the host tier agrees with the device-tier hit. + + The two runs of the prompt are separated by enough filler to evict it, + so the second one is served by the host->device load kernel rather than + by the radix tree; load_back_tokens_total is what tells the two apart. + """ + # Radix reuse is page-granular, so the prompt must span several + # --page-size 16 pages; an 11-token prompt reports 0 cached whatever the + # KV path does. + prompt = ( + "The answer to life, the universe, and everything has been debated " + "by philosophers, novelists, and astronomers alike. " * 8 + ) + " The most widely cited answer is" + prime_cache(self.base_url, self.model, prompt, max_tokens=COMPARE_TOKENS) + first, first_cached = complete( + self.base_url, + self.model, + prompt, + max_tokens=COMPARE_TOKENS, + want_cached=True, + ) + # 191-token prompt + 10 x 128 tokens of filler against a 1024-token + # device pool, so the prompt's pages are evicted to the host tier. + for i in range(10): + complete( + self.base_url, + self.model, + f"Unrelated filler request {i}: " + "lorem ipsum " * 60, + max_tokens=16, + ) + loaded_before = load_back_tokens(self.base_url) + second, cached = complete( + self.base_url, + self.model, + prompt, + max_tokens=COMPARE_TOKENS, + want_cached=True, + ) + loaded_after = wait_load_back_tokens(self.base_url, loaded_before) + self.assertGreater( + loaded_after, + loaded_before, + "no tokens loaded back from the host tier -> the repeat was served " + "from the device pool, so the assertions below say nothing about the " + "device<->host path", + ) + self.assertGreater( + cached, 0, "repeat reported 0 cached tokens -> no prefix reuse" + ) + self.assertEqual( + cached, + first_cached, + f"reuse boundary moved between the two repeats " + f"({first_cached} then {cached} cached tokens)", + ) + spl = shared_prefix_len(second, first) + self.assertEqual( + second, + first, + f"repeat diverged after {spl} chars " + f"(first={first!r} second={second!r}) -> device<->host reuse " + f"corrupted the KV", + ) + + +@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") +class TestNoStorageDirectLayerFirst(_NoStorageServer): + """direct/layer_first: device<->host reuse with no storage tier.""" + + io_backend = "direct" + mem_layout = "layer_first" + + +@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") +class TestNoStorageKernelPageFirst(_NoStorageServer): + """kernel/page_first: device<->host reuse with no storage tier.""" + + io_backend = "kernel" + mem_layout = "page_first" + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/registered/unit/mem_cache/test_hicache_host_register.py b/test/registered/unit/mem_cache/test_hicache_host_register.py index 11f46b734b..893c65986e 100644 --- a/test/registered/unit/mem_cache/test_hicache_host_register.py +++ b/test/registered/unit/mem_cache/test_hicache_host_register.py @@ -20,6 +20,9 @@ from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, _cuda_host_register, _cuda_host_unregister, + alloc_with_host_register, + alloc_with_pin_memory, + alloc_with_pin_memory_xpu, ) from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost @@ -547,6 +550,33 @@ class TestHiCacheHostRegister(unittest.TestCase): for ptr, _, _ in cudart.registrations: self.assertEqual((ptr - base) % page_copy_bytes, 0) + def test_null_pinned_allocation_raises(self): + """On XPU a null pinned buffer must fail at allocation, not at first transfer.""" + gib = 1024**3 + + def alloc(buffer): + with mock.patch.object(torch, "empty", return_value=buffer): + return alloc_with_pin_memory_xpu( + (buffer.numel(),), + dtype=torch.uint8, + device="cpu", + pin_memory=True, + allocator=None, + ) + + with self.assertRaisesRegex(RuntimeError, r"2\.0 GiB .* null pointer"): + alloc(_FakeBuffer(0, 2 * gib)) + # An empty tensor has data_ptr() == 0 on every backend and is not a failure. + empty = _FakeBuffer(0, 0) + self.assertIs(alloc(empty), empty) + + def test_only_xpu_uses_the_null_checked_allocator(self): + self.assertIs(ALLOC_MEMORY_FUNCS["xpu"], alloc_with_pin_memory_xpu) + for device in ("npu", "musa"): + self.assertIs(ALLOC_MEMORY_FUNCS[device], alloc_with_pin_memory) + # CUDA and ROCm ("cuda") keep the cudaHostRegister default. + self.assertIs(ALLOC_MEMORY_FUNCS.default_factory(), alloc_with_host_register) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_hicache_kvcacheio_gate.py b/test/registered/unit/mem_cache/test_hicache_kvcacheio_gate.py new file mode 100644 index 0000000000..03ed988e75 --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_kvcacheio_gate.py @@ -0,0 +1,312 @@ +import ast +import importlib +import sys +import types +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from sglang.srt import runtime_context +from sglang.srt import utils as srt_utils +from sglang.srt.mem_cache.pool_host import _kvcacheio +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +# The ops `_kvcacheio` binds on XPU, read from its `_bind("...")` calls: off XPU +# (CPU CI) the module binds nothing. +ALL_OPS = sorted( + node.args[0].value + for node in ast.walk(ast.parse(Path(_kvcacheio.__file__).read_text())) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_bind" +) + +_MEM_CACHE = Path(_kvcacheio.__file__).resolve().parents[1] +_POOL_FILES = [ + _MEM_CACHE / "memory_pool_host.py", + *(_MEM_CACHE / "pool_host").glob("*.py"), +] +# Pools that call kvcacheio ops but refuse unsupported devices on their own terms. +_OWN_DEVICE_GATE = { + "MambaPoolHost": "refuses XPU at construction", + "UnifiedPageEnvelopeHostPool": "imports kvcacheio itself and raises off CUDA/ROCm", +} + + +def _shared_table(pool: str, name: str): + table = getattr(_kvcacheio, name, None) + if table is None: + raise AssertionError( + f"{pool}._kvcacheio_ops = {name}: a shared op table must be defined " + "in pool_host/_kvcacheio.py" + ) + return table + + +def _is_xpu_guard(test: ast.expr) -> bool: + """`_is_xpu`, or an `and` whose first operand is `_is_xpu`.""" + if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.And): + test = test.values[0] + return isinstance(test, ast.Name) and test.id == "_is_xpu" + + +class _PoolClass: + def __init__(self, node: ast.ClassDef): + self.bases = [b.id for b in node.bases if isinstance(b, ast.Name)] + self.calls = { + n.func.id + for n in ast.walk(node) + if isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) + and n.func.id in ALL_OPS + } + self.own_table = None + self.table_is_xpu_only = False + for stmt, guarded in self._class_level(node.body): + if ( + isinstance(stmt, ast.Assign) + and len(stmt.targets) == 1 + and isinstance(stmt.targets[0], ast.Name) + and stmt.targets[0].id == "_kvcacheio_ops" + ): + self.table_is_xpu_only = guarded + self.own_table = ( + _shared_table(node.name, stmt.value.id) + if isinstance(stmt.value, ast.Name) + else ast.literal_eval(stmt.value) + ) + + @staticmethod + def _class_level(body): + """Class-body statements, looking inside `if _is_xpu:` blocks.""" + for stmt in body: + if isinstance(stmt, ast.If) and _is_xpu_guard(stmt.test): + yield from ((inner, True) for inner in stmt.body) + else: + yield stmt, False + + +def _parse_pool_classes() -> dict[str, _PoolClass]: + classes = {} + for path in _POOL_FILES: + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node, ast.ClassDef): + classes[node.name] = _PoolClass(node) + return classes + + +_POOLS = _parse_pool_classes() + + +def _table(name: str): + """The `_kvcacheio_ops` a pool class declares or inherits, else None.""" + cls = _POOLS[name] + if cls.own_table is not None: + return cls.own_table + for base in cls.bases: + if base in _POOLS and (table := _table(base)) is not None: + return table + return None + + +def _table_ops(table) -> set[str]: + return {op for layouts in table.values() for ops in layouts.values() for op in ops} + + +class TestHostPoolOpTables(unittest.TestCase): + """Each host pool's `_kvcacheio_ops` must match the ops its dispatch calls.""" + + def test_every_called_op_is_declared(self): + # A pool calling an op its table omits skips the construction check for it. + checked = 0 + for name, cls in _POOLS.items(): + if not cls.calls or name in _OWN_DEVICE_GATE: + continue + table = _table(name) + with self.subTest(pool=name): + self.assertIsNotNone(table, "calls kvcacheio ops but has no table") + self.assertLessEqual(cls.calls, _table_ops(table)) + checked += 1 + self.assertGreaterEqual(checked, 8) + + def test_no_declared_op_is_stale(self): + # A table entry the dispatch never calls refuses a pool that would work. + for name, cls in _POOLS.items(): + if cls.own_table is None: + continue + with self.subTest(pool=name): + self.assertLessEqual(_table_ops(cls.own_table), cls.calls) + + +def _require_calls(tree: ast.Module) -> list[ast.Call]: + return [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "require_kvcacheio_ops" + ] + + +class TestRequireIsXpuOnly(unittest.TestCase): + """CUDA/ROCm/NPU/CPU never reach the construction check.""" + + def test_every_call_is_under_an_is_xpu_branch(self): + calls = 0 + for path in _POOL_FILES: + tree = ast.parse(path.read_text()) + parents = { + child: node + for node in ast.walk(tree) + for child in ast.iter_child_nodes(node) + } + for call in _require_calls(tree): + calls += 1 + branch = parents[call] + while not isinstance(branch, (ast.If, ast.FunctionDef, ast.Module)): + branch = parents[branch] + with self.subTest(file=path.name, line=call.lineno): + self.assertIsInstance(branch, ast.If, "not under any if") + self.assertTrue( + _is_xpu_guard(branch.test), + f"guarded by `{ast.unparse(branch.test)}`, not `_is_xpu`", + ) + self.assertGreaterEqual(calls, 6) + + def test_every_table_is_xpu_only(self): + tables = 0 + for name, cls in _POOLS.items(): + if cls.own_table is None: + continue + tables += 1 + with self.subTest(pool=name): + self.assertTrue( + cls.table_is_xpu_only, "_kvcacheio_ops is not under `if _is_xpu:`" + ) + self.assertGreaterEqual(tables, 7) + + def test_imported_at_module_level(self): + # The call is guarded by `_is_xpu` alone, so an import under + # `elif _is_xpu:` would be skipped when CUDA and XPU are both visible. + for path in _POOL_FILES: + tree = ast.parse(path.read_text()) + if not _require_calls(tree): + continue + imported = { + alias.name + for node in tree.body + if isinstance(node, ast.ImportFrom) + for alias in node.names + } + with self.subTest(file=path.name): + self.assertIn("require_kvcacheio_ops", imported) + + +def _reload_gate(): + # reload re-runs the module in its existing namespace and never deletes + # names, so drop the ops an earlier load bound (an XPU box binds them at + # import) before binding them again. + for name in ALL_OPS: + vars(_kvcacheio).pop(name, None) + importlib.reload(_kvcacheio) + + +class TestKvcacheioGate(unittest.TestCase): + """On XPU the gate binds or refuses per op, as the sgl-kernel-xpu build allows.""" + + PAIR = _table("MHATokenToKVPoolHost") + SINGLE = _table("MLATokenToKVPoolHost") + K_ONLY = _table("MHATokenToKOnlyPoolHost") + + def tearDown(self): + # Rebind against the real device and sgl_kernel for the tests after this one. + _reload_gate() + + def _reload(self, *, device, kvcacheio_ops): + """Reload the gate as if on `device` with a kvcacheio exposing `kvcacheio_ops`. + + `kvcacheio_ops=None` makes `import sgl_kernel.kvcacheio` fail. + """ + if kvcacheio_ops is None: + modules = {"sgl_kernel.kvcacheio": None} + else: + fake = types.ModuleType("sgl_kernel.kvcacheio") + for name in kvcacheio_ops: + setattr(fake, name, mock.Mock(name=name)) + parent = types.ModuleType("sgl_kernel") + parent.kvcacheio = fake + modules = {"sgl_kernel": parent, "sgl_kernel.kvcacheio": fake} + with ( + mock.patch.dict(sys.modules, modules), + mock.patch.object(srt_utils, "is_xpu", lambda: device == "xpu"), + ): + _reload_gate() + + def _require(self, *, io_backend, layout, ops): + memory = SimpleNamespace(hicache_io_backend=io_backend) + with mock.patch.object(runtime_context, "get_memory", return_value=memory): + _kvcacheio.require_kvcacheio_ops(pool="TestPool", layout=layout, ops=ops) + + def _assert_never_reads_server_args(self): + with mock.patch.object( + runtime_context, "get_memory", side_effect=AssertionError("read") + ): + for table in (self.PAIR, self.SINGLE): + _kvcacheio.require_kvcacheio_ops( + pool="TestPool", layout="layer_first", ops=table + ) + + def test_xpu_complete_build_binds_all_ops(self): + self._reload(device="xpu", kvcacheio_ops=ALL_OPS) + self.assertEqual(_kvcacheio.missing_ops, {}) + self.assertIsNone(_kvcacheio.unavailable_reason) + self._assert_never_reads_server_args() + + def test_xpu_missing_op_refuses_only_pools_that_use_it(self): + self._reload( + device="xpu", + kvcacheio_ops=[n for n in ALL_OPS if n != "transfer_kv_all_layer_mla"], + ) + self.assertEqual(list(_kvcacheio.missing_ops), ["transfer_kv_all_layer_mla"]) + self.assertIsNone(_kvcacheio.unavailable_reason) + + for io_backend, layouts in self.PAIR.items(): + for layout in layouts: + self._require(io_backend=io_backend, layout=layout, ops=self.PAIR) + self._require(io_backend="kernel", layout="page_first", ops=self.SINGLE) + # K-only backs up layer_first per layer, so it never calls the all-layer op. + self._require(io_backend="kernel", layout="layer_first", ops=self.K_ONLY) + with self.assertRaisesRegex(ValueError, "Pick another io backend or layout"): + self._require(io_backend="kernel", layout="layer_first", ops=self.SINGLE) + with self.assertRaisesRegex(RuntimeError, "transfer_kv_all_layer_mla"): + _kvcacheio.transfer_kv_all_layer_mla() + + def test_xpu_without_kvcacheio_refuses_every_pool(self): + self._reload(device="xpu", kvcacheio_ops=None) + self.assertEqual(sorted(_kvcacheio.missing_ops), ALL_OPS) + self.assertIsNotNone(_kvcacheio.unavailable_reason) + for ops in (self.PAIR, self.SINGLE): + with self.assertRaisesRegex(ValueError, "Launch without HiCache"): + self._require(io_backend="direct", layout="layer_first", ops=ops) + + def test_other_devices_bind_nothing(self): + # CUDA/ROCm import kvcacheio in the pool modules; the gate stays inert. + for device in ("cuda", "hip", "npu", "cpu"): + with self.subTest(device=device): + self._reload( + device=device, + kvcacheio_ops=[ + n for n in ALL_OPS if n != "transfer_kv_all_layer_mla" + ], + ) + self.assertEqual(_kvcacheio.missing_ops, {}) + self.assertIsNone(_kvcacheio.unavailable_reason) + self.assertFalse(hasattr(_kvcacheio, "transfer_kv_per_layer")) + self._assert_never_reads_server_args() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_hicache_mtp_state.py b/test/registered/unit/mem_cache/test_hicache_mtp_state.py index fb3e0937b9..f76b8f809f 100644 --- a/test/registered/unit/mem_cache/test_hicache_mtp_state.py +++ b/test/registered/unit/mem_cache/test_hicache_mtp_state.py @@ -134,6 +134,8 @@ def test_ascend_packed_state_roundtrip_uses_buffer_descriptors( monkeypatch, temporal_width ): monkeypatch.setenv("SGLANG_NPU_HICACHE_MAMBA_IO", "sync") + # CPU buffers: keep the XPU refusal off when this runs on an XPU host. + monkeypatch.setattr("sglang.srt.mem_cache.pool_host.mamba._is_xpu", False) infos = tuple( MambaStateBufferInfo( temporal=torch.arange(layers * 8 * temporal_width).reshape( diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index a8cc98cbc0..cffaf022ff 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -1399,5 +1399,67 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): self.assertEqual(captured["host_indices"].device.type, "cpu") +class _IndexTensorStub: + """Stands in for an index tensor resident on ``device_type``; CPU CI has no xpu.""" + + def __init__(self, device_type: str): + self.device = torch.device(device_type) + self.is_cuda = device_type == "cuda" + self.recorded = [] + + def record_stream(self, stream): + self.recorded.append(stream) + + +def _record_stream_transfer(host_indices, device_indices) -> L2Transfer: + return L2Transfer( + host_pool=None, + device_pool=None, + host_indices=host_indices, + device_indices=device_indices, + ) + + +class TestL2TransferRecordStream(CustomTestCase): + """XPU index tensors must be kept alive across an L2 copy, like CUDA ones.""" + + def test_xpu_indices_are_kept_alive(self): + host = _IndexTensorStub("cpu") + device = _IndexTensorStub("xpu") + stream = object() + + L2TransferEngine._record_stream([_record_stream_transfer(host, device)], stream) + + self.assertEqual( + device.recorded, + [stream], + "xpu index tensor was not pinned to the transfer stream; it can be " + "freed and reused while the copy is in flight", + ) + self.assertEqual( + host.recorded, [], "record_stream is meaningless for a host tensor" + ) + + def test_cuda_indices_are_kept_alive(self): + host = _IndexTensorStub("cpu") + device = _IndexTensorStub("cuda") + stream = object() + + L2TransferEngine._record_stream([_record_stream_transfer(host, device)], stream) + + self.assertEqual(device.recorded, [stream]) + self.assertEqual(host.recorded, []) + + def test_other_accelerators_are_unchanged(self): + # "npu" needs torch_npu to parse; mps stands in for any other accelerator. + device = _IndexTensorStub("mps") + + L2TransferEngine._record_stream( + [_record_stream_transfer(_IndexTensorStub("cpu"), device)], object() + ) + + self.assertEqual(device.recorded, []) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 713bc3dae3..3572f6f608 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -9,6 +9,7 @@ from types import SimpleNamespace import torch +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer from sglang.srt.mem_cache.memory_pool import ( MambaPool, @@ -22,6 +23,7 @@ from sglang.srt.mem_cache.memory_pool_host import ( LogicalHostPool, ) from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry, base +from sglang.srt.mem_cache.pool_host import mamba as mamba_pool_host from sglang.srt.mem_cache.pool_host.dsa import ( DSAIndexerPoolHost, make_dsa_indexer_pool_decl, @@ -610,14 +612,16 @@ class TestLazyHostPoolRelease(CustomTestCase): pool._slot_siblings = [] (info,) = pool.get_device_pool_infos() info.buffer_info.validate(layer_ids=info.layer_ids) - host = MambaPoolHost( - info.buffer_info, - host_to_device_ratio=2, - host_size=0, - device_capacity=4, - layout="page_first", - pin_memory=False, - ) + # CPU buffers: keep the XPU refusal off when this runs on an XPU host. + with unittest.mock.patch.object(mamba_pool_host, "_is_xpu", False): + host = MambaPoolHost( + info.buffer_info, + host_to_device_ratio=2, + host_size=0, + device_capacity=4, + layout="page_first", + pin_memory=False, + ) try: self.assertEqual(info.layer_ids, ()) self.assertEqual(host.num_mamba_layers, 0) @@ -936,5 +940,24 @@ class TestDSAIndexerPoolDecl(CustomTestCase): self.assertEqual(mirror._draft_host_layer(stub.layer_num), 2) +class TestMambaHostPoolOnXpu(CustomTestCase): + def test_refused_at_construction(self): + state = MambaStateBufferInfo( + temporal=torch.zeros(1, 4, 2, 2), conv=(torch.zeros(1, 4, 3, 2),) + ) + with ( + unittest.mock.patch.object(mamba_pool_host, "_is_xpu", True), + self.assertRaisesRegex(ValueError, "hybrid Mamba models on XPU"), + ): + MambaPoolHost( + state, + host_to_device_ratio=2, + host_size=0, + device_capacity=4, + layout="page_first", + pin_memory=False, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/xpu/test_hicache_transfer_round_trip_xpu.py b/test/registered/xpu/test_hicache_transfer_round_trip_xpu.py new file mode 100644 index 0000000000..2d2eb49aaa --- /dev/null +++ b/test/registered/xpu/test_hicache_transfer_round_trip_xpu.py @@ -0,0 +1,169 @@ +"""Device<->host KV transfer on Intel XPU is an identity, per kernel pair. + +backup_from_device_all_layer then load_to_device_per_layer must return the exact +bytes, which the e2e tests cannot check since they compare greedy continuations. + +Kernel pairs covered (see pool_host/mha.py and pool_host/mla.py, +load_to_device_per_layer / backup_from_device_all_layer): + + io_backend mem_layout load kernel backup kernel + ---------- ------------------ ---------------------------------- ---------------------------------- + kernel layer_first transfer_kv_per_layer transfer_kv_all_layer + kernel page_first transfer_kv_per_layer_pf_lf transfer_kv_all_layer_lf_pf + kernel page_head transfer_kv_per_layer_ph_lf transfer_kv_all_layer_lf_ph + direct layer_first transfer_kv_direct transfer_kv_direct + direct page_first_direct transfer_kv_per_layer_direct_pf_lf transfer_kv_all_layer_direct_lf_pf + kernel layer_first (MLA) transfer_kv_per_layer_mla transfer_kv_all_layer_mla + kernel page_first (MLA) transfer_kv_per_layer_mla_pf_lf transfer_kv_all_layer_mla_lf_pf +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, MLATokenToKVPool +from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost +from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase + +register_xpu_ci(est_time=30, suite="stage-a-test-1-gpu-xpu") + +XPU_AVAILABLE = torch.xpu.is_available() if hasattr(torch, "xpu") else False + +DEVICE = "xpu" +PAGE_SIZE = 16 +NUM_PAGES = 8 +POOL_TOKENS = PAGE_SIZE * NUM_PAGES +LAYER_NUM = 4 +DTYPE = torch.bfloat16 + + +def _fill_random(buffers) -> list[torch.Tensor]: + """Randomize each buffer and return clones to compare against.""" + for buf in buffers: + buf.copy_(torch.randn(buf.shape, dtype=torch.float32, device=buf.device)) + return [buf.clone() for buf in buffers] + + +def _transfer_indices(io_backend: str): + """One page-aligned run of tokens, host slots shifted by a page. + + The shift keeps host and device indices from coinciding, so a kernel that + ignores one of the two index tensors still round-trips wrong. Placement + mirrors CacheController.move_indices: the kernel backend wants both tensors + on the device, the direct backend wants device_indices on the host. + """ + n = PAGE_SIZE * (NUM_PAGES - 1) + device_indices = torch.arange(n, dtype=torch.int64, device=DEVICE) + host_indices = torch.arange(PAGE_SIZE, PAGE_SIZE + n, dtype=torch.int64) + if io_backend == "kernel": + return host_indices.to(DEVICE), device_indices, n + return host_indices, device_indices.cpu(), n + + +@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") +class TestHiCacheTransferRoundTripXpu(CustomTestCase): + def _assert_round_trip(self, host, device_pool, device_buffers, io_backend): + reference = _fill_random(device_buffers) + host_indices, device_indices, n = _transfer_indices(io_backend) + + host.backup_from_device_all_layer( + device_pool, host_indices, device_indices, io_backend + ) + torch.xpu.synchronize() + for buf in device_buffers: + buf.zero_() + torch.xpu.synchronize() + for layer_id in range(LAYER_NUM): + host.load_to_device_per_layer( + device_pool, host_indices, device_indices, layer_id, io_backend + ) + torch.xpu.synchronize() + + for i, (got, want) in enumerate(zip(device_buffers, reference)): + differing = int((got[:n] != want[:n]).sum()) + self.assertEqual( + differing, + 0, + f"buffer {i}: {differing}/{got[:n].numel()} elements changed across " + f"the {io_backend} round trip -> the transfer is not an identity", + ) + + def _mha_round_trip(self, io_backend: str, layout: str): + device_pool = MHATokenToKVPool( + size=POOL_TOKENS, + page_size=PAGE_SIZE, + dtype=DTYPE, + head_num=2, + head_dim=128, + layer_num=LAYER_NUM, + device=DEVICE, + enable_memory_saver=False, + enable_alt_stream=False, + ) + host = MHATokenToKVPoolHost( + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, + host_to_device_ratio=2.0, + host_size=0, + page_size=PAGE_SIZE, + layout=layout, + device_capacity_tokens=device_pool.size, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, + ) + buffers = [device_pool.k_buffer[i] for i in range(LAYER_NUM)] + buffers += [device_pool.v_buffer[i] for i in range(LAYER_NUM)] + self._assert_round_trip(host, device_pool, buffers, io_backend) + + def _mla_round_trip(self, io_backend: str, layout: str): + device_pool = MLATokenToKVPool( + size=POOL_TOKENS, + page_size=PAGE_SIZE, + dtype=DTYPE, + kv_lora_rank=512, + qk_rope_head_dim=64, + layer_num=LAYER_NUM, + device=DEVICE, + enable_memory_saver=False, + ) + host = MLATokenToKVPoolHost( + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, + host_to_device_ratio=2.0, + host_size=0, + page_size=PAGE_SIZE, + layout=layout, + device_capacity_tokens=device_pool.size, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + target_layer_num=device_pool.layer_num, + ) + # MLA keeps the latent KV in one buffer per layer, not a K/V pair. + buffers = [device_pool.kv_buffer[i] for i in range(LAYER_NUM)] + self._assert_round_trip(host, device_pool, buffers, io_backend) + + def test_mha_kernel_layer_first(self): + self._mha_round_trip("kernel", "layer_first") + + def test_mha_kernel_page_first(self): + self._mha_round_trip("kernel", "page_first") + + def test_mha_kernel_page_head(self): + self._mha_round_trip("kernel", "page_head") + + def test_mha_direct_layer_first(self): + self._mha_round_trip("direct", "layer_first") + + def test_mha_direct_page_first_direct(self): + self._mha_round_trip("direct", "page_first_direct") + + def test_mla_kernel_layer_first(self): + self._mla_round_trip("kernel", "layer_first") + + def test_mla_kernel_page_first(self): + self._mla_round_trip("kernel", "page_first") + + +if __name__ == "__main__": + unittest.main(verbosity=2)