[Feat] [Intel] hicache: Enable L1<->L2 support on XPU (#32503)
This commit is contained in:
@@ -197,5 +197,7 @@ class L2TransferEngine:
|
||||
# Keep temporary translated indices alive until the transfer completes.
|
||||
tensors.extend(resolved)
|
||||
for indices in tensors:
|
||||
if indices is not None and indices.is_cuda:
|
||||
if indices is not None and (
|
||||
indices.is_cuda or indices.device.type == "xpu"
|
||||
):
|
||||
indices.record_stream(stream)
|
||||
|
||||
@@ -13,6 +13,11 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
transfer_hicache_all_layer_mla_staged_lf_pf as jit_transfer_hicache_all_layer_mla_staged_lf_pf,
|
||||
)
|
||||
from sglang.kernels.ops.kvcache.hisparse import transfer_cache_dsv4_mla
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
MLA_KVCACHEIO_OPS,
|
||||
require_kvcacheio_ops,
|
||||
unavailable_stub,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
|
||||
|
||||
@@ -21,11 +26,22 @@ _is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
_is_mps = is_mps()
|
||||
_dsv4_mla_unavailable_reason: Optional[str] = None
|
||||
if _is_xpu:
|
||||
from sgl_kernel import transfer_cache_dsv4_mla
|
||||
try:
|
||||
from sgl_kernel import transfer_cache_dsv4_mla
|
||||
except ImportError as e:
|
||||
# Only DSV4 needs this op; a wheel without it must not break scheduler import.
|
||||
_dsv4_mla_unavailable_reason = (
|
||||
f"sgl_kernel.transfer_cache_dsv4_mla failed to import ({e}); XPU "
|
||||
"takes it from the out-of-tree sgl-kernel-xpu wheel"
|
||||
)
|
||||
transfer_cache_dsv4_mla = unavailable_stub(
|
||||
"transfer_cache_dsv4_mla", _dsv4_mla_unavailable_reason
|
||||
)
|
||||
else:
|
||||
from sglang.kernels.ops.kvcache.hisparse import transfer_cache_dsv4_mla
|
||||
if _is_cuda or _is_hip or _is_xpu:
|
||||
if _is_cuda or _is_hip:
|
||||
from sgl_kernel.kvcacheio import (
|
||||
transfer_kv_all_layer_direct_lf_pf,
|
||||
transfer_kv_all_layer_mla,
|
||||
@@ -35,6 +51,16 @@ if _is_cuda or _is_hip or _is_xpu:
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
)
|
||||
elif _is_xpu:
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
transfer_kv_all_layer_direct_lf_pf,
|
||||
transfer_kv_all_layer_mla,
|
||||
transfer_kv_all_layer_mla_lf_pf,
|
||||
transfer_kv_direct,
|
||||
transfer_kv_per_layer_direct_pf_lf,
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
)
|
||||
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange
|
||||
@@ -223,6 +249,18 @@ class DeepSeekV4PagedHostPool(HiSparseHostPoolMixin, HostKVCache):
|
||||
allocator_type: str = "default",
|
||||
page_aligned_only: bool = False,
|
||||
):
|
||||
if _is_xpu:
|
||||
require_kvcacheio_ops(
|
||||
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
|
||||
)
|
||||
# Token-granular transfers go through transfer_cache_dsv4_mla whatever
|
||||
# the io backend and layout, so it is checked here, not in the table.
|
||||
if not page_aligned_only and _dsv4_mla_unavailable_reason is not None:
|
||||
raise ValueError(
|
||||
f"HiCache cannot build the {type(self).__name__} host pool here: "
|
||||
f"{_dsv4_mla_unavailable_reason}. Launch without HiCache "
|
||||
"(--enable-hierarchical-cache)."
|
||||
)
|
||||
self.pool_name = pool_name
|
||||
self.layer_num = len(device_buffers)
|
||||
self.item_bytes = item_bytes
|
||||
@@ -455,6 +493,11 @@ class DeepSeekV4PagedHostPool(HiSparseHostPoolMixin, HostKVCache):
|
||||
self.num_release_slots += len(indices_cpu)
|
||||
return len(indices)
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
_kvcacheio_ops = MLA_KVCACHEIO_OPS
|
||||
|
||||
def backup_from_device_all_layer(
|
||||
self, device_pool, host_indices, device_indices, io_backend
|
||||
):
|
||||
@@ -734,6 +777,10 @@ class DeepSeekV4StateHostPool(HostKVCache):
|
||||
):
|
||||
if any(pool is None for pool in state_pools):
|
||||
raise ValueError(f"{pool_name} state_pools must not contain None")
|
||||
if _is_xpu:
|
||||
require_kvcacheio_ops(
|
||||
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
|
||||
)
|
||||
|
||||
self.pool_name = pool_name
|
||||
self.state_pools = state_pools
|
||||
@@ -956,6 +1003,11 @@ class DeepSeekV4StateHostPool(HostKVCache):
|
||||
f"{self.pool_name} reuses SWA transfer indices and has no free list"
|
||||
)
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
_kvcacheio_ops = MLA_KVCACHEIO_OPS
|
||||
|
||||
def backup_from_device_all_layer(
|
||||
self, device_pool, host_indices, device_indices, io_backend
|
||||
):
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""`sgl_kernel.kvcacheio` KV-transfer ops for XPU, from the sgl-kernel-xpu wheel.
|
||||
|
||||
CUDA/ROCm keep importing `sgl_kernel.kvcacheio` directly in each pool module;
|
||||
only XPU imports the ops from here. The XPU wheel can lag the CUDA op set, so
|
||||
each op is bound on its own: a missing one becomes a stub naming it, is recorded
|
||||
in `missing_ops`, and host pools refuse at construction via
|
||||
`require_kvcacheio_ops`. `unavailable_reason` is set when the module itself
|
||||
does not import. Off XPU nothing is bound and `missing_ops` stays empty.
|
||||
`hisparse_memory_pool.py` still imports `transfer_kv_all_layer_mla` directly,
|
||||
so on XPU a wheel without that op or without the module fails at import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
|
||||
from sglang.srt.utils import is_xpu
|
||||
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
|
||||
def unavailable_stub(name: str, reason: str):
|
||||
"""A stand-in for one missing op that names it when called."""
|
||||
|
||||
def _raise(*args, **kwargs):
|
||||
raise RuntimeError(
|
||||
f"HiCache host<->device KV transfer needs {name}, which is not "
|
||||
f"available here: {reason}"
|
||||
)
|
||||
|
||||
return _raise
|
||||
|
||||
|
||||
# The table shared by the pools that move one latent buffer per layer
|
||||
# (MLA, DSA indexer, DeepSeek-V4). Read-only, since four classes share it.
|
||||
MLA_KVCACHEIO_OPS = MappingProxyType(
|
||||
{
|
||||
"kernel": MappingProxyType(
|
||||
{
|
||||
"layer_first": (
|
||||
"transfer_kv_per_layer_mla",
|
||||
"transfer_kv_all_layer_mla",
|
||||
),
|
||||
"page_first": (
|
||||
"transfer_kv_per_layer_mla_pf_lf",
|
||||
"transfer_kv_all_layer_mla_lf_pf",
|
||||
),
|
||||
}
|
||||
),
|
||||
"direct": MappingProxyType(
|
||||
{
|
||||
"layer_first": ("transfer_kv_direct",),
|
||||
"page_first_direct": (
|
||||
"transfer_kv_per_layer_direct_pf_lf",
|
||||
"transfer_kv_all_layer_direct_lf_pf",
|
||||
),
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def require_kvcacheio_ops(
|
||||
*, pool: str, layout: str, ops: Mapping[str, Mapping[str, tuple[str, ...]]]
|
||||
) -> None:
|
||||
"""Fail at host-pool construction if this build lacks an op the pool will call.
|
||||
|
||||
`ops` is the pool's own `{io_backend: {layout: op names}}` table, declared
|
||||
next to the transfer methods that dispatch to those ops.
|
||||
"""
|
||||
if not missing_ops:
|
||||
return
|
||||
# Only read when an op is missing, so pools stay constructible without
|
||||
# published server args wherever the build is complete.
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
|
||||
io_backend = get_memory().hicache_io_backend
|
||||
missing = [
|
||||
name for name in ops.get(io_backend, {}).get(layout, ()) if name in missing_ops
|
||||
]
|
||||
if missing:
|
||||
if unavailable_reason is None:
|
||||
fix = "Pick another io backend or layout, or launch without HiCache"
|
||||
else:
|
||||
fix = "Launch without HiCache"
|
||||
raise ValueError(
|
||||
f"HiCache cannot build the {pool} host pool for --hicache-io-backend "
|
||||
f"{io_backend} --hicache-mem-layout {layout} here: "
|
||||
f"{missing_ops[missing[0]]}. {fix} (--enable-hierarchical-cache)."
|
||||
)
|
||||
|
||||
|
||||
missing_ops: dict[str, str] = {}
|
||||
unavailable_reason: str | None = None
|
||||
if _is_xpu:
|
||||
try:
|
||||
import sgl_kernel.kvcacheio as _kvcacheio
|
||||
except ImportError as e:
|
||||
_kvcacheio = None
|
||||
unavailable_reason = (
|
||||
f"sgl_kernel.kvcacheio failed to import ({e}); XPU takes it from "
|
||||
"the out-of-tree sgl-kernel-xpu wheel"
|
||||
)
|
||||
|
||||
def _bind(name: str):
|
||||
# getattr, not a from-import: the op set varies by sgl-kernel-xpu wheel,
|
||||
# and an absent op must stub only itself.
|
||||
op = None if _kvcacheio is None else getattr(_kvcacheio, name, None)
|
||||
if op is not None:
|
||||
return op
|
||||
reason = unavailable_reason or (
|
||||
f"this sgl-kernel-xpu build has no sgl_kernel.kvcacheio.{name}"
|
||||
)
|
||||
missing_ops[name] = reason
|
||||
return unavailable_stub(name, reason)
|
||||
|
||||
transfer_kv_all_layer = _bind("transfer_kv_all_layer")
|
||||
transfer_kv_all_layer_direct_lf_pf = _bind("transfer_kv_all_layer_direct_lf_pf")
|
||||
transfer_kv_all_layer_lf_pf = _bind("transfer_kv_all_layer_lf_pf")
|
||||
transfer_kv_all_layer_lf_ph = _bind("transfer_kv_all_layer_lf_ph")
|
||||
transfer_kv_all_layer_mla = _bind("transfer_kv_all_layer_mla")
|
||||
transfer_kv_all_layer_mla_lf_pf = _bind("transfer_kv_all_layer_mla_lf_pf")
|
||||
transfer_kv_direct = _bind("transfer_kv_direct")
|
||||
transfer_kv_per_layer = _bind("transfer_kv_per_layer")
|
||||
transfer_kv_per_layer_direct_pf_lf = _bind("transfer_kv_per_layer_direct_pf_lf")
|
||||
transfer_kv_per_layer_mla = _bind("transfer_kv_per_layer_mla")
|
||||
transfer_kv_per_layer_mla_pf_lf = _bind("transfer_kv_per_layer_mla_pf_lf")
|
||||
transfer_kv_per_layer_pf_lf = _bind("transfer_kv_per_layer_pf_lf")
|
||||
transfer_kv_per_layer_ph_lf = _bind("transfer_kv_per_layer_ph_lf")
|
||||
@@ -365,6 +365,35 @@ def alloc_with_pin_memory(
|
||||
return buffer
|
||||
|
||||
|
||||
def alloc_with_pin_memory_xpu(
|
||||
dims: tuple,
|
||||
dtype: torch.dtype,
|
||||
device: str,
|
||||
pin_memory: bool,
|
||||
allocator: None,
|
||||
registration_granularity_bytes: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""`alloc_with_pin_memory`, failing on a null pinned buffer.
|
||||
|
||||
torch-xpu returns `data_ptr() == 0` from a failed pinned allocation instead of
|
||||
raising; an empty tensor has a null pointer on every backend and is fine.
|
||||
"""
|
||||
buffer = alloc_with_pin_memory(
|
||||
dims=dims,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
allocator=allocator,
|
||||
)
|
||||
if pin_memory and buffer.numel() > 0 and buffer.data_ptr() == 0:
|
||||
n_bytes = buffer.numel() * buffer.element_size()
|
||||
raise RuntimeError(
|
||||
f"Pinned host memory allocation of {n_bytes / (1024**3):.1f} GiB "
|
||||
f"({n_bytes} bytes) returned a null pointer"
|
||||
)
|
||||
return buffer
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _resolve_device_accessible_ptr_fn():
|
||||
try:
|
||||
@@ -426,6 +455,6 @@ ALLOC_MEMORY_FUNCS = defaultdict(
|
||||
{
|
||||
"npu": alloc_with_pin_memory,
|
||||
"musa": alloc_with_pin_memory,
|
||||
"xpu": alloc_with_pin_memory,
|
||||
"xpu": alloc_with_pin_memory_xpu,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -17,6 +17,10 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
MLA_KVCACHEIO_OPS,
|
||||
require_kvcacheio_ops,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
HostKVCache,
|
||||
@@ -50,6 +54,16 @@ if _is_cuda or _is_hip:
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
)
|
||||
elif _is_xpu:
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
transfer_kv_all_layer_direct_lf_pf,
|
||||
transfer_kv_all_layer_mla,
|
||||
transfer_kv_all_layer_mla_lf_pf,
|
||||
transfer_kv_direct,
|
||||
transfer_kv_per_layer_direct_pf_lf,
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -142,6 +156,12 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
allocator_type: str = "default",
|
||||
is_dummy: bool = False,
|
||||
):
|
||||
if _is_xpu and not is_dummy:
|
||||
require_kvcacheio_ops(
|
||||
pool=type(self).__name__,
|
||||
layout=anchor_host.layout,
|
||||
ops=self._kvcacheio_ops,
|
||||
)
|
||||
self._is_dummy = is_dummy
|
||||
self._destroyed = False
|
||||
self.decl = decl
|
||||
@@ -348,6 +368,11 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
)
|
||||
return host_page_indices, device_page_indices
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
_kvcacheio_ops = MLA_KVCACHEIO_OPS
|
||||
|
||||
def load_to_device_per_layer(
|
||||
self,
|
||||
device_pool,
|
||||
|
||||
@@ -19,11 +19,12 @@ from sglang.srt.mem_cache.pool_host.common import (
|
||||
ALLOC_MEMORY_FUNCS,
|
||||
get_allocator_from_storage,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
transfer_state_per_layer_direct_pf_lf = None
|
||||
transfer_state_all_layer_direct_lf_pf = None
|
||||
transfer_mamba_state = None
|
||||
@@ -104,6 +105,15 @@ class MambaPoolHost(HostKVCache):
|
||||
"page_first_direct",
|
||||
], f"Unsupported layout: {layout}"
|
||||
|
||||
if _is_xpu:
|
||||
raise ValueError(
|
||||
"HiCache does not support hybrid Mamba models on XPU: the Mamba KV "
|
||||
"transfer kernels are JIT-built from CUDA source. "
|
||||
"Launch without HiCache host offload "
|
||||
"(--enable-hierarchical-cache, "
|
||||
"--disaggregation-decode-retraction-backup=host_pool)."
|
||||
)
|
||||
|
||||
self.layout = layout
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.mem_cache.device_pool_info import (
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
MHATokenToKOnlyPool,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import require_kvcacheio_ops
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
HostKVCache,
|
||||
@@ -71,6 +72,21 @@ if _is_cuda or _is_hip:
|
||||
transfer_kv_per_layer_pf_lf,
|
||||
transfer_kv_per_layer_ph_lf,
|
||||
)
|
||||
elif _is_xpu:
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
transfer_kv_all_layer,
|
||||
transfer_kv_all_layer_direct_lf_pf,
|
||||
transfer_kv_all_layer_lf_pf,
|
||||
transfer_kv_all_layer_lf_ph,
|
||||
transfer_kv_all_layer_mla_lf_pf,
|
||||
transfer_kv_direct,
|
||||
transfer_kv_per_layer,
|
||||
transfer_kv_per_layer_direct_pf_lf,
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
transfer_kv_per_layer_pf_lf,
|
||||
transfer_kv_per_layer_ph_lf,
|
||||
)
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange
|
||||
|
||||
@@ -136,6 +152,10 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
pool_label: str = "kv",
|
||||
head_group_num: int = 1,
|
||||
):
|
||||
if _is_xpu:
|
||||
require_kvcacheio_ops(
|
||||
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
|
||||
)
|
||||
infos = (buffer_info, *draft_buffer_infos)
|
||||
for info in infos:
|
||||
if (
|
||||
@@ -375,6 +395,30 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
def v_buffer(self):
|
||||
return self.kv_buffer[1]
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
_kvcacheio_ops = {
|
||||
"kernel": {
|
||||
"layer_first": ("transfer_kv_per_layer", "transfer_kv_all_layer"),
|
||||
"page_first": (
|
||||
"transfer_kv_per_layer_pf_lf",
|
||||
"transfer_kv_all_layer_lf_pf",
|
||||
),
|
||||
"page_head": (
|
||||
"transfer_kv_per_layer_ph_lf",
|
||||
"transfer_kv_all_layer_lf_ph",
|
||||
),
|
||||
},
|
||||
"direct": {
|
||||
"layer_first": ("transfer_kv_direct",),
|
||||
"page_first_direct": (
|
||||
"transfer_kv_per_layer_direct_pf_lf",
|
||||
"transfer_kv_all_layer_direct_lf_pf",
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def load_to_device_per_layer(
|
||||
self,
|
||||
device_pool,
|
||||
@@ -870,6 +914,10 @@ class MHATokenToKOnlyPoolHost(HostKVCache):
|
||||
allocator_type: str = "default",
|
||||
):
|
||||
self._destroyed = False
|
||||
if _is_xpu:
|
||||
require_kvcacheio_ops(
|
||||
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
|
||||
)
|
||||
self.device_pool = device_pool
|
||||
self.page_size = anchor_host.page_size
|
||||
self.layout = layout
|
||||
@@ -979,6 +1027,27 @@ class MHATokenToKOnlyPoolHost(HostKVCache):
|
||||
def get_hybrid_pool_buffer(self):
|
||||
return [self.k_buffer]
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
# layer_first backs up one layer at a time.
|
||||
_kvcacheio_ops = {
|
||||
"kernel": {
|
||||
"layer_first": ("transfer_kv_per_layer_mla",),
|
||||
"page_first": (
|
||||
"transfer_kv_per_layer_mla_pf_lf",
|
||||
"transfer_kv_all_layer_mla_lf_pf",
|
||||
),
|
||||
},
|
||||
"direct": {
|
||||
"layer_first": ("transfer_kv_direct",),
|
||||
"page_first_direct": (
|
||||
"transfer_kv_per_layer_direct_pf_lf",
|
||||
"transfer_kv_all_layer_direct_lf_pf",
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def load_to_device_per_layer(
|
||||
self,
|
||||
device_pool,
|
||||
@@ -1352,6 +1421,24 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
|
||||
"Use a backend that does not use this interface (e.g. mooncake, simm)."
|
||||
)
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
_kvcacheio_ops = {
|
||||
"kernel": {
|
||||
"page_first": (
|
||||
"transfer_kv_per_layer_mla_pf_lf",
|
||||
"transfer_kv_all_layer_mla_lf_pf",
|
||||
),
|
||||
},
|
||||
"direct": {
|
||||
"page_first_direct": (
|
||||
"transfer_kv_per_layer_direct_pf_lf",
|
||||
"transfer_kv_all_layer_direct_lf_pf",
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
def load_to_device_per_layer(
|
||||
self,
|
||||
device_pool,
|
||||
|
||||
@@ -21,6 +21,10 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
|
||||
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo, MLASplitBuffers
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
MLA_KVCACHEIO_OPS,
|
||||
require_kvcacheio_ops,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
HostKVCache,
|
||||
@@ -47,7 +51,7 @@ _is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
_is_mps = is_mps()
|
||||
if _is_cuda or _is_hip or _is_xpu:
|
||||
if _is_cuda or _is_hip:
|
||||
from sgl_kernel.kvcacheio import (
|
||||
transfer_kv_all_layer_direct_lf_pf,
|
||||
transfer_kv_all_layer_mla,
|
||||
@@ -57,6 +61,16 @@ if _is_cuda or _is_hip or _is_xpu:
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
)
|
||||
elif _is_xpu:
|
||||
from sglang.srt.mem_cache.pool_host._kvcacheio import (
|
||||
transfer_kv_all_layer_direct_lf_pf,
|
||||
transfer_kv_all_layer_mla,
|
||||
transfer_kv_all_layer_mla_lf_pf,
|
||||
transfer_kv_direct,
|
||||
transfer_kv_per_layer_direct_pf_lf,
|
||||
transfer_kv_per_layer_mla,
|
||||
transfer_kv_per_layer_mla_pf_lf,
|
||||
)
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange
|
||||
|
||||
@@ -106,6 +120,10 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
):
|
||||
self._is_dummy = isinstance(buffer_info, MLAHostAllocation)
|
||||
self.override_kv_cache_dim = override_kv_cache_dim
|
||||
if _is_xpu and not self._is_dummy:
|
||||
require_kvcacheio_ops(
|
||||
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
|
||||
)
|
||||
self.dtype = buffer_info.dtype
|
||||
self.layout = layout
|
||||
self.kv_lora_rank = buffer_info.kv_lora_rank
|
||||
@@ -668,6 +686,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"offload.kv_exchange_copy failed with code {ret}")
|
||||
|
||||
if _is_xpu:
|
||||
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
|
||||
# checked at construction; keep in step with their branches.
|
||||
_kvcacheio_ops = MLA_KVCACHEIO_OPS
|
||||
|
||||
def load_to_device_per_layer(
|
||||
self,
|
||||
device_pool,
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Shared helpers for the Intel XPU HiCache E2E tests."""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import requests
|
||||
import torch
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
find_available_port,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
XPU_AVAILABLE = torch.xpu.is_available() if hasattr(torch, "xpu") else False
|
||||
|
||||
# Device KV pool, in tokens. --mem-fraction-static cannot size it small enough
|
||||
# to evict anything on a 22 GB Arc; 1024 tokens (64 pages at --page-size 16) is
|
||||
# below every scenario's prompt set, so the reload has to come from the host.
|
||||
EVICT_DEVICE_POOL_TOKENS = 1024
|
||||
# Keeps the host tier (~230 MB) well clear of every working set, so the prefix
|
||||
# under test is not evicted from the host too and recomputed.
|
||||
EVICT_HICACHE_RATIO = 8
|
||||
# Tokens compared golden-vs-restore. A restored prefix sits on different pages,
|
||||
# so bf16 reduction order eventually flips a near-tie token even on an exact
|
||||
# transfer. Measured on Arc (Qwen2.5-1.5B, page_size 16): an exact restore holds
|
||||
# for ~30 tokens, so 16 leaves margin. Exact identity is checked in
|
||||
# test_hicache_transfer_round_trip_xpu.py.
|
||||
COMPARE_TOKENS = 16
|
||||
|
||||
|
||||
def resolve_base_url() -> str:
|
||||
"""A probed-free port, so back-to-back launches in one file cannot collide."""
|
||||
default_port = int(DEFAULT_URL_FOR_TEST.rsplit(":", 1)[1])
|
||||
return f"http://127.0.0.1:{find_available_port(default_port)}"
|
||||
|
||||
|
||||
def launch_server(model: str, base_url: str, other_args: list, env_extra=None):
|
||||
"""Launch a server with env_extra layered over the current environment."""
|
||||
return popen_launch_server(
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
env={**os.environ, **(env_extra or {})},
|
||||
)
|
||||
|
||||
|
||||
def complete(base_url, model, prompt, max_tokens=32, timeout=90, want_cached=False):
|
||||
resp = requests.post(
|
||||
f"{base_url}/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0.0,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
text = data["choices"][0]["text"]
|
||||
if not want_cached:
|
||||
return text
|
||||
cached = 0
|
||||
usage = data.get("usage") or {}
|
||||
details = usage.get("prompt_tokens_details") or {}
|
||||
if details:
|
||||
cached = details.get("cached_tokens", 0) or 0
|
||||
return text, cached
|
||||
|
||||
|
||||
def prime_cache(base_url, model, prompt, max_tokens=COMPARE_TOKENS, timeout=90) -> None:
|
||||
"""Serve `prompt` once so the next identical request is a cache hit.
|
||||
|
||||
The golden must be a cache hit too: cold and prefix-reuse prefill are
|
||||
different numeric paths and diverge within COMPARE_TOKENS on Arc.
|
||||
"""
|
||||
complete(base_url, model, prompt, max_tokens=max_tokens, timeout=timeout)
|
||||
|
||||
|
||||
def load_back_tokens(base_url) -> float:
|
||||
"""Tokens loaded host (L2) -> device (L1) since launch, all pools summed.
|
||||
|
||||
Requires --enable-metrics. The only evidence the load kernel ran: output
|
||||
agreement and cached_tokens > 0 are both satisfied by a device-tier hit.
|
||||
"""
|
||||
body = requests.get(f"{base_url}/metrics", timeout=30).text
|
||||
return sum(
|
||||
float(line.rsplit(" ", 1)[1])
|
||||
for line in body.splitlines()
|
||||
if line.startswith("sglang:load_back_tokens_total")
|
||||
)
|
||||
|
||||
|
||||
def wait_load_back_tokens(base_url, above: float, timeout_s: float = 20.0) -> float:
|
||||
"""Poll load_back_tokens_total until it passes `above`, then return it.
|
||||
|
||||
The counter is incremented when the scheduler reaps the load ack
|
||||
(hiradix_cache.py load_back bookkeeping), which can trail the HTTP response
|
||||
by a loop iteration, so a bare read right after the completion races.
|
||||
Returns the last value observed either way; the caller does the asserting.
|
||||
"""
|
||||
deadline = time.time() + timeout_s
|
||||
seen = load_back_tokens(base_url)
|
||||
while seen <= above and time.time() < deadline:
|
||||
time.sleep(0.5)
|
||||
seen = load_back_tokens(base_url)
|
||||
return seen
|
||||
|
||||
|
||||
def shared_prefix_len(a: str, b: str) -> int:
|
||||
n = min(len(a), len(b))
|
||||
i = 0
|
||||
while i < n and a[i] == b[i]:
|
||||
i += 1
|
||||
return i
|
||||
Reference in New Issue
Block a user