[Feat] [Intel] hicache: Enable L1<->L2 support on XPU (#32503)

This commit is contained in:
kpjeeja
2026-10-20 10:42:21 +08:00
committed by GitHub
parent 5e630fa571
commit 8683f8ad4a
17 changed files with 1498 additions and 14 deletions
+3 -1
View File
@@ -197,5 +197,7 @@ class L2TransferEngine:
# Keep temporary translated indices alive until the transfer completes.
tensors.extend(resolved)
for indices in tensors:
if indices is not None and indices.is_cuda:
if indices is not None and (
indices.is_cuda or indices.device.type == "xpu"
):
indices.record_stream(stream)
@@ -13,6 +13,11 @@ from sglang.kernels.ops.kvcache.hicache import (
transfer_hicache_all_layer_mla_staged_lf_pf as jit_transfer_hicache_all_layer_mla_staged_lf_pf,
)
from sglang.kernels.ops.kvcache.hisparse import transfer_cache_dsv4_mla
from sglang.srt.mem_cache.pool_host._kvcacheio import (
MLA_KVCACHEIO_OPS,
require_kvcacheio_ops,
unavailable_stub,
)
from sglang.srt.platforms import current_platform
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
@@ -21,11 +26,22 @@ _is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
_is_mps = is_mps()
_dsv4_mla_unavailable_reason: Optional[str] = None
if _is_xpu:
from sgl_kernel import transfer_cache_dsv4_mla
try:
from sgl_kernel import transfer_cache_dsv4_mla
except ImportError as e:
# Only DSV4 needs this op; a wheel without it must not break scheduler import.
_dsv4_mla_unavailable_reason = (
f"sgl_kernel.transfer_cache_dsv4_mla failed to import ({e}); XPU "
"takes it from the out-of-tree sgl-kernel-xpu wheel"
)
transfer_cache_dsv4_mla = unavailable_stub(
"transfer_cache_dsv4_mla", _dsv4_mla_unavailable_reason
)
else:
from sglang.kernels.ops.kvcache.hisparse import transfer_cache_dsv4_mla
if _is_cuda or _is_hip or _is_xpu:
if _is_cuda or _is_hip:
from sgl_kernel.kvcacheio import (
transfer_kv_all_layer_direct_lf_pf,
transfer_kv_all_layer_mla,
@@ -35,6 +51,16 @@ if _is_cuda or _is_hip or _is_xpu:
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
)
elif _is_xpu:
from sglang.srt.mem_cache.pool_host._kvcacheio import (
transfer_kv_all_layer_direct_lf_pf,
transfer_kv_all_layer_mla,
transfer_kv_all_layer_mla_lf_pf,
transfer_kv_direct,
transfer_kv_per_layer_direct_pf_lf,
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
)
if _is_npu:
from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange
@@ -223,6 +249,18 @@ class DeepSeekV4PagedHostPool(HiSparseHostPoolMixin, HostKVCache):
allocator_type: str = "default",
page_aligned_only: bool = False,
):
if _is_xpu:
require_kvcacheio_ops(
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
)
# Token-granular transfers go through transfer_cache_dsv4_mla whatever
# the io backend and layout, so it is checked here, not in the table.
if not page_aligned_only and _dsv4_mla_unavailable_reason is not None:
raise ValueError(
f"HiCache cannot build the {type(self).__name__} host pool here: "
f"{_dsv4_mla_unavailable_reason}. Launch without HiCache "
"(--enable-hierarchical-cache)."
)
self.pool_name = pool_name
self.layer_num = len(device_buffers)
self.item_bytes = item_bytes
@@ -455,6 +493,11 @@ class DeepSeekV4PagedHostPool(HiSparseHostPoolMixin, HostKVCache):
self.num_release_slots += len(indices_cpu)
return len(indices)
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
_kvcacheio_ops = MLA_KVCACHEIO_OPS
def backup_from_device_all_layer(
self, device_pool, host_indices, device_indices, io_backend
):
@@ -734,6 +777,10 @@ class DeepSeekV4StateHostPool(HostKVCache):
):
if any(pool is None for pool in state_pools):
raise ValueError(f"{pool_name} state_pools must not contain None")
if _is_xpu:
require_kvcacheio_ops(
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
)
self.pool_name = pool_name
self.state_pools = state_pools
@@ -956,6 +1003,11 @@ class DeepSeekV4StateHostPool(HostKVCache):
f"{self.pool_name} reuses SWA transfer indices and has no free list"
)
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
_kvcacheio_ops = MLA_KVCACHEIO_OPS
def backup_from_device_all_layer(
self, device_pool, host_indices, device_indices, io_backend
):
@@ -0,0 +1,130 @@
"""`sgl_kernel.kvcacheio` KV-transfer ops for XPU, from the sgl-kernel-xpu wheel.
CUDA/ROCm keep importing `sgl_kernel.kvcacheio` directly in each pool module;
only XPU imports the ops from here. The XPU wheel can lag the CUDA op set, so
each op is bound on its own: a missing one becomes a stub naming it, is recorded
in `missing_ops`, and host pools refuse at construction via
`require_kvcacheio_ops`. `unavailable_reason` is set when the module itself
does not import. Off XPU nothing is bound and `missing_ops` stays empty.
`hisparse_memory_pool.py` still imports `transfer_kv_all_layer_mla` directly,
so on XPU a wheel without that op or without the module fails at import.
"""
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from sglang.srt.utils import is_xpu
_is_xpu = is_xpu()
def unavailable_stub(name: str, reason: str):
"""A stand-in for one missing op that names it when called."""
def _raise(*args, **kwargs):
raise RuntimeError(
f"HiCache host<->device KV transfer needs {name}, which is not "
f"available here: {reason}"
)
return _raise
# The table shared by the pools that move one latent buffer per layer
# (MLA, DSA indexer, DeepSeek-V4). Read-only, since four classes share it.
MLA_KVCACHEIO_OPS = MappingProxyType(
{
"kernel": MappingProxyType(
{
"layer_first": (
"transfer_kv_per_layer_mla",
"transfer_kv_all_layer_mla",
),
"page_first": (
"transfer_kv_per_layer_mla_pf_lf",
"transfer_kv_all_layer_mla_lf_pf",
),
}
),
"direct": MappingProxyType(
{
"layer_first": ("transfer_kv_direct",),
"page_first_direct": (
"transfer_kv_per_layer_direct_pf_lf",
"transfer_kv_all_layer_direct_lf_pf",
),
}
),
}
)
def require_kvcacheio_ops(
*, pool: str, layout: str, ops: Mapping[str, Mapping[str, tuple[str, ...]]]
) -> None:
"""Fail at host-pool construction if this build lacks an op the pool will call.
`ops` is the pool's own `{io_backend: {layout: op names}}` table, declared
next to the transfer methods that dispatch to those ops.
"""
if not missing_ops:
return
# Only read when an op is missing, so pools stay constructible without
# published server args wherever the build is complete.
from sglang.srt.runtime_context import get_memory
io_backend = get_memory().hicache_io_backend
missing = [
name for name in ops.get(io_backend, {}).get(layout, ()) if name in missing_ops
]
if missing:
if unavailable_reason is None:
fix = "Pick another io backend or layout, or launch without HiCache"
else:
fix = "Launch without HiCache"
raise ValueError(
f"HiCache cannot build the {pool} host pool for --hicache-io-backend "
f"{io_backend} --hicache-mem-layout {layout} here: "
f"{missing_ops[missing[0]]}. {fix} (--enable-hierarchical-cache)."
)
missing_ops: dict[str, str] = {}
unavailable_reason: str | None = None
if _is_xpu:
try:
import sgl_kernel.kvcacheio as _kvcacheio
except ImportError as e:
_kvcacheio = None
unavailable_reason = (
f"sgl_kernel.kvcacheio failed to import ({e}); XPU takes it from "
"the out-of-tree sgl-kernel-xpu wheel"
)
def _bind(name: str):
# getattr, not a from-import: the op set varies by sgl-kernel-xpu wheel,
# and an absent op must stub only itself.
op = None if _kvcacheio is None else getattr(_kvcacheio, name, None)
if op is not None:
return op
reason = unavailable_reason or (
f"this sgl-kernel-xpu build has no sgl_kernel.kvcacheio.{name}"
)
missing_ops[name] = reason
return unavailable_stub(name, reason)
transfer_kv_all_layer = _bind("transfer_kv_all_layer")
transfer_kv_all_layer_direct_lf_pf = _bind("transfer_kv_all_layer_direct_lf_pf")
transfer_kv_all_layer_lf_pf = _bind("transfer_kv_all_layer_lf_pf")
transfer_kv_all_layer_lf_ph = _bind("transfer_kv_all_layer_lf_ph")
transfer_kv_all_layer_mla = _bind("transfer_kv_all_layer_mla")
transfer_kv_all_layer_mla_lf_pf = _bind("transfer_kv_all_layer_mla_lf_pf")
transfer_kv_direct = _bind("transfer_kv_direct")
transfer_kv_per_layer = _bind("transfer_kv_per_layer")
transfer_kv_per_layer_direct_pf_lf = _bind("transfer_kv_per_layer_direct_pf_lf")
transfer_kv_per_layer_mla = _bind("transfer_kv_per_layer_mla")
transfer_kv_per_layer_mla_pf_lf = _bind("transfer_kv_per_layer_mla_pf_lf")
transfer_kv_per_layer_pf_lf = _bind("transfer_kv_per_layer_pf_lf")
transfer_kv_per_layer_ph_lf = _bind("transfer_kv_per_layer_ph_lf")
@@ -365,6 +365,35 @@ def alloc_with_pin_memory(
return buffer
def alloc_with_pin_memory_xpu(
dims: tuple,
dtype: torch.dtype,
device: str,
pin_memory: bool,
allocator: None,
registration_granularity_bytes: int | None = None,
) -> torch.Tensor:
"""`alloc_with_pin_memory`, failing on a null pinned buffer.
torch-xpu returns `data_ptr() == 0` from a failed pinned allocation instead of
raising; an empty tensor has a null pointer on every backend and is fine.
"""
buffer = alloc_with_pin_memory(
dims=dims,
dtype=dtype,
device=device,
pin_memory=pin_memory,
allocator=allocator,
)
if pin_memory and buffer.numel() > 0 and buffer.data_ptr() == 0:
n_bytes = buffer.numel() * buffer.element_size()
raise RuntimeError(
f"Pinned host memory allocation of {n_bytes / (1024**3):.1f} GiB "
f"({n_bytes} bytes) returned a null pointer"
)
return buffer
@lru_cache(maxsize=1)
def _resolve_device_accessible_ptr_fn():
try:
@@ -426,6 +455,6 @@ ALLOC_MEMORY_FUNCS = defaultdict(
{
"npu": alloc_with_pin_memory,
"musa": alloc_with_pin_memory,
"xpu": alloc_with_pin_memory,
"xpu": alloc_with_pin_memory_xpu,
},
)
@@ -17,6 +17,10 @@ from sglang.kernels.ops.kvcache.hicache import (
)
from sglang.srt.mem_cache.hicache_storage import PoolName
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
from sglang.srt.mem_cache.pool_host._kvcacheio import (
MLA_KVCACHEIO_OPS,
require_kvcacheio_ops,
)
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
HostKVCache,
@@ -50,6 +54,16 @@ if _is_cuda or _is_hip:
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
)
elif _is_xpu:
from sglang.srt.mem_cache.pool_host._kvcacheio import (
transfer_kv_all_layer_direct_lf_pf,
transfer_kv_all_layer_mla,
transfer_kv_all_layer_mla_lf_pf,
transfer_kv_direct,
transfer_kv_per_layer_direct_pf_lf,
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
)
logger = logging.getLogger(__name__)
@@ -142,6 +156,12 @@ class DSAIndexerPoolHost(HostKVCache):
allocator_type: str = "default",
is_dummy: bool = False,
):
if _is_xpu and not is_dummy:
require_kvcacheio_ops(
pool=type(self).__name__,
layout=anchor_host.layout,
ops=self._kvcacheio_ops,
)
self._is_dummy = is_dummy
self._destroyed = False
self.decl = decl
@@ -348,6 +368,11 @@ class DSAIndexerPoolHost(HostKVCache):
)
return host_page_indices, device_page_indices
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
_kvcacheio_ops = MLA_KVCACHEIO_OPS
def load_to_device_per_layer(
self,
device_pool,
+11 -1
View File
@@ -19,11 +19,12 @@ from sglang.srt.mem_cache.pool_host.common import (
ALLOC_MEMORY_FUNCS,
get_allocator_from_storage,
)
from sglang.srt.utils import is_cuda, is_hip, is_npu
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
transfer_state_per_layer_direct_pf_lf = None
transfer_state_all_layer_direct_lf_pf = None
transfer_mamba_state = None
@@ -104,6 +105,15 @@ class MambaPoolHost(HostKVCache):
"page_first_direct",
], f"Unsupported layout: {layout}"
if _is_xpu:
raise ValueError(
"HiCache does not support hybrid Mamba models on XPU: the Mamba KV "
"transfer kernels are JIT-built from CUDA source. "
"Launch without HiCache host offload "
"(--enable-hierarchical-cache, "
"--disaggregation-decode-retraction-backup=host_pool)."
)
self.layout = layout
self.pin_memory = pin_memory
self.device = device
@@ -35,6 +35,7 @@ from sglang.srt.mem_cache.device_pool_info import (
from sglang.srt.mem_cache.memory_pool import (
MHATokenToKOnlyPool,
)
from sglang.srt.mem_cache.pool_host._kvcacheio import require_kvcacheio_ops
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
HostKVCache,
@@ -71,6 +72,21 @@ if _is_cuda or _is_hip:
transfer_kv_per_layer_pf_lf,
transfer_kv_per_layer_ph_lf,
)
elif _is_xpu:
from sglang.srt.mem_cache.pool_host._kvcacheio import (
transfer_kv_all_layer,
transfer_kv_all_layer_direct_lf_pf,
transfer_kv_all_layer_lf_pf,
transfer_kv_all_layer_lf_ph,
transfer_kv_all_layer_mla_lf_pf,
transfer_kv_direct,
transfer_kv_per_layer,
transfer_kv_per_layer_direct_pf_lf,
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
transfer_kv_per_layer_pf_lf,
transfer_kv_per_layer_ph_lf,
)
if _is_npu:
from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange
@@ -136,6 +152,10 @@ class MHATokenToKVPoolHost(HostKVCache):
pool_label: str = "kv",
head_group_num: int = 1,
):
if _is_xpu:
require_kvcacheio_ops(
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
)
infos = (buffer_info, *draft_buffer_infos)
for info in infos:
if (
@@ -375,6 +395,30 @@ class MHATokenToKVPoolHost(HostKVCache):
def v_buffer(self):
return self.kv_buffer[1]
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
_kvcacheio_ops = {
"kernel": {
"layer_first": ("transfer_kv_per_layer", "transfer_kv_all_layer"),
"page_first": (
"transfer_kv_per_layer_pf_lf",
"transfer_kv_all_layer_lf_pf",
),
"page_head": (
"transfer_kv_per_layer_ph_lf",
"transfer_kv_all_layer_lf_ph",
),
},
"direct": {
"layer_first": ("transfer_kv_direct",),
"page_first_direct": (
"transfer_kv_per_layer_direct_pf_lf",
"transfer_kv_all_layer_direct_lf_pf",
),
},
}
def load_to_device_per_layer(
self,
device_pool,
@@ -870,6 +914,10 @@ class MHATokenToKOnlyPoolHost(HostKVCache):
allocator_type: str = "default",
):
self._destroyed = False
if _is_xpu:
require_kvcacheio_ops(
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
)
self.device_pool = device_pool
self.page_size = anchor_host.page_size
self.layout = layout
@@ -979,6 +1027,27 @@ class MHATokenToKOnlyPoolHost(HostKVCache):
def get_hybrid_pool_buffer(self):
return [self.k_buffer]
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
# layer_first backs up one layer at a time.
_kvcacheio_ops = {
"kernel": {
"layer_first": ("transfer_kv_per_layer_mla",),
"page_first": (
"transfer_kv_per_layer_mla_pf_lf",
"transfer_kv_all_layer_mla_lf_pf",
),
},
"direct": {
"layer_first": ("transfer_kv_direct",),
"page_first_direct": (
"transfer_kv_per_layer_direct_pf_lf",
"transfer_kv_all_layer_direct_lf_pf",
),
},
}
def load_to_device_per_layer(
self,
device_pool,
@@ -1352,6 +1421,24 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
"Use a backend that does not use this interface (e.g. mooncake, simm)."
)
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
_kvcacheio_ops = {
"kernel": {
"page_first": (
"transfer_kv_per_layer_mla_pf_lf",
"transfer_kv_all_layer_mla_lf_pf",
),
},
"direct": {
"page_first_direct": (
"transfer_kv_per_layer_direct_pf_lf",
"transfer_kv_all_layer_direct_lf_pf",
),
},
}
def load_to_device_per_layer(
self,
device_pool,
+24 -1
View File
@@ -21,6 +21,10 @@ from sglang.kernels.ops.kvcache.hicache import (
)
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo, MLASplitBuffers
from sglang.srt.mem_cache.pool_host._kvcacheio import (
MLA_KVCACHEIO_OPS,
require_kvcacheio_ops,
)
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
HostKVCache,
@@ -47,7 +51,7 @@ _is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
_is_mps = is_mps()
if _is_cuda or _is_hip or _is_xpu:
if _is_cuda or _is_hip:
from sgl_kernel.kvcacheio import (
transfer_kv_all_layer_direct_lf_pf,
transfer_kv_all_layer_mla,
@@ -57,6 +61,16 @@ if _is_cuda or _is_hip or _is_xpu:
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
)
elif _is_xpu:
from sglang.srt.mem_cache.pool_host._kvcacheio import (
transfer_kv_all_layer_direct_lf_pf,
transfer_kv_all_layer_mla,
transfer_kv_all_layer_mla_lf_pf,
transfer_kv_direct,
transfer_kv_per_layer_direct_pf_lf,
transfer_kv_per_layer_mla,
transfer_kv_per_layer_mla_pf_lf,
)
if _is_npu:
from sgl_kernel_npu.kvcacheio import TransferDirection, transfer_kv_dim_exchange
@@ -106,6 +120,10 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
):
self._is_dummy = isinstance(buffer_info, MLAHostAllocation)
self.override_kv_cache_dim = override_kv_cache_dim
if _is_xpu and not self._is_dummy:
require_kvcacheio_ops(
pool=type(self).__name__, layout=layout, ops=self._kvcacheio_ops
)
self.dtype = buffer_info.dtype
self.layout = layout
self.kv_lora_rank = buffer_info.kv_lora_rank
@@ -668,6 +686,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
if ret != 0:
raise RuntimeError(f"offload.kv_exchange_copy failed with code {ret}")
if _is_xpu:
# {io_backend: {layout: kvcacheio ops}} the transfer methods below call,
# checked at construction; keep in step with their branches.
_kvcacheio_ops = MLA_KVCACHEIO_OPS
def load_to_device_per_layer(
self,
device_pool,
+118
View File
@@ -0,0 +1,118 @@
"""Shared helpers for the Intel XPU HiCache E2E tests."""
import os
import time
import requests
import torch
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
find_available_port,
popen_launch_server,
)
XPU_AVAILABLE = torch.xpu.is_available() if hasattr(torch, "xpu") else False
# Device KV pool, in tokens. --mem-fraction-static cannot size it small enough
# to evict anything on a 22 GB Arc; 1024 tokens (64 pages at --page-size 16) is
# below every scenario's prompt set, so the reload has to come from the host.
EVICT_DEVICE_POOL_TOKENS = 1024
# Keeps the host tier (~230 MB) well clear of every working set, so the prefix
# under test is not evicted from the host too and recomputed.
EVICT_HICACHE_RATIO = 8
# Tokens compared golden-vs-restore. A restored prefix sits on different pages,
# so bf16 reduction order eventually flips a near-tie token even on an exact
# transfer. Measured on Arc (Qwen2.5-1.5B, page_size 16): an exact restore holds
# for ~30 tokens, so 16 leaves margin. Exact identity is checked in
# test_hicache_transfer_round_trip_xpu.py.
COMPARE_TOKENS = 16
def resolve_base_url() -> str:
"""A probed-free port, so back-to-back launches in one file cannot collide."""
default_port = int(DEFAULT_URL_FOR_TEST.rsplit(":", 1)[1])
return f"http://127.0.0.1:{find_available_port(default_port)}"
def launch_server(model: str, base_url: str, other_args: list, env_extra=None):
"""Launch a server with env_extra layered over the current environment."""
return popen_launch_server(
model=model,
base_url=base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env={**os.environ, **(env_extra or {})},
)
def complete(base_url, model, prompt, max_tokens=32, timeout=90, want_cached=False):
resp = requests.post(
f"{base_url}/v1/completions",
json={
"model": model,
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": 0.0,
},
timeout=timeout,
)
resp.raise_for_status()
data = resp.json()
text = data["choices"][0]["text"]
if not want_cached:
return text
cached = 0
usage = data.get("usage") or {}
details = usage.get("prompt_tokens_details") or {}
if details:
cached = details.get("cached_tokens", 0) or 0
return text, cached
def prime_cache(base_url, model, prompt, max_tokens=COMPARE_TOKENS, timeout=90) -> None:
"""Serve `prompt` once so the next identical request is a cache hit.
The golden must be a cache hit too: cold and prefix-reuse prefill are
different numeric paths and diverge within COMPARE_TOKENS on Arc.
"""
complete(base_url, model, prompt, max_tokens=max_tokens, timeout=timeout)
def load_back_tokens(base_url) -> float:
"""Tokens loaded host (L2) -> device (L1) since launch, all pools summed.
Requires --enable-metrics. The only evidence the load kernel ran: output
agreement and cached_tokens > 0 are both satisfied by a device-tier hit.
"""
body = requests.get(f"{base_url}/metrics", timeout=30).text
return sum(
float(line.rsplit(" ", 1)[1])
for line in body.splitlines()
if line.startswith("sglang:load_back_tokens_total")
)
def wait_load_back_tokens(base_url, above: float, timeout_s: float = 20.0) -> float:
"""Poll load_back_tokens_total until it passes `above`, then return it.
The counter is incremented when the scheduler reaps the load ack
(hiradix_cache.py load_back bookkeeping), which can trail the HTTP response
by a loop iteration, so a bare read right after the completion races.
Returns the last value observed either way; the caller does the asserting.
"""
deadline = time.time() + timeout_s
seen = load_back_tokens(base_url)
while seen <= above and time.time() < deadline:
time.sleep(0.5)
seen = load_back_tokens(base_url)
return seen
def shared_prefix_len(a: str, b: str) -> int:
n = min(len(a), len(b))
i = 0
while i < n and a[i] == b[i]:
i += 1
return i