[mem_cache] Remove unused helpers in mem_cache, storage backends, and metrics (#40963)

This commit is contained in:
Liangsheng Yin
2026-09-23 18:16:44 -07:00
committed by GitHub
parent 81cb895b85
commit 621136e094
16 changed files with 3 additions and 267 deletions
@@ -922,14 +922,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.physical_to_virtual,
)
def bind_pages(
self, virtual_pages: torch.Tensor, physical_pages: torch.Tensor
) -> None:
"""Page-granular alias of ``bind``."""
with record_function("MultiEndedAlloc.bind_pages"):
self.bind(virtual_pages, physical_pages)
# -- fused take_physical_pages + bind_pages --
# -- fused take_physical_pages + bind --
def _alloc_bind_fast_or_slow(
self, v_pages: torch.Tensor, N: int
@@ -1690,16 +1683,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
t.numel() for t in ready_tensors
)
def maybe_drain_pending_reuse(self) -> None:
"""Public scheduler hook (once per step): flow fired compaction-src pages
back into `_free_phys_pages` for immediate reuse without waiting for `_flush`.
"""
if not self.lazy_compaction:
return
if not self._pending_reuse:
return
self._drain_pending_reuse(urgent=False)
def _topmost_survivor(
self,
start_hint: Optional[int] = None,
@@ -2572,33 +2555,6 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
return 0
return self._flush(urgent=False)
def backup_state(self):
# Span-aware snapshot (the base backs up `watermark_physical`, meaningless
# here). Spec decode is asserted off under unified today.
return (
self.low_wm_page,
self.high_wm_page,
self._free_phys_pages.clone(),
(len(self.free_virtual_ids) if self.is_id_owner else None),
len(self._inverse_history),
)
def restore_state(self, state):
low_wm, high_wm, holes, _n_free_virtual, n_inverse = state
self.low_wm_page = low_wm
self.high_wm_page = high_wm
self._free_phys_pages = holes
new_entries = self._inverse_history[n_inverse:]
if new_entries:
logger.warning(
"FloatMultiEndedAllocator.restore_state: %d relocation(s) inside "
"a backup window (sub_pool=%s) — float moves are not reversible.",
len(new_entries),
self.sub_pool_name,
)
del self._inverse_history[n_inverse:]
return new_entries
def compact_holes(self, *, retreat_side: str) -> int:
"""Close ALL interior holes by packing live pages toward the side
OPPOSITE ``retreat_side`` (order-preserving), shrinking the span on
@@ -1804,14 +1804,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
return self.request_window.buffer(self._swa_local_layer_id(layer_id))
return self.swa_kv_pool.kv_buffer[self._swa_local_layer_id(layer_id)]
def get_swa_key_buffer(self, layer_id: int) -> torch.Tensor:
self.wait_layer_transfer(layer_id)
if self.request_window is not None:
return self.get_swa_raw_buffer(layer_id).view(
self.request_window.state.dtype
)
return self.swa_kv_pool.get_key_buffer(self._swa_local_layer_id(layer_id))
def set_swa_key_buffer(
self,
layer_id: int,
@@ -457,13 +457,6 @@ class HiCacheFile(HiCacheStorage):
return self._get_suffixed_key(key)
return self._get_suffixed_key(f"{key}.{component_name}")
def _get_component_path(
self, key: str, component_name: Optional[str] = None
) -> str:
return os.path.join(
self.file_path, f"{self._get_component_key(key, component_name)}.bin"
)
def _scan_existing_files_to_metadata_cache(self) -> None:
try:
names = os.listdir(self.file_path)
@@ -130,16 +130,6 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
loc = self.translate_loc_to_hisparse_device(loc)
return super().get_mla_kv_buffer(layer, loc, dst_dtype)
def transfer_values_on_device(self, dst_indices, src_indices):
transfer_kv_all_layer_mla(
src_layers=self.data_ptrs,
dst_layers=self.data_ptrs,
src_indices=src_indices,
dst_indices=dst_indices,
item_size=self.bytes_per_token,
num_layers=self.layer_num,
)
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
raise NotImplementedError("HiSparseDevicePool does not support get_cpu_copy")
@@ -226,28 +216,6 @@ class HiSparseMHAMainPool(MHATokenToKVPool):
translated = self.translate_loc_to_hisparse_device(raw_loc)
super().set_kv_buffer(layer, translated, cache_k, cache_v, *args, **kwargs)
def transfer_values_on_device(
self,
dst_indices: torch.Tensor,
src_indices: torch.Tensor,
) -> None:
transfer_kv_all_layer_mla(
src_layers=self.k_data_ptrs,
dst_layers=self.k_data_ptrs,
src_indices=src_indices,
dst_indices=dst_indices,
item_size=self.bytes_per_token_k,
num_layers=self.layer_num,
)
transfer_kv_all_layer_mla(
src_layers=self.v_data_ptrs,
dst_layers=self.v_data_ptrs,
src_indices=src_indices,
dst_indices=dst_indices,
item_size=self.bytes_per_token_v,
num_layers=self.layer_num,
)
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
raise NotImplementedError("HiSparseMHAMainPool does not support get_cpu_copy")
@@ -25,7 +25,6 @@ class KVCacheBuildResult:
from typing import TYPE_CHECKING
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.hybrid_arch import (
glm5_next_config,
hybrid_gdn_config,
@@ -117,29 +116,6 @@ def prepare_hicache_staging(
)
def get_draft_kv_pool(
*,
draft_worker: BaseTpWorker,
spec_algorithm: SpeculativeAlgorithm,
server_args: ServerArgs,
):
"""Return the draft token-to-KV pool for the current draft worker,
or None when no draft KV pool is available."""
if draft_worker is None or spec_algorithm.is_ngram():
return None
# V2 draft workers exist only on their hosting PP stage; other ranks own no
# nested draft worker or draft KV pool.
if draft_worker.draft_worker is None:
return None
if resolving_view(server_args).enable_multi_layer_eagle:
draft_runner = draft_worker.draft_worker.draft_runner_list[0]
else:
draft_runner = draft_worker.draft_worker.draft_runner
return draft_runner.token_to_kv_pool
def maybe_register_hicache_draft(
*,
tree_cache,
@@ -66,7 +66,6 @@ from sglang.srt.mem_cache.memory_pool import (
HybridReqToTokenPool,
KVCache,
MHATokenToKVPool,
MHATokenToKVPoolFP4,
MHATokenToKVPoolMXFP8,
MiniMaxSparseKVPool,
MLATokenToKVPool,
@@ -1981,26 +1980,6 @@ class KVCacheConfigurator:
)
return token_to_kv_pool
def _build_mha_fp4_kv_pool(self, *, max_total_num_tokens: int) -> KVCache:
token_to_kv_pool = MHATokenToKVPoolFP4(
max_total_num_tokens,
page_size=self.pool_page_size,
dtype=self.kv_cache_dtype,
head_num=self.model_config.get_num_kv_heads(
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
v_head_dim=self.model_config.v_head_dim,
layer_num=self.layer_info.num_effective_layers,
device=self.device,
enable_memory_saver=get_exec().features.enable_memory_saver,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
enable_alt_stream=not get_disagg().enable_pdmux,
enable_kv_cache_copy=(get_spec().speculative_algorithm is not None),
)
return token_to_kv_pool
def _build_mha_kv_pool(
self, *, max_total_num_tokens: int, mha_pool_class: type, quant_method=None
) -> KVCache:
@@ -96,12 +96,6 @@ class KVIndexTable(msgspec.Struct, frozen=True):
is_translated: bool # entries are already kernel-facing ids
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
def sliding_window_read_ids(self) -> torch.Tensor:
"""Which array a sliding-window gather reads: the parallel swa array
when translated, else the full-attention array, which the caller maps
through the pool's own full->swa map."""
return self.sliding_window_ids if self.is_translated else self.ids
class KVIndexTranslator:
"""Built once per ModelRunner."""
@@ -2,7 +2,7 @@ import json
import logging
import os
import time
from typing import Any, List, Optional, Tuple
from typing import Any, List, Optional
import eic
import torch
@@ -146,12 +146,6 @@ class FlexibleKVCacheMemoryPool:
return
self.free_data_addr.add(self.data_ptr_to_index[data_ptr])
def check_data_ptr_allocated(self, data_ptr):
return data_ptr in self.data_ptr_to_index
def left_count(self):
return len(self.free_data_addr)
class EICStorage(HiCacheStorage):
def __init__(
@@ -435,16 +429,6 @@ class EICStorage(HiCacheStorage):
def clear(self) -> None:
return
# Not used for now
def _filter_kv_cache(self, total_len) -> Tuple[int, int]:
mean_len = total_len // self.world_size
remainder = total_len % self.world_size
tp_keys_len = mean_len + (1 if self.rank < remainder else 0)
start = self.rank * mean_len + min(self.rank, remainder)
end = start + tp_keys_len
logger.debug(f"start: {start}, end: {end}, tp_keys_len: {tp_keys_len}")
return start, end
def zero_copy_batch_set(self, keys: List[str], values: List[torch.Tensor]) -> bool:
logger.debug(f"eic zero copy set {len(keys)} keys")
if len(keys) == 0:
@@ -24,7 +24,6 @@ This file provides:
from __future__ import annotations
import ctypes
import errno
import logging
import os
import pickle
@@ -478,25 +477,6 @@ def eventfd(initval: int = 0, flags: int = 0) -> int:
return fd
def eventfd_write(fd: int, val: int) -> None:
v = ctypes.c_uint64(val)
n = _libc.write(fd, ctypes.byref(v), ctypes.sizeof(v))
if n != ctypes.sizeof(v):
err = ctypes.get_errno()
raise OSError(err, f"eventfd write failed: {os.strerror(err)}")
def eventfd_read(fd: int) -> int:
v = ctypes.c_uint64()
n = _libc.read(fd, ctypes.byref(v), ctypes.sizeof(v))
if n != ctypes.sizeof(v):
err = ctypes.get_errno()
if err == errno.EAGAIN:
return 0
raise OSError(err, f"eventfd read failed: {os.strerror(err)}")
return v.value
def send_fds(sock: socket.socket, fds: list, extra_data: bytes = b"x") -> None:
"""SCM_RIGHTS-send a list of file descriptors over a UDS socket."""
fds_packed = struct.pack(f"{len(fds)}i", *fds)
@@ -587,22 +587,6 @@ class FlexKVConnector:
completed_handles = self._sync_ctx.scatter(completed_handles)
return completed_handles
def wait_store(self, handle: CacheRequestHandle, timeout: float = 30.0) -> bool:
"""Block until a single store task identified by ``handle`` finishes."""
fkv_task_id = self._inflight_stores.pop(handle, -1)
if fkv_task_id < 0:
return True
if not self._sync_ctx.is_sync_leader or self.kv_manager is None:
return True
try:
resp = self.kv_manager.wait([fkv_task_id], timeout=timeout)
except Exception as exc: # noqa: BLE001
logger.warning("[FlexKV] wait_store: %s", exc)
return False
return (
fkv_task_id in resp and resp[fkv_task_id].status == KVResponseStatus.SUCCESS
)
# ------------------------------------------------------------------
# Public API — prefetch
# ------------------------------------------------------------------
@@ -196,10 +196,6 @@ class TensorcastStore(HiCacheStorage):
return 0
return self._publish_if_available(hit_pages, fallback=0)
def _build_page_fragments(self, keys: list[str]) -> tuple[PageFragment, ...]:
registered = self._require_registered()
return _expand_page_fragments(registered, keys)
def _build_artifact_specs(self, keys: list[str]) -> tuple[ByteArtifactSpec, ...]:
registered = self._require_registered()
return _expand_artifact_specs(registered, keys)
@@ -457,10 +457,6 @@ class UMBPDirectLinker(UnifiedCacheLinker):
# every layer, and a layer is read back as a byte range inside it.
return component_keys, multiplier
def _object_keys(self, transfer: PoolTransfer) -> list[str]:
keys, _ = self._object_keys_for_pages(list(transfer.keys or []), transfer)
return keys
def _page_exists(self, page_keys: list[str], transfer: PoolTransfer) -> list[bool]:
entry = self.pools[transfer.name]
objects_per_page = 1 if entry.packed else len(entry.components)
@@ -154,15 +154,6 @@ class MHASubPoolSpec(SubPoolSpec):
def page_bytes(self, page_size: int) -> int:
return page_size * self.entry_bytes()
def layer_k_offset_in_page(self, layer_id: int, page_size: int) -> int:
return layer_id * page_size * (self.k_row_bytes() + self.v_row_bytes())
def layer_v_offset_in_page(self, layer_id: int, page_size: int) -> int:
return (
self.layer_k_offset_in_page(layer_id, page_size)
+ page_size * self.k_row_bytes()
)
def view_tail_pad_bytes(self, page_size: int) -> int:
return page_size * self.entry_bytes()
@@ -310,15 +310,6 @@ class UnifiedRadixCache(BasePrefixCache):
if not reduced and self.tp_world_size > 1:
torch.distributed.all_reduce(tensor, op=op, group=self.tp_group)
def _barrier_attn_groups(self):
waited = False
for group in (self.attn_cp_group, self.attn_tp_group):
if group is not None and torch.distributed.get_world_size(group=group) > 1:
torch.distributed.barrier(group=group)
waited = True
if not waited and self.tp_world_size > 1:
torch.distributed.barrier(group=self.tp_group)
def _drain_async_work(self):
"""
Block until all outstanding async sends are consumed, then clear.
@@ -1463,31 +1463,6 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
self.last_log_time = time.perf_counter()
def log_grammar_stats(self, grammar_stats) -> None:
if grammar_stats.compilation_time is not None:
self._log_histogram(
self.grammar_compilation_time, grammar_stats.compilation_time
)
if grammar_stats.schema_count is not None:
self._log_histogram(self.grammar_schema_count, grammar_stats.schema_count)
if grammar_stats.ebnf_size is not None:
self._log_histogram(self.grammar_ebnf_size, grammar_stats.ebnf_size)
tree_times = grammar_stats.tree_traversal_time
if tree_times:
max_time = max(tree_times)
avg_time = sum(tree_times) / len(tree_times)
self._log_histogram(self.grammar_tree_traversal_time_max, max_time)
self._log_histogram(self.grammar_tree_traversal_time_avg, avg_time)
if grammar_stats.is_cache_hit:
self.num_grammar_cache_hit.labels(**self.labels).inc(1)
if grammar_stats.is_grammar_aborted:
self.num_grammar_aborted.labels(**self.labels).inc(1)
if grammar_stats.num_timeout > 0:
self.num_grammar_timeout.labels(**self.labels).inc(
grammar_stats.num_timeout
)
self.num_grammar_total.labels(**self.labels).inc(1)
def emit_constants(
self,
max_total_num_tokens: int,
@@ -1847,25 +1822,6 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
**labels, is_streaming="true" if stream else "false"
).observe(value)
def check_time_to_first_token_straggler(self, value: float) -> bool:
# Injected backends (e.g. Ray) route metrics out of process and can't
# introspect prometheus_client buckets here.
if self._histogram_cls is not None:
return False
his = self.histogram_time_to_first_token.labels(
**self.labels, is_streaming="true"
)
total_observations = sum(bucket._value for bucket in his._buckets)
if total_observations < 100:
return False
p99_threshold = total_observations * 0.99
cumulative_count = 0
for i, bucket in enumerate(his._buckets):
cumulative_count += bucket._value
if cumulative_count > p99_threshold:
return value >= his._upper_bounds[i]
return False
def observe_inter_token_latency(
self, labels: Dict[str, str], internval: float, num_new_tokens: int
):