[mem_cache] Remove unused helpers in mem_cache, storage backends, and metrics (#40963)
This commit is contained in:
@@ -922,14 +922,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.physical_to_virtual,
|
||||
)
|
||||
|
||||
def bind_pages(
|
||||
self, virtual_pages: torch.Tensor, physical_pages: torch.Tensor
|
||||
) -> None:
|
||||
"""Page-granular alias of ``bind``."""
|
||||
with record_function("MultiEndedAlloc.bind_pages"):
|
||||
self.bind(virtual_pages, physical_pages)
|
||||
|
||||
# -- fused take_physical_pages + bind_pages --
|
||||
# -- fused take_physical_pages + bind --
|
||||
|
||||
def _alloc_bind_fast_or_slow(
|
||||
self, v_pages: torch.Tensor, N: int
|
||||
@@ -1690,16 +1683,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
t.numel() for t in ready_tensors
|
||||
)
|
||||
|
||||
def maybe_drain_pending_reuse(self) -> None:
|
||||
"""Public scheduler hook (once per step): flow fired compaction-src pages
|
||||
back into `_free_phys_pages` for immediate reuse without waiting for `_flush`.
|
||||
"""
|
||||
if not self.lazy_compaction:
|
||||
return
|
||||
if not self._pending_reuse:
|
||||
return
|
||||
self._drain_pending_reuse(urgent=False)
|
||||
|
||||
def _topmost_survivor(
|
||||
self,
|
||||
start_hint: Optional[int] = None,
|
||||
@@ -2572,33 +2555,6 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
|
||||
return 0
|
||||
return self._flush(urgent=False)
|
||||
|
||||
def backup_state(self):
|
||||
# Span-aware snapshot (the base backs up `watermark_physical`, meaningless
|
||||
# here). Spec decode is asserted off under unified today.
|
||||
return (
|
||||
self.low_wm_page,
|
||||
self.high_wm_page,
|
||||
self._free_phys_pages.clone(),
|
||||
(len(self.free_virtual_ids) if self.is_id_owner else None),
|
||||
len(self._inverse_history),
|
||||
)
|
||||
|
||||
def restore_state(self, state):
|
||||
low_wm, high_wm, holes, _n_free_virtual, n_inverse = state
|
||||
self.low_wm_page = low_wm
|
||||
self.high_wm_page = high_wm
|
||||
self._free_phys_pages = holes
|
||||
new_entries = self._inverse_history[n_inverse:]
|
||||
if new_entries:
|
||||
logger.warning(
|
||||
"FloatMultiEndedAllocator.restore_state: %d relocation(s) inside "
|
||||
"a backup window (sub_pool=%s) — float moves are not reversible.",
|
||||
len(new_entries),
|
||||
self.sub_pool_name,
|
||||
)
|
||||
del self._inverse_history[n_inverse:]
|
||||
return new_entries
|
||||
|
||||
def compact_holes(self, *, retreat_side: str) -> int:
|
||||
"""Close ALL interior holes by packing live pages toward the side
|
||||
OPPOSITE ``retreat_side`` (order-preserving), shrinking the span on
|
||||
|
||||
@@ -1804,14 +1804,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
return self.request_window.buffer(self._swa_local_layer_id(layer_id))
|
||||
return self.swa_kv_pool.kv_buffer[self._swa_local_layer_id(layer_id)]
|
||||
|
||||
def get_swa_key_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
self.wait_layer_transfer(layer_id)
|
||||
if self.request_window is not None:
|
||||
return self.get_swa_raw_buffer(layer_id).view(
|
||||
self.request_window.state.dtype
|
||||
)
|
||||
return self.swa_kv_pool.get_key_buffer(self._swa_local_layer_id(layer_id))
|
||||
|
||||
def set_swa_key_buffer(
|
||||
self,
|
||||
layer_id: int,
|
||||
|
||||
@@ -457,13 +457,6 @@ class HiCacheFile(HiCacheStorage):
|
||||
return self._get_suffixed_key(key)
|
||||
return self._get_suffixed_key(f"{key}.{component_name}")
|
||||
|
||||
def _get_component_path(
|
||||
self, key: str, component_name: Optional[str] = None
|
||||
) -> str:
|
||||
return os.path.join(
|
||||
self.file_path, f"{self._get_component_key(key, component_name)}.bin"
|
||||
)
|
||||
|
||||
def _scan_existing_files_to_metadata_cache(self) -> None:
|
||||
try:
|
||||
names = os.listdir(self.file_path)
|
||||
|
||||
@@ -130,16 +130,6 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
|
||||
loc = self.translate_loc_to_hisparse_device(loc)
|
||||
return super().get_mla_kv_buffer(layer, loc, dst_dtype)
|
||||
|
||||
def transfer_values_on_device(self, dst_indices, src_indices):
|
||||
transfer_kv_all_layer_mla(
|
||||
src_layers=self.data_ptrs,
|
||||
dst_layers=self.data_ptrs,
|
||||
src_indices=src_indices,
|
||||
dst_indices=dst_indices,
|
||||
item_size=self.bytes_per_token,
|
||||
num_layers=self.layer_num,
|
||||
)
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
||||
raise NotImplementedError("HiSparseDevicePool does not support get_cpu_copy")
|
||||
|
||||
@@ -226,28 +216,6 @@ class HiSparseMHAMainPool(MHATokenToKVPool):
|
||||
translated = self.translate_loc_to_hisparse_device(raw_loc)
|
||||
super().set_kv_buffer(layer, translated, cache_k, cache_v, *args, **kwargs)
|
||||
|
||||
def transfer_values_on_device(
|
||||
self,
|
||||
dst_indices: torch.Tensor,
|
||||
src_indices: torch.Tensor,
|
||||
) -> None:
|
||||
transfer_kv_all_layer_mla(
|
||||
src_layers=self.k_data_ptrs,
|
||||
dst_layers=self.k_data_ptrs,
|
||||
src_indices=src_indices,
|
||||
dst_indices=dst_indices,
|
||||
item_size=self.bytes_per_token_k,
|
||||
num_layers=self.layer_num,
|
||||
)
|
||||
transfer_kv_all_layer_mla(
|
||||
src_layers=self.v_data_ptrs,
|
||||
dst_layers=self.v_data_ptrs,
|
||||
src_indices=src_indices,
|
||||
dst_indices=dst_indices,
|
||||
item_size=self.bytes_per_token_v,
|
||||
num_layers=self.layer_num,
|
||||
)
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
||||
raise NotImplementedError("HiSparseMHAMainPool does not support get_cpu_copy")
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ class KVCacheBuildResult:
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.hybrid_arch import (
|
||||
glm5_next_config,
|
||||
hybrid_gdn_config,
|
||||
@@ -117,29 +116,6 @@ def prepare_hicache_staging(
|
||||
)
|
||||
|
||||
|
||||
def get_draft_kv_pool(
|
||||
*,
|
||||
draft_worker: BaseTpWorker,
|
||||
spec_algorithm: SpeculativeAlgorithm,
|
||||
server_args: ServerArgs,
|
||||
):
|
||||
"""Return the draft token-to-KV pool for the current draft worker,
|
||||
or None when no draft KV pool is available."""
|
||||
if draft_worker is None or spec_algorithm.is_ngram():
|
||||
return None
|
||||
|
||||
# V2 draft workers exist only on their hosting PP stage; other ranks own no
|
||||
# nested draft worker or draft KV pool.
|
||||
if draft_worker.draft_worker is None:
|
||||
return None
|
||||
|
||||
if resolving_view(server_args).enable_multi_layer_eagle:
|
||||
draft_runner = draft_worker.draft_worker.draft_runner_list[0]
|
||||
else:
|
||||
draft_runner = draft_worker.draft_worker.draft_runner
|
||||
return draft_runner.token_to_kv_pool
|
||||
|
||||
|
||||
def maybe_register_hicache_draft(
|
||||
*,
|
||||
tree_cache,
|
||||
|
||||
@@ -66,7 +66,6 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridReqToTokenPool,
|
||||
KVCache,
|
||||
MHATokenToKVPool,
|
||||
MHATokenToKVPoolFP4,
|
||||
MHATokenToKVPoolMXFP8,
|
||||
MiniMaxSparseKVPool,
|
||||
MLATokenToKVPool,
|
||||
@@ -1981,26 +1980,6 @@ class KVCacheConfigurator:
|
||||
)
|
||||
return token_to_kv_pool
|
||||
|
||||
def _build_mha_fp4_kv_pool(self, *, max_total_num_tokens: int) -> KVCache:
|
||||
token_to_kv_pool = MHATokenToKVPoolFP4(
|
||||
max_total_num_tokens,
|
||||
page_size=self.pool_page_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
head_num=self.model_config.get_num_kv_heads(
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
),
|
||||
head_dim=self.model_config.head_dim,
|
||||
v_head_dim=self.model_config.v_head_dim,
|
||||
layer_num=self.layer_info.num_effective_layers,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
enable_alt_stream=not get_disagg().enable_pdmux,
|
||||
enable_kv_cache_copy=(get_spec().speculative_algorithm is not None),
|
||||
)
|
||||
return token_to_kv_pool
|
||||
|
||||
def _build_mha_kv_pool(
|
||||
self, *, max_total_num_tokens: int, mha_pool_class: type, quant_method=None
|
||||
) -> KVCache:
|
||||
|
||||
@@ -96,12 +96,6 @@ class KVIndexTable(msgspec.Struct, frozen=True):
|
||||
is_translated: bool # entries are already kernel-facing ids
|
||||
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
|
||||
|
||||
def sliding_window_read_ids(self) -> torch.Tensor:
|
||||
"""Which array a sliding-window gather reads: the parallel swa array
|
||||
when translated, else the full-attention array, which the caller maps
|
||||
through the pool's own full->swa map."""
|
||||
return self.sliding_window_ids if self.is_translated else self.ids
|
||||
|
||||
|
||||
class KVIndexTranslator:
|
||||
"""Built once per ModelRunner."""
|
||||
|
||||
@@ -2,7 +2,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import eic
|
||||
import torch
|
||||
@@ -146,12 +146,6 @@ class FlexibleKVCacheMemoryPool:
|
||||
return
|
||||
self.free_data_addr.add(self.data_ptr_to_index[data_ptr])
|
||||
|
||||
def check_data_ptr_allocated(self, data_ptr):
|
||||
return data_ptr in self.data_ptr_to_index
|
||||
|
||||
def left_count(self):
|
||||
return len(self.free_data_addr)
|
||||
|
||||
|
||||
class EICStorage(HiCacheStorage):
|
||||
def __init__(
|
||||
@@ -435,16 +429,6 @@ class EICStorage(HiCacheStorage):
|
||||
def clear(self) -> None:
|
||||
return
|
||||
|
||||
# Not used for now
|
||||
def _filter_kv_cache(self, total_len) -> Tuple[int, int]:
|
||||
mean_len = total_len // self.world_size
|
||||
remainder = total_len % self.world_size
|
||||
tp_keys_len = mean_len + (1 if self.rank < remainder else 0)
|
||||
start = self.rank * mean_len + min(self.rank, remainder)
|
||||
end = start + tp_keys_len
|
||||
logger.debug(f"start: {start}, end: {end}, tp_keys_len: {tp_keys_len}")
|
||||
return start, end
|
||||
|
||||
def zero_copy_batch_set(self, keys: List[str], values: List[torch.Tensor]) -> bool:
|
||||
logger.debug(f"eic zero copy set {len(keys)} keys")
|
||||
if len(keys) == 0:
|
||||
|
||||
@@ -24,7 +24,6 @@ This file provides:
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import pickle
|
||||
@@ -478,25 +477,6 @@ def eventfd(initval: int = 0, flags: int = 0) -> int:
|
||||
return fd
|
||||
|
||||
|
||||
def eventfd_write(fd: int, val: int) -> None:
|
||||
v = ctypes.c_uint64(val)
|
||||
n = _libc.write(fd, ctypes.byref(v), ctypes.sizeof(v))
|
||||
if n != ctypes.sizeof(v):
|
||||
err = ctypes.get_errno()
|
||||
raise OSError(err, f"eventfd write failed: {os.strerror(err)}")
|
||||
|
||||
|
||||
def eventfd_read(fd: int) -> int:
|
||||
v = ctypes.c_uint64()
|
||||
n = _libc.read(fd, ctypes.byref(v), ctypes.sizeof(v))
|
||||
if n != ctypes.sizeof(v):
|
||||
err = ctypes.get_errno()
|
||||
if err == errno.EAGAIN:
|
||||
return 0
|
||||
raise OSError(err, f"eventfd read failed: {os.strerror(err)}")
|
||||
return v.value
|
||||
|
||||
|
||||
def send_fds(sock: socket.socket, fds: list, extra_data: bytes = b"x") -> None:
|
||||
"""SCM_RIGHTS-send a list of file descriptors over a UDS socket."""
|
||||
fds_packed = struct.pack(f"{len(fds)}i", *fds)
|
||||
|
||||
@@ -587,22 +587,6 @@ class FlexKVConnector:
|
||||
completed_handles = self._sync_ctx.scatter(completed_handles)
|
||||
return completed_handles
|
||||
|
||||
def wait_store(self, handle: CacheRequestHandle, timeout: float = 30.0) -> bool:
|
||||
"""Block until a single store task identified by ``handle`` finishes."""
|
||||
fkv_task_id = self._inflight_stores.pop(handle, -1)
|
||||
if fkv_task_id < 0:
|
||||
return True
|
||||
if not self._sync_ctx.is_sync_leader or self.kv_manager is None:
|
||||
return True
|
||||
try:
|
||||
resp = self.kv_manager.wait([fkv_task_id], timeout=timeout)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("[FlexKV] wait_store: %s", exc)
|
||||
return False
|
||||
return (
|
||||
fkv_task_id in resp and resp[fkv_task_id].status == KVResponseStatus.SUCCESS
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API — prefetch
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -196,10 +196,6 @@ class TensorcastStore(HiCacheStorage):
|
||||
return 0
|
||||
return self._publish_if_available(hit_pages, fallback=0)
|
||||
|
||||
def _build_page_fragments(self, keys: list[str]) -> tuple[PageFragment, ...]:
|
||||
registered = self._require_registered()
|
||||
return _expand_page_fragments(registered, keys)
|
||||
|
||||
def _build_artifact_specs(self, keys: list[str]) -> tuple[ByteArtifactSpec, ...]:
|
||||
registered = self._require_registered()
|
||||
return _expand_artifact_specs(registered, keys)
|
||||
|
||||
@@ -457,10 +457,6 @@ class UMBPDirectLinker(UnifiedCacheLinker):
|
||||
# every layer, and a layer is read back as a byte range inside it.
|
||||
return component_keys, multiplier
|
||||
|
||||
def _object_keys(self, transfer: PoolTransfer) -> list[str]:
|
||||
keys, _ = self._object_keys_for_pages(list(transfer.keys or []), transfer)
|
||||
return keys
|
||||
|
||||
def _page_exists(self, page_keys: list[str], transfer: PoolTransfer) -> list[bool]:
|
||||
entry = self.pools[transfer.name]
|
||||
objects_per_page = 1 if entry.packed else len(entry.components)
|
||||
|
||||
@@ -154,15 +154,6 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
def page_bytes(self, page_size: int) -> int:
|
||||
return page_size * self.entry_bytes()
|
||||
|
||||
def layer_k_offset_in_page(self, layer_id: int, page_size: int) -> int:
|
||||
return layer_id * page_size * (self.k_row_bytes() + self.v_row_bytes())
|
||||
|
||||
def layer_v_offset_in_page(self, layer_id: int, page_size: int) -> int:
|
||||
return (
|
||||
self.layer_k_offset_in_page(layer_id, page_size)
|
||||
+ page_size * self.k_row_bytes()
|
||||
)
|
||||
|
||||
def view_tail_pad_bytes(self, page_size: int) -> int:
|
||||
return page_size * self.entry_bytes()
|
||||
|
||||
|
||||
@@ -310,15 +310,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if not reduced and self.tp_world_size > 1:
|
||||
torch.distributed.all_reduce(tensor, op=op, group=self.tp_group)
|
||||
|
||||
def _barrier_attn_groups(self):
|
||||
waited = False
|
||||
for group in (self.attn_cp_group, self.attn_tp_group):
|
||||
if group is not None and torch.distributed.get_world_size(group=group) > 1:
|
||||
torch.distributed.barrier(group=group)
|
||||
waited = True
|
||||
if not waited and self.tp_world_size > 1:
|
||||
torch.distributed.barrier(group=self.tp_group)
|
||||
|
||||
def _drain_async_work(self):
|
||||
"""
|
||||
Block until all outstanding async sends are consumed, then clear.
|
||||
|
||||
@@ -1463,31 +1463,6 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
|
||||
self.last_log_time = time.perf_counter()
|
||||
|
||||
def log_grammar_stats(self, grammar_stats) -> None:
|
||||
if grammar_stats.compilation_time is not None:
|
||||
self._log_histogram(
|
||||
self.grammar_compilation_time, grammar_stats.compilation_time
|
||||
)
|
||||
if grammar_stats.schema_count is not None:
|
||||
self._log_histogram(self.grammar_schema_count, grammar_stats.schema_count)
|
||||
if grammar_stats.ebnf_size is not None:
|
||||
self._log_histogram(self.grammar_ebnf_size, grammar_stats.ebnf_size)
|
||||
tree_times = grammar_stats.tree_traversal_time
|
||||
if tree_times:
|
||||
max_time = max(tree_times)
|
||||
avg_time = sum(tree_times) / len(tree_times)
|
||||
self._log_histogram(self.grammar_tree_traversal_time_max, max_time)
|
||||
self._log_histogram(self.grammar_tree_traversal_time_avg, avg_time)
|
||||
if grammar_stats.is_cache_hit:
|
||||
self.num_grammar_cache_hit.labels(**self.labels).inc(1)
|
||||
if grammar_stats.is_grammar_aborted:
|
||||
self.num_grammar_aborted.labels(**self.labels).inc(1)
|
||||
if grammar_stats.num_timeout > 0:
|
||||
self.num_grammar_timeout.labels(**self.labels).inc(
|
||||
grammar_stats.num_timeout
|
||||
)
|
||||
self.num_grammar_total.labels(**self.labels).inc(1)
|
||||
|
||||
def emit_constants(
|
||||
self,
|
||||
max_total_num_tokens: int,
|
||||
@@ -1847,25 +1822,6 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
|
||||
**labels, is_streaming="true" if stream else "false"
|
||||
).observe(value)
|
||||
|
||||
def check_time_to_first_token_straggler(self, value: float) -> bool:
|
||||
# Injected backends (e.g. Ray) route metrics out of process and can't
|
||||
# introspect prometheus_client buckets here.
|
||||
if self._histogram_cls is not None:
|
||||
return False
|
||||
his = self.histogram_time_to_first_token.labels(
|
||||
**self.labels, is_streaming="true"
|
||||
)
|
||||
total_observations = sum(bucket._value for bucket in his._buckets)
|
||||
if total_observations < 100:
|
||||
return False
|
||||
p99_threshold = total_observations * 0.99
|
||||
cumulative_count = 0
|
||||
for i, bucket in enumerate(his._buckets):
|
||||
cumulative_count += bucket._value
|
||||
if cumulative_count > p99_threshold:
|
||||
return value >= his._upper_bounds[i]
|
||||
return False
|
||||
|
||||
def observe_inter_token_latency(
|
||||
self, labels: Dict[str, str], internval: float, num_new_tokens: int
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user