diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 3d337b2587..5e924d376f 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -922,14 +922,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self.physical_to_virtual, ) - def bind_pages( - self, virtual_pages: torch.Tensor, physical_pages: torch.Tensor - ) -> None: - """Page-granular alias of ``bind``.""" - with record_function("MultiEndedAlloc.bind_pages"): - self.bind(virtual_pages, physical_pages) - - # -- fused take_physical_pages + bind_pages -- + # -- fused take_physical_pages + bind -- def _alloc_bind_fast_or_slow( self, v_pages: torch.Tensor, N: int @@ -1690,16 +1683,6 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): t.numel() for t in ready_tensors ) - def maybe_drain_pending_reuse(self) -> None: - """Public scheduler hook (once per step): flow fired compaction-src pages - back into `_free_phys_pages` for immediate reuse without waiting for `_flush`. - """ - if not self.lazy_compaction: - return - if not self._pending_reuse: - return - self._drain_pending_reuse(urgent=False) - def _topmost_survivor( self, start_hint: Optional[int] = None, @@ -2572,33 +2555,6 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): return 0 return self._flush(urgent=False) - def backup_state(self): - # Span-aware snapshot (the base backs up `watermark_physical`, meaningless - # here). Spec decode is asserted off under unified today. - return ( - self.low_wm_page, - self.high_wm_page, - self._free_phys_pages.clone(), - (len(self.free_virtual_ids) if self.is_id_owner else None), - len(self._inverse_history), - ) - - def restore_state(self, state): - low_wm, high_wm, holes, _n_free_virtual, n_inverse = state - self.low_wm_page = low_wm - self.high_wm_page = high_wm - self._free_phys_pages = holes - new_entries = self._inverse_history[n_inverse:] - if new_entries: - logger.warning( - "FloatMultiEndedAllocator.restore_state: %d relocation(s) inside " - "a backup window (sub_pool=%s) — float moves are not reversible.", - len(new_entries), - self.sub_pool_name, - ) - del self._inverse_history[n_inverse:] - return new_entries - def compact_holes(self, *, retreat_side: str) -> int: """Close ALL interior holes by packing live pages toward the side OPPOSITE ``retreat_side`` (order-preserving), shrinking the span on diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 7b5f012801..d733331098 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -1804,14 +1804,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): return self.request_window.buffer(self._swa_local_layer_id(layer_id)) return self.swa_kv_pool.kv_buffer[self._swa_local_layer_id(layer_id)] - def get_swa_key_buffer(self, layer_id: int) -> torch.Tensor: - self.wait_layer_transfer(layer_id) - if self.request_window is not None: - return self.get_swa_raw_buffer(layer_id).view( - self.request_window.state.dtype - ) - return self.swa_kv_pool.get_key_buffer(self._swa_local_layer_id(layer_id)) - def set_swa_key_buffer( self, layer_id: int, diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index 1450da14fd..2b4fd2f8f7 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -457,13 +457,6 @@ class HiCacheFile(HiCacheStorage): return self._get_suffixed_key(key) return self._get_suffixed_key(f"{key}.{component_name}") - def _get_component_path( - self, key: str, component_name: Optional[str] = None - ) -> str: - return os.path.join( - self.file_path, f"{self._get_component_key(key, component_name)}.bin" - ) - def _scan_existing_files_to_metadata_cache(self) -> None: try: names = os.listdir(self.file_path) diff --git a/python/sglang/srt/mem_cache/hisparse_memory_pool.py b/python/sglang/srt/mem_cache/hisparse_memory_pool.py index 7e6ff2534a..210b691a2c 100644 --- a/python/sglang/srt/mem_cache/hisparse_memory_pool.py +++ b/python/sglang/srt/mem_cache/hisparse_memory_pool.py @@ -130,16 +130,6 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool): loc = self.translate_loc_to_hisparse_device(loc) return super().get_mla_kv_buffer(layer, loc, dst_dtype) - def transfer_values_on_device(self, dst_indices, src_indices): - transfer_kv_all_layer_mla( - src_layers=self.data_ptrs, - dst_layers=self.data_ptrs, - src_indices=src_indices, - dst_indices=dst_indices, - item_size=self.bytes_per_token, - num_layers=self.layer_num, - ) - def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None): raise NotImplementedError("HiSparseDevicePool does not support get_cpu_copy") @@ -226,28 +216,6 @@ class HiSparseMHAMainPool(MHATokenToKVPool): translated = self.translate_loc_to_hisparse_device(raw_loc) super().set_kv_buffer(layer, translated, cache_k, cache_v, *args, **kwargs) - def transfer_values_on_device( - self, - dst_indices: torch.Tensor, - src_indices: torch.Tensor, - ) -> None: - transfer_kv_all_layer_mla( - src_layers=self.k_data_ptrs, - dst_layers=self.k_data_ptrs, - src_indices=src_indices, - dst_indices=dst_indices, - item_size=self.bytes_per_token_k, - num_layers=self.layer_num, - ) - transfer_kv_all_layer_mla( - src_layers=self.v_data_ptrs, - dst_layers=self.v_data_ptrs, - src_indices=src_indices, - dst_indices=dst_indices, - item_size=self.bytes_per_token_v, - num_layers=self.layer_num, - ) - def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None): raise NotImplementedError("HiSparseMHAMainPool does not support get_cpu_copy") diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 196dc7199c..70c0df78d2 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -25,7 +25,6 @@ class KVCacheBuildResult: from typing import TYPE_CHECKING -from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.hybrid_arch import ( glm5_next_config, hybrid_gdn_config, @@ -117,29 +116,6 @@ def prepare_hicache_staging( ) -def get_draft_kv_pool( - *, - draft_worker: BaseTpWorker, - spec_algorithm: SpeculativeAlgorithm, - server_args: ServerArgs, -): - """Return the draft token-to-KV pool for the current draft worker, - or None when no draft KV pool is available.""" - if draft_worker is None or spec_algorithm.is_ngram(): - return None - - # V2 draft workers exist only on their hosting PP stage; other ranks own no - # nested draft worker or draft KV pool. - if draft_worker.draft_worker is None: - return None - - if resolving_view(server_args).enable_multi_layer_eagle: - draft_runner = draft_worker.draft_worker.draft_runner_list[0] - else: - draft_runner = draft_worker.draft_worker.draft_runner - return draft_runner.token_to_kv_pool - - def maybe_register_hicache_draft( *, tree_cache, diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index fed1680a2b..23fffa68d7 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -66,7 +66,6 @@ from sglang.srt.mem_cache.memory_pool import ( HybridReqToTokenPool, KVCache, MHATokenToKVPool, - MHATokenToKVPoolFP4, MHATokenToKVPoolMXFP8, MiniMaxSparseKVPool, MLATokenToKVPool, @@ -1981,26 +1980,6 @@ class KVCacheConfigurator: ) return token_to_kv_pool - def _build_mha_fp4_kv_pool(self, *, max_total_num_tokens: int) -> KVCache: - token_to_kv_pool = MHATokenToKVPoolFP4( - max_total_num_tokens, - page_size=self.pool_page_size, - dtype=self.kv_cache_dtype, - head_num=self.model_config.get_num_kv_heads( - get_parallel().attn_tp_size, get_parallel().attn_dcp_size - ), - head_dim=self.model_config.head_dim, - v_head_dim=self.model_config.v_head_dim, - layer_num=self.layer_info.num_effective_layers, - device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, - start_layer=self.layer_info.start_layer, - end_layer=self.layer_info.end_layer, - enable_alt_stream=not get_disagg().enable_pdmux, - enable_kv_cache_copy=(get_spec().speculative_algorithm is not None), - ) - return token_to_kv_pool - def _build_mha_kv_pool( self, *, max_total_num_tokens: int, mha_pool_class: type, quant_method=None ) -> KVCache: diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 5fbb2e589e..6c42daf64c 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -96,12 +96,6 @@ class KVIndexTable(msgspec.Struct, frozen=True): is_translated: bool # entries are already kernel-facing ids sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array - def sliding_window_read_ids(self) -> torch.Tensor: - """Which array a sliding-window gather reads: the parallel swa array - when translated, else the full-attention array, which the caller maps - through the pool's own full->swa map.""" - return self.sliding_window_ids if self.is_translated else self.ids - class KVIndexTranslator: """Built once per ModelRunner.""" diff --git a/python/sglang/srt/mem_cache/storage/eic/eic_storage.py b/python/sglang/srt/mem_cache/storage/eic/eic_storage.py index f9f4f2be21..58746c4a89 100644 --- a/python/sglang/srt/mem_cache/storage/eic/eic_storage.py +++ b/python/sglang/srt/mem_cache/storage/eic/eic_storage.py @@ -2,7 +2,7 @@ import json import logging import os import time -from typing import Any, List, Optional, Tuple +from typing import Any, List, Optional import eic import torch @@ -146,12 +146,6 @@ class FlexibleKVCacheMemoryPool: return self.free_data_addr.add(self.data_ptr_to_index[data_ptr]) - def check_data_ptr_allocated(self, data_ptr): - return data_ptr in self.data_ptr_to_index - - def left_count(self): - return len(self.free_data_addr) - class EICStorage(HiCacheStorage): def __init__( @@ -435,16 +429,6 @@ class EICStorage(HiCacheStorage): def clear(self) -> None: return - # Not used for now - def _filter_kv_cache(self, total_len) -> Tuple[int, int]: - mean_len = total_len // self.world_size - remainder = total_len % self.world_size - tp_keys_len = mean_len + (1 if self.rank < remainder else 0) - start = self.rank * mean_len + min(self.rank, remainder) - end = start + tp_keys_len - logger.debug(f"start: {start}, end: {end}, tp_keys_len: {tp_keys_len}") - return start, end - def zero_copy_batch_set(self, keys: List[str], values: List[torch.Tensor]) -> bool: logger.debug(f"eic zero copy set {len(keys)} keys") if len(keys) == 0: diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_comm.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_comm.py index fa03412129..659e338b18 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_comm.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_comm.py @@ -24,7 +24,6 @@ This file provides: from __future__ import annotations import ctypes -import errno import logging import os import pickle @@ -478,25 +477,6 @@ def eventfd(initval: int = 0, flags: int = 0) -> int: return fd -def eventfd_write(fd: int, val: int) -> None: - v = ctypes.c_uint64(val) - n = _libc.write(fd, ctypes.byref(v), ctypes.sizeof(v)) - if n != ctypes.sizeof(v): - err = ctypes.get_errno() - raise OSError(err, f"eventfd write failed: {os.strerror(err)}") - - -def eventfd_read(fd: int) -> int: - v = ctypes.c_uint64() - n = _libc.read(fd, ctypes.byref(v), ctypes.sizeof(v)) - if n != ctypes.sizeof(v): - err = ctypes.get_errno() - if err == errno.EAGAIN: - return 0 - raise OSError(err, f"eventfd read failed: {os.strerror(err)}") - return v.value - - def send_fds(sock: socket.socket, fds: list, extra_data: bytes = b"x") -> None: """SCM_RIGHTS-send a list of file descriptors over a UDS socket.""" fds_packed = struct.pack(f"{len(fds)}i", *fds) diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py index 0f9f204c7c..64141c4bf5 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py @@ -587,22 +587,6 @@ class FlexKVConnector: completed_handles = self._sync_ctx.scatter(completed_handles) return completed_handles - def wait_store(self, handle: CacheRequestHandle, timeout: float = 30.0) -> bool: - """Block until a single store task identified by ``handle`` finishes.""" - fkv_task_id = self._inflight_stores.pop(handle, -1) - if fkv_task_id < 0: - return True - if not self._sync_ctx.is_sync_leader or self.kv_manager is None: - return True - try: - resp = self.kv_manager.wait([fkv_task_id], timeout=timeout) - except Exception as exc: # noqa: BLE001 - logger.warning("[FlexKV] wait_store: %s", exc) - return False - return ( - fkv_task_id in resp and resp[fkv_task_id].status == KVResponseStatus.SUCCESS - ) - # ------------------------------------------------------------------ # Public API — prefetch # ------------------------------------------------------------------ diff --git a/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py b/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py index c1487b5a6e..3e3d3bd3b8 100644 --- a/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py +++ b/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py @@ -196,10 +196,6 @@ class TensorcastStore(HiCacheStorage): return 0 return self._publish_if_available(hit_pages, fallback=0) - def _build_page_fragments(self, keys: list[str]) -> tuple[PageFragment, ...]: - registered = self._require_registered() - return _expand_page_fragments(registered, keys) - def _build_artifact_specs(self, keys: list[str]) -> tuple[ByteArtifactSpec, ...]: registered = self._require_registered() return _expand_artifact_specs(registered, keys) diff --git a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py index 6f81ab9616..542835eb2f 100644 --- a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py @@ -457,10 +457,6 @@ class UMBPDirectLinker(UnifiedCacheLinker): # every layer, and a layer is read back as a byte range inside it. return component_keys, multiplier - def _object_keys(self, transfer: PoolTransfer) -> list[str]: - keys, _ = self._object_keys_for_pages(list(transfer.keys or []), transfer) - return keys - def _page_exists(self, page_keys: list[str], transfer: PoolTransfer) -> list[bool]: entry = self.pools[transfer.name] objects_per_page = 1 if entry.packed else len(entry.components) diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 3cca20a7db..ad3cc0642b 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -154,15 +154,6 @@ class MHASubPoolSpec(SubPoolSpec): def page_bytes(self, page_size: int) -> int: return page_size * self.entry_bytes() - def layer_k_offset_in_page(self, layer_id: int, page_size: int) -> int: - return layer_id * page_size * (self.k_row_bytes() + self.v_row_bytes()) - - def layer_v_offset_in_page(self, layer_id: int, page_size: int) -> int: - return ( - self.layer_k_offset_in_page(layer_id, page_size) - + page_size * self.k_row_bytes() - ) - def view_tail_pad_bytes(self, page_size: int) -> int: return page_size * self.entry_bytes() diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index cd45906334..3a69aff3c2 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -310,15 +310,6 @@ class UnifiedRadixCache(BasePrefixCache): if not reduced and self.tp_world_size > 1: torch.distributed.all_reduce(tensor, op=op, group=self.tp_group) - def _barrier_attn_groups(self): - waited = False - for group in (self.attn_cp_group, self.attn_tp_group): - if group is not None and torch.distributed.get_world_size(group=group) > 1: - torch.distributed.barrier(group=group) - waited = True - if not waited and self.tp_world_size > 1: - torch.distributed.barrier(group=self.tp_group) - def _drain_async_work(self): """ Block until all outstanding async sends are consumed, then clear. diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 0f4a92ca69..4307f511ea 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1463,31 +1463,6 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): self.last_log_time = time.perf_counter() - def log_grammar_stats(self, grammar_stats) -> None: - if grammar_stats.compilation_time is not None: - self._log_histogram( - self.grammar_compilation_time, grammar_stats.compilation_time - ) - if grammar_stats.schema_count is not None: - self._log_histogram(self.grammar_schema_count, grammar_stats.schema_count) - if grammar_stats.ebnf_size is not None: - self._log_histogram(self.grammar_ebnf_size, grammar_stats.ebnf_size) - tree_times = grammar_stats.tree_traversal_time - if tree_times: - max_time = max(tree_times) - avg_time = sum(tree_times) / len(tree_times) - self._log_histogram(self.grammar_tree_traversal_time_max, max_time) - self._log_histogram(self.grammar_tree_traversal_time_avg, avg_time) - if grammar_stats.is_cache_hit: - self.num_grammar_cache_hit.labels(**self.labels).inc(1) - if grammar_stats.is_grammar_aborted: - self.num_grammar_aborted.labels(**self.labels).inc(1) - if grammar_stats.num_timeout > 0: - self.num_grammar_timeout.labels(**self.labels).inc( - grammar_stats.num_timeout - ) - self.num_grammar_total.labels(**self.labels).inc(1) - def emit_constants( self, max_total_num_tokens: int, @@ -1847,25 +1822,6 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): **labels, is_streaming="true" if stream else "false" ).observe(value) - def check_time_to_first_token_straggler(self, value: float) -> bool: - # Injected backends (e.g. Ray) route metrics out of process and can't - # introspect prometheus_client buckets here. - if self._histogram_cls is not None: - return False - his = self.histogram_time_to_first_token.labels( - **self.labels, is_streaming="true" - ) - total_observations = sum(bucket._value for bucket in his._buckets) - if total_observations < 100: - return False - p99_threshold = total_observations * 0.99 - cumulative_count = 0 - for i, bucket in enumerate(his._buckets): - cumulative_count += bucket._value - if cumulative_count > p99_threshold: - return value >= his._upper_bounds[i] - return False - def observe_inter_token_latency( self, labels: Dict[str, str], internval: float, num_new_tokens: int ): diff --git a/test/registered/unit/mem_cache/test_multi_ended_allocator.py b/test/registered/unit/mem_cache/test_multi_ended_allocator.py index 2418d9bea3..1f6b874cb5 100644 --- a/test/registered/unit/mem_cache/test_multi_ended_allocator.py +++ b/test/registered/unit/mem_cache/test_multi_ended_allocator.py @@ -2170,7 +2170,7 @@ class TestLazyCompaction(unittest.TestCase): class TestO3FusedAllocBind(unittest.TestCase): - """Fused take_physical_pages + bind_pages via `_alloc_bind_fast_or_slow`. + """Fused take_physical_pages + bind via `_alloc_bind_fast_or_slow`. GPU-only: the fused kernel is Triton.""" def setUp(self):