diff --git a/.github/labeler.yml b/.github/labeler.yml index 0e511e4fae..16ff878feb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -114,7 +114,6 @@ unified-radix-cache: - changed-files: - any-glob-to-any-file: - 'python/sglang/srt/mem_cache/unified_cache/**/*' - - 'python/sglang/srt/mem_cache/cpp_radix_tree/**/*' - 'python/sglang/srt/mem_cache/**/*radix*' - 'python/sglang/srt/mem_cache/chunk_cache.py' - 'python/sglang/srt/mem_cache/evict_policy.py' diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 84e05966c1..4260f6c958 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -1419,11 +1419,6 @@ SGLang supports various environment variables that can be used to configure its Use the unified radix-tree cache implementation. false - - SGLANG_EXPERIMENTAL_CPP_RADIX_TREE - Use the experimental C++ radix-tree implementation. - false - SGLANG_RADIX_FORCE_MISS Force radix-cache misses (debugging/benchmarking). diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index b497b0c3ce..d173f3ca8e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -662,7 +662,6 @@ class Envs: # =================================================================== # Radix and sparse KV caches # =================================================================== - SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False) SGLANG_RADIX_FORCE_MISS = EnvBool(False) SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD = EnvInt(8192) SGLANG_MAX_KV_CHUNK_CAPACITY = EnvInt(128 * 1024) diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/.clang-format b/python/sglang/srt/mem_cache/cpp_radix_tree/.clang-format deleted file mode 120000 index 7d1cf7aa84..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/.clang-format +++ /dev/null @@ -1 +0,0 @@ -../../../kernels/aot/.clang-format \ No newline at end of file diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/common.h b/python/sglang/srt/mem_cache/cpp_radix_tree/common.h deleted file mode 100644 index 72c2c78be5..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/common.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include - -namespace radix_tree_v2 { - -using token_t = std::int32_t; -using token_vec_t = std::vector; -using token_slice = std::span; -using NodeHandle = std::size_t; -using IOTicket = std::uint32_t; - -inline void _assert( - bool condition, - const char* message = "Assertion failed", - std::source_location loc = std::source_location::current()) { - if (!condition) [[unlikely]] { - std::string msg = message; - msg = msg + " at " + loc.file_name() + ":" + std::to_string(loc.line()) + " in " + loc.function_name(); - throw std::runtime_error(msg); - } -} - -} // namespace radix_tree_v2 diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/radix_tree.py b/python/sglang/srt/mem_cache/cpp_radix_tree/radix_tree.py deleted file mode 100644 index 592727aac4..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/radix_tree.py +++ /dev/null @@ -1,182 +0,0 @@ -from __future__ import annotations - -import os -from typing import TYPE_CHECKING, List, Optional, Tuple - -import torch -from torch.utils.cpp_extension import load - -_abs_path = os.path.dirname(os.path.abspath(__file__)) -radix_tree_cpp = load( - name="radix_tree_cpp", - sources=[ - f"{_abs_path}/tree_v2_binding.cpp", - f"{_abs_path}/tree_v2_debug.cpp", - f"{_abs_path}/tree_v2.cpp", - ], - extra_cflags=["-O3", "-std=c++20"], -) - -if TYPE_CHECKING: - - class TreeNodeCpp: - """ - A placeholder for the TreeNode class. Cannot be constructed elsewhere. - """ - - class IOHandle: - """ - A placeholder for the IOHandle class. Cannot be constructed elsewhere. - """ - - class RadixTreeCpp: - def __init__( - self, - disabled: bool, - host_size: Optional[int], - page_size: int, - write_through_threshold: int, - ): - """ - Initializes the RadixTreeCpp instance. - Args: - disabled (bool): If True, the radix tree is disabled. - host_size (Optional[int]): Size of the radix tree on the CPU. None means no CPU tree. - page_size (int): Size of the page for the radix tree. - write_through_threshold (int): Threshold for writing through from GPU to CPU. - """ - self.tree = radix_tree_cpp.RadixTree( # type: ignore - disabled, host_size, page_size, write_through_threshold - ) - - def match_prefix( - self, prefix: List[int] - ) -> Tuple[List[torch.Tensor], int, TreeNodeCpp, TreeNodeCpp]: - """ - Matches a prefix in the radix tree. - Args: - prefix (List[int]): The prefix to match. - Returns: - Tuple[List[torch.Tensor], TreeNodeCpp, TreeNodeCpp]: - 0. A list of indices that is matched by the prefix on the GPU. - 1. Sum length of the indices matched on the CPU. - 2. The last node of the prefix matched on the GPU. - 3. The last node of the prefix matched on the CPU. - """ - return self.tree.match_prefix(prefix) - - def evict(self, num_tokens: int) -> List[torch.Tensor]: - """ - Evicts a number of tokens from the radix tree. - Args: - num_tokens (int): The number of tokens to evict. - Returns: - List[torch.Tensor]: A list of indices that were evicted. - """ - return self.tree.evict(num_tokens) - - def lock_ref(self, handle: TreeNodeCpp, lock: bool) -> None: - """ - Locks or unlocks a reference to a tree node. - After locking, the node will not be evicted from the radix tree. - Args: - handle (TreeNodeCpp): The tree node to lock or unlock. - lock (bool): If True, locks the node; if False, unlocks it. - """ - return self.tree.lock_ref(handle, lock) - - def writing_through( - self, key: List[int], indices: torch.Tensor - ) -> Tuple[List[Tuple[IOHandle, torch.Tensor, torch.Tensor]], int]: - """ - Inserts a key-value pair into the radix tree and perform write-through check. - Args: - key (List[int]): The key to insert. - indices (torch.Tensor): The value associated with the key. - Returns: - Tuple[List[Tuple[IOHandle, torch.Tensor, torch.Tensor]], int]: - 0. A list of (IOHandle, device indices, host indices) tuples. - These IOhandles require write-through to the CPU in python side. - 1. The number of indices that are matched on device. - """ - return self.tree.writing_through(key, indices) - - def loading_onboard( - self, - host_node: TreeNodeCpp, - new_device_indices: torch.Tensor, - ) -> Tuple[IOHandle, List[torch.Tensor]]: - """ - Updates the device indices of tree nodes within a range on the tree. - Args: - host_node (TreeNodeCpp): The tree node on the host, must be descendant of device_node. - new_device_indices (torch.Tensor): The new device indices to set. - The length of this tensor must be exactly host indices length. - Returns: - Tuple[IOHandle, List[torch.Tensor]]: - 0. An IOHandle that requires loading to the CPU in python side. - 1. A list of host indices corresponding to the new device indices. - """ - return self.tree.loading_onboard(host_node, new_device_indices) - - def commit_writing_through(self, handle: IOHandle, success: bool) -> None: - """ - Commits the write-through process for a tree node. - Args: - handle (IOHandle): The IOHandle to commit. - success (bool): If True, commits the write-through; if False, just indicates failure. - """ - return self.tree.commit_writing_through(handle, success) - - def commit_loading_onboard(self, handle: IOHandle, success: bool) -> None: - """ - Commits the load onboard process for tree nodes within a range on the tree. - Args: - handle (IOHandle): The IOHandle to commit. - success (bool): If True, commits the load-onboard; if False, just indicates failure. - """ - return self.tree.commit_loading_onboard(handle, success) - - def evictable_size(self) -> int: - """ - Returns the size of the evictable part of the radix tree. - This is the size of the part that can be evicted from the GPU (ref_count = 0). - Returns: - int: The size of the evictable part. - """ - return self.tree.evictable_size() - - def protected_size(self) -> int: - """ - Returns the size of the protected part of the radix tree. - This is the size of the part that cannot be evicted from the GPU (ref_count > 0). - Returns: - int: The size of the protected part. - """ - return self.tree.protected_size() - - def total_size(self) -> int: - """ - Returns the total size of the radix tree (including CPU nodes). - Returns: - int: The total size of the radix tree. - """ - return self.tree.total_size() - - def reset(self) -> None: - """ - Resets the radix tree, clearing all nodes and indices. - """ - return self.tree.reset() - - def debug_print(self) -> None: - """ - Prints the internal state of the radix tree for debugging purposes. - """ - return self.tree.debug_print() - -else: - # Real implementation of the classes for runtime - RadixTreeCpp = radix_tree_cpp.RadixTree - TreeNodeCpp = object - IOHandle = object diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.cpp b/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.cpp deleted file mode 100644 index 2a54332218..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "tree_v2.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "tree_v2_impl.h" -#include "tree_v2_node.h" - -namespace radix_tree_v2 { - -static NodeHandle node2id(TreeNode* node) { - return node->node_id; -} - -// compare function for the TreeNode pointers based on their time -// we use LRU, so we want to evict the least recently used nodes -// since std::priority_queue is a max-heap, we need to reverse the comparison -static constexpr auto cmp = [](TreeNode* lhs, TreeNode* rhs) { return lhs->time() > rhs->time(); }; - -RadixTree::RadixTree(bool disabled, std::optional host_size, std::size_t page_size, std::size_t threshold) - : m_impl(std::make_unique(disabled, host_size.has_value(), page_size, host_size.value_or(0), threshold)) {} - -RadixTree::~RadixTree() = default; - -std::tuple, std::size_t, NodeHandle, NodeHandle> -RadixTree::match_prefix(const token_vec_t& _key) { - if (m_impl->disabled) return {}; - - const auto key = token_slice{_key.data(), m_impl->align(_key.size())}; - const auto [host_node, _] = m_impl->tree_walk(key); - - // walk up to the first non-evicted node - std::size_t host_hit_length = 0; - const auto device_node = host_node; - - // collect all the device indices - std::vector indices{}; - walk_to_root(device_node, [&](TreeNode* n) { indices.push_back(n->device_indices()); }); - std::reverse(indices.begin(), indices.end()); - - return {std::move(indices), host_hit_length, node2id(device_node), node2id(host_node)}; -} - -std::vector RadixTree::evict(std::size_t num_tokens) { - if (m_impl->disabled || num_tokens == 0) return {}; - auto heap = std::priority_queue{cmp, m_impl->collect_leaves_device()}; - std::vector evicted_values; - // evict nodes until we reach the desired number of tokens - std::size_t num_evict = 0; - while (num_evict < num_tokens && !heap.empty()) { - const auto node = heap.top(); - heap.pop(); - // when ref_count == 0, can't be writing through - _assert(node->on_gpu() && node->ref_count == 0); - if (!node->is_io_free()) continue; // skip nodes that are undergoing IO (i.e. indices protected) - evicted_values.push_back(node->device_indices()); - num_evict += node->length(); - const auto parent = node->parent(); - m_impl->remove_device_node(node); - if (parent->is_leaf_device() && parent->ref_count == 0) - heap.push(parent); // push parent to the heap if it is now a free leaf - } - - return evicted_values; -} - -std::tuple>, std::size_t> -RadixTree::writing_through(const token_vec_t& _key, at::Tensor value) { - if (m_impl->disabled) return {}; - _assert(_key.size() == std::size_t(value.size(0)), "Key and value must have the same size"); - - // just align the key to the page size, clip the unaligned tail - const auto key = token_slice{_key.data(), m_impl->align(_key.size())}; - - // walk the tree to find the right place to insert - const auto [host_node, host_prefix_length] = m_impl->tree_walk(key); - - // insert and create a new node if the remaining part of the key is not empty - if (host_prefix_length != key.size()) { - m_impl->create_device_node( - host_node, - {key.begin() + host_prefix_length, key.end()}, - value.slice(/*dim=*/0, host_prefix_length, key.size())); - } - - // add the hit count for the device node - walk_to_root(host_node, [&](TreeNode* n) { n->hit_count++; }); - - std::vector> result; - - // don't write through if hicache is disabled (no host memory), fast path - if (!m_impl->use_hicache) return {std::move(result), host_prefix_length}; - throw std::runtime_error("Not implemented yet"); -} - -std::tuple> RadixTree::loading_onboard(NodeHandle, at::Tensor) { - if (m_impl->disabled) return {}; - throw std::runtime_error("Not implemented yet"); -} - -void RadixTree::commit_writing_through(IOTicket, bool) { - if (m_impl->disabled) return; - throw std::runtime_error("Not implemented yet"); -} - -void RadixTree::commit_loading_onboard(IOTicket, bool) { - if (m_impl->disabled) return; - throw std::runtime_error("Not implemented yet"); -} - -void RadixTree::reset() { - m_impl->reset(); -} - -void RadixTree::lock_ref(NodeHandle node_id, bool increment) { - if (m_impl->disabled) return; - m_impl->lock_ref(node_id, increment); -} - -std::size_t RadixTree::evictable_size() const { - return m_impl->evictable_size(); -} - -std::size_t RadixTree::protected_size() const { - return m_impl->protected_size(); -} - -std::size_t RadixTree::total_size() const { - return m_impl->total_size(); -} - -} // namespace radix_tree_v2 diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.h b/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.h deleted file mode 100644 index 68da9b9e17..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once -#include -#include - -#include -#include -#include -#include -#include - -#include "common.h" - -namespace radix_tree_v2 { - -struct RadixTree { - public: - RadixTree(bool disabled, std::optional host_size, std::size_t page_size, std::size_t threshold); - ~RadixTree(); - - // Trees should not be copied or moved, as they manage their own memory and state. - RadixTree(const RadixTree&) = delete; - RadixTree(RadixTree&&) = delete; - RadixTree& operator=(const RadixTree&) = delete; - RadixTree& operator=(RadixTree&&) = delete; - - /// @return (device indices that are matched, host indices length, device node, host node) - std::tuple, std::size_t, NodeHandle, NodeHandle> match_prefix(const token_vec_t& key); - /// @return Device indices that need to be evicted (on python side). - std::vector evict(std::size_t num_tokens); - /// @brief (Un-)Lock a node. - void lock_ref(NodeHandle node_id, bool increment /* increment or decrement */); - /// @brief Update new key-value pair and try to perform write-through. - std::tuple>, std::size_t> - writing_through(const token_vec_t& key, at::Tensor value); - /// @brief Load to device from host within a range of nodes. - std::tuple> loading_onboard(NodeHandle host_id, at::Tensor indices); - /// @brief Commit a transaction of write-through. - void commit_writing_through(IOTicket ticket, bool success); - /// @brief Commit a transaction of load onboard. - void commit_loading_onboard(IOTicket ticket, bool success); - /// @brief Clear and reset the tree. - void reset(); - - /// @return How many size are still evictable (on device + not locked). - std::size_t evictable_size() const; - /// @return How many size are protected (locked). - std::size_t protected_size() const; - /// @return How many size are used on device. - std::size_t total_size() const; - - /// @brief Print debug information of the tree. - void debug_print() const; - - private: - struct Impl; - std::unique_ptr m_impl; -}; - -} // namespace radix_tree_v2 diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_binding.cpp b/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_binding.cpp deleted file mode 100644 index 81069e4fe4..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_binding.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include - -#include -#include - -#include "tree_v2.h" - -PYBIND11_MODULE(radix_tree_cpp, m) { - using namespace radix_tree_v2; - namespace py = pybind11; - py::class_(m, "RadixTree") - .def( - py::init, std::size_t, std::size_t>(), - py::arg("disabled"), - py::arg("host_size"), - py::arg("page_size"), - py::arg("write_through_threshold")) - .def("match_prefix", &RadixTree::match_prefix) - .def("evict", &RadixTree::evict) - .def("lock_ref", &RadixTree::lock_ref) - .def("evictable_size", &RadixTree::evictable_size) - .def("protected_size", &RadixTree::protected_size) - .def("total_size", &RadixTree::total_size) - .def("writing_through", &RadixTree::writing_through) - .def("loading_onboard", &RadixTree::loading_onboard) - .def("commit_writing_through", &RadixTree::commit_writing_through) - .def("commit_loading_onboard", &RadixTree::commit_loading_onboard) - .def("reset", &RadixTree::reset) - .def("debug_print", &RadixTree::debug_print); -} diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_debug.cpp b/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_debug.cpp deleted file mode 100644 index 89b6290b1f..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_debug.cpp +++ /dev/null @@ -1,194 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "tree_v2.h" -#include "tree_v2_impl.h" - -namespace radix_tree_v2 { - -void RadixTree::debug_print() const { - m_impl->debug_print(std::clog); -} - -static constexpr auto npos = std::size_t(-1); - -void RadixTree::Impl::debug_print(std::ostream& os) const { - static constexpr auto _check = [](bool condition, auto msg, std::size_t id = npos) { - if (!condition) { - std::string suffix = id == npos ? "" : " [id = " + std::to_string(id) + "]"; - throw std::runtime_error(std::string("RadixTree::debug_print failed: ") + msg + suffix); - } - }; - - static constexpr auto _print_node = [](TreeNode* node, std::size_t depth, std::ostream& os) { - const auto length = node->length(); - os << node->node_id << " [depth = " << depth << "] [len = " << length << "]"; - - // placement status - if (node->on_both()) { - os << " [cpu + gpu]"; - } else if (node->on_gpu()) { - os << " [gpu]"; - } else if (node->on_cpu()) { - os << " [cpu]"; - } else { - _check(false, "Node is not on GPU or CPU", node->node_id); - } - - // IO status - if (node->is_io_free()) { - os << " [io = free]"; - } else if (node->is_io_device_to_host()) { - os << " [io = gpu -> cpu]"; - } else if (node->is_io_host_to_device()) { - os << " [io = cpu -> gpu]"; - } else { - _check(false, "Node is in unknown IO state", node->node_id); - } - - os << " [rc = " << node->ref_count << "]"; - os << " [hit = " << node->hit_count << "]"; - }; - - static constexpr auto _print_indices = [](at::Tensor indices, std::ostream& os) { - if (!indices.defined()) { - os << "[[N/A]]"; - return indices; - } - indices = indices.to(c10::kCPU, c10::kLong, false, false, c10::MemoryFormat::Contiguous); - const auto length = indices.numel(); - os << "["; - auto* data_ptr = indices.data_ptr(); - for (const auto i : c10::irange(indices.size(0))) { - os << data_ptr[i]; - if (i != length - 1) os << ", "; - } - os << "]"; - return indices; - }; - - os << "Evictable size: " << evictable_size() << std::endl; - os << "Protected size: " << protected_size() << std::endl; - os << "Total size: " << const_cast(this)->total_size() << std::endl; - std::vector> stack; - auto root = const_cast(&m_root); - os << root->node_id << " [root]" << std::endl; - for (const auto& [key, child] : *root) { - stack.push_back({child.get(), root, key}); - } - - std::unordered_map depth_map; - std::string indent_buffer; - depth_map[root] = 0; - std::vector visited_id; - std::size_t evictable_size_real = 0; - while (!stack.empty()) { - const auto [node, parent, key] = stack.back(); - stack.pop_back(); - visited_id.push_back(node->node_id); - const auto nid = node->node_id; - _check(node != nullptr, "Node is null", nid); - _check(node->on_gpu() || node->on_cpu(), "Node is not on GPU or CPU", nid); - _check(node->parent() == parent, "Parent is not correct", nid); - _check(key.size() == page_size && node->diff_key(key, 0) == page_size, "Key is not correct", nid); - _check(depth_map.count(node) == 0, "Node is visited twice", nid); - _check(m_node_map.count(nid) == 1, "Node is not in the map", nid); - _check(m_node_map.at(nid) == node, "Node in the map is not the same as the one in the stack", nid); - _check(!node->on_gpu() || parent->is_root() || parent->on_gpu(), "Node on GPU must have a GPU/root parent", nid); - if (!node->is_io_free()) { - _check(node->ref_count > 0, "Node is in IO state but not protected", nid); - _check(node->on_both(), "Node in IO state must be on both CPU and GPU", nid); - } - - if (node->on_gpu() && node->ref_count == 0) { - evictable_size_real += node->length(); - } - - const auto depth = (depth_map[node] = depth_map[parent] + 1); - indent_buffer.resize(depth * 2, ' '); - os << indent_buffer; - _print_node(node, depth, os); - os << std::endl; - for (const auto& [key, child] : *node) { - stack.push_back({child.get(), node, key}); - } - } - - _check(evictable_size_real == evictable_size(), "Evictable size is wrong"); - _check(m_node_map.count(root->node_id) == 1, "Root node is not in the map"); - _check(m_node_map.at(root->node_id) == root, "Root node in the map is not correct"); - - std::sort(visited_id.begin(), visited_id.end()); - if (visited_id.size() != m_node_map.size() - 1) { - // Some error in the tree, not all nodes are visited - std::string id_list; - id_list += "(visited: "; - id_list += std::to_string(root->node_id) + " "; - for (const auto& id : visited_id) { - id_list += std::to_string(id) + " "; - } - id_list += "), (in map: "; - for (const auto& [id, _] : m_node_map) { - id_list += std::to_string(id) + " "; - } - id_list += ")"; - _check(false, "Not all nodes are visited " + id_list); - } - - static const auto kSGLANG_RADIX_CPP_DEBUG_LIMIT = [] { - const char* env = std::getenv("SGLANG_RADIX_CPP_DEBUG_LIMIT"); - const std::size_t default_limit = 16; - if (env != nullptr) { - try { - return static_cast(std::stoull(env)); - } catch (const std::exception& e) { - std::cerr << "Invalid SGLANG_RADIX_CPP_DEBUG_LIMIT value: " << env // - << ". Using default value =" << default_limit << std::endl; - } - } - return default_limit; - }(); - - for (const auto nid : visited_id) { - const auto node = m_node_map.at(nid); - // print key and indices - const auto& key = node->_unsafe_tokens(); - if (key.size() > kSGLANG_RADIX_CPP_DEBUG_LIMIT) { - os << "Node " << nid << ": key is too long (" << key.size() << " tokens), skipping..." << std::endl; - continue; - } - os << "Node " << nid << ": key = ["; - for (const auto& i : c10::irange(key.size())) { - os << key[i]; - if (i != key.size() - 1) os << ", "; - } - - _check(key.size() % page_size == 0, "Misaligned key", nid); - - os << "] device_indices = "; - const auto device_indices = _print_indices(node->device_indices(), os); - if (device_indices.defined()) { - std::size_t length = device_indices.numel(); - _check(device_indices.dim() == 1, "Device indices must be 1D tensor", nid); - _check(length == node->length(), "Wrong device indices size", nid); - } - - os << " host_indices = "; - const auto host_indices = _print_indices(node->host_indices(), os); - if (host_indices.defined()) { - std::size_t length = host_indices.numel(); - _check(host_indices.dim() == 1, "Host indices must be 1D tensor", nid); - _check(length == node->length(), "Wrong host indices size", nid); - } - os << std::endl; - } -} - -} // namespace radix_tree_v2 diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_impl.h b/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_impl.h deleted file mode 100644 index 1f4a23cd57..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_impl.h +++ /dev/null @@ -1,286 +0,0 @@ -#pragma once -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "tree_v2.h" -#include "tree_v2_node.h" - -namespace radix_tree_v2 { - -using node_iterator_t = typename TreeNode::iterator_t; - -struct RadixTree::Impl { - public: - Impl(bool disabled, bool use_hicache, std::size_t page_size, std::size_t host_size, std::size_t threshold) - : m_root(/*node_id_=*/0), - m_evictable_size(0), - m_protected_size(0), - m_cached_vec(), - m_node_map(), - m_node_counter(1), // start from 1 to avoid confusion with root node - disabled(disabled), - use_hicache(use_hicache), - page_size(page_size), - threshold(threshold) { - _assert(page_size > 0, "Page size must be greater than zero"); - _assert(use_hicache == (host_size > 0), "Hierarchical cache is enabled iff host size > 0"); - m_root.ref_count = 1; // root node is always protected - m_cached_vec.reserve(page_size); // to avoid repeated allocations - m_node_map[m_root.node_id] = &m_root; // add root to the map - } - - TreeNode* split_node(node_iterator_t iterator, std::size_t prefix_length) { - // from `parent -> old_node` to `parent-> new_node -> old_node` - // the prefix part of the old node is moved to the new node - auto old_node_ptr = std::move(iterator->second); - auto new_node_ptr = std::make_unique(m_node_counter++); - auto* old_node = old_node_ptr.get(); - auto* new_node = new_node_ptr.get(); - auto* parent = old_node->parent(); - // set up data structures - split_prefix(new_node, old_node, prefix_length); - // set up parent-child relationship - add_child(new_node, std::move(old_node_ptr)); - add_child(parent, std::move(new_node_ptr), iterator); - m_node_map[new_node->node_id] = new_node; // add to the map - return new_node; - } - - // node: x -> [GPU] - TreeNode* create_device_node(TreeNode* parent, token_vec_t vec, at::Tensor indices) { - auto new_node_ptr = std::make_unique(m_node_counter++); - auto new_node = new_node_ptr.get(); - new_node_ptr->_unsafe_tokens() = std::move(vec); - new_node_ptr->_unsafe_device_indices() = std::move(indices); - m_evictable_size += new_node_ptr->length(); - add_child(parent, std::move(new_node_ptr)); - m_node_map[new_node->node_id] = new_node; // add to the map - return new_node; - } - - // node: [GPU] -> x - void remove_device_node(TreeNode* node) { - _assert(node->on_gpu_only() && node->ref_count == 0); - m_evictable_size -= node->length(); - node->parent()->erase_child(get_key(node)); - m_node_map.erase(node->node_id); // remove from the map - } - - /** - * @brief Walk the tree to find the node that matches the key. - * If the key partially matches a node, it will split that node. - * @return A pair containing the last node that matches the key and - * the total prefix length matched (on gpu and cpu) so far. - */ - std::pair tree_walk(token_slice key) { - _assert(key.size() % page_size == 0, "Key should be page-aligned"); - - std::size_t total_prefix_length = 0; - TreeNode* node = &m_root; - - const auto now = std::chrono::steady_clock::now(); - while (key.size() > 0) { - const auto iterator = node->find_child(get_key(key)); - if (iterator == node->end()) break; - - // walk to the child node - node = iterator->second.get(); - - // at least `page_size` tokens are matched, and there may be more tokens to match - // the return value prefix_length is no less than `page_size` - const auto prefix_length = align(node->diff_key(key, page_size) + page_size); - total_prefix_length += prefix_length; - - // split the node if the prefix is not the whole token vector - if (prefix_length < node->length()) { - return {split_node(iterator, prefix_length), total_prefix_length}; - } - - // we have matched the whole key, continue to the next node - node->access(now); - key = key.subspan(prefix_length); - } - - return {node, total_prefix_length}; - } - - std::vector collect_leaves() const { - std::vector leaves; - std::vector stack = {}; - - auto process_node = [&](TreeNode* node) { - if (node->is_leaf()) { - if (node->ref_count == 0) { - leaves.push_back(node); - } - } else { - stack.push_back(node); - } - }; - for (const auto& [_, child] : m_root) { - process_node(child.get()); - } - while (!stack.empty()) { - const auto node = stack.back(); - stack.pop_back(); - for (const auto& [_, child] : *node) { - process_node(child.get()); - } - } - return leaves; - } - - std::vector collect_leaves_device() const { - // for non-hicache, every leaf device node is a leaf node (since no backup on host) - if (!use_hicache) return collect_leaves(); - std::vector leaves; - std::vector stack = {}; - - auto process_node = [&](TreeNode* node) { - if (!node->on_gpu()) return; // skip nodes that are not on GPU - - if (node->is_leaf_device()) { - if (node->ref_count == 0) { - leaves.push_back(node); - } - } else { - stack.push_back(node); - } - }; - for (const auto& [_, child] : m_root) { - process_node(child.get()); - } - while (!stack.empty()) { - const auto node = stack.back(); - stack.pop_back(); - for (const auto& [_, child] : *node) { - process_node(child.get()); - } - } - - return leaves; - } - - void lock_ref(TreeNode* node, bool increment) { - if (node->is_root()) return; // skip root node - _assert(node->on_gpu(), "Cannot lock reference on an evicted node"); - if (increment) - walk_to_root(node, [this](TreeNode* n) { - if (n->ref_count == 0) { - m_evictable_size -= n->length(); - m_protected_size += n->length(); - } - n->ref_count++; - }); - else - walk_to_root(node, [this](TreeNode* n) { - _assert(n->ref_count != 0, "Cannot decrement reference count = zero"); - n->ref_count--; - if (n->ref_count == 0) { - m_protected_size -= n->length(); - m_evictable_size += n->length(); - } - }); - } - - void lock_ref(NodeHandle node_ptr, bool increment) { - return lock_ref(id2node(node_ptr), increment); - } - - void lock(TreeNode* node) { - return lock_ref(node, /*increment=*/true); - } - - void unlock(TreeNode* node) { - return lock_ref(node, /*increment=*/false); - } - - std::size_t total_size() const { - std::size_t size = 0; - std::vector stack = {&m_root}; - while (!stack.empty()) { - auto* node = stack.back(); - stack.pop_back(); - size += node->length(); - for (const auto& [_, child] : *node) - stack.push_back(child.get()); - } - return size; - } - - std::size_t evictable_size() const { - return m_evictable_size; - } - - std::size_t protected_size() const { - return m_protected_size; - } - - std::size_t align(std::size_t size) const { - return (size / page_size) * page_size; // align to page size - } - - TreeNode* id2node(NodeHandle node_id) const { - const auto iterator = m_node_map.find(node_id); - _assert(iterator != m_node_map.end(), "Node not found in the map"); - return iterator->second; - } - - void reset() { - _assert(m_root.ref_count == 1, "Root node must be protected during reset"); - m_node_counter = 1; // reset node counter - m_root.root_reset(); - m_evictable_size = 0; - m_protected_size = 0; - m_node_map.clear(); - m_node_map[m_root.node_id] = &m_root; // re-add root to the map - } - - void debug_print(std::ostream& os) const; - - private: - // some auxiliary functions - token_vec_t& get_key(token_slice tokens) { - _assert(tokens.size() >= page_size, "Key should be at least page-sized"); - tokens = tokens.subspan(0, page_size); - m_cached_vec.assign(tokens.begin(), tokens.end()); - return m_cached_vec; - } - - // justify for _unsafe call: we need to read the key part of the tokens - token_vec_t& get_key(TreeNode* node) { - return get_key(node->_unsafe_tokens()); - } - - void add_child(TreeNode* parent, std::unique_ptr&& child) { - parent->add_child(get_key(child.get()), std::move(child)); - } - - void add_child(TreeNode* parent, std::unique_ptr&& child, node_iterator_t it) { - parent->add_child(it, std::move(child)); - } - - TreeNode m_root; // root node of the tree - std::size_t m_evictable_size; // number of evictable tokens on GPU (lock ref = 0) - std::size_t m_protected_size; // number of protected tokens on GPU (lock ref > 0) - token_vec_t m_cached_vec; // cached vector of tokens for the current operation - std::unordered_map m_node_map; // map of node keys to nodes - std::size_t m_node_counter; // counter for node IDs - - public: - // some public constant configurations (without m_ prefix) - const bool disabled; // whether the cache is enabled, or just a temporary cache - const bool use_hicache; // whether to use the HiCache for this tree - const std::size_t page_size; // size of each page in the cache - const std::size_t threshold; // threshold for write_through -}; - -} // namespace radix_tree_v2 diff --git a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_node.h b/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_node.h deleted file mode 100644 index 4eac86ea44..0000000000 --- a/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_node.h +++ /dev/null @@ -1,257 +0,0 @@ -#pragma once -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" - -namespace radix_tree_v2 { - -struct std_vector_hash { - // see https://stackoverflow.com/questions/20511347/a-good-hash-function-for-a-vector - std::size_t operator()(const token_vec_t& vec) const { - std::size_t hash = 0; - for (const auto& token : vec) { - hash ^= token + 0x9e3779b9 + (hash << 6) + (hash >> 2); - } - return hash; - } -}; - -struct TreeNode { - public: - using childern_map_t = std::unordered_map, std_vector_hash>; - using iterator_t = typename childern_map_t::iterator; - using const_iterator_t = typename childern_map_t::const_iterator; - using timestamp_t = std::chrono::steady_clock::time_point; - - TreeNode(std::size_t node_id_) - : ref_count(0), - hit_count(0), - m_io_locked(std::nullopt), - m_io_status(IOStatus::None), - m_io_ticket(), - m_tokens(), - m_device_indices(), - m_host_indices(), - m_parent(), - m_children(), - m_last_access_time(std::chrono::steady_clock::now()), - node_id(node_id_) {} - - void access(timestamp_t time = std::chrono::steady_clock::now()) { - m_last_access_time = time; - } - - bool is_root() const { - return m_parent == nullptr; - } - - timestamp_t time() const { - return m_last_access_time; - } - - bool on_gpu() const { - return m_device_indices.defined(); - } - - bool on_cpu() const { - return m_host_indices.defined(); - } - - bool on_gpu_only() const { - return on_gpu() && !on_cpu(); - } - - bool on_cpu_only() const { - return !on_gpu() && on_cpu(); - } - - bool on_both() const { - return on_gpu() && on_cpu(); - } - - std::size_t length() const { - return m_tokens.size(); - } - - bool is_leaf() const { - return m_children.empty(); - } - - bool is_leaf_device() const { - for (const auto& [_, child] : m_children) - if (child->on_gpu()) return false; // at least one child is on the device - return true; - } - - void add_child(const token_vec_t& v, std::unique_ptr&& child) { - child->m_parent = this; - m_children[v] = std::move(child); - } - - void add_child(iterator_t it, std::unique_ptr&& child) { - child->m_parent = this; - it->second = std::move(child); - } - - void erase_child(const token_vec_t& v) { - _assert(m_children.erase(v) > 0, "Child node not found"); - } - - iterator_t find_child(const token_vec_t& v) { - return m_children.find(v); - } - - iterator_t begin() { - return m_children.begin(); - } - - iterator_t end() { - return m_children.end(); - } - - const_iterator_t begin() const { - return m_children.begin(); - } - - const_iterator_t end() const { - return m_children.end(); - } - - TreeNode* parent() { - return m_parent; - } - - // set up all data structures except for parent-child relationship - friend void split_prefix(TreeNode* new_node, TreeNode* old_node, std::size_t prefix_length) { - auto tokens = std::move(old_node->m_tokens); - _assert(0 < prefix_length && prefix_length < tokens.size(), "Invalid prefix size for split"); - - // set up tokens - old_node->m_tokens = token_vec_t(tokens.begin() + prefix_length, tokens.end()); - new_node->m_tokens = std::move(tokens); - new_node->m_tokens.resize(prefix_length); - - // set up values - const int64_t new_size = new_node->length(); - const int64_t old_size = old_node->length(); - if (old_node->m_device_indices.defined()) { - auto new_indices = old_node->m_device_indices.split_with_sizes({new_size, old_size}); - new_node->m_device_indices = std::move(new_indices[0]); - old_node->m_device_indices = std::move(new_indices[1]); - } - if (old_node->m_host_indices.defined()) { - auto new_indices = old_node->m_host_indices.split_with_sizes({new_size, old_size}); - new_node->m_host_indices = std::move(new_indices[0]); - old_node->m_host_indices = std::move(new_indices[1]); - } - - // set up ref counts and hit counts - new_node->ref_count = old_node->ref_count; - new_node->hit_count = old_node->hit_count; - - // If the old node (child) was locked for IO, the new node (parent) does not need - // to be locked, since it is naturally protected by the child node's lock. - if (old_node->m_io_locked.has_value()) { - new_node->m_io_locked = false; - new_node->m_io_status = old_node->m_io_status; - new_node->m_io_ticket = old_node->m_io_ticket; - } - } - - /// @return The first index in `m_tokens` that differs from `key`. - std::size_t diff_key(token_slice key, std::size_t offset) const { - const auto a = token_slice{key}.subspan(offset); - const auto b = token_slice{m_tokens}.subspan(offset); - const auto [it_a, it_b] = std::ranges::mismatch(a, b); - return it_a - a.begin(); // return the index of the first differing token - } - - at::Tensor device_indices() const { - return m_device_indices; - } - at::Tensor host_indices() const { - return m_host_indices; - } - - // visiting tokens are always unsafe (use `diff_key` instead) - token_vec_t& _unsafe_tokens() { - return m_tokens; - } - at::Tensor& _unsafe_device_indices() { - return m_device_indices; - } - at::Tensor& _unsafe_host_indices() { - return m_host_indices; - } - - bool is_io_free() const { - return m_io_status == IOStatus::None; - } - - bool is_io_device_to_host() const { - return m_io_status == IOStatus::DeviceToHost; - } - - bool is_io_host_to_device() const { - return m_io_status == IOStatus::HostToDevice; - } - - void root_reset() { - _assert(is_root(), "Only root node can call root_reset"); - _assert( - m_io_status == IOStatus::None && m_io_locked == std::nullopt, - "IO operation in progress, cannot reset root node"); - _assert(this->m_tokens.empty(), "Root node tokens should be empty on reset"); - _assert( - !this->m_device_indices.defined() && !this->m_host_indices.defined(), - "Root node indices should be always be empty and never assigned"); - m_children.clear(); - this->access(); - } - - public: - std::size_t ref_count; - std::size_t hit_count; - - private: - enum class IOStatus : std::uint8_t { - None, - HostToDevice, - DeviceToHost, - }; - - std::optional m_io_locked; // whether the node is locked in IO operation - IOStatus m_io_status; - IOTicket m_io_ticket; - - token_vec_t m_tokens; - at::Tensor m_device_indices; // indices of device value - at::Tensor m_host_indices; // indices of host value - TreeNode* m_parent; - childern_map_t m_children; - timestamp_t m_last_access_time; - - public: - const std::size_t node_id; // unique ID for the node -}; - -template -inline TreeNode* walk_to_root(TreeNode* t, const F& f) { - while (!t->is_root()) { - f(t); - t = t->parent(); - } - return t; // return the root node -} - -} // namespace radix_tree_v2 diff --git a/python/sglang/srt/mem_cache/radix_cache_cpp.py b/python/sglang/srt/mem_cache/radix_cache_cpp.py deleted file mode 100644 index 168df6eb10..0000000000 --- a/python/sglang/srt/mem_cache/radix_cache_cpp.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import annotations - -import logging -import time -from typing import TYPE_CHECKING, List, Optional, Set - -import torch - -from sglang.srt.mem_cache.base_prefix_cache import ( - BasePrefixCache, - DecLockRefParams, - DecLockRefResult, - EvictParams, - EvictResult, - IncLockRefResult, - MatchPrefixParams, - MatchResult, -) -from sglang.srt.mem_cache.cpp_radix_tree.radix_tree import ( - IOHandle, - RadixTreeCpp, - TreeNodeCpp, -) -from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.runtime_context import ( - get_memory, -) - -if TYPE_CHECKING: - from sglang.srt.managers.schedule_batch import Req - from sglang.srt.mem_cache.cache_init_params import CacheInitParams - from sglang.srt.server_args import ServerArgs - - -logger = logging.getLogger(__name__) - - -class RadixCacheCpp(BasePrefixCache): - @staticmethod - def _reject_cache_salt(cache_salt: Optional[str]) -> None: - if cache_salt is not None: - raise ValueError( - "cache_salt is not supported by the experimental C++ radix tree" - ) - - def __init__( - self, - params: CacheInitParams, - server_args: ServerArgs, - enable_write_cancel: bool = False, - ): - self.disable = params.disable - self.enable_write_cancel = enable_write_cancel - - assert params.enable_kv_cache_events is False, ( - "HiRadixCache does not support kv cache events yet" - ) - - # record the nodes with ongoing write through - self.ongoing_write_through: Set[IOHandle] = set() - # record the node segments with ongoing load back - self.ongoing_load_back: Set[IOHandle] = set() - # todo: dynamically adjust the threshold - self.write_through_threshold = ( - 1 if get_memory().hicache_write_policy == "write_through" else 2 - ) - self.token_to_kv_pool_allocator = params.token_to_kv_pool_allocator - self.device = self.token_to_kv_pool_allocator.device - self.req_to_token_pool = params.req_to_token_pool - self.page_size = params.page_size - self.kv_cache = self.token_to_kv_pool_allocator.get_kvcache() - - self.tp_group = params.tp_cache_group - - if params.enable_metrics: - self.init_metrics_collector() - - if not get_memory().enable_hierarchical_cache: - self.tree = RadixTreeCpp( - disabled=self.disable, - page_size=self.page_size, - host_size=None, # no host cache, this should be removed in the future - write_through_threshold=self.write_through_threshold, - ) - self.cache_controller = None - return # early return if hicache is not used - - raise NotImplementedError("Host cache is not supported yet") - - def _merge_tensor(self, l: List[torch.Tensor]) -> torch.Tensor: - """ - Merge a list of tensors into a single tensor. - Args: - l (List[torch.Tensor]): List of tensors to merge. - Returns: - torch.Tensor: Merged tensor. - """ - if len(l) == 0: - return torch.empty(0, dtype=torch.int64, device=self.device) - elif len(l) == 1: - return l[0] - else: - return torch.cat(l) - - def reset(self): - if self.cache_controller is not None: - # need to clear the acks before resetting the cache controller - raise NotImplementedError("Host cache is not supported yet") - self.tree.reset() - - def match_prefix(self, params: MatchPrefixParams) -> MatchResult: - key = params.key - self._reject_cache_salt(key.cache_salt) - device_indices_vec, host_indices_length, node_gpu, node_cpu = ( - self.tree.match_prefix(key.raw_token_ids()) - ) - return MatchResult( - device_indices=self._merge_tensor(device_indices_vec), - last_device_node=node_gpu, - last_host_node=node_cpu, - best_match_node=node_cpu, - host_hit_length=host_indices_length, - ) - - def _insert(self, key: RadixKey, value: torch.Tensor) -> int: - """ - Insert a key-value pair into the radix tree. - Args: - key (RadixKey): The key to insert, represented as a RadixKey. - value (torch.Tensor): The value to associate with the key. - Returns: - int: Number of device indices that were already present in the tree before the insertion. - """ - ongoing_write, length = self.tree.writing_through(key.token_ids, value) - if self.cache_controller is None: - assert len(ongoing_write) == 0, "Implementation error" - return length - - raise NotImplementedError("Host cache is not supported yet") - - def dec_lock_ref( - self, node: TreeNodeCpp, params: Optional[DecLockRefParams] = None - ) -> DecLockRefResult: - """ - Decrement the reference count of a node to root of the radix tree. - Args: - node (TreeNodeCpp): The handle of the node to decrement the reference count for. - """ - self.tree.lock_ref(node, False) # do not increment - return DecLockRefResult() - - def inc_lock_ref(self, node: TreeNodeCpp) -> IncLockRefResult: - """ - Increment the reference count of from a node to root of the radix tree. - Args: - node (TreeNodeCpp): The handle of the node to increment the reference count for. - """ - self.tree.lock_ref(node, True) - return IncLockRefResult() - - def evict(self, params: EvictParams) -> EvictResult: - start_time = time.perf_counter() - num_tokens = params.num_tokens - evicted_device_indices = self.tree.evict(num_tokens) - - num_evicted = 0 - for indice in evicted_device_indices: - num_evicted += len(indice) - self.token_to_kv_pool_allocator.free(indice) - - self.update_eviction_metrics(num_evicted, start_time) - return EvictResult(num_tokens_evicted=num_evicted) - - def evictable_size(self): - return self.tree.evictable_size() - - def protected_size(self): - return self.tree.protected_size() - - def total_size(self): - return self.tree.total_size() - - def cache_finished_req( - self, req: Req, is_insert: bool = True, *, owned_kv_len: int - ): - """Cache request when it finishes.""" - self._reject_cache_salt(req.cache_salt) - assert req.kv.holds_kv - token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :owned_kv_len - ].to(dtype=torch.int64, copy=True) - - # NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned - # it will automatically align them, but length of them should be equal - old_prefix_len = len(req.prefix_indices) // self.page_size * self.page_size - page_aligned_overall_len = owned_kv_len // self.page_size * self.page_size - - if is_insert: - new_prefix_len = self._insert( - RadixKey(token_ids, req.extra_key), kv_indices - ) - # NOTE: kv_indices[:old_prefix_len] == req.prefix_indices - assert old_prefix_len <= new_prefix_len, "Wrong prefix indices" - # Free duplicates that were already in the pool - if old_prefix_len < new_prefix_len: - self.token_to_kv_pool_allocator.free( - kv_indices[old_prefix_len:new_prefix_len] - ) - else: - self.token_to_kv_pool_allocator.free( - kv_indices[old_prefix_len:page_aligned_overall_len] - ) - - # need to free the unaligned part, since it cannot be inserted into the radix tree - if page_aligned_overall_len < owned_kv_len: - # NOTE: sglang PagedAllocator support unaligned free (which will automatically align it) - self.token_to_kv_pool_allocator.free(kv_indices[page_aligned_overall_len:]) - - # Remove req slot release the cache lock - self.dec_lock_ref(req.last_node) - - def cache_unfinished_req(self, req: Req, chunked=False): - """Cache request when it is unfinished.""" - self._reject_cache_salt(req.cache_salt) - assert req.kv.holds_kv - token_ids = req.get_fill_ids() - prefill_len = len(token_ids) # prefill only (maybe chunked) - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :prefill_len - ].to(dtype=torch.int64, copy=True) - - # NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned - # it will automatically align them, but length of them should be equal - old_prefix_len = len(req.prefix_indices) // self.page_size * self.page_size - new_prefix_len = self._insert(RadixKey(token_ids, req.extra_key), kv_indices) - - # NOTE: kv_indices[:old_prefix_len] == req.prefix_indices - assert old_prefix_len <= new_prefix_len, "Wrong prefix indices" - - # TODO(dark): optimize the `insert` and `match` (e.g. merge into 1 function) - # The prefix indices need to updated to reuse the kv indices in the pool - new_indices_vec, _, new_last_node, _ = self.tree.match_prefix( - RadixKey(token_ids, req.extra_key).token_ids - ) - new_indices = self._merge_tensor(new_indices_vec) - assert new_prefix_len <= len(new_indices) - - # KVCache between old & new is newly generated, but already exists in the pool - # we need to free this newly generated kv indices and reuse the indices in the pool - if old_prefix_len < new_prefix_len: - self.token_to_kv_pool_allocator.free( - kv_indices[old_prefix_len:new_prefix_len] - ) - reused_indices = new_indices[old_prefix_len:new_prefix_len] - self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, old_prefix_len:new_prefix_len - ] = reused_indices - - if req.last_node != new_last_node: - self.dec_lock_ref(req.last_node) - self.inc_lock_ref(new_last_node) - - # NOTE: there might be unaligned tail, so we may need to append it - assert len(new_indices) <= prefill_len < len(new_indices) + self.page_size - if self.page_size != 1 and len(new_indices) < prefill_len: - req.prefix_indices = torch.cat( - [new_indices, kv_indices[len(new_indices) :]] - ) - else: - req.prefix_indices = new_indices - req.last_node = new_last_node - - def pretty_print(self): - return self.tree.debug_print() diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index de35ad0143..6c3655f180 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -14,7 +14,6 @@ import logging from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, Optional -from sglang.srt.environ import envs from sglang.srt.hardware_backend.mlx.runtime import use_mlx from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.cache_init_params import CacheInitParams @@ -101,13 +100,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: return SWAChunkCache(params) - if envs.SGLANG_EXPERIMENTAL_CPP_RADIX_TREE.get(): - # lazy import to avoid JIT overhead - from sglang.srt.mem_cache.radix_cache_cpp import RadixCacheCpp - - logger.info("Using experimental C++ radix tree implementation.") - return RadixCacheCpp(params=params, server_args=server_args) - if get_memory().enable_unified_cache_external_linker: return _create_unified_radix_cache(ctx, server_args, params) diff --git a/test/registered/unit/mem_cache/test_radix_cache_cpp_unit.py b/test/registered/unit/mem_cache/test_radix_cache_cpp_unit.py deleted file mode 100644 index 49fdf0b837..0000000000 --- a/test/registered/unit/mem_cache/test_radix_cache_cpp_unit.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Unit tests for fail-closed C++ radix-cache request validation.""" - -import importlib -import sys -import types -import unittest -from unittest.mock import patch - -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=11, suite="base-a-test-cpu") - - -class TestRadixCacheCppCacheSalt(CustomTestCase): - def test_cache_salt_is_rejected_without_loading_cpp_extension(self): - extension_name = "sglang.srt.mem_cache.cpp_radix_tree.radix_tree" - module_name = "sglang.srt.mem_cache.radix_cache_cpp" - fake_extension = types.ModuleType(extension_name) - fake_extension.IOHandle = object - fake_extension.RadixTreeCpp = object - fake_extension.TreeNodeCpp = object - - original_module = sys.modules.pop(module_name, None) - try: - with patch.dict(sys.modules, {extension_name: fake_extension}): - module = importlib.import_module(module_name) - module.RadixCacheCpp._reject_cache_salt(None) - with self.assertRaisesRegex(ValueError, "experimental C\\+\\+"): - module.RadixCacheCpp._reject_cache_salt("tenant-a") - finally: - sys.modules.pop(module_name, None) - if original_module is not None: - sys.modules[module_name] = original_module - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index 47bb5671cf..b1398f3049 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -214,30 +214,6 @@ class TestDefaultRadixCacheFactory(CustomTestCase): PureSWAChunkCache.assert_called_once_with(ctx.params) self.assertIs(result, PureSWAChunkCache.return_value) - def test_cpp_radix_cache_when_env_flag_set(self): - ctx = _make_ctx( - self, - ) - # `radix_cache_cpp` requires ninja + C++ extension to import, so - # we inject a stand-in module rather than letting patch() trigger - # the real import. - fake_module = MagicMock() - with ( - patch( - "sglang.srt.mem_cache.registry.envs.SGLANG_EXPERIMENTAL_CPP_RADIX_TREE.get", - return_value=True, - ), - patch.dict( - "sys.modules", - {"sglang.srt.mem_cache.radix_cache_cpp": fake_module}, - ), - ): - result = default_radix_cache_factory(ctx) - fake_module.RadixCacheCpp.assert_called_once_with( - params=ctx.params, server_args=ctx.server_args - ) - self.assertIs(result, fake_module.RadixCacheCpp.return_value) - def test_unified_radix_cache_is_the_default(self): ctx = _make_ctx( self,