[Radix Cache] Add Rust TreeCore backend with shared parity tests (#32710)

Co-authored-by: alphabetc1 <2508695655@qq.com>
Co-authored-by: ispobock <ispobaoke@gmail.com>
This commit is contained in:
Jialin Ouyang
2026-09-01 00:26:20 +08:00
committed by GitHub
co-authored by alphabetc1 ispobock
parent 52e1c24744
commit 9cf157c252
72 changed files with 39973 additions and 396 deletions
@@ -0,0 +1,228 @@
"""Test-only inspection adapter for the Rust Unified TreeCore."""
from __future__ import annotations
from typing import Optional
import torch
from unified_tree_core_inspection_interface import (
UnifiedTreeCoreInspectionInterface,
)
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams, MatchResult
from sglang.srt.mem_cache.rust_tree_core.adapter import (
RustUnifiedTreeCore,
_fill_evict_result,
_match_result_from_binding,
_radix_key_buffer,
)
from sglang.srt.mem_cache.rust_tree_core.extension import load_tree_core_extension
from sglang.srt.mem_cache.unified_cache.components import ComponentType, EvictLayer
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
BaseEvictionResult,
NodeId,
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(
est_time=0, suite="base-a-test-cpu", disabled="Rust TreeCore test inspector"
)
_inspection_bindings = load_tree_core_extension(inspection=True)
class RustUnifiedTreeCoreInspector(
RustUnifiedTreeCore, UnifiedTreeCoreInspectionInterface
):
"""Rust TreeCore variant used by the shared backend-conformance tests.
The production adapter deliberately implements only
``UnifiedTreeCoreInterface``. These forwarding methods keep white-box state
controls in test code while the binding returns snapshots rather than Rust
iterators across the Python boundary.
"""
_bindings = _inspection_bindings
def contains_node(self, node_id: NodeId) -> bool:
return self._binding.inspect_contains_node(node_id)
def get_parent_node_id(self, node_id: NodeId) -> Optional[NodeId]:
return self._binding.inspect_get_parent_node_id(node_id)
def get_child_node_ids(self, node_id: NodeId) -> list[NodeId]:
return self._binding.inspect_get_child_node_ids(node_id)
def get_node_key_length(self, node_id: NodeId) -> int:
return self._binding.inspect_get_node_key_length(node_id)
def get_node_token_ids(self, node_id: NodeId) -> list[int]:
return self._binding.inspect_get_node_token_ids(node_id)
def is_node_key_bigram(self, node_id: NodeId) -> bool:
return self._binding.inspect_is_node_key_bigram(node_id)
def get_component_host_value(
self, node_id: NodeId, component_type: ComponentType
) -> Optional[torch.Tensor]:
return self._binding.inspect_get_component_host_value(
node_id, int(component_type)
)
def get_component_device_lock_ref(
self, node_id: NodeId, component_type: ComponentType
) -> int:
return self._binding.inspect_get_component_device_lock_ref(
node_id, int(component_type)
)
def get_node_hit_count(self, node_id: NodeId) -> int:
return self._binding.inspect_get_node_hit_count(node_id)
def get_write_through_pending_id(self, node_id: NodeId) -> Optional[int]:
return self._binding.inspect_get_write_through_pending_id(node_id)
def is_node_in_device_lru(
self, node_id: NodeId, component_type: ComponentType
) -> bool:
return self._binding.inspect_is_node_in_device_lru(node_id, int(component_type))
def is_node_in_host_lru(
self, node_id: NodeId, component_type: ComponentType
) -> bool:
return self._binding.inspect_is_node_in_host_lru(node_id, int(component_type))
def get_component_device_lru_node_ids(
self, component_type: ComponentType
) -> list[NodeId]:
return self._binding.inspect_get_component_device_lru_node_ids(
int(component_type)
)
def is_device_evictable_leaf(self, node_id: NodeId) -> bool:
return self._binding.inspect_is_device_evictable_leaf(node_id)
def is_host_evictable_leaf(self, node_id: NodeId) -> bool:
return self._binding.inspect_is_host_evictable_leaf(node_id)
def is_device_leaf(self, node_id: NodeId) -> bool:
return self._binding.inspect_is_device_leaf(node_id)
def get_all_node_ids(self) -> list[NodeId]:
return self._binding.inspect_get_all_node_ids()
def component_protected_size(self, component_type: ComponentType) -> int:
return self._binding.inspect_component_protected_size(int(component_type))
def set_node_hash_values(
self, node_id: NodeId, hash_values: Optional[list[str]]
) -> None:
self._binding.inspect_set_node_hash_values(node_id, hash_values)
def set_component_device_value_raw(
self,
node_id: NodeId,
component_type: ComponentType,
value: Optional[torch.Tensor],
) -> None:
self._binding.inspect_set_component_device_value_raw(
node_id, int(component_type), value
)
def set_component_host_value_raw(
self,
node_id: NodeId,
component_type: ComponentType,
value: Optional[torch.Tensor],
) -> None:
self._binding.inspect_set_component_host_value_raw(
node_id, int(component_type), value
)
def set_component_device_lock_ref(
self, node_id: NodeId, component_type: ComponentType, lock_ref: int
) -> None:
assert lock_ref >= 0
self._binding.inspect_set_component_device_lock_ref(
node_id, int(component_type), lock_ref
)
def remove_node_from_device_lru(
self, node_id: NodeId, component_type: ComponentType
) -> None:
self._binding.inspect_remove_node_from_device_lru(node_id, int(component_type))
def insert_node_into_host_lru(
self, node_id: NodeId, component_type: ComponentType
) -> None:
self._binding.inspect_insert_node_into_host_lru(node_id, int(component_type))
def set_component_evictable_size(
self, component_type: ComponentType, value: int
) -> None:
assert value >= 0
self._binding.inspect_set_component_evictable_size(int(component_type), value)
def set_component_protected_size(
self, component_type: ComponentType, value: int
) -> None:
assert value >= 0
self._binding.inspect_set_component_protected_size(int(component_type), value)
def update_duplicate_tracking(self, node_id: NodeId) -> None:
self._binding.inspect_update_duplicate_tracking(node_id)
def advance_insert_walk_once(self) -> None:
self._binding.inspect_advance_insert_walk_once()
def evict_component(
self,
node_id: NodeId,
component_type: ComponentType,
target: EvictLayer,
) -> BaseEvictionResult:
binding_result = self._binding.inspect_evict_component(
node_id, int(component_type), int(target)
)
return _fill_evict_result(binding_result, BaseEvictionResult())
def validate_cascade_evict(
self,
node_id: NodeId,
component_type: ComponentType,
target: EvictLayer,
) -> None:
self._binding.inspect_validate_cascade_evict(
node_id, int(component_type), int(target)
)
def cleanup_tombstone_ancestors(self, node_id: NodeId) -> BaseEvictionResult:
binding_result = self._binding.inspect_cleanup_tombstone_ancestors(node_id)
return _fill_evict_result(binding_result, BaseEvictionResult())
def finalize_component_match_result(
self,
component_type: ComponentType,
result: MatchResult,
params: MatchPrefixParams,
value_chunks: list[torch.Tensor],
best_value_len: int,
) -> MatchResult:
binding_result = self._binding.inspect_finalize_component_match_result(
int(component_type),
result,
_radix_key_buffer(params.key),
params.key.extra_key,
params.key.cache_salt,
value_chunks,
best_value_len,
)
return _match_result_from_binding(binding_result)._replace(
cache_protected_len=result.cache_protected_len,
cache_actions=result.cache_actions,
)
def build_backup_node_ids(
self, node_id: NodeId, write_back: bool = False
) -> list[NodeId]:
return self._binding.inspect_build_backup_node_ids(node_id, write_back)
@@ -0,0 +1,149 @@
"""Smoke tests for the in-tree Rust TreeCore backend (``rust``).
Requires a Rust toolchain: the extension builds with cargo on first use.
"""
import shutil
from array import array
import pytest
import torch
from unified_tree_core_inspection_interface import UnifiedTreeCoreInspectionInterface
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=90, suite="base-a-test-cpu")
if shutil.which("cargo") is None:
pytest.skip("the rust backend builds with cargo", allow_module_level=True)
from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.tree_core_registry import create_tree_core
def _tree_core():
return create_tree_core(
"rust",
CacheInitParams(
disable=False,
req_to_token_pool=None,
token_to_kv_pool_allocator=None,
page_size=1,
tree_components=(ComponentType.FULL,),
),
components={},
)
def _key(token_ids, extra_key=None):
return RadixKey(array("q", token_ids), extra_key=extra_key)
def _pump_insert(core, params):
step = core.begin_insert(params)
while step.result is None:
step = core.resume_insert()
core.end_insert()
return step.result
def test_registry_resolves_the_rust_backend_lazily():
core = _tree_core()
assert type(core).__name__ == "RustUnifiedTreeCore"
assert not isinstance(core, UnifiedTreeCoreInspectionInterface)
assert not any(name.startswith("inspect_") for name in dir(core._binding))
def test_insert_then_match_round_trips():
core = _tree_core()
_pump_insert(
core,
InsertParams(
key=_key([1, 2, 3]), value=torch.tensor([10, 11, 12], dtype=torch.int64)
),
)
matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3])))
assert matched.device_indices.tolist() == [10, 11, 12]
def test_lock_moves_tokens_between_evictable_and_protected():
core = _tree_core()
_pump_insert(
core,
InsertParams(key=_key([1, 2]), value=torch.tensor([10, 11], dtype=torch.int64)),
)
matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2])))
core.inc_lock_ref(matched.best_match_node)
assert core.protected_size() == 2
assert core.evictable_size() == 0
core.dec_lock_ref(matched.best_match_node)
assert core.evictable_size() == 2
def test_namespaces_isolate_the_same_tokens():
core = _tree_core()
_pump_insert(
core,
InsertParams(
key=_key([1, 2], extra_key="chat"),
value=torch.tensor([20, 21], dtype=torch.int64),
),
)
salted = core.match_prefix(MatchPrefixParams(key=_key([1, 2], extra_key="chat")))
assert salted.device_indices.tolist() == [20, 21]
unsalted = core.match_prefix(MatchPrefixParams(key=_key([1, 2])))
assert unsalted.device_indices.numel() == 0
def test_backfill_hashes_existing_nodes_in_parent_order():
expected = _tree_core()
expected.enable_storage = True
_pump_insert(
expected,
InsertParams(key=_key([1, 2]), value=torch.tensor([10, 11], dtype=torch.int64)),
)
_pump_insert(
expected,
InsertParams(
key=_key([1, 2, 3, 4]),
value=torch.tensor([10, 11, 12, 13], dtype=torch.int64),
),
)
late = _tree_core()
_pump_insert(
late,
InsertParams(key=_key([1, 2]), value=torch.tensor([10, 11], dtype=torch.int64)),
)
_pump_insert(
late,
InsertParams(
key=_key([1, 2, 3, 4]),
value=torch.tensor([10, 11, 12, 13], dtype=torch.int64),
),
)
parent = late.match_prefix(MatchPrefixParams(key=_key([1, 2]))).best_match_node
child = late.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4]))).best_match_node
expected_parent = expected.match_prefix(
MatchPrefixParams(key=_key([1, 2]))
).best_match_node
expected_child = expected.match_prefix(
MatchPrefixParams(key=_key([1, 2, 3, 4]))
).best_match_node
assert late.get_hash_values(parent) == []
assert late.get_hash_values(child) == []
assert late.backfill_missing_hash_values() == 2
assert late.get_hash_values(parent) == expected.get_hash_values(expected_parent)
assert late.get_hash_values(child) == expected.get_hash_values(expected_child)
assert late.backfill_missing_hash_values() == 0
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,29 @@
"""Run the UnifiedRadixCache benchmark/fuzz suite with the Rust TreeCore."""
import unittest
import test_unified_radix_cache_bench as shared_suite
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=25, stage="base-b", runner_config="1-gpu-small")
class RustBackendSuite(unittest.TestSuite):
"""Scope the backend override to this suite and restore it afterward."""
def run(self, result, debug=False):
previous = shared_suite._TREE_CORE_TEST_BACKEND
shared_suite._TREE_CORE_TEST_BACKEND = "rust"
try:
return super().run(result, debug)
finally:
shared_suite._TREE_CORE_TEST_BACKEND = previous
def load_tests(loader, standard_tests, pattern):
return RustBackendSuite(loader.loadTestsFromModule(shared_suite))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,30 @@
"""Run the shared UnifiedRadixCache unit suite with the Rust TreeCore."""
import unittest
import test_unified_radix_cache_unittest as shared_suite
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=180, stage="base-b", runner_config="1-gpu-small")
class RustBackendSuite(unittest.TestSuite):
"""Scope the test backend to this suite without polluting discovery."""
def run(self, result, debug=False):
previous = shared_suite._TREE_CORE_TEST_BACKEND
shared_suite._TREE_CORE_TEST_BACKEND = "rust"
try:
return super().run(result, debug)
finally:
shared_suite._TREE_CORE_TEST_BACKEND = previous
def load_tests(loader, standard_tests, pattern):
"""Reuse the exact cache-level suite while swapping only its test factory."""
return RustBackendSuite(loader.loadTestsFromModule(shared_suite))
if __name__ == "__main__":
unittest.main()
@@ -14,7 +14,7 @@ import sys
import time
import unittest
from array import array
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from typing import Callable
@@ -40,6 +40,7 @@ from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
from sglang.srt.utils import get_device
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=25, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=25, suite="stage-b-test-1-gpu-small-amd")
@@ -59,6 +60,7 @@ _BENCH_KV_SIZE = 500_000
_BENCH_CHUNK_LEN = 256
_DEFAULT_COMPONENTS = (ComponentType.FULL, ComponentType.MAMBA)
_TREE_CORE_TEST_BACKEND: str | None = None
@contextmanager
@@ -226,16 +228,22 @@ def create_bench_cache(
# --- tree ---
if tree_cls is None:
tree_cls = UnifiedRadixCache
tree = tree_cls(
params=CacheInitParams(
req_to_token_pool=req_to_token_pool,
token_to_kv_pool_allocator=allocator,
page_size=page_size,
disable=False,
tree_components=components if tree_cls is UnifiedRadixCache else None,
sliding_window_size=sliding_window_size if has_swa else None,
)
backend_override = (
envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(_TREE_CORE_TEST_BACKEND)
if _TREE_CORE_TEST_BACKEND is not None and tree_cls is UnifiedRadixCache
else nullcontext()
)
with backend_override:
tree = tree_cls(
params=CacheInitParams(
req_to_token_pool=req_to_token_pool,
token_to_kv_pool_allocator=allocator,
page_size=page_size,
disable=False,
tree_components=components if tree_cls is UnifiedRadixCache else None,
sliding_window_size=sliding_window_size if has_swa else None,
)
)
_rid = [0]
@@ -780,6 +788,10 @@ class _BenchSuite:
verify=True,
page_size=cfg["page_size"],
)
backend = (
_TREE_CORE_TEST_BACKEND or envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
)
print(f"[{backend}] {r.report()}")
self.assertGreater(r.num_ops, 0)
self.assertGreater(r.ops_per_sec, 0)
@@ -803,7 +815,7 @@ for _cfg in _CI_BENCH_CONFIGS:
_name = f"TestBench_{_cfg['label']}"
globals()[_name] = type(
_name,
(_BenchSuite, unittest.TestCase),
(_BenchSuite, CustomTestCase),
{"bench_cfg": _cfg},
)
globals()[_name].__module__ = __name__
File diff suppressed because it is too large Load Diff
@@ -205,6 +205,11 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
# ==== Targeted white-box operations ====
@abstractmethod
def advance_insert_walk_once(self) -> None:
"""Advance one suspended insert walk step without flushing its actions."""
...
@abstractmethod
def evict_component(
self,
@@ -14,6 +14,7 @@ from sglang.srt.mem_cache.unified_cache.components import ComponentType, EvictLa
from sglang.srt.mem_cache.unified_cache.unified_tree_core import (
UnifiedLRUList,
UnifiedTreeCore,
_InsertPhase,
)
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
BaseEvictionResult,
@@ -195,6 +196,15 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
"""Refresh duplicate-host tracking for the node."""
self._update_duplicate_tracking(self.node_by_id(node_id))
def advance_insert_walk_once(self) -> None:
"""Advance one suspended insert walk step without flushing its actions."""
state = self._ongoing_insert_walk_state
if state is None:
raise RuntimeError("no in-flight insert")
if state.phase is not _InsertPhase.WALK:
raise RuntimeError("in-flight insert is not in walk phase")
self._insert_walk_step(state)
def evict_component(
self,
node_id: NodeId,