[HiCache] fix compatibility bugs with eagle and HiCacheStorage (#19570)
Co-authored-by: maoyuhan <susanmao1997@outlook.com>
This commit is contained in:
@@ -39,6 +39,7 @@ from sglang.srt.mem_cache.radix_cache import (
|
|||||||
compute_node_hash_values,
|
compute_node_hash_values,
|
||||||
split_node_hash_value,
|
split_node_hash_value,
|
||||||
)
|
)
|
||||||
|
from sglang.srt.mem_cache.utils import convert_to_bigram_key
|
||||||
from sglang.srt.observability.metrics_collector import StorageMetricsCollector
|
from sglang.srt.observability.metrics_collector import StorageMetricsCollector
|
||||||
from sglang.srt.utils import bind_to_closest_numa_node_cuda
|
from sglang.srt.utils import bind_to_closest_numa_node_cuda
|
||||||
|
|
||||||
@@ -1298,6 +1299,11 @@ class HiRadixCache(RadixCache):
|
|||||||
last_hash: Optional[str] = None,
|
last_hash: Optional[str] = None,
|
||||||
prefix_keys: Optional[List[str]] = None,
|
prefix_keys: Optional[List[str]] = None,
|
||||||
):
|
):
|
||||||
|
new_input_tokens = (
|
||||||
|
convert_to_bigram_key(new_input_tokens)
|
||||||
|
if self.is_eagle
|
||||||
|
else new_input_tokens
|
||||||
|
)
|
||||||
# align the number of fetching tokens to the page size
|
# align the number of fetching tokens to the page size
|
||||||
prefetch_length = len(new_input_tokens) - (
|
prefetch_length = len(new_input_tokens) - (
|
||||||
len(new_input_tokens) % self.page_size
|
len(new_input_tokens) % self.page_size
|
||||||
|
|||||||
Reference in New Issue
Block a user