minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[HiCache] Optimize HiCache hash generation with bulk token byte conversion (#28287)

Browse Source
This commit is contained in:
huangtingwei
2026-07-01 15:44:23 +08:00
committed by GitHub
parent df0dfbaa45
commit 5e1ccd9320
7 changed files with 675 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/cache_controller.py
+5 -11
View File
@@ -1002,18 +1002,12 @@ class HiCacheController:
storage_query_count = 0
hash_value = []
page_hashes = self.get_hash_str(
tokens_to_fetch, last_hash, page_size=self.page_size
)
for start in range(
0, len(tokens_to_fetch), self.page_size * STORAGE_BATCH_SIZE
):
end = min(start + self.page_size * STORAGE_BATCH_SIZE, len(tokens_to_fetch))
batch_tokens = tokens_to_fetch[start:end]
batch_hashes = []
for i in range(0, len(batch_tokens), self.page_size):
last_hash = self.get_hash_str(
batch_tokens[i : i + self.page_size], last_hash
)
batch_hashes.append(last_hash)
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
hash_value.extend(batch_hashes[:hit_page_num])