minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix CPP Radix Cache and add test to CI (#11645)

Browse Source
This commit is contained in:
cctry
2025-11-11 11:51:37 -08:00
committed by GitHub
parent e8114102fa
commit e0e6a6efb2
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/mem_cache/radix_cache_cpp.py
+4 -2
View File
@@ -167,7 +167,7 @@ class RadixCacheCpp(BasePrefixCache):
token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len] token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len]
kv_indices = self.req_to_token_pool.req_to_token[ kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, :kv_committed_len req.req_pool_idx, :kv_committed_len
] ].to(dtype=torch.int64, copy=True)
# NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned # NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned
# it will automatically align them, but length of them should be equal # it will automatically align them, but length of them should be equal
@@ -204,7 +204,9 @@ class RadixCacheCpp(BasePrefixCache):
assert req.req_pool_idx is not None assert req.req_pool_idx is not None
token_ids = req.fill_ids token_ids = req.fill_ids
prefill_len = len(token_ids) # prefill only (maybe chunked) prefill_len = len(token_ids) # prefill only (maybe chunked)
kv_indices = self.req_to_token_pool.req_to_token[req.req_pool_idx, :prefill_len] kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, :prefill_len
].to(dtype=torch.int64, copy=True)
# NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned # NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned
# it will automatically align them, but length of them should be equal # it will automatically align them, but length of them should be equal
test/srt/run_suite.py
+1
View File
@@ -217,6 +217,7 @@ suites = {
TestFile("openai_server/features/test_lora_openai_compatible.py", 150), TestFile("openai_server/features/test_lora_openai_compatible.py", 150),
TestFile("batch_invariant/test_batch_invariant_ops.py", 10), TestFile("batch_invariant/test_batch_invariant_ops.py", 10),
TestFile("test_deepseek_v3_deterministic.py", 240), TestFile("test_deepseek_v3_deterministic.py", 240),
TestFile("test_cpp_radix_cache.py", 60),
], ],
"nightly-4-gpu-b200": [ "nightly-4-gpu-b200": [
TestFile("test_fp4_moe.py", 300), TestFile("test_fp4_moe.py", 300),