[Scheduler] Derive prefix KV indices from last_node at allocation; drop Req.prefix_indices (#42825)

This commit is contained in:
Liangsheng Yin
2026-10-07 16:03:09 -07:00
committed by GitHub
parent e9240b3243
commit 5349d3e9d2
40 changed files with 160 additions and 272 deletions
+2 -5
View File
@@ -438,11 +438,8 @@ def prepare_extend_inputs_for_correctness_test(
req.full_untruncated_fill_ids.extend(input_ids[i][bench_args.cut_len :])
if model_runner is not None:
# Use req.kv.req_pool_idx instead of i to handle slot 0 padding correctly
req.set_prefix_indices(
model_runner.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : bench_args.cut_len
].to(torch.int64)
)
# The cut prefix is already in the request's row from the first extend.
req.prefix_len = bench_args.cut_len
req.logprob_start_len = -1
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
return reqs
+2 -4
View File
@@ -145,7 +145,7 @@ def _bootstrap_addr(req: Req) -> str:
def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
"""Start a decode-radix request that owns its whole KV row at the root."""
req.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
req.prefix_len = 0
req.last_node = tree_cache.root_node_handle(req.extra_key)
req.last_host_node = req.last_node
req.best_match_node = req.last_node
@@ -2204,9 +2204,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
# Decode-disagg allocates before batch assembly, so it binds the prefix
# here instead of in init_next_round_input.
req.set_prefix_indices(
prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64)
)
req.prefix_len = prefix_len
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
self.tree_cache.maybe_hand_to_session(req)
@@ -211,7 +211,7 @@ class DecodeHiCacheTransferMixin:
return False
self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle)
# Re-match: req.last_node / prefix_indices updated to current device state.
# Re-match: req.last_node / prefix_len updated to current device state.
rematch = match_kv_cache(
dr.req,
self.tree_cache,
@@ -316,11 +316,7 @@ class DecodeHiCacheTransferMixin:
),
decode_req.hicache_restored_kv_indices,
)
req.set_prefix_indices(
torch.cat(
[prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices]
)
)
req.prefix_len = prefix_match.decode_prefix_len
req.last_node = restore_lock.node
req.lock = restore_lock
# Prevent abort cleanup from releasing the transferred lock.
+3 -12
View File
@@ -1176,14 +1176,9 @@ class Req(ReqDllmMixin):
self.mm_video_tokens: int = 0
# Prefix info
# Tokens [0, prefix_len) are already in the row or come from a match;
# the next extend starts at prefix_len.
# Tokens [0, prefix_len) are already in the row, or, before allocation,
# on the path to last_node; the next extend starts at prefix_len.
self.prefix_len: int = 0
# The matched prefix's KV indices, consumed when the next allocation
# writes them into the row; None once written.
self.prefix_indices: Optional[torch.Tensor] = torch.empty(
(0,), dtype=torch.int64
)
# TODO(ispobock): rename to last_device_node
self.last_node: Any = None
self.last_host_node: Any = None
@@ -1571,10 +1566,6 @@ class Req(ReqDllmMixin):
def set_extend_range(self, start: int, end: int) -> None:
self.extend_range = Range(start, end)
def set_prefix_indices(self, prefix_indices: torch.Tensor) -> None:
self.prefix_indices = prefix_indices
self.prefix_len = len(prefix_indices)
def get_fill_ids(self) -> array:
return self.full_untruncated_fill_ids[: self.extend_range.end]
@@ -1934,7 +1925,7 @@ class Req(ReqDllmMixin):
# since we are tracking the total number of retractions for each request.
self.retraction_count += 1
self.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
self.prefix_len = 0
self.routed_experts = None
self.indexer_topk = None
self.last_node = None
@@ -1412,7 +1412,7 @@ class PrefillAdder:
)
if isinstance(admission, AddReqResult):
return admission
req.set_prefix_indices(torch.cat([req.prefix_indices, new_indices]))
req.prefix_len += len(new_indices)
req.kv.cache_protected_len = req.prefix_len
# Sharded pools cannot load host KV; reserve scratch after all other gates.
+14 -12
View File
@@ -48,6 +48,8 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_EMPTY_PREFIX = torch.empty((0,), dtype=torch.int64)
def write_cache_indices(
out_cache_loc: torch.Tensor,
@@ -348,6 +350,17 @@ def _alloc_page_size(batch: ScheduleBatch) -> int:
return batch.tree_cache.page_size
def _prefix_kv_indices(batch: ScheduleBatch, req: Req) -> torch.Tensor:
# A request holding its row (a later chunk, a borrowed session record)
# already has its prefix there; a new one reads its match off the tree.
if req.kv.holds_kv:
row = batch.req_to_token_pool.req_to_token[req.kv.req_pool_idx]
return row[: req.prefix_len].to(torch.int64)
if req.prefix_len == 0:
return _EMPTY_PREFIX
return batch.tree_cache.prefix_device_indices(req)
def alloc_for_extend(
batch: ScheduleBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
@@ -361,17 +374,7 @@ def alloc_for_extend(
# free out-of-window swa tokens
batch.maybe_evict_swa()
# A request holding its row (a later chunk, a borrowed session record)
# already has its prefix there; a new one writes its matched prefix.
req_to_token = batch.req_to_token_pool.req_to_token
prefix_tensors = [
(
req_to_token[r.kv.req_pool_idx, : r.prefix_len].to(torch.int64)
if r.kv.holds_kv
else r.prefix_indices
)
for r in batch.reqs
]
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
reuse_kv = None
if batch.is_dllm():
@@ -465,7 +468,6 @@ def alloc_for_extend(
for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()):
req.kv.kv_allocated_len = seq_len
req.kv.kv_committed_len = seq_len
req.prefix_indices = None
batch.tree_cache.maybe_hand_to_session(req)
return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu
@@ -528,6 +528,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
if lock is not None:
self.dec_lock_ref(lock.node, lock.receipt)
def prefix_device_indices(self, req: Req) -> torch.Tensor:
"""KV indices of req's matched prefix, read off the path to its locked
match node; valid from match until allocation writes them into the row."""
raise NotImplementedError
def maybe_hand_to_session(self, req: Req) -> None:
"""A cache that keeps records across requests (a streaming session) takes
the just-allocated row and the request's tree lock; the request borrows it."""
@@ -1249,12 +1249,7 @@ class BufferModePipeline:
)
self._refetch_staged(f)
return False
root_id = self._cache.tree_core.empty_match_result.last_device_node
full_indices = self._cache.tree_core.collect_full_device_indices(
node_id, root_id
)[:matched_len]
assert len(full_indices) == matched_len
req.set_prefix_indices(full_indices)
req.prefix_len = matched_len
req.last_node = node_id
req.kv.cache_protected_len = matched_len
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
@@ -1441,6 +1436,7 @@ class BufferModePipeline:
splice_base = plan.device_prefix_len
assert req.prefix_len == splice_base
prefix_indices = self._cache.prefix_device_indices(req)
trim_tokens = splice_base - f.matched_len
assert trim_tokens % cache.page_size == 0, (
f"staged splice trim not page-aligned req={req.rid}: "
@@ -1497,7 +1493,7 @@ class BufferModePipeline:
anchor_parts = []
host_parts = []
for repair_start, repair_end_ in repair_ranges:
anchor_parts.append(req.prefix_indices[repair_start:repair_end_])
anchor_parts.append(prefix_indices[repair_start:repair_end_])
host_parts.append(
slice(repair_start - window_start, repair_end_ - window_start)
)
@@ -1560,16 +1556,14 @@ class BufferModePipeline:
key,
repair_start,
repair_end_,
req.prefix_indices[repair_start:repair_end_],
prefix_indices[repair_start:repair_end_],
):
cache._apply_cache_action(action)
elif swa_dev is not None:
# Register the window's FULL->SWA translation now (attention reads
# through it). Keep SWA slots another request may still hold; their
# redundant H2D destinations are reclaimed at the transfer ack.
full_window = torch.cat([req.prefix_indices, device_indices])[
-len(swa_dev) :
]
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
allocator = cache.token_to_kv_pool_allocator
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
missing = old_swa <= 0
@@ -1619,7 +1613,7 @@ class BufferModePipeline:
insert_result = cache.insert(
InsertParams(
key=key,
value=torch.cat([req.prefix_indices, device_indices]),
value=torch.cat([prefix_indices, device_indices]),
prev_prefix_len=splice_base,
component_evicted_seqlens={
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
+1 -1
View File
@@ -209,7 +209,7 @@ def match_kv_cache(
match_result = zero_match_result(
tree_cache, match_result, extra_key=req.extra_key
)
req.set_prefix_indices(match_result.device_indices)
req.prefix_len = len(match_result.device_indices)
(
req.last_node,
req.last_host_node,
@@ -624,6 +624,17 @@ class RadixCache(BasePrefixCache):
##### Internal Helper Functions #####
def prefix_device_indices(self, req: Req) -> torch.Tensor:
values = []
node = req.last_node
while node is not self.root_node:
values.append(node.value)
node = node.parent
values.reverse()
path = torch.cat(values)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
def _match_prefix_helper(self, node: TreeNode, key: RadixKey):
access_time = time.monotonic()
node.last_access_time = access_time
@@ -87,6 +87,20 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
# LMCache rebuilds lookup keys from tokens, so any L1 node is valid.
return True
def prefix_device_indices(self, req: Req) -> torch.Tensor:
# match_prefix may append loaded slots not yet published to the tree,
# so the path alone can be shorter than the prefix.
root = self.root_node_handle(req.extra_key)
path = self.tree_core.collect_full_device_indices(req.last_node, root)
indices = path[: req.prefix_len]
missing = req.prefix_len - len(indices)
if missing > 0:
load = self._external_flows[req.rid].load
skip = max(len(indices) - load.local_hit_tokens, 0)
indices = torch.cat([indices, load.device_indices[skip : skip + missing]])
assert len(indices) == req.prefix_len, (req.rid, len(indices), req.prefix_len)
return indices
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
"""Report LMCache hits through host fields until GPU slots are ready."""
requested_key_len = len(params.key)
@@ -338,7 +338,7 @@ class UnifiedCacheLinkerWrapper:
# Insert the newly loaded tail into the tree.
prefix_indices = torch.cat(
[req.prefix_indices.to(torch.int64), full_transfer.device_indices]
[cache.prefix_device_indices(req), full_transfer.device_indices]
)
mamba_transfer = next(
(
@@ -3801,6 +3801,12 @@ class UnifiedRadixCache(BasePrefixCache):
# Internal callers (and the session sentinel / None) pass a non-int through.
return node_handle
def prefix_device_indices(self, req: Req) -> torch.Tensor:
root = self.root_node_handle(req.extra_key)
path = self.tree_core.collect_full_device_indices(req.last_node, root)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
"""The root's NodeId -- URC match results carry NodeIds."""
return self.tree_core.root_node_handle(extra_key)
@@ -96,9 +96,9 @@ class TestSWABasic(ScriptedTestCase):
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
for _ in range(400):
if r.is_chunking:
assert len(r.req.prefix_indices) <= r.req.kv.kv_committed_len, (
assert r.req.prefix_len <= r.req.kv.kv_committed_len, (
f"prefix_indices must be bounded by kv_committed_len, "
f"got prefix_indices_len={len(r.req.prefix_indices)}, "
f"got prefix_indices_len={r.req.prefix_len}, "
f"kv_committed_len={r.req.kv.kv_committed_len}"
)
if r.finished:
@@ -303,7 +303,7 @@ class TestRadixNoTailChunked(ScriptedTestCase):
req = s.chunked_req
if req is not None and req.rid == r.rid:
observed_mid_chunk = True
prefix_len: int = len(req.prefix_indices)
prefix_len: int = req.prefix_len
protected_len: int = req.kv.cache_protected_len
assert prefix_len == protected_len, (
f"page_size=1 must take the no-tail else branch: "
@@ -459,7 +459,7 @@ class TestRadixPartialPage(ScriptedTestCase):
for _ in range(800):
req = s.chunked_req
if req is not None and req.rid == r.rid:
prefix_len: int = len(req.prefix_indices)
prefix_len: int = req.prefix_len
protected_len: int = req.kv.cache_protected_len
assert prefix_len >= protected_len, (
f"len(prefix_indices)={prefix_len} dropped below "
@@ -148,7 +148,7 @@ class TestRegressionBasic(ScriptedTestCase):
r2 = t.start_req(prompt_len=DEFAULT_CHUNK_SIZE, max_new_tokens=2)
yield from run_until(r2, lambda h: h.status == "waiting")
r1_prefix = len(r1.req.prefix_indices)
r1_prefix = r1.req.prefix_len
assert r1.is_chunking, "r1 must still be the in-flight chunked req"
assert r1_prefix > 0, "r1 must hold a committed prefix as the chunked req"
@@ -516,9 +516,9 @@ class TestRegressionGptOss(ScriptedTestCase):
committed = r.req.kv.kv_committed_len
assert committed > 0
assert len(r.req.prefix_indices) <= committed, (
assert r.req.prefix_len <= committed, (
f"checkpoint over-read past kv_committed_len: "
f"prefix_indices_len={len(r.req.prefix_indices)}, "
f"prefix_indices_len={r.req.prefix_len}, "
f"kv_committed_len={committed}"
)
yield from run_until_finished(r)
@@ -23,7 +23,7 @@ def _load_inquirer_pending_for_rid(t: ScriptedContext, rid: str) -> int:
s = t.scheduler
chunked = s.chunked_req
if chunked is not None and chunked.rid == rid:
return chunked.seqlen - len(chunked.prefix_indices)
return chunked.seqlen - chunked.prefix_len
for req in s.waiting_queue:
if req.rid == rid:
return req.seqlen
@@ -226,9 +226,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
)
for _ in range(DEFAULT_MAX_STEPS):
if r.is_chunking:
assert len(r.req.prefix_indices) <= r.req.kv.kv_committed_len, (
assert r.req.prefix_len <= r.req.kv.kv_committed_len, (
f"streaming-session chunked stash must stay bounded by "
f"kv_committed_len; prefix_indices_len={len(r.req.prefix_indices)}, "
f"kv_committed_len; prefix_indices_len={r.req.prefix_len}, "
f"kv_committed_len={r.req.kv.kv_committed_len}"
)
if r.finished:
@@ -329,7 +329,7 @@ class TestSpecialCaseBasic(ScriptedTestCase):
if r.is_chunking and chunked is not None and chunked.rid == r.rid:
saw_chunking = True
pending = _load_inquirer_pending_for_rid(t, r.rid)
expected = chunked.seqlen - len(chunked.prefix_indices)
expected = chunked.seqlen - chunked.prefix_len
assert pending == expected, (
f"load_inquirer chunked contribution must equal the prefix-"
f"subtracting formula seqlen - len(prefix_indices) = {expected}; "
@@ -339,7 +339,7 @@ class TestSpecialCaseBasic(ScriptedTestCase):
f"chunked contribution must never exceed its full seqlen "
f"{chunked.seqlen}; got {pending} — dual-queue dedup violated"
)
if len(chunked.prefix_indices) > 0:
if chunked.prefix_len > 0:
saw_dedup = pending < chunked.seqlen
if r.finished:
break
@@ -373,7 +373,7 @@ class TestSpecialCaseBasic(ScriptedTestCase):
"test requires an empty waiting_queue so the chunked req is "
f"the sole pending-token contributor; got {len(s.waiting_queue)}"
)
expected = chunked.seqlen - len(chunked.prefix_indices)
expected = chunked.seqlen - chunked.prefix_len
observed = s.load_inquirer._get_num_pending_tokens()
assert observed == expected, (
f"chunked contribution must equal remainder "
@@ -478,13 +478,12 @@ class TestSpecialCaseBasic(ScriptedTestCase):
):
saw_mid_chunk = True
assert (
req.extend_range.end
== len(req.prefix_indices) + req.extend_range.length
req.extend_range.end == req.prefix_len + req.extend_range.length
), (
f"init_next_round_input must rebuild fill_ids to the committed "
f"prefix plus the in-flight chunk; "
f"fill_ids_len={req.extend_range.end}, "
f"prefix_indices_len={len(req.prefix_indices)}, "
f"prefix_indices_len={req.prefix_len}, "
f"extend_input_len={req.extend_range.length}, "
f"chunks_done={r.chunks_done}"
)
+1 -3
View File
@@ -113,9 +113,7 @@ class TestForwardSplitPrefill(CustomTestCase):
)
req.full_untruncated_fill_ids = req.origin_input_ids
req.logprob_start_len = -1
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
reqs.append(req)
# Create dummy tree_cache for tests (no prefix caching, just allocation)
@@ -665,7 +665,6 @@ class TestDisaggregationWire(unittest.TestCase):
def test_prebuilt_skips_unused_prompt_tensor(self):
req = SimpleNamespace(
kv=ReqKvInfo(req_pool_idx=0),
prefix_indices=[0, 1],
prefix_len=2,
extend_range=SimpleNamespace(length=3),
origin_input_ids=[0, 1, 2, 3, 4],
@@ -39,7 +39,7 @@ def _make_mock_req(
req_pool_idx: int,
kv_committed_len: int,
kv_allocated_len: int,
prefix_indices_len: int = 0,
prefix_len: int = 0,
rid: int = 0,
origin_len: int = 0,
):
@@ -54,8 +54,7 @@ def _make_mock_req(
kv_committed_len=kv_committed_len,
kv_allocated_len=kv_allocated_len,
)
req.prefix_indices = list(range(prefix_indices_len))
req.prefix_len = len(req.prefix_indices)
req.prefix_len = prefix_len
req.owned_kv_len = lambda: req.kv.kv_committed_len
return req
@@ -203,7 +202,7 @@ class TestReleaseFinishedReq(unittest.TestCase):
req_pool_idx=0,
kv_committed_len=20,
kv_allocated_len=20,
prefix_indices_len=5,
prefix_len=5,
)
req.kv.cache_protected_len = 5
lock = req.lock
@@ -27,8 +27,7 @@ class _Req:
self.full_untruncated_fill_ids = self.origin_input_ids + array(
"q", [0] * block_size
)
self.prefix_indices = torch.arange(context_len, dtype=torch.int64)
self.prefix_len = len(self.prefix_indices)
self.prefix_len = context_len
self.dllm_block_offset = context_len
self.dllm_incomplete_ids = array("q")
self.dllm_algo_state = None
@@ -202,8 +201,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase):
adder._check_prefill_tile_budget = Mock(return_value=None)
req = _Req(context_len=300, block_size=256, prefill=True)
req.host_hit_length = req.storage_hit_length = 0
req.prefix_indices = torch.empty(0, dtype=torch.int64)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 0
admission = adder._select_prefill_admission(
req,
total_tokens=1024,
@@ -228,8 +226,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase):
adder._update_prefill_budget = Mock()
req = _Req(context_len=10, block_size=4, prefill=True)
req.prefix_indices = torch.arange(7)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 7
result = PrefillAdder.add_dllm_staging_req(adder, req)
@@ -167,8 +167,7 @@ class _FakeRunner:
class _FakeReq:
def __init__(self, rid, req_pool_idx=0):
self.rid = rid
self.prefix_indices = torch.empty(0, dtype=torch.long)
self.prefix_len = len(self.prefix_indices)
self.prefix_len = 0
self.fill_ids = [0]
self.kv = ReqKvInfo(req_pool_idx=req_pool_idx)
# Mirrors Req's chunk-finality contract read by
@@ -43,9 +43,7 @@ class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase):
class TestSchedulerLoadInquirer(unittest.TestCase):
def make_inquirer(self, waiting_queue_prefix_matched):
waiting_req = SimpleNamespace(seqlen=100, num_matched_prefix_tokens=20)
chunked_req = SimpleNamespace(
seqlen=50, prefix_indices=range(10), prefix_len=10
)
chunked_req = SimpleNamespace(seqlen=50, prefix_len=10)
return SimpleNamespace(
disaggregation_mode=DisaggregationMode.NULL,
get_waiting_queue=lambda: [waiting_req],
@@ -53,8 +53,7 @@ def _track_seqlen(
sampling_params=sampling_params,
vocab_size=128,
)
req.prefix_indices = torch.arange(prefix_len, dtype=torch.int64)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = prefix_len
req.set_extend_range(prefix_len, prefix_len + extend_len)
req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64)
req.kv.mamba_next_track_idx = 0
@@ -1,4 +1,3 @@
import functools
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -124,8 +123,7 @@ class TestPrefillAdder(CustomTestCase):
req.rid = str(rid)
req.cache_request_handle = CacheRequestHandle(req.rid, 0)
req.priority = priority
req.set_prefix_indices = functools.partial(Req.set_prefix_indices, req)
req.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
req.prefix_len = 0
req.full_untruncated_fill_ids = []
req.output_ids = [0] * output_len
req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens)
@@ -261,8 +259,7 @@ class TestPrefillAdder(CustomTestCase):
adder = self.create_shortest_prefill_adder(chunk_tokens=512)
req = self.create_shared_req("host-miss")
req.full_untruncated_fill_ids = list(range(1024))
req.prefix_indices = torch.empty(0, dtype=torch.int64)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 0
req.host_hit_length = 768
req.best_match_node = req.last_node
req.needs_host_load_back.return_value = True
@@ -843,8 +840,7 @@ class TestPrefillAdder(CustomTestCase):
def create_sharded_req(self, rid, *, prefix_len=12, extend_len=4):
req = self.create_shared_req(rid, max_new_tokens=1)
req.prefix_indices = list(range(prefix_len))
req.prefix_len = len(req.prefix_indices)
req.prefix_len = prefix_len
req.full_untruncated_fill_ids = list(range(prefix_len + extend_len))
return req
@@ -1043,8 +1039,7 @@ class TestPrefillAdder(CustomTestCase):
)
req = self.create_mock_req("chunked", priority=0, max_new_tokens=128)
req.prefix_indices = []
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 0
req.full_untruncated_fill_ids = list(range(extend_input_len))
# set_extend_range is the only writer of extend_range; the production
# path reads req.extend_range.length right after calling it, so the mock
@@ -1156,8 +1151,7 @@ class TestPrefillAdder(CustomTestCase):
req = self.create_mock_req(
"resume", priority=0, max_new_tokens=40, output_len=10
)
req.prefix_indices = list(range(PREFIX))
req.prefix_len = len(req.prefix_indices)
req.prefix_len = PREFIX
req.full_untruncated_fill_ids = list(range(PREFIX + EXTEND))
req.host_hit_length = 0
req.swa_host_hit_length = 0
@@ -1207,8 +1201,7 @@ class TestPrefillAdder(CustomTestCase):
self.mock_token_allocator, self.mock_tree_cache
)
req = self.create_mock_req("dropped-fetch", priority=0, max_new_tokens=8)
req.prefix_indices = torch.empty(0, dtype=torch.int64)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 0
req.full_untruncated_fill_ids = list(range(SPAN))
req.host_hit_length = HOST_HIT
req.swa_host_hit_length = WINDOW
@@ -1250,7 +1243,7 @@ class TestPrefillAdder(CustomTestCase):
# budget. A successful load must not run admission gates again.
_, admitted, req = run(HOST_HIT + 4, remaining_after_load=0)
self.assertEqual(len(admitted), 1)
self.assertEqual(len(req.prefix_indices), HOST_HIT + 4)
self.assertEqual(req.prefix_len, HOST_HIT + 4)
self.assertEqual(req.kv.cache_protected_len, HOST_HIT + 4)
req.set_extend_range.assert_called_once_with(HOST_HIT + 4, SPAN)
# A partial FULL load stays fatal.
@@ -1441,8 +1434,7 @@ class TestPrefillAdder(CustomTestCase):
def _create_host_hit_req(self, *, prefix_len=0, host_hit=8192, tail=1024):
req = self._create_delayer_req(prefix_len + host_hit + tail)
req.prefix_indices = torch.arange(prefix_len)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = prefix_len
req.host_hit_length = host_hit
req.needs_host_load_back.return_value = True
req.best_match_node = req.last_node
@@ -208,7 +208,6 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase):
req_pool_idx=1,
cache_protected_len=2,
),
prefix_indices=torch.tensor([8, 9], dtype=torch.int64),
prefix_len=2,
priority=3,
extra_key=None,
@@ -242,7 +242,6 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase):
req_with_image = types.SimpleNamespace(
rid="img",
multimodal_inputs=types.SimpleNamespace(num_image_tokens=2),
prefix_indices=[],
prefix_len=0,
extend_range=Range(0, 5),
logprob_start_len=0,
@@ -250,7 +249,6 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase):
req_text_only = types.SimpleNamespace(
rid="txt",
multimodal_inputs=None,
prefix_indices=[],
prefix_len=0,
extend_range=Range(0, 4),
logprob_start_len=0,
@@ -139,8 +139,7 @@ class TestShortestPrefillFirst(CustomTestCase):
req = _make_req(rid, "", list(range(uncached + cached)))
req.full_untruncated_fill_ids = req.origin_input_ids[:]
req.num_matched_prefix_tokens = cached
req.prefix_indices = list(range(cached))
req.prefix_len = len(req.prefix_indices)
req.prefix_len = cached
req.time_stats.wait_queue_entry_time = arrived
return req
@@ -25,7 +25,7 @@ def _make_req(
*,
req_pool_idx: int,
fill_ids: list,
prefix_indices: torch.Tensor,
prefix_len: int,
extend_input_len: int,
fill_len: int,
) -> Req:
@@ -34,8 +34,7 @@ def _make_req(
req.origin_input_ids = array("q", fill_ids)
req.output_ids = array("q")
req.full_untruncated_fill_ids = array("q", fill_ids)
req.prefix_indices = prefix_indices
req.prefix_len = len(req.prefix_indices)
req.prefix_len = prefix_len
req.extend_range = Range(fill_len - extend_input_len, fill_len)
req.inflight_middle_chunks = 0
req.host_hit_length = 0
@@ -56,8 +55,6 @@ def _make_req(
def _make_req_to_token_pool(num_slots: int, max_context: int) -> SimpleNamespace:
# Slot s contains a recognizable fingerprint [s*1000, s*1000+1, ...]
# so we can tell a corrupted prefix_indices from a healthy one by content.
pool = SimpleNamespace()
pool.req_to_token = (
torch.arange(max_context, dtype=torch.int32).unsqueeze(0).repeat(num_slots, 1)
@@ -119,11 +116,8 @@ def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler:
class TestStashGatePreservesPrefix(CustomTestCase):
"""Consumer side: real ChunkCache.checkpoint mutates
req.prefix_indices iff stash actually runs, so prefix_indices content
is the bug-detection signal. The stash gate is content-based:
`fill_len > len(prefix_indices)` means there is freshly computed KV to
cache; otherwise the chunk was parked and stashing must be skipped."""
"""The stash gate advances prefix_len iff `fill_len > prefix_len`, i.e. the
chunk computed new KV; a parked chunk must be left untouched."""
POOL_IDX = 4
INITIAL_PREFIX_LEN = 8 # what was really cached last iter
@@ -134,35 +128,31 @@ class TestStashGatePreservesPrefix(CustomTestCase):
def _build(self, *, fill_len: int):
pool = _make_req_to_token_pool(self.NUM_SLOTS, self.MAX_CONTEXT)
cache = _make_chunk_cache(pool)
initial_prefix = pool.req_to_token[self.POOL_IDX, : self.INITIAL_PREFIX_LEN].to(
dtype=torch.int64, copy=True
)
req = _make_req(
req_pool_idx=self.POOL_IDX,
fill_ids=list(range(self.POST_RESET_FILL_LEN)),
prefix_indices=initial_prefix,
prefix_len=self.INITIAL_PREFIX_LEN,
extend_input_len=fill_len - self.INITIAL_PREFIX_LEN,
fill_len=fill_len,
)
s = _scheduler_for_get_next_batch(tree_cache=cache, chunked_req=req)
return s, req, initial_prefix, pool
return s, req, pool
def test_parked_chunked_req_keeps_its_prefix(self):
# A parked chunk has fill_len == prefix_len: no new KV was computed,
# so the gate must skip stash and leave the prefix intact.
s, req, initial_prefix, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN)
s, req, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN)
Scheduler.get_next_batch_to_run(
s, running_batch=s.running_batch, last_batch=s.last_batch
)
self.assertEqual(req.prefix_len, self.INITIAL_PREFIX_LEN)
self.assertTrue(torch.equal(req.prefix_indices, initial_prefix))
def test_scheduled_chunked_req_advances_prefix_via_real_stash(self):
# Symmetric guard against over-gating: when fill_len has advanced past
# the cached prefix, stash must run and advance prefix_len.
s, req, _, pool = self._build(fill_len=self.POST_RESET_FILL_LEN)
s, req, _ = self._build(fill_len=self.POST_RESET_FILL_LEN)
Scheduler.get_next_batch_to_run(
s, running_batch=s.running_batch, last_batch=s.last_batch
@@ -100,8 +100,7 @@ class MockReq:
self.lock = lock
self.extra_key = None
self.cache_salt = None
self.prefix_indices = torch.empty(0, dtype=torch.int64)
self.prefix_len = len(self.prefix_indices)
self.prefix_len = 0
self.priority = 0
self.kv = ReqKvInfo(
req_pool_idx=req_pool_idx,
@@ -61,11 +61,14 @@ class _FakeTreeCache:
def maybe_hand_to_session(self, req):
pass
def prefix_device_indices(self, req):
return req.tree_prefix
def _make_req(rid, prefix, block_size, *, req_pool_idx=None, reuse=False):
return SimpleNamespace(
rid=rid,
prefix_indices=torch.tensor(prefix, dtype=torch.int32),
tree_prefix=torch.tensor(prefix, dtype=torch.int32),
prefix_len=len(prefix),
dllm_incomplete_ids=array("q", range(block_size)) if reuse else array("q"),
inflight_middle_chunks=1 if req_pool_idx is not None else 0,
@@ -87,10 +90,7 @@ def _remove_allocated_req_slots(pool, *reqs):
def _make_batch(pool, allocator, reqs, extend_lens):
seq_lens_cpu = torch.tensor(
[
len(req.prefix_indices) + extend_len
for req, extend_len in zip(reqs, extend_lens)
],
[req.prefix_len + extend_len for req, extend_len in zip(reqs, extend_lens)],
dtype=torch.int64,
)
return SimpleNamespace(
@@ -99,7 +99,7 @@ def _make_batch(pool, allocator, reqs, extend_lens):
req_to_token_pool=pool,
token_to_kv_pool_allocator=allocator,
tree_cache=_FakeTreeCache(allocator),
prefix_lens=[len(req.prefix_indices) for req in reqs],
prefix_lens=[req.prefix_len for req in reqs],
extend_lens=extend_lens,
seq_lens=seq_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
@@ -110,8 +110,8 @@ def _make_batch(pool, allocator, reqs, extend_lens):
def _seed_retained_block(pool, req, values):
prefix_len = len(req.prefix_indices)
pool.req_to_token[req.kv.req_pool_idx, :prefix_len] = req.prefix_indices
prefix_len = req.prefix_len
pool.req_to_token[req.kv.req_pool_idx, :prefix_len] = req.tree_prefix
pool.req_to_token[req.kv.req_pool_idx, prefix_len : prefix_len + len(values)] = (
torch.tensor(values, dtype=torch.int32)
)
@@ -9,7 +9,6 @@ from unittest import mock
import torch
from sglang.srt.managers.cache_controller import CacheOperation, HiCacheController
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache import l2_transfer as transfer_module
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
@@ -35,11 +34,6 @@ from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
class _ReqStub(SimpleNamespace):
set_prefix_indices = Req.set_prefix_indices
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
MEMORY_POOL_HOST_MODULE = "sglang.srt.mem_cache.memory_pool_host"
@@ -316,10 +310,9 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase):
pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices(
0, 6
)
req = _ReqStub(
req = SimpleNamespace(
rid="r",
cache_request_handle=handle,
prefix_indices=_indices(0, 0),
prefix_len=0,
kv=SimpleNamespace(cache_protected_len=0),
)
@@ -333,10 +326,9 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase):
pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices(
0, 6
)
req = _ReqStub(
req = SimpleNamespace(
rid="r",
cache_request_handle=handle,
prefix_indices=_indices(0, 0),
prefix_len=0,
kv=SimpleNamespace(cache_protected_len=0),
)
@@ -247,12 +247,6 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase):
req.set_extend_range = set_extend_range
def set_prefix_indices(prefix_indices):
req.prefix_indices = prefix_indices
req.prefix_len = len(prefix_indices)
req.set_prefix_indices = set_prefix_indices
class ReqToTokenPool:
def __init__(self):
self.writes = []
@@ -784,8 +784,7 @@ class _GraftReq:
self.kv = ReqKvInfo(req_pool_idx=req_pool_idx)
self.extra_key = None
self.cache_salt = None
self.prefix_indices = torch.empty(0, dtype=torch.int64)
self.prefix_len = len(self.prefix_indices)
self.prefix_len = 0
self.last_node = None
self.priority = 0
self.kv_rotation_base = None
@@ -204,9 +204,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
for page_size in (4, 64):
with self.subTest(page_size=page_size):
allocator, req, adder = self._new_admission(page_size, pool_pages=6)
req.prefix_indices = allocator.alloc(page_size)
req.prefix_len = len(req.prefix_indices)
self.assertIsNotNone(req.prefix_indices)
req.prefix_len = len(allocator.alloc(page_size))
self.assertTrue(allocator.can_reserve(page_size + 2, page_size + 2))
self.assertIsNone(adder.add_chunked_req(req))
@@ -219,7 +217,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
allocator, req, adder = self._new_admission(
page_size, pool_pages=5, ignore_eos=True
)
self.assertEqual(len(req.prefix_indices), 0)
self.assertEqual(req.prefix_len, 0)
self.assertTrue(allocator.can_reserve(2 * page_size + 2, 2 * page_size))
adder.add_one_req(
@@ -16,7 +16,6 @@ from types import SimpleNamespace
import torch
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.base_prefix_cache import (
InsertParams,
MatchPrefixParams,
@@ -33,7 +32,6 @@ class _StubReq:
self.output_ids = array("q")
self.extra_key = None
self.cache_salt = None
self.prefix_indices = None
self.prefix_len = 0
self.last_node = None
self.last_host_node = None
@@ -43,8 +41,6 @@ class _StubReq:
self.swa_branching_seqlen = None
self.kv = SimpleNamespace(cache_protected_len=None)
set_prefix_indices = Req.set_prefix_indices
def _compute_max_prefix_len(self, input_len):
return max(input_len - 1, 0)
@@ -117,14 +113,14 @@ class TestMatchKvCacheForceMiss(unittest.TestCase):
baseline_req = _StubReq([10, 11, 12, 13, 99, 100])
with envs.SGLANG_RADIX_FORCE_MISS.override(False):
match_kv_cache(baseline_req, tree)
self.assertGreater(int(baseline_req.prefix_indices.numel()), 0)
self.assertGreater(baseline_req.prefix_len, 0)
self.assertIsNot(baseline_req.last_node, tree.root_node)
# With the flag, the same lookup is forced to miss.
forced_req = _StubReq([10, 11, 12, 13, 99, 100])
with envs.SGLANG_RADIX_FORCE_MISS.override(True):
match_kv_cache(forced_req, tree)
self.assertEqual(int(forced_req.prefix_indices.numel()), 0)
self.assertEqual(forced_req.prefix_len, 0)
self.assertIs(forced_req.last_node, tree.root_node)
self.assertIs(forced_req.last_host_node, tree.root_node)
self.assertEqual(forced_req.host_hit_length, 0)
@@ -11,7 +11,7 @@ from unittest.mock import Mock
import torch
from sglang.srt.managers.schedule_batch import Req, split_cached_prefix_by_tier
from sglang.srt.managers.schedule_batch import split_cached_prefix_by_tier
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
InitLoadBackParams,
@@ -33,11 +33,6 @@ from sglang.srt.mem_cache.unified_radix_cache import (
from sglang.srt.mem_cache.utils import get_hash_str
from sglang.test.ci.ci_register import register_cpu_ci
class _ReqStub(SimpleNamespace):
set_prefix_indices = Req.set_prefix_indices
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
_REQ = CacheRequestHandle("r", 0)
@@ -52,6 +47,7 @@ def _staged_fixture(full_match=2):
prefetch_anchor_info=lambda node: (None, None),
match_full_device_prefix=Mock(return_value=(full_match, 1, full_match)),
collect_full_device_indices=Mock(return_value=torch.arange(8)),
root_node_handle=lambda extra_key=None: 0,
inc_full_pin=Mock(),
dec_full_pin=Mock(),
empty_match_result=SimpleNamespace(
@@ -114,10 +110,9 @@ def _staged_fixture(full_match=2):
hash_values=["a", "b", "c"],
operation_id=1,
)
req = _ReqStub(
req = SimpleNamespace(
rid="r",
cache_request_handle=_REQ,
prefix_indices=torch.arange(2),
prefix_len=2,
last_node=1,
kv=SimpleNamespace(cache_protected_len=2),
@@ -207,8 +202,7 @@ def _two_rank_retry_trace(rank, rendezvous):
1,
full_match,
)
req.prefix_indices = torch.arange(full_match)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = full_match
if cache.buffer_pipeline.prepare_staged_prefetch(req):
assert (
cache.init_load_back(
@@ -309,8 +303,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
len(RadixKey(held.key_tokens, is_bigram=bigram)), 10
)
# A joint match counts bigrams, not their extra raw boundary token.
req.prefix_indices = torch.arange(10)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 10
req.kv.cache_protected_len = 10
self.assertTrue(pipeline.prepare_staged_prefetch(req))
self.assertFalse(pipeline.has_staged(req.cache_request_handle))
@@ -352,8 +345,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
# A twin made [0, 6) jointly reusable while the fetch was held.
cache.tree_core.match_full_device_prefix.return_value = (8, 1, 8)
req.prefix_indices = torch.arange(6)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 6
self.assertTrue(pipeline.prepare_staged_prefetch(req))
self.assertEqual(
split_cached_prefix_by_tier(
@@ -390,13 +382,12 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
cache.tree_core.match_full_device_prefix.assert_called_once()
# A twin finished first: the next pass's joint match runs past the
# staged span and is kept as is (a shrink would strand its recompute).
req.prefix_indices = torch.arange(12)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 12
req.last_node = 9
req.kv.cache_protected_len = 12
self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req))
self.assertIsNone(req.staged_prefetch_plan)
self.assertEqual(len(req.prefix_indices), 12)
self.assertEqual(req.prefix_len, 12)
self.assertEqual((req.last_node, req.kv.cache_protected_len), (9, 12))
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0))
self.assertFalse(pipeline.has_staged(req.cache_request_handle))
@@ -425,8 +416,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
anchor = pipeline.anchor_locks[req.cache_request_handle]
cache.tree_core.match_full_device_prefix.reset_mock()
for attempt in range(1, 4):
req.prefix_indices = torch.arange(2)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 2
self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req))
self.assertIsNone(
cache.init_load_back(
@@ -467,16 +457,14 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
# Tree changes occur while queued, before the next preparation pass.
cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6)
req.prefix_indices = torch.arange(2)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 2
self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req))
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4))
self.assertTrue(pipeline.has_staged(req.cache_request_handle))
self.assertEqual(cache.storage_prefetch_retries.pop_ready([req], 0, 8), [])
cache.tree_core.match_full_device_prefix.return_value = (0, 0, 0)
req.prefix_indices = torch.arange(0)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 0
self.assertFalse(cache.buffer_pipeline.prepare_staged_prefetch(req))
self.assertFalse(pipeline.has_staged(req.cache_request_handle))
self.assertEqual(
@@ -528,8 +516,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
pipeline.max_staged_admission_defers = 3
params = lambda: InitLoadBackParams(None, req.host_hit_length, req=req)
for attempt in range(1, 4):
req.prefix_indices = torch.arange(2)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 2
self.assertTrue(pipeline.prepare_staged_prefetch(req))
self.assertIsNone(cache.init_load_back(params()))
self.assertEqual(pipeline.has_staged(req.cache_request_handle), attempt < 3)
@@ -1296,6 +1296,7 @@ def test_linker_load_preserves_swa_boundaries(
)
),
resolve_node_handle=lambda node_id: SimpleNamespace(id=0),
prefix_device_indices=lambda req: torch.empty(0, dtype=torch.int64),
)
wrapper = UnifiedCacheLinkerWrapper(cache, _FakeLinker())
wrapper.hit_markers["rid"] = ExternalCacheHitMarker(
@@ -1315,7 +1316,6 @@ def test_linker_load_preserves_swa_boundaries(
req = SimpleNamespace(
rid="rid",
kv=kv,
prefix_indices=torch.empty(0, dtype=torch.int64),
prefix_len=0,
last_node=0,
priority=0,
@@ -634,9 +634,7 @@ def bench_release(
req.origin_input_ids = array("q", seq)
req.output_ids = array("q")
req.full_untruncated_fill_ids = array("q", seq)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
req.last_node = node
req.kv.cache_protected_len = matched_len
req.kv.kv_committed_len = len(seq)
@@ -1,7 +1,6 @@
"""Unit tests for UnifiedRadixCache"""
import atexit
import functools
import json
import shutil
import sys
@@ -122,11 +121,6 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.mem_cache_utils import finish_req
from sglang.test.test_utils import CustomTestCase
class _ReqStub(SimpleNamespace):
set_prefix_indices = Req.set_prefix_indices
register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small-amd")
@@ -1602,8 +1596,7 @@ class UnifiedRadixCacheSuite:
return req
def _apply_match_to_req(self, req, match):
req.prefix_indices = match.device_indices
req.prefix_len = len(req.prefix_indices)
req.prefix_len = len(match.device_indices)
req.last_node = match.last_device_node
req.last_host_node = match.last_host_node
req.best_match_node = match.best_match_node
@@ -1860,9 +1853,7 @@ class UnifiedRadixCacheSuite:
req.lock = None
req.extra_key = None
req.full_untruncated_fill_ids = array("q", input_ids + output_ids)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
if self.cfg.has_mamba:
req.kv.mamba_last_track_seqlen = kv_len
@@ -1908,9 +1899,7 @@ class UnifiedRadixCacheSuite:
req.origin_input_ids = array("q", prompt_ids)
req.output_ids = array("q", output_ids)
req.full_untruncated_fill_ids = array("q", prompt_ids + output_ids)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
kv_len = req.extend_range.end
kv_indices = self._alloc(allocator, kv_len)
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices)
@@ -1983,9 +1972,7 @@ class UnifiedRadixCacheSuite:
req.lock = None
req.extra_key = None
req.full_untruncated_fill_ids = array("q", tokens)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
avail_before = allocator.available_size()
release_kv_cache(req, cache, checkpoint=False)
@@ -2003,9 +1990,7 @@ class UnifiedRadixCacheSuite:
req.origin_input_ids = array("q", tokens)
req.output_ids = array("q")
req.full_untruncated_fill_ids = array("q", tokens)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
kv_len = len(tokens)
kv_indices = self._alloc(allocator, kv_len)
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices)
@@ -2149,9 +2134,7 @@ class UnifiedRadixCacheSuite:
req.lock = None
req.extra_key = None
req.full_untruncated_fill_ids = array("q", input_ids)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
if self.cfg.has_mamba:
req.kv.mamba_last_track_seqlen = kv_len
@@ -3452,7 +3435,6 @@ class UnifiedRadixCacheSuite:
cache,
req_id,
prefix_len=None,
prefix_indices=None,
timeout: float = 10.0,
*,
extra_key=None,
@@ -3476,20 +3458,7 @@ class UnifiedRadixCacheSuite:
req.cache_request_handle = req_id
req.extra_key = extra_key
req.cache_salt = cache_salt
if prefix_indices is not None:
# Spliceable mid-anchor consumption publishes value=cat(prefix,
# fill) — the real device prefix is required (zeros would insert
# bogus slots into the tree).
assert len(prefix_indices) == prefix_len
req.prefix_indices = prefix_indices
req.prefix_len = len(req.prefix_indices)
else:
req.prefix_indices = torch.zeros(
prefix_len,
dtype=torch.int64,
device=cache.tree_core.empty_match_result.device_indices.device,
)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = prefix_len
req.last_node = cache.root_node_handle() if last_node is None else last_node
joint = cache.match_prefix(
MatchPrefixParams(
@@ -3498,8 +3467,7 @@ class UnifiedRadixCacheSuite:
)
)
)
req.prefix_indices = joint.device_indices
req.prefix_len = len(req.prefix_indices)
req.prefix_len = len(joint.device_indices)
joint_covers_span = len(joint.device_indices) >= len(f.key_tokens)
adder = PrefillAdder.__new__(PrefillAdder)
adder.tree_cache = cache
@@ -4517,12 +4485,7 @@ class UnifiedRadixCacheSuite:
req.extra_key = None
req.cache_salt = None
req.last_node = cons.root_node_handle()
req.prefix_indices = torch.zeros(
held.matched_len,
dtype=torch.int64,
device=cons.tree_core.empty_match_result.device_indices.device,
)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = held.matched_len
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
spliced, _last = cons.init_load_back(
InitLoadBackParams(
@@ -4707,7 +4670,6 @@ class UnifiedRadixCacheSuite:
cons2,
anchored_req,
prefix_len=len(prefix),
prefix_indices=prefix_match.device_indices,
extra_key=extra_key,
cache_salt=cache_salt,
last_node=anchor,
@@ -4886,10 +4848,9 @@ class UnifiedRadixCacheSuite:
self.assertIsNotNone(cons_alloc.swa_attn_allocator.alloc(hold // ps * ps))
# The adder's SWA gate for this request (_swa_budget_for_req).
req = _ReqStub(
req = SimpleNamespace(
rid=req_id.rid,
cache_request_handle=req_id,
prefix_indices=cons.tree_core.empty_match_result.device_indices,
prefix_len=len(cons.tree_core.empty_match_result.device_indices),
kv=SimpleNamespace(cache_protected_len=0),
)
@@ -4911,12 +4872,7 @@ class UnifiedRadixCacheSuite:
req.extra_key = None
req.cache_salt = None
req.last_node = cons.root_node_handle()
req.prefix_indices = torch.zeros(
held.matched_len,
dtype=torch.int64,
device=cons.tree_core.empty_match_result.device_indices.device,
)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = held.matched_len
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
spliced, last_node = cons.init_load_back(
InitLoadBackParams(
@@ -5141,7 +5097,7 @@ class UnifiedRadixCacheSuite:
return_req=True,
)
self.assertEqual(int(spliced.numel()), 0)
self.assertEqual(len(req.prefix_indices), len(seq), "FULL rematch was lost")
self.assertEqual(req.prefix_len, len(seq), "FULL rematch was lost")
self.assertEqual(len(dispatched["device"]), window_tokens)
self.assertEqual(cons_alloc.full_available_size(), full_avail0)
k, v = self._snapshot_full_kv(cons_alloc, masked_full)
@@ -5219,7 +5175,7 @@ class UnifiedRadixCacheSuite:
spliced, req = self._consume_staged_prefetch(
cons, req_id, prefix_len=0, return_req=True
)
self.assertEqual(len(req.prefix_indices), split_at)
self.assertEqual(req.prefix_len, split_at)
self.assertEqual(int(spliced.numel()), page_size)
self.assertEqual(cons_alloc.full_available_size(), full_avail0 - page_size)
k, v = self._snapshot_full_kv(cons_alloc, head_full)
@@ -5351,12 +5307,7 @@ class UnifiedRadixCacheSuite:
req.cache_request_handle = req_id
req.extra_key = None
req.cache_salt = None
req.prefix_indices = torch.zeros(
0,
dtype=torch.int64,
device=cons.tree_core.empty_match_result.device_indices.device,
)
req.prefix_len = len(req.prefix_indices)
req.prefix_len = 0
req.last_node = cons.root_node_handle()
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
with mock.patch.object(cons.cache_controller, "load", adversarial_load):
@@ -5442,10 +5393,9 @@ class UnifiedRadixCacheSuite:
self.assertGreater(aux_max, len(seq) - len(head))
# The surfaced host hit is the splice-able tail, not the full span.
req = _ReqStub(
req = SimpleNamespace(
rid=req_id.rid,
cache_request_handle=req_id,
prefix_indices=sib.device_indices,
prefix_len=len(sib.device_indices),
kv=SimpleNamespace(cache_protected_len=len(sib.device_indices)),
)
@@ -5453,9 +5403,7 @@ class UnifiedRadixCacheSuite:
self.assertEqual(req.host_hit_length, len(seq) - len(head))
self.assertTrue(cons.buffer_pipeline.has_staged(req_id))
spliced = self._consume_staged_prefetch(
cons, req_id, prefix_len=len(head), prefix_indices=sib.device_indices
)
spliced = self._consume_staged_prefetch(cons, req_id, prefix_len=len(head))
self.assertEqual(int(spliced.numel()), len(seq) - len(head))
m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq))))
@@ -5513,11 +5461,9 @@ class UnifiedRadixCacheSuite:
req = mock.Mock(
rid=req_id.rid,
cache_request_handle=req_id,
prefix_indices=live.device_indices,
prefix_len=len(live.device_indices),
last_node=live.last_device_node,
)
req.set_prefix_indices = functools.partial(Req.set_prefix_indices, req)
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0))
self.assertFalse(cons.buffer_pipeline.has_staged(req_id))
@@ -5672,7 +5618,6 @@ class UnifiedRadixCacheSuite:
spliced = self._consume_staged_prefetch(
cons2,
CacheRequestHandle("window-req", 0),
prefix_indices=m.device_indices,
)
self.assertEqual(int(spliced.numel()), len(seq_ab) - len(seq_a))
self.assertEqual(
@@ -7687,7 +7632,7 @@ class UnifiedRadixCacheSuite:
)
self.assertEqual(new_node, leaf)
self.assertEqual(len(torch.cat([req.prefix_indices, new_indices])), len(tokens))
self.assertEqual(req.prefix_len + len(new_indices), len(tokens))
self.assertIsNotNone(_device_value(cache, leaf, ComponentType.MAMBA))
self._finish_pending_loads(cache)
self._release_ongoing_load_back_locks(cache)
@@ -7729,7 +7674,7 @@ class UnifiedRadixCacheSuite:
self.assertEqual(new_node, leaf)
self.assertEqual(new_indices.tolist(), leaf_full.tolist())
self.assertEqual(len(torch.cat([req.prefix_indices, new_indices])), len(tokens))
self.assertEqual(req.prefix_len + len(new_indices), len(tokens))
self.assertEqual(
_device_value(cache, leaf, ComponentType.FULL).tolist(),
leaf_full.tolist(),
@@ -8017,7 +7962,7 @@ class UnifiedRadixCacheSuite:
sampling_params=SamplingParams(temperature=0, max_new_tokens=1),
)
req.init_next_round_input(cache)
self.assertEqual((len(req.prefix_indices), req.host_hit_length), (0, 0))
self.assertEqual((req.prefix_len, req.host_hit_length), (0, 0))
self.assertEqual(req.mamba_host_hit_length, 1)
adder = PrefillAdder(
page_size=1,
@@ -8034,7 +7979,7 @@ class UnifiedRadixCacheSuite:
# 18-token demand would reject and scheduler cleanup would free the CoW
# destination even though the queued H2D still targets it.
self.assertEqual(adder.can_run_list, [req])
self.assertEqual(len(req.prefix_indices), 12)
self.assertEqual(req.prefix_len, 12)
self.assertEqual(req.kv.cache_protected_len, 12)
slot = req.kv.mamba_pool_idx.unsqueeze(0)
other_slots = pool.mamba_allocator.alloc(pool.mamba_allocator.available_size())