[Scheduler] Derive prefix KV indices from last_node at allocation; drop Req.prefix_indices (#42825)
This commit is contained in:
@@ -438,11 +438,8 @@ def prepare_extend_inputs_for_correctness_test(
|
||||
req.full_untruncated_fill_ids.extend(input_ids[i][bench_args.cut_len :])
|
||||
if model_runner is not None:
|
||||
# Use req.kv.req_pool_idx instead of i to handle slot 0 padding correctly
|
||||
req.set_prefix_indices(
|
||||
model_runner.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : bench_args.cut_len
|
||||
].to(torch.int64)
|
||||
)
|
||||
# The cut prefix is already in the request's row from the first extend.
|
||||
req.prefix_len = bench_args.cut_len
|
||||
req.logprob_start_len = -1
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
return reqs
|
||||
|
||||
@@ -145,7 +145,7 @@ def _bootstrap_addr(req: Req) -> str:
|
||||
|
||||
def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
"""Start a decode-radix request that owns its whole KV row at the root."""
|
||||
req.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
|
||||
req.prefix_len = 0
|
||||
req.last_node = tree_cache.root_node_handle(req.extra_key)
|
||||
req.last_host_node = req.last_node
|
||||
req.best_match_node = req.last_node
|
||||
@@ -2204,9 +2204,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
|
||||
# Decode-disagg allocates before batch assembly, so it binds the prefix
|
||||
# here instead of in init_next_round_input.
|
||||
req.set_prefix_indices(
|
||||
prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64)
|
||||
)
|
||||
req.prefix_len = prefix_len
|
||||
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
|
||||
self.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
|
||||
@@ -211,7 +211,7 @@ class DecodeHiCacheTransferMixin:
|
||||
return False
|
||||
self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle)
|
||||
|
||||
# Re-match: req.last_node / prefix_indices updated to current device state.
|
||||
# Re-match: req.last_node / prefix_len updated to current device state.
|
||||
rematch = match_kv_cache(
|
||||
dr.req,
|
||||
self.tree_cache,
|
||||
@@ -316,11 +316,7 @@ class DecodeHiCacheTransferMixin:
|
||||
),
|
||||
decode_req.hicache_restored_kv_indices,
|
||||
)
|
||||
req.set_prefix_indices(
|
||||
torch.cat(
|
||||
[prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices]
|
||||
)
|
||||
)
|
||||
req.prefix_len = prefix_match.decode_prefix_len
|
||||
req.last_node = restore_lock.node
|
||||
req.lock = restore_lock
|
||||
# Prevent abort cleanup from releasing the transferred lock.
|
||||
|
||||
@@ -1176,14 +1176,9 @@ class Req(ReqDllmMixin):
|
||||
self.mm_video_tokens: int = 0
|
||||
|
||||
# Prefix info
|
||||
# Tokens [0, prefix_len) are already in the row or come from a match;
|
||||
# the next extend starts at prefix_len.
|
||||
# Tokens [0, prefix_len) are already in the row, or, before allocation,
|
||||
# on the path to last_node; the next extend starts at prefix_len.
|
||||
self.prefix_len: int = 0
|
||||
# The matched prefix's KV indices, consumed when the next allocation
|
||||
# writes them into the row; None once written.
|
||||
self.prefix_indices: Optional[torch.Tensor] = torch.empty(
|
||||
(0,), dtype=torch.int64
|
||||
)
|
||||
# TODO(ispobock): rename to last_device_node
|
||||
self.last_node: Any = None
|
||||
self.last_host_node: Any = None
|
||||
@@ -1571,10 +1566,6 @@ class Req(ReqDllmMixin):
|
||||
def set_extend_range(self, start: int, end: int) -> None:
|
||||
self.extend_range = Range(start, end)
|
||||
|
||||
def set_prefix_indices(self, prefix_indices: torch.Tensor) -> None:
|
||||
self.prefix_indices = prefix_indices
|
||||
self.prefix_len = len(prefix_indices)
|
||||
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
@@ -1934,7 +1925,7 @@ class Req(ReqDllmMixin):
|
||||
# since we are tracking the total number of retractions for each request.
|
||||
self.retraction_count += 1
|
||||
|
||||
self.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
|
||||
self.prefix_len = 0
|
||||
self.routed_experts = None
|
||||
self.indexer_topk = None
|
||||
self.last_node = None
|
||||
|
||||
@@ -1412,7 +1412,7 @@ class PrefillAdder:
|
||||
)
|
||||
if isinstance(admission, AddReqResult):
|
||||
return admission
|
||||
req.set_prefix_indices(torch.cat([req.prefix_indices, new_indices]))
|
||||
req.prefix_len += len(new_indices)
|
||||
req.kv.cache_protected_len = req.prefix_len
|
||||
|
||||
# Sharded pools cannot load host KV; reserve scratch after all other gates.
|
||||
|
||||
@@ -48,6 +48,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EMPTY_PREFIX = torch.empty((0,), dtype=torch.int64)
|
||||
|
||||
|
||||
def write_cache_indices(
|
||||
out_cache_loc: torch.Tensor,
|
||||
@@ -348,6 +350,17 @@ def _alloc_page_size(batch: ScheduleBatch) -> int:
|
||||
return batch.tree_cache.page_size
|
||||
|
||||
|
||||
def _prefix_kv_indices(batch: ScheduleBatch, req: Req) -> torch.Tensor:
|
||||
# A request holding its row (a later chunk, a borrowed session record)
|
||||
# already has its prefix there; a new one reads its match off the tree.
|
||||
if req.kv.holds_kv:
|
||||
row = batch.req_to_token_pool.req_to_token[req.kv.req_pool_idx]
|
||||
return row[: req.prefix_len].to(torch.int64)
|
||||
if req.prefix_len == 0:
|
||||
return _EMPTY_PREFIX
|
||||
return batch.tree_cache.prefix_device_indices(req)
|
||||
|
||||
|
||||
def alloc_for_extend(
|
||||
batch: ScheduleBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
@@ -361,17 +374,7 @@ def alloc_for_extend(
|
||||
# free out-of-window swa tokens
|
||||
batch.maybe_evict_swa()
|
||||
|
||||
# A request holding its row (a later chunk, a borrowed session record)
|
||||
# already has its prefix there; a new one writes its matched prefix.
|
||||
req_to_token = batch.req_to_token_pool.req_to_token
|
||||
prefix_tensors = [
|
||||
(
|
||||
req_to_token[r.kv.req_pool_idx, : r.prefix_len].to(torch.int64)
|
||||
if r.kv.holds_kv
|
||||
else r.prefix_indices
|
||||
)
|
||||
for r in batch.reqs
|
||||
]
|
||||
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
|
||||
|
||||
reuse_kv = None
|
||||
if batch.is_dllm():
|
||||
@@ -465,7 +468,6 @@ def alloc_for_extend(
|
||||
for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()):
|
||||
req.kv.kv_allocated_len = seq_len
|
||||
req.kv.kv_committed_len = seq_len
|
||||
req.prefix_indices = None
|
||||
batch.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu
|
||||
|
||||
@@ -528,6 +528,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
if lock is not None:
|
||||
self.dec_lock_ref(lock.node, lock.receipt)
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
"""KV indices of req's matched prefix, read off the path to its locked
|
||||
match node; valid from match until allocation writes them into the row."""
|
||||
raise NotImplementedError
|
||||
|
||||
def maybe_hand_to_session(self, req: Req) -> None:
|
||||
"""A cache that keeps records across requests (a streaming session) takes
|
||||
the just-allocated row and the request's tree lock; the request borrows it."""
|
||||
|
||||
@@ -1249,12 +1249,7 @@ class BufferModePipeline:
|
||||
)
|
||||
self._refetch_staged(f)
|
||||
return False
|
||||
root_id = self._cache.tree_core.empty_match_result.last_device_node
|
||||
full_indices = self._cache.tree_core.collect_full_device_indices(
|
||||
node_id, root_id
|
||||
)[:matched_len]
|
||||
assert len(full_indices) == matched_len
|
||||
req.set_prefix_indices(full_indices)
|
||||
req.prefix_len = matched_len
|
||||
req.last_node = node_id
|
||||
req.kv.cache_protected_len = matched_len
|
||||
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
|
||||
@@ -1441,6 +1436,7 @@ class BufferModePipeline:
|
||||
|
||||
splice_base = plan.device_prefix_len
|
||||
assert req.prefix_len == splice_base
|
||||
prefix_indices = self._cache.prefix_device_indices(req)
|
||||
trim_tokens = splice_base - f.matched_len
|
||||
assert trim_tokens % cache.page_size == 0, (
|
||||
f"staged splice trim not page-aligned req={req.rid}: "
|
||||
@@ -1497,7 +1493,7 @@ class BufferModePipeline:
|
||||
anchor_parts = []
|
||||
host_parts = []
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
anchor_parts.append(req.prefix_indices[repair_start:repair_end_])
|
||||
anchor_parts.append(prefix_indices[repair_start:repair_end_])
|
||||
host_parts.append(
|
||||
slice(repair_start - window_start, repair_end_ - window_start)
|
||||
)
|
||||
@@ -1560,16 +1556,14 @@ class BufferModePipeline:
|
||||
key,
|
||||
repair_start,
|
||||
repair_end_,
|
||||
req.prefix_indices[repair_start:repair_end_],
|
||||
prefix_indices[repair_start:repair_end_],
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
elif swa_dev is not None:
|
||||
# Register the window's FULL->SWA translation now (attention reads
|
||||
# through it). Keep SWA slots another request may still hold; their
|
||||
# redundant H2D destinations are reclaimed at the transfer ack.
|
||||
full_window = torch.cat([req.prefix_indices, device_indices])[
|
||||
-len(swa_dev) :
|
||||
]
|
||||
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
|
||||
missing = old_swa <= 0
|
||||
@@ -1619,7 +1613,7 @@ class BufferModePipeline:
|
||||
insert_result = cache.insert(
|
||||
InsertParams(
|
||||
key=key,
|
||||
value=torch.cat([req.prefix_indices, device_indices]),
|
||||
value=torch.cat([prefix_indices, device_indices]),
|
||||
prev_prefix_len=splice_base,
|
||||
component_evicted_seqlens={
|
||||
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
|
||||
|
||||
@@ -209,7 +209,7 @@ def match_kv_cache(
|
||||
match_result = zero_match_result(
|
||||
tree_cache, match_result, extra_key=req.extra_key
|
||||
)
|
||||
req.set_prefix_indices(match_result.device_indices)
|
||||
req.prefix_len = len(match_result.device_indices)
|
||||
(
|
||||
req.last_node,
|
||||
req.last_host_node,
|
||||
|
||||
@@ -624,6 +624,17 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
##### Internal Helper Functions #####
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
values = []
|
||||
node = req.last_node
|
||||
while node is not self.root_node:
|
||||
values.append(node.value)
|
||||
node = node.parent
|
||||
values.reverse()
|
||||
path = torch.cat(values)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
|
||||
def _match_prefix_helper(self, node: TreeNode, key: RadixKey):
|
||||
access_time = time.monotonic()
|
||||
node.last_access_time = access_time
|
||||
|
||||
@@ -87,6 +87,20 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
# LMCache rebuilds lookup keys from tokens, so any L1 node is valid.
|
||||
return True
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
# match_prefix may append loaded slots not yet published to the tree,
|
||||
# so the path alone can be shorter than the prefix.
|
||||
root = self.root_node_handle(req.extra_key)
|
||||
path = self.tree_core.collect_full_device_indices(req.last_node, root)
|
||||
indices = path[: req.prefix_len]
|
||||
missing = req.prefix_len - len(indices)
|
||||
if missing > 0:
|
||||
load = self._external_flows[req.rid].load
|
||||
skip = max(len(indices) - load.local_hit_tokens, 0)
|
||||
indices = torch.cat([indices, load.device_indices[skip : skip + missing]])
|
||||
assert len(indices) == req.prefix_len, (req.rid, len(indices), req.prefix_len)
|
||||
return indices
|
||||
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
"""Report LMCache hits through host fields until GPU slots are ready."""
|
||||
requested_key_len = len(params.key)
|
||||
|
||||
@@ -338,7 +338,7 @@ class UnifiedCacheLinkerWrapper:
|
||||
|
||||
# Insert the newly loaded tail into the tree.
|
||||
prefix_indices = torch.cat(
|
||||
[req.prefix_indices.to(torch.int64), full_transfer.device_indices]
|
||||
[cache.prefix_device_indices(req), full_transfer.device_indices]
|
||||
)
|
||||
mamba_transfer = next(
|
||||
(
|
||||
|
||||
@@ -3801,6 +3801,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# Internal callers (and the session sentinel / None) pass a non-int through.
|
||||
return node_handle
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
root = self.root_node_handle(req.extra_key)
|
||||
path = self.tree_core.collect_full_device_indices(req.last_node, root)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
|
||||
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
|
||||
"""The root's NodeId -- URC match results carry NodeIds."""
|
||||
return self.tree_core.root_node_handle(extra_key)
|
||||
|
||||
@@ -96,9 +96,9 @@ class TestSWABasic(ScriptedTestCase):
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
for _ in range(400):
|
||||
if r.is_chunking:
|
||||
assert len(r.req.prefix_indices) <= r.req.kv.kv_committed_len, (
|
||||
assert r.req.prefix_len <= r.req.kv.kv_committed_len, (
|
||||
f"prefix_indices must be bounded by kv_committed_len, "
|
||||
f"got prefix_indices_len={len(r.req.prefix_indices)}, "
|
||||
f"got prefix_indices_len={r.req.prefix_len}, "
|
||||
f"kv_committed_len={r.req.kv.kv_committed_len}"
|
||||
)
|
||||
if r.finished:
|
||||
|
||||
@@ -303,7 +303,7 @@ class TestRadixNoTailChunked(ScriptedTestCase):
|
||||
req = s.chunked_req
|
||||
if req is not None and req.rid == r.rid:
|
||||
observed_mid_chunk = True
|
||||
prefix_len: int = len(req.prefix_indices)
|
||||
prefix_len: int = req.prefix_len
|
||||
protected_len: int = req.kv.cache_protected_len
|
||||
assert prefix_len == protected_len, (
|
||||
f"page_size=1 must take the no-tail else branch: "
|
||||
@@ -459,7 +459,7 @@ class TestRadixPartialPage(ScriptedTestCase):
|
||||
for _ in range(800):
|
||||
req = s.chunked_req
|
||||
if req is not None and req.rid == r.rid:
|
||||
prefix_len: int = len(req.prefix_indices)
|
||||
prefix_len: int = req.prefix_len
|
||||
protected_len: int = req.kv.cache_protected_len
|
||||
assert prefix_len >= protected_len, (
|
||||
f"len(prefix_indices)={prefix_len} dropped below "
|
||||
|
||||
@@ -148,7 +148,7 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
r2 = t.start_req(prompt_len=DEFAULT_CHUNK_SIZE, max_new_tokens=2)
|
||||
yield from run_until(r2, lambda h: h.status == "waiting")
|
||||
|
||||
r1_prefix = len(r1.req.prefix_indices)
|
||||
r1_prefix = r1.req.prefix_len
|
||||
assert r1.is_chunking, "r1 must still be the in-flight chunked req"
|
||||
assert r1_prefix > 0, "r1 must hold a committed prefix as the chunked req"
|
||||
|
||||
@@ -516,9 +516,9 @@ class TestRegressionGptOss(ScriptedTestCase):
|
||||
committed = r.req.kv.kv_committed_len
|
||||
assert committed > 0
|
||||
|
||||
assert len(r.req.prefix_indices) <= committed, (
|
||||
assert r.req.prefix_len <= committed, (
|
||||
f"checkpoint over-read past kv_committed_len: "
|
||||
f"prefix_indices_len={len(r.req.prefix_indices)}, "
|
||||
f"prefix_indices_len={r.req.prefix_len}, "
|
||||
f"kv_committed_len={committed}"
|
||||
)
|
||||
yield from run_until_finished(r)
|
||||
|
||||
@@ -23,7 +23,7 @@ def _load_inquirer_pending_for_rid(t: ScriptedContext, rid: str) -> int:
|
||||
s = t.scheduler
|
||||
chunked = s.chunked_req
|
||||
if chunked is not None and chunked.rid == rid:
|
||||
return chunked.seqlen - len(chunked.prefix_indices)
|
||||
return chunked.seqlen - chunked.prefix_len
|
||||
for req in s.waiting_queue:
|
||||
if req.rid == rid:
|
||||
return req.seqlen
|
||||
@@ -226,9 +226,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
)
|
||||
for _ in range(DEFAULT_MAX_STEPS):
|
||||
if r.is_chunking:
|
||||
assert len(r.req.prefix_indices) <= r.req.kv.kv_committed_len, (
|
||||
assert r.req.prefix_len <= r.req.kv.kv_committed_len, (
|
||||
f"streaming-session chunked stash must stay bounded by "
|
||||
f"kv_committed_len; prefix_indices_len={len(r.req.prefix_indices)}, "
|
||||
f"kv_committed_len; prefix_indices_len={r.req.prefix_len}, "
|
||||
f"kv_committed_len={r.req.kv.kv_committed_len}"
|
||||
)
|
||||
if r.finished:
|
||||
@@ -329,7 +329,7 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
if r.is_chunking and chunked is not None and chunked.rid == r.rid:
|
||||
saw_chunking = True
|
||||
pending = _load_inquirer_pending_for_rid(t, r.rid)
|
||||
expected = chunked.seqlen - len(chunked.prefix_indices)
|
||||
expected = chunked.seqlen - chunked.prefix_len
|
||||
assert pending == expected, (
|
||||
f"load_inquirer chunked contribution must equal the prefix-"
|
||||
f"subtracting formula seqlen - len(prefix_indices) = {expected}; "
|
||||
@@ -339,7 +339,7 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
f"chunked contribution must never exceed its full seqlen "
|
||||
f"{chunked.seqlen}; got {pending} — dual-queue dedup violated"
|
||||
)
|
||||
if len(chunked.prefix_indices) > 0:
|
||||
if chunked.prefix_len > 0:
|
||||
saw_dedup = pending < chunked.seqlen
|
||||
if r.finished:
|
||||
break
|
||||
@@ -373,7 +373,7 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
"test requires an empty waiting_queue so the chunked req is "
|
||||
f"the sole pending-token contributor; got {len(s.waiting_queue)}"
|
||||
)
|
||||
expected = chunked.seqlen - len(chunked.prefix_indices)
|
||||
expected = chunked.seqlen - chunked.prefix_len
|
||||
observed = s.load_inquirer._get_num_pending_tokens()
|
||||
assert observed == expected, (
|
||||
f"chunked contribution must equal remainder "
|
||||
@@ -478,13 +478,12 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
):
|
||||
saw_mid_chunk = True
|
||||
assert (
|
||||
req.extend_range.end
|
||||
== len(req.prefix_indices) + req.extend_range.length
|
||||
req.extend_range.end == req.prefix_len + req.extend_range.length
|
||||
), (
|
||||
f"init_next_round_input must rebuild fill_ids to the committed "
|
||||
f"prefix plus the in-flight chunk; "
|
||||
f"fill_ids_len={req.extend_range.end}, "
|
||||
f"prefix_indices_len={len(req.prefix_indices)}, "
|
||||
f"prefix_indices_len={req.prefix_len}, "
|
||||
f"extend_input_len={req.extend_range.length}, "
|
||||
f"chunks_done={r.chunks_done}"
|
||||
)
|
||||
|
||||
@@ -113,9 +113,7 @@ class TestForwardSplitPrefill(CustomTestCase):
|
||||
)
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids
|
||||
req.logprob_start_len = -1
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
reqs.append(req)
|
||||
|
||||
# Create dummy tree_cache for tests (no prefix caching, just allocation)
|
||||
|
||||
@@ -665,7 +665,6 @@ class TestDisaggregationWire(unittest.TestCase):
|
||||
def test_prebuilt_skips_unused_prompt_tensor(self):
|
||||
req = SimpleNamespace(
|
||||
kv=ReqKvInfo(req_pool_idx=0),
|
||||
prefix_indices=[0, 1],
|
||||
prefix_len=2,
|
||||
extend_range=SimpleNamespace(length=3),
|
||||
origin_input_ids=[0, 1, 2, 3, 4],
|
||||
|
||||
@@ -39,7 +39,7 @@ def _make_mock_req(
|
||||
req_pool_idx: int,
|
||||
kv_committed_len: int,
|
||||
kv_allocated_len: int,
|
||||
prefix_indices_len: int = 0,
|
||||
prefix_len: int = 0,
|
||||
rid: int = 0,
|
||||
origin_len: int = 0,
|
||||
):
|
||||
@@ -54,8 +54,7 @@ def _make_mock_req(
|
||||
kv_committed_len=kv_committed_len,
|
||||
kv_allocated_len=kv_allocated_len,
|
||||
)
|
||||
req.prefix_indices = list(range(prefix_indices_len))
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = prefix_len
|
||||
req.owned_kv_len = lambda: req.kv.kv_committed_len
|
||||
return req
|
||||
|
||||
@@ -203,7 +202,7 @@ class TestReleaseFinishedReq(unittest.TestCase):
|
||||
req_pool_idx=0,
|
||||
kv_committed_len=20,
|
||||
kv_allocated_len=20,
|
||||
prefix_indices_len=5,
|
||||
prefix_len=5,
|
||||
)
|
||||
req.kv.cache_protected_len = 5
|
||||
lock = req.lock
|
||||
|
||||
@@ -27,8 +27,7 @@ class _Req:
|
||||
self.full_untruncated_fill_ids = self.origin_input_ids + array(
|
||||
"q", [0] * block_size
|
||||
)
|
||||
self.prefix_indices = torch.arange(context_len, dtype=torch.int64)
|
||||
self.prefix_len = len(self.prefix_indices)
|
||||
self.prefix_len = context_len
|
||||
self.dllm_block_offset = context_len
|
||||
self.dllm_incomplete_ids = array("q")
|
||||
self.dllm_algo_state = None
|
||||
@@ -202,8 +201,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase):
|
||||
adder._check_prefill_tile_budget = Mock(return_value=None)
|
||||
req = _Req(context_len=300, block_size=256, prefill=True)
|
||||
req.host_hit_length = req.storage_hit_length = 0
|
||||
req.prefix_indices = torch.empty(0, dtype=torch.int64)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 0
|
||||
admission = adder._select_prefill_admission(
|
||||
req,
|
||||
total_tokens=1024,
|
||||
@@ -228,8 +226,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase):
|
||||
adder._update_prefill_budget = Mock()
|
||||
|
||||
req = _Req(context_len=10, block_size=4, prefill=True)
|
||||
req.prefix_indices = torch.arange(7)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 7
|
||||
|
||||
result = PrefillAdder.add_dllm_staging_req(adder, req)
|
||||
|
||||
|
||||
@@ -167,8 +167,7 @@ class _FakeRunner:
|
||||
class _FakeReq:
|
||||
def __init__(self, rid, req_pool_idx=0):
|
||||
self.rid = rid
|
||||
self.prefix_indices = torch.empty(0, dtype=torch.long)
|
||||
self.prefix_len = len(self.prefix_indices)
|
||||
self.prefix_len = 0
|
||||
self.fill_ids = [0]
|
||||
self.kv = ReqKvInfo(req_pool_idx=req_pool_idx)
|
||||
# Mirrors Req's chunk-finality contract read by
|
||||
|
||||
@@ -43,9 +43,7 @@ class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase):
|
||||
class TestSchedulerLoadInquirer(unittest.TestCase):
|
||||
def make_inquirer(self, waiting_queue_prefix_matched):
|
||||
waiting_req = SimpleNamespace(seqlen=100, num_matched_prefix_tokens=20)
|
||||
chunked_req = SimpleNamespace(
|
||||
seqlen=50, prefix_indices=range(10), prefix_len=10
|
||||
)
|
||||
chunked_req = SimpleNamespace(seqlen=50, prefix_len=10)
|
||||
return SimpleNamespace(
|
||||
disaggregation_mode=DisaggregationMode.NULL,
|
||||
get_waiting_queue=lambda: [waiting_req],
|
||||
|
||||
@@ -53,8 +53,7 @@ def _track_seqlen(
|
||||
sampling_params=sampling_params,
|
||||
vocab_size=128,
|
||||
)
|
||||
req.prefix_indices = torch.arange(prefix_len, dtype=torch.int64)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = prefix_len
|
||||
req.set_extend_range(prefix_len, prefix_len + extend_len)
|
||||
req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64)
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import functools
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -124,8 +123,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
req.rid = str(rid)
|
||||
req.cache_request_handle = CacheRequestHandle(req.rid, 0)
|
||||
req.priority = priority
|
||||
req.set_prefix_indices = functools.partial(Req.set_prefix_indices, req)
|
||||
req.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
|
||||
req.prefix_len = 0
|
||||
req.full_untruncated_fill_ids = []
|
||||
req.output_ids = [0] * output_len
|
||||
req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens)
|
||||
@@ -261,8 +259,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
adder = self.create_shortest_prefill_adder(chunk_tokens=512)
|
||||
req = self.create_shared_req("host-miss")
|
||||
req.full_untruncated_fill_ids = list(range(1024))
|
||||
req.prefix_indices = torch.empty(0, dtype=torch.int64)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 0
|
||||
req.host_hit_length = 768
|
||||
req.best_match_node = req.last_node
|
||||
req.needs_host_load_back.return_value = True
|
||||
@@ -843,8 +840,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
|
||||
def create_sharded_req(self, rid, *, prefix_len=12, extend_len=4):
|
||||
req = self.create_shared_req(rid, max_new_tokens=1)
|
||||
req.prefix_indices = list(range(prefix_len))
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = prefix_len
|
||||
req.full_untruncated_fill_ids = list(range(prefix_len + extend_len))
|
||||
return req
|
||||
|
||||
@@ -1043,8 +1039,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
)
|
||||
|
||||
req = self.create_mock_req("chunked", priority=0, max_new_tokens=128)
|
||||
req.prefix_indices = []
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 0
|
||||
req.full_untruncated_fill_ids = list(range(extend_input_len))
|
||||
# set_extend_range is the only writer of extend_range; the production
|
||||
# path reads req.extend_range.length right after calling it, so the mock
|
||||
@@ -1156,8 +1151,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
req = self.create_mock_req(
|
||||
"resume", priority=0, max_new_tokens=40, output_len=10
|
||||
)
|
||||
req.prefix_indices = list(range(PREFIX))
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = PREFIX
|
||||
req.full_untruncated_fill_ids = list(range(PREFIX + EXTEND))
|
||||
req.host_hit_length = 0
|
||||
req.swa_host_hit_length = 0
|
||||
@@ -1207,8 +1201,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.mock_token_allocator, self.mock_tree_cache
|
||||
)
|
||||
req = self.create_mock_req("dropped-fetch", priority=0, max_new_tokens=8)
|
||||
req.prefix_indices = torch.empty(0, dtype=torch.int64)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 0
|
||||
req.full_untruncated_fill_ids = list(range(SPAN))
|
||||
req.host_hit_length = HOST_HIT
|
||||
req.swa_host_hit_length = WINDOW
|
||||
@@ -1250,7 +1243,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
# budget. A successful load must not run admission gates again.
|
||||
_, admitted, req = run(HOST_HIT + 4, remaining_after_load=0)
|
||||
self.assertEqual(len(admitted), 1)
|
||||
self.assertEqual(len(req.prefix_indices), HOST_HIT + 4)
|
||||
self.assertEqual(req.prefix_len, HOST_HIT + 4)
|
||||
self.assertEqual(req.kv.cache_protected_len, HOST_HIT + 4)
|
||||
req.set_extend_range.assert_called_once_with(HOST_HIT + 4, SPAN)
|
||||
# A partial FULL load stays fatal.
|
||||
@@ -1441,8 +1434,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
|
||||
def _create_host_hit_req(self, *, prefix_len=0, host_hit=8192, tail=1024):
|
||||
req = self._create_delayer_req(prefix_len + host_hit + tail)
|
||||
req.prefix_indices = torch.arange(prefix_len)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = prefix_len
|
||||
req.host_hit_length = host_hit
|
||||
req.needs_host_load_back.return_value = True
|
||||
req.best_match_node = req.last_node
|
||||
|
||||
@@ -208,7 +208,6 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase):
|
||||
req_pool_idx=1,
|
||||
cache_protected_len=2,
|
||||
),
|
||||
prefix_indices=torch.tensor([8, 9], dtype=torch.int64),
|
||||
prefix_len=2,
|
||||
priority=3,
|
||||
extra_key=None,
|
||||
|
||||
@@ -242,7 +242,6 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase):
|
||||
req_with_image = types.SimpleNamespace(
|
||||
rid="img",
|
||||
multimodal_inputs=types.SimpleNamespace(num_image_tokens=2),
|
||||
prefix_indices=[],
|
||||
prefix_len=0,
|
||||
extend_range=Range(0, 5),
|
||||
logprob_start_len=0,
|
||||
@@ -250,7 +249,6 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase):
|
||||
req_text_only = types.SimpleNamespace(
|
||||
rid="txt",
|
||||
multimodal_inputs=None,
|
||||
prefix_indices=[],
|
||||
prefix_len=0,
|
||||
extend_range=Range(0, 4),
|
||||
logprob_start_len=0,
|
||||
|
||||
@@ -139,8 +139,7 @@ class TestShortestPrefillFirst(CustomTestCase):
|
||||
req = _make_req(rid, "", list(range(uncached + cached)))
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids[:]
|
||||
req.num_matched_prefix_tokens = cached
|
||||
req.prefix_indices = list(range(cached))
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = cached
|
||||
req.time_stats.wait_queue_entry_time = arrived
|
||||
return req
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def _make_req(
|
||||
*,
|
||||
req_pool_idx: int,
|
||||
fill_ids: list,
|
||||
prefix_indices: torch.Tensor,
|
||||
prefix_len: int,
|
||||
extend_input_len: int,
|
||||
fill_len: int,
|
||||
) -> Req:
|
||||
@@ -34,8 +34,7 @@ def _make_req(
|
||||
req.origin_input_ids = array("q", fill_ids)
|
||||
req.output_ids = array("q")
|
||||
req.full_untruncated_fill_ids = array("q", fill_ids)
|
||||
req.prefix_indices = prefix_indices
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = prefix_len
|
||||
req.extend_range = Range(fill_len - extend_input_len, fill_len)
|
||||
req.inflight_middle_chunks = 0
|
||||
req.host_hit_length = 0
|
||||
@@ -56,8 +55,6 @@ def _make_req(
|
||||
|
||||
|
||||
def _make_req_to_token_pool(num_slots: int, max_context: int) -> SimpleNamespace:
|
||||
# Slot s contains a recognizable fingerprint [s*1000, s*1000+1, ...]
|
||||
# so we can tell a corrupted prefix_indices from a healthy one by content.
|
||||
pool = SimpleNamespace()
|
||||
pool.req_to_token = (
|
||||
torch.arange(max_context, dtype=torch.int32).unsqueeze(0).repeat(num_slots, 1)
|
||||
@@ -119,11 +116,8 @@ def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler:
|
||||
|
||||
|
||||
class TestStashGatePreservesPrefix(CustomTestCase):
|
||||
"""Consumer side: real ChunkCache.checkpoint mutates
|
||||
req.prefix_indices iff stash actually runs, so prefix_indices content
|
||||
is the bug-detection signal. The stash gate is content-based:
|
||||
`fill_len > len(prefix_indices)` means there is freshly computed KV to
|
||||
cache; otherwise the chunk was parked and stashing must be skipped."""
|
||||
"""The stash gate advances prefix_len iff `fill_len > prefix_len`, i.e. the
|
||||
chunk computed new KV; a parked chunk must be left untouched."""
|
||||
|
||||
POOL_IDX = 4
|
||||
INITIAL_PREFIX_LEN = 8 # what was really cached last iter
|
||||
@@ -134,35 +128,31 @@ class TestStashGatePreservesPrefix(CustomTestCase):
|
||||
def _build(self, *, fill_len: int):
|
||||
pool = _make_req_to_token_pool(self.NUM_SLOTS, self.MAX_CONTEXT)
|
||||
cache = _make_chunk_cache(pool)
|
||||
initial_prefix = pool.req_to_token[self.POOL_IDX, : self.INITIAL_PREFIX_LEN].to(
|
||||
dtype=torch.int64, copy=True
|
||||
)
|
||||
req = _make_req(
|
||||
req_pool_idx=self.POOL_IDX,
|
||||
fill_ids=list(range(self.POST_RESET_FILL_LEN)),
|
||||
prefix_indices=initial_prefix,
|
||||
prefix_len=self.INITIAL_PREFIX_LEN,
|
||||
extend_input_len=fill_len - self.INITIAL_PREFIX_LEN,
|
||||
fill_len=fill_len,
|
||||
)
|
||||
s = _scheduler_for_get_next_batch(tree_cache=cache, chunked_req=req)
|
||||
return s, req, initial_prefix, pool
|
||||
return s, req, pool
|
||||
|
||||
def test_parked_chunked_req_keeps_its_prefix(self):
|
||||
# A parked chunk has fill_len == prefix_len: no new KV was computed,
|
||||
# so the gate must skip stash and leave the prefix intact.
|
||||
s, req, initial_prefix, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN)
|
||||
s, req, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN)
|
||||
|
||||
Scheduler.get_next_batch_to_run(
|
||||
s, running_batch=s.running_batch, last_batch=s.last_batch
|
||||
)
|
||||
|
||||
self.assertEqual(req.prefix_len, self.INITIAL_PREFIX_LEN)
|
||||
self.assertTrue(torch.equal(req.prefix_indices, initial_prefix))
|
||||
|
||||
def test_scheduled_chunked_req_advances_prefix_via_real_stash(self):
|
||||
# Symmetric guard against over-gating: when fill_len has advanced past
|
||||
# the cached prefix, stash must run and advance prefix_len.
|
||||
s, req, _, pool = self._build(fill_len=self.POST_RESET_FILL_LEN)
|
||||
s, req, _ = self._build(fill_len=self.POST_RESET_FILL_LEN)
|
||||
|
||||
Scheduler.get_next_batch_to_run(
|
||||
s, running_batch=s.running_batch, last_batch=s.last_batch
|
||||
|
||||
@@ -100,8 +100,7 @@ class MockReq:
|
||||
self.lock = lock
|
||||
self.extra_key = None
|
||||
self.cache_salt = None
|
||||
self.prefix_indices = torch.empty(0, dtype=torch.int64)
|
||||
self.prefix_len = len(self.prefix_indices)
|
||||
self.prefix_len = 0
|
||||
self.priority = 0
|
||||
self.kv = ReqKvInfo(
|
||||
req_pool_idx=req_pool_idx,
|
||||
|
||||
@@ -61,11 +61,14 @@ class _FakeTreeCache:
|
||||
def maybe_hand_to_session(self, req):
|
||||
pass
|
||||
|
||||
def prefix_device_indices(self, req):
|
||||
return req.tree_prefix
|
||||
|
||||
|
||||
def _make_req(rid, prefix, block_size, *, req_pool_idx=None, reuse=False):
|
||||
return SimpleNamespace(
|
||||
rid=rid,
|
||||
prefix_indices=torch.tensor(prefix, dtype=torch.int32),
|
||||
tree_prefix=torch.tensor(prefix, dtype=torch.int32),
|
||||
prefix_len=len(prefix),
|
||||
dllm_incomplete_ids=array("q", range(block_size)) if reuse else array("q"),
|
||||
inflight_middle_chunks=1 if req_pool_idx is not None else 0,
|
||||
@@ -87,10 +90,7 @@ def _remove_allocated_req_slots(pool, *reqs):
|
||||
|
||||
def _make_batch(pool, allocator, reqs, extend_lens):
|
||||
seq_lens_cpu = torch.tensor(
|
||||
[
|
||||
len(req.prefix_indices) + extend_len
|
||||
for req, extend_len in zip(reqs, extend_lens)
|
||||
],
|
||||
[req.prefix_len + extend_len for req, extend_len in zip(reqs, extend_lens)],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
@@ -99,7 +99,7 @@ def _make_batch(pool, allocator, reqs, extend_lens):
|
||||
req_to_token_pool=pool,
|
||||
token_to_kv_pool_allocator=allocator,
|
||||
tree_cache=_FakeTreeCache(allocator),
|
||||
prefix_lens=[len(req.prefix_indices) for req in reqs],
|
||||
prefix_lens=[req.prefix_len for req in reqs],
|
||||
extend_lens=extend_lens,
|
||||
seq_lens=seq_lens_cpu,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
@@ -110,8 +110,8 @@ def _make_batch(pool, allocator, reqs, extend_lens):
|
||||
|
||||
|
||||
def _seed_retained_block(pool, req, values):
|
||||
prefix_len = len(req.prefix_indices)
|
||||
pool.req_to_token[req.kv.req_pool_idx, :prefix_len] = req.prefix_indices
|
||||
prefix_len = req.prefix_len
|
||||
pool.req_to_token[req.kv.req_pool_idx, :prefix_len] = req.tree_prefix
|
||||
pool.req_to_token[req.kv.req_pool_idx, prefix_len : prefix_len + len(values)] = (
|
||||
torch.tensor(values, dtype=torch.int32)
|
||||
)
|
||||
|
||||
@@ -9,7 +9,6 @@ from unittest import mock
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import CacheOperation, HiCacheController
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache import l2_transfer as transfer_module
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
||||
@@ -35,11 +34,6 @@ from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class _ReqStub(SimpleNamespace):
|
||||
set_prefix_indices = Req.set_prefix_indices
|
||||
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
MEMORY_POOL_HOST_MODULE = "sglang.srt.mem_cache.memory_pool_host"
|
||||
@@ -316,10 +310,9 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase):
|
||||
pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices(
|
||||
0, 6
|
||||
)
|
||||
req = _ReqStub(
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
cache_request_handle=handle,
|
||||
prefix_indices=_indices(0, 0),
|
||||
prefix_len=0,
|
||||
kv=SimpleNamespace(cache_protected_len=0),
|
||||
)
|
||||
@@ -333,10 +326,9 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase):
|
||||
pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices(
|
||||
0, 6
|
||||
)
|
||||
req = _ReqStub(
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
cache_request_handle=handle,
|
||||
prefix_indices=_indices(0, 0),
|
||||
prefix_len=0,
|
||||
kv=SimpleNamespace(cache_protected_len=0),
|
||||
)
|
||||
|
||||
@@ -247,12 +247,6 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase):
|
||||
|
||||
req.set_extend_range = set_extend_range
|
||||
|
||||
def set_prefix_indices(prefix_indices):
|
||||
req.prefix_indices = prefix_indices
|
||||
req.prefix_len = len(prefix_indices)
|
||||
|
||||
req.set_prefix_indices = set_prefix_indices
|
||||
|
||||
class ReqToTokenPool:
|
||||
def __init__(self):
|
||||
self.writes = []
|
||||
|
||||
@@ -784,8 +784,7 @@ class _GraftReq:
|
||||
self.kv = ReqKvInfo(req_pool_idx=req_pool_idx)
|
||||
self.extra_key = None
|
||||
self.cache_salt = None
|
||||
self.prefix_indices = torch.empty(0, dtype=torch.int64)
|
||||
self.prefix_len = len(self.prefix_indices)
|
||||
self.prefix_len = 0
|
||||
self.last_node = None
|
||||
self.priority = 0
|
||||
self.kv_rotation_base = None
|
||||
|
||||
@@ -204,9 +204,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
|
||||
for page_size in (4, 64):
|
||||
with self.subTest(page_size=page_size):
|
||||
allocator, req, adder = self._new_admission(page_size, pool_pages=6)
|
||||
req.prefix_indices = allocator.alloc(page_size)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
self.assertIsNotNone(req.prefix_indices)
|
||||
req.prefix_len = len(allocator.alloc(page_size))
|
||||
self.assertTrue(allocator.can_reserve(page_size + 2, page_size + 2))
|
||||
|
||||
self.assertIsNone(adder.add_chunked_req(req))
|
||||
@@ -219,7 +217,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
|
||||
allocator, req, adder = self._new_admission(
|
||||
page_size, pool_pages=5, ignore_eos=True
|
||||
)
|
||||
self.assertEqual(len(req.prefix_indices), 0)
|
||||
self.assertEqual(req.prefix_len, 0)
|
||||
self.assertTrue(allocator.can_reserve(2 * page_size + 2, 2 * page_size))
|
||||
|
||||
adder.add_one_req(
|
||||
|
||||
@@ -16,7 +16,6 @@ from types import SimpleNamespace
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
InsertParams,
|
||||
MatchPrefixParams,
|
||||
@@ -33,7 +32,6 @@ class _StubReq:
|
||||
self.output_ids = array("q")
|
||||
self.extra_key = None
|
||||
self.cache_salt = None
|
||||
self.prefix_indices = None
|
||||
self.prefix_len = 0
|
||||
self.last_node = None
|
||||
self.last_host_node = None
|
||||
@@ -43,8 +41,6 @@ class _StubReq:
|
||||
self.swa_branching_seqlen = None
|
||||
self.kv = SimpleNamespace(cache_protected_len=None)
|
||||
|
||||
set_prefix_indices = Req.set_prefix_indices
|
||||
|
||||
def _compute_max_prefix_len(self, input_len):
|
||||
return max(input_len - 1, 0)
|
||||
|
||||
@@ -117,14 +113,14 @@ class TestMatchKvCacheForceMiss(unittest.TestCase):
|
||||
baseline_req = _StubReq([10, 11, 12, 13, 99, 100])
|
||||
with envs.SGLANG_RADIX_FORCE_MISS.override(False):
|
||||
match_kv_cache(baseline_req, tree)
|
||||
self.assertGreater(int(baseline_req.prefix_indices.numel()), 0)
|
||||
self.assertGreater(baseline_req.prefix_len, 0)
|
||||
self.assertIsNot(baseline_req.last_node, tree.root_node)
|
||||
|
||||
# With the flag, the same lookup is forced to miss.
|
||||
forced_req = _StubReq([10, 11, 12, 13, 99, 100])
|
||||
with envs.SGLANG_RADIX_FORCE_MISS.override(True):
|
||||
match_kv_cache(forced_req, tree)
|
||||
self.assertEqual(int(forced_req.prefix_indices.numel()), 0)
|
||||
self.assertEqual(forced_req.prefix_len, 0)
|
||||
self.assertIs(forced_req.last_node, tree.root_node)
|
||||
self.assertIs(forced_req.last_host_node, tree.root_node)
|
||||
self.assertEqual(forced_req.host_hit_length, 0)
|
||||
|
||||
@@ -11,7 +11,7 @@ from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import Req, split_cached_prefix_by_tier
|
||||
from sglang.srt.managers.schedule_batch import split_cached_prefix_by_tier
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestHandle,
|
||||
InitLoadBackParams,
|
||||
@@ -33,11 +33,6 @@ from sglang.srt.mem_cache.unified_radix_cache import (
|
||||
from sglang.srt.mem_cache.utils import get_hash_str
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
|
||||
class _ReqStub(SimpleNamespace):
|
||||
set_prefix_indices = Req.set_prefix_indices
|
||||
|
||||
|
||||
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
|
||||
|
||||
_REQ = CacheRequestHandle("r", 0)
|
||||
@@ -52,6 +47,7 @@ def _staged_fixture(full_match=2):
|
||||
prefetch_anchor_info=lambda node: (None, None),
|
||||
match_full_device_prefix=Mock(return_value=(full_match, 1, full_match)),
|
||||
collect_full_device_indices=Mock(return_value=torch.arange(8)),
|
||||
root_node_handle=lambda extra_key=None: 0,
|
||||
inc_full_pin=Mock(),
|
||||
dec_full_pin=Mock(),
|
||||
empty_match_result=SimpleNamespace(
|
||||
@@ -114,10 +110,9 @@ def _staged_fixture(full_match=2):
|
||||
hash_values=["a", "b", "c"],
|
||||
operation_id=1,
|
||||
)
|
||||
req = _ReqStub(
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
cache_request_handle=_REQ,
|
||||
prefix_indices=torch.arange(2),
|
||||
prefix_len=2,
|
||||
last_node=1,
|
||||
kv=SimpleNamespace(cache_protected_len=2),
|
||||
@@ -207,8 +202,7 @@ def _two_rank_retry_trace(rank, rendezvous):
|
||||
1,
|
||||
full_match,
|
||||
)
|
||||
req.prefix_indices = torch.arange(full_match)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = full_match
|
||||
if cache.buffer_pipeline.prepare_staged_prefetch(req):
|
||||
assert (
|
||||
cache.init_load_back(
|
||||
@@ -309,8 +303,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
len(RadixKey(held.key_tokens, is_bigram=bigram)), 10
|
||||
)
|
||||
# A joint match counts bigrams, not their extra raw boundary token.
|
||||
req.prefix_indices = torch.arange(10)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 10
|
||||
req.kv.cache_protected_len = 10
|
||||
self.assertTrue(pipeline.prepare_staged_prefetch(req))
|
||||
self.assertFalse(pipeline.has_staged(req.cache_request_handle))
|
||||
@@ -352,8 +345,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
|
||||
# A twin made [0, 6) jointly reusable while the fetch was held.
|
||||
cache.tree_core.match_full_device_prefix.return_value = (8, 1, 8)
|
||||
req.prefix_indices = torch.arange(6)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 6
|
||||
self.assertTrue(pipeline.prepare_staged_prefetch(req))
|
||||
self.assertEqual(
|
||||
split_cached_prefix_by_tier(
|
||||
@@ -390,13 +382,12 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
cache.tree_core.match_full_device_prefix.assert_called_once()
|
||||
# A twin finished first: the next pass's joint match runs past the
|
||||
# staged span and is kept as is (a shrink would strand its recompute).
|
||||
req.prefix_indices = torch.arange(12)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 12
|
||||
req.last_node = 9
|
||||
req.kv.cache_protected_len = 12
|
||||
self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
self.assertIsNone(req.staged_prefetch_plan)
|
||||
self.assertEqual(len(req.prefix_indices), 12)
|
||||
self.assertEqual(req.prefix_len, 12)
|
||||
self.assertEqual((req.last_node, req.kv.cache_protected_len), (9, 12))
|
||||
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0))
|
||||
self.assertFalse(pipeline.has_staged(req.cache_request_handle))
|
||||
@@ -425,8 +416,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
anchor = pipeline.anchor_locks[req.cache_request_handle]
|
||||
cache.tree_core.match_full_device_prefix.reset_mock()
|
||||
for attempt in range(1, 4):
|
||||
req.prefix_indices = torch.arange(2)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 2
|
||||
self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
self.assertIsNone(
|
||||
cache.init_load_back(
|
||||
@@ -467,16 +457,14 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
|
||||
# Tree changes occur while queued, before the next preparation pass.
|
||||
cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6)
|
||||
req.prefix_indices = torch.arange(2)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 2
|
||||
self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4))
|
||||
self.assertTrue(pipeline.has_staged(req.cache_request_handle))
|
||||
self.assertEqual(cache.storage_prefetch_retries.pop_ready([req], 0, 8), [])
|
||||
|
||||
cache.tree_core.match_full_device_prefix.return_value = (0, 0, 0)
|
||||
req.prefix_indices = torch.arange(0)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 0
|
||||
self.assertFalse(cache.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
self.assertFalse(pipeline.has_staged(req.cache_request_handle))
|
||||
self.assertEqual(
|
||||
@@ -528,8 +516,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
pipeline.max_staged_admission_defers = 3
|
||||
params = lambda: InitLoadBackParams(None, req.host_hit_length, req=req)
|
||||
for attempt in range(1, 4):
|
||||
req.prefix_indices = torch.arange(2)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 2
|
||||
self.assertTrue(pipeline.prepare_staged_prefetch(req))
|
||||
self.assertIsNone(cache.init_load_back(params()))
|
||||
self.assertEqual(pipeline.has_staged(req.cache_request_handle), attempt < 3)
|
||||
|
||||
@@ -1296,6 +1296,7 @@ def test_linker_load_preserves_swa_boundaries(
|
||||
)
|
||||
),
|
||||
resolve_node_handle=lambda node_id: SimpleNamespace(id=0),
|
||||
prefix_device_indices=lambda req: torch.empty(0, dtype=torch.int64),
|
||||
)
|
||||
wrapper = UnifiedCacheLinkerWrapper(cache, _FakeLinker())
|
||||
wrapper.hit_markers["rid"] = ExternalCacheHitMarker(
|
||||
@@ -1315,7 +1316,6 @@ def test_linker_load_preserves_swa_boundaries(
|
||||
req = SimpleNamespace(
|
||||
rid="rid",
|
||||
kv=kv,
|
||||
prefix_indices=torch.empty(0, dtype=torch.int64),
|
||||
prefix_len=0,
|
||||
last_node=0,
|
||||
priority=0,
|
||||
|
||||
@@ -634,9 +634,7 @@ def bench_release(
|
||||
req.origin_input_ids = array("q", seq)
|
||||
req.output_ids = array("q")
|
||||
req.full_untruncated_fill_ids = array("q", seq)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
req.last_node = node
|
||||
req.kv.cache_protected_len = matched_len
|
||||
req.kv.kv_committed_len = len(seq)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Unit tests for UnifiedRadixCache"""
|
||||
|
||||
import atexit
|
||||
import functools
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
@@ -122,11 +121,6 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.mem_cache_utils import finish_req
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class _ReqStub(SimpleNamespace):
|
||||
set_prefix_indices = Req.set_prefix_indices
|
||||
|
||||
|
||||
register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
@@ -1602,8 +1596,7 @@ class UnifiedRadixCacheSuite:
|
||||
return req
|
||||
|
||||
def _apply_match_to_req(self, req, match):
|
||||
req.prefix_indices = match.device_indices
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = len(match.device_indices)
|
||||
req.last_node = match.last_device_node
|
||||
req.last_host_node = match.last_host_node
|
||||
req.best_match_node = match.best_match_node
|
||||
@@ -1860,9 +1853,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.lock = None
|
||||
req.extra_key = None
|
||||
req.full_untruncated_fill_ids = array("q", input_ids + output_ids)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
if self.cfg.has_mamba:
|
||||
req.kv.mamba_last_track_seqlen = kv_len
|
||||
|
||||
@@ -1908,9 +1899,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.origin_input_ids = array("q", prompt_ids)
|
||||
req.output_ids = array("q", output_ids)
|
||||
req.full_untruncated_fill_ids = array("q", prompt_ids + output_ids)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
kv_len = req.extend_range.end
|
||||
kv_indices = self._alloc(allocator, kv_len)
|
||||
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices)
|
||||
@@ -1983,9 +1972,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.lock = None
|
||||
req.extra_key = None
|
||||
req.full_untruncated_fill_ids = array("q", tokens)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
|
||||
avail_before = allocator.available_size()
|
||||
release_kv_cache(req, cache, checkpoint=False)
|
||||
@@ -2003,9 +1990,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.origin_input_ids = array("q", tokens)
|
||||
req.output_ids = array("q")
|
||||
req.full_untruncated_fill_ids = array("q", tokens)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
kv_len = len(tokens)
|
||||
kv_indices = self._alloc(allocator, kv_len)
|
||||
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices)
|
||||
@@ -2149,9 +2134,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.lock = None
|
||||
req.extra_key = None
|
||||
req.full_untruncated_fill_ids = array("q", input_ids)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
if self.cfg.has_mamba:
|
||||
req.kv.mamba_last_track_seqlen = kv_len
|
||||
|
||||
@@ -3452,7 +3435,6 @@ class UnifiedRadixCacheSuite:
|
||||
cache,
|
||||
req_id,
|
||||
prefix_len=None,
|
||||
prefix_indices=None,
|
||||
timeout: float = 10.0,
|
||||
*,
|
||||
extra_key=None,
|
||||
@@ -3476,20 +3458,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.cache_request_handle = req_id
|
||||
req.extra_key = extra_key
|
||||
req.cache_salt = cache_salt
|
||||
if prefix_indices is not None:
|
||||
# Spliceable mid-anchor consumption publishes value=cat(prefix,
|
||||
# fill) — the real device prefix is required (zeros would insert
|
||||
# bogus slots into the tree).
|
||||
assert len(prefix_indices) == prefix_len
|
||||
req.prefix_indices = prefix_indices
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
else:
|
||||
req.prefix_indices = torch.zeros(
|
||||
prefix_len,
|
||||
dtype=torch.int64,
|
||||
device=cache.tree_core.empty_match_result.device_indices.device,
|
||||
)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = prefix_len
|
||||
req.last_node = cache.root_node_handle() if last_node is None else last_node
|
||||
joint = cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
@@ -3498,8 +3467,7 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
)
|
||||
)
|
||||
req.prefix_indices = joint.device_indices
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = len(joint.device_indices)
|
||||
joint_covers_span = len(joint.device_indices) >= len(f.key_tokens)
|
||||
adder = PrefillAdder.__new__(PrefillAdder)
|
||||
adder.tree_cache = cache
|
||||
@@ -4517,12 +4485,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.extra_key = None
|
||||
req.cache_salt = None
|
||||
req.last_node = cons.root_node_handle()
|
||||
req.prefix_indices = torch.zeros(
|
||||
held.matched_len,
|
||||
dtype=torch.int64,
|
||||
device=cons.tree_core.empty_match_result.device_indices.device,
|
||||
)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = held.matched_len
|
||||
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
spliced, _last = cons.init_load_back(
|
||||
InitLoadBackParams(
|
||||
@@ -4707,7 +4670,6 @@ class UnifiedRadixCacheSuite:
|
||||
cons2,
|
||||
anchored_req,
|
||||
prefix_len=len(prefix),
|
||||
prefix_indices=prefix_match.device_indices,
|
||||
extra_key=extra_key,
|
||||
cache_salt=cache_salt,
|
||||
last_node=anchor,
|
||||
@@ -4886,10 +4848,9 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertIsNotNone(cons_alloc.swa_attn_allocator.alloc(hold // ps * ps))
|
||||
|
||||
# The adder's SWA gate for this request (_swa_budget_for_req).
|
||||
req = _ReqStub(
|
||||
req = SimpleNamespace(
|
||||
rid=req_id.rid,
|
||||
cache_request_handle=req_id,
|
||||
prefix_indices=cons.tree_core.empty_match_result.device_indices,
|
||||
prefix_len=len(cons.tree_core.empty_match_result.device_indices),
|
||||
kv=SimpleNamespace(cache_protected_len=0),
|
||||
)
|
||||
@@ -4911,12 +4872,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.extra_key = None
|
||||
req.cache_salt = None
|
||||
req.last_node = cons.root_node_handle()
|
||||
req.prefix_indices = torch.zeros(
|
||||
held.matched_len,
|
||||
dtype=torch.int64,
|
||||
device=cons.tree_core.empty_match_result.device_indices.device,
|
||||
)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = held.matched_len
|
||||
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
spliced, last_node = cons.init_load_back(
|
||||
InitLoadBackParams(
|
||||
@@ -5141,7 +5097,7 @@ class UnifiedRadixCacheSuite:
|
||||
return_req=True,
|
||||
)
|
||||
self.assertEqual(int(spliced.numel()), 0)
|
||||
self.assertEqual(len(req.prefix_indices), len(seq), "FULL rematch was lost")
|
||||
self.assertEqual(req.prefix_len, len(seq), "FULL rematch was lost")
|
||||
self.assertEqual(len(dispatched["device"]), window_tokens)
|
||||
self.assertEqual(cons_alloc.full_available_size(), full_avail0)
|
||||
k, v = self._snapshot_full_kv(cons_alloc, masked_full)
|
||||
@@ -5219,7 +5175,7 @@ class UnifiedRadixCacheSuite:
|
||||
spliced, req = self._consume_staged_prefetch(
|
||||
cons, req_id, prefix_len=0, return_req=True
|
||||
)
|
||||
self.assertEqual(len(req.prefix_indices), split_at)
|
||||
self.assertEqual(req.prefix_len, split_at)
|
||||
self.assertEqual(int(spliced.numel()), page_size)
|
||||
self.assertEqual(cons_alloc.full_available_size(), full_avail0 - page_size)
|
||||
k, v = self._snapshot_full_kv(cons_alloc, head_full)
|
||||
@@ -5351,12 +5307,7 @@ class UnifiedRadixCacheSuite:
|
||||
req.cache_request_handle = req_id
|
||||
req.extra_key = None
|
||||
req.cache_salt = None
|
||||
req.prefix_indices = torch.zeros(
|
||||
0,
|
||||
dtype=torch.int64,
|
||||
device=cons.tree_core.empty_match_result.device_indices.device,
|
||||
)
|
||||
req.prefix_len = len(req.prefix_indices)
|
||||
req.prefix_len = 0
|
||||
req.last_node = cons.root_node_handle()
|
||||
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
with mock.patch.object(cons.cache_controller, "load", adversarial_load):
|
||||
@@ -5442,10 +5393,9 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertGreater(aux_max, len(seq) - len(head))
|
||||
|
||||
# The surfaced host hit is the splice-able tail, not the full span.
|
||||
req = _ReqStub(
|
||||
req = SimpleNamespace(
|
||||
rid=req_id.rid,
|
||||
cache_request_handle=req_id,
|
||||
prefix_indices=sib.device_indices,
|
||||
prefix_len=len(sib.device_indices),
|
||||
kv=SimpleNamespace(cache_protected_len=len(sib.device_indices)),
|
||||
)
|
||||
@@ -5453,9 +5403,7 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertEqual(req.host_hit_length, len(seq) - len(head))
|
||||
self.assertTrue(cons.buffer_pipeline.has_staged(req_id))
|
||||
|
||||
spliced = self._consume_staged_prefetch(
|
||||
cons, req_id, prefix_len=len(head), prefix_indices=sib.device_indices
|
||||
)
|
||||
spliced = self._consume_staged_prefetch(cons, req_id, prefix_len=len(head))
|
||||
self.assertEqual(int(spliced.numel()), len(seq) - len(head))
|
||||
|
||||
m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq))))
|
||||
@@ -5513,11 +5461,9 @@ class UnifiedRadixCacheSuite:
|
||||
req = mock.Mock(
|
||||
rid=req_id.rid,
|
||||
cache_request_handle=req_id,
|
||||
prefix_indices=live.device_indices,
|
||||
prefix_len=len(live.device_indices),
|
||||
last_node=live.last_device_node,
|
||||
)
|
||||
req.set_prefix_indices = functools.partial(Req.set_prefix_indices, req)
|
||||
self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req))
|
||||
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0))
|
||||
self.assertFalse(cons.buffer_pipeline.has_staged(req_id))
|
||||
@@ -5672,7 +5618,6 @@ class UnifiedRadixCacheSuite:
|
||||
spliced = self._consume_staged_prefetch(
|
||||
cons2,
|
||||
CacheRequestHandle("window-req", 0),
|
||||
prefix_indices=m.device_indices,
|
||||
)
|
||||
self.assertEqual(int(spliced.numel()), len(seq_ab) - len(seq_a))
|
||||
self.assertEqual(
|
||||
@@ -7687,7 +7632,7 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
|
||||
self.assertEqual(new_node, leaf)
|
||||
self.assertEqual(len(torch.cat([req.prefix_indices, new_indices])), len(tokens))
|
||||
self.assertEqual(req.prefix_len + len(new_indices), len(tokens))
|
||||
self.assertIsNotNone(_device_value(cache, leaf, ComponentType.MAMBA))
|
||||
self._finish_pending_loads(cache)
|
||||
self._release_ongoing_load_back_locks(cache)
|
||||
@@ -7729,7 +7674,7 @@ class UnifiedRadixCacheSuite:
|
||||
|
||||
self.assertEqual(new_node, leaf)
|
||||
self.assertEqual(new_indices.tolist(), leaf_full.tolist())
|
||||
self.assertEqual(len(torch.cat([req.prefix_indices, new_indices])), len(tokens))
|
||||
self.assertEqual(req.prefix_len + len(new_indices), len(tokens))
|
||||
self.assertEqual(
|
||||
_device_value(cache, leaf, ComponentType.FULL).tolist(),
|
||||
leaf_full.tolist(),
|
||||
@@ -8017,7 +7962,7 @@ class UnifiedRadixCacheSuite:
|
||||
sampling_params=SamplingParams(temperature=0, max_new_tokens=1),
|
||||
)
|
||||
req.init_next_round_input(cache)
|
||||
self.assertEqual((len(req.prefix_indices), req.host_hit_length), (0, 0))
|
||||
self.assertEqual((req.prefix_len, req.host_hit_length), (0, 0))
|
||||
self.assertEqual(req.mamba_host_hit_length, 1)
|
||||
adder = PrefillAdder(
|
||||
page_size=1,
|
||||
@@ -8034,7 +7979,7 @@ class UnifiedRadixCacheSuite:
|
||||
# 18-token demand would reject and scheduler cleanup would free the CoW
|
||||
# destination even though the queued H2D still targets it.
|
||||
self.assertEqual(adder.can_run_list, [req])
|
||||
self.assertEqual(len(req.prefix_indices), 12)
|
||||
self.assertEqual(req.prefix_len, 12)
|
||||
self.assertEqual(req.kv.cache_protected_len, 12)
|
||||
slot = req.kv.mamba_pool_idx.unsqueeze(0)
|
||||
other_slots = pool.mamba_allocator.alloc(pool.mamba_allocator.available_size())
|
||||
|
||||
Reference in New Issue
Block a user