diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index 31a5f7c71d..236dfd91b8 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -438,11 +438,8 @@ def prepare_extend_inputs_for_correctness_test( req.full_untruncated_fill_ids.extend(input_ids[i][bench_args.cut_len :]) if model_runner is not None: # Use req.kv.req_pool_idx instead of i to handle slot 0 padding correctly - req.set_prefix_indices( - model_runner.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, : bench_args.cut_len - ].to(torch.int64) - ) + # The cut prefix is already in the request's row from the first extend. + req.prefix_len = bench_args.cut_len req.logprob_start_len = -1 req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) return reqs diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index db9c0f1e52..464be0aca1 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -145,7 +145,7 @@ def _bootstrap_addr(req: Req) -> str: def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None: """Start a decode-radix request that owns its whole KV row at the root.""" - req.set_prefix_indices(torch.empty((0,), dtype=torch.int64)) + req.prefix_len = 0 req.last_node = tree_cache.root_node_handle(req.extra_key) req.last_host_node = req.last_node req.best_match_node = req.last_node @@ -2204,9 +2204,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids # Decode-disagg allocates before batch assembly, so it binds the prefix # here instead of in init_next_round_input. - req.set_prefix_indices( - prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64) - ) + req.prefix_len = prefix_len req.set_extend_range(total_prefix_len, req.kv.kv_committed_len) self.tree_cache.maybe_hand_to_session(req) diff --git a/python/sglang/srt/disaggregation/decode_hicache_mixin.py b/python/sglang/srt/disaggregation/decode_hicache_mixin.py index 86cbb475ab..2538f30819 100644 --- a/python/sglang/srt/disaggregation/decode_hicache_mixin.py +++ b/python/sglang/srt/disaggregation/decode_hicache_mixin.py @@ -211,7 +211,7 @@ class DecodeHiCacheTransferMixin: return False self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle) - # Re-match: req.last_node / prefix_indices updated to current device state. + # Re-match: req.last_node / prefix_len updated to current device state. rematch = match_kv_cache( dr.req, self.tree_cache, @@ -316,11 +316,7 @@ class DecodeHiCacheTransferMixin: ), decode_req.hicache_restored_kv_indices, ) - req.set_prefix_indices( - torch.cat( - [prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices] - ) - ) + req.prefix_len = prefix_match.decode_prefix_len req.last_node = restore_lock.node req.lock = restore_lock # Prevent abort cleanup from releasing the transferred lock. diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 2aef3c5793..dbcb736d00 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1176,14 +1176,9 @@ class Req(ReqDllmMixin): self.mm_video_tokens: int = 0 # Prefix info - # Tokens [0, prefix_len) are already in the row or come from a match; - # the next extend starts at prefix_len. + # Tokens [0, prefix_len) are already in the row, or, before allocation, + # on the path to last_node; the next extend starts at prefix_len. self.prefix_len: int = 0 - # The matched prefix's KV indices, consumed when the next allocation - # writes them into the row; None once written. - self.prefix_indices: Optional[torch.Tensor] = torch.empty( - (0,), dtype=torch.int64 - ) # TODO(ispobock): rename to last_device_node self.last_node: Any = None self.last_host_node: Any = None @@ -1571,10 +1566,6 @@ class Req(ReqDllmMixin): def set_extend_range(self, start: int, end: int) -> None: self.extend_range = Range(start, end) - def set_prefix_indices(self, prefix_indices: torch.Tensor) -> None: - self.prefix_indices = prefix_indices - self.prefix_len = len(prefix_indices) - def get_fill_ids(self) -> array: return self.full_untruncated_fill_ids[: self.extend_range.end] @@ -1934,7 +1925,7 @@ class Req(ReqDllmMixin): # since we are tracking the total number of retractions for each request. self.retraction_count += 1 - self.set_prefix_indices(torch.empty((0,), dtype=torch.int64)) + self.prefix_len = 0 self.routed_experts = None self.indexer_topk = None self.last_node = None diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 78dec2d4b7..b15c226f3b 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -1412,7 +1412,7 @@ class PrefillAdder: ) if isinstance(admission, AddReqResult): return admission - req.set_prefix_indices(torch.cat([req.prefix_indices, new_indices])) + req.prefix_len += len(new_indices) req.kv.cache_protected_len = req.prefix_len # Sharded pools cannot load host KV; reserve scratch after all other gates. diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index 02292cfc11..1012d7f124 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -48,6 +48,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_EMPTY_PREFIX = torch.empty((0,), dtype=torch.int64) + def write_cache_indices( out_cache_loc: torch.Tensor, @@ -348,6 +350,17 @@ def _alloc_page_size(batch: ScheduleBatch) -> int: return batch.tree_cache.page_size +def _prefix_kv_indices(batch: ScheduleBatch, req: Req) -> torch.Tensor: + # A request holding its row (a later chunk, a borrowed session record) + # already has its prefix there; a new one reads its match off the tree. + if req.kv.holds_kv: + row = batch.req_to_token_pool.req_to_token[req.kv.req_pool_idx] + return row[: req.prefix_len].to(torch.int64) + if req.prefix_len == 0: + return _EMPTY_PREFIX + return batch.tree_cache.prefix_device_indices(req) + + def alloc_for_extend( batch: ScheduleBatch, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: @@ -361,17 +374,7 @@ def alloc_for_extend( # free out-of-window swa tokens batch.maybe_evict_swa() - # A request holding its row (a later chunk, a borrowed session record) - # already has its prefix there; a new one writes its matched prefix. - req_to_token = batch.req_to_token_pool.req_to_token - prefix_tensors = [ - ( - req_to_token[r.kv.req_pool_idx, : r.prefix_len].to(torch.int64) - if r.kv.holds_kv - else r.prefix_indices - ) - for r in batch.reqs - ] + prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs] reuse_kv = None if batch.is_dllm(): @@ -465,7 +468,6 @@ def alloc_for_extend( for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()): req.kv.kv_allocated_len = seq_len req.kv.kv_committed_len = seq_len - req.prefix_indices = None batch.tree_cache.maybe_hand_to_session(req) return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 7bed675515..a871059f62 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -528,6 +528,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait): if lock is not None: self.dec_lock_ref(lock.node, lock.receipt) + def prefix_device_indices(self, req: Req) -> torch.Tensor: + """KV indices of req's matched prefix, read off the path to its locked + match node; valid from match until allocation writes them into the row.""" + raise NotImplementedError + def maybe_hand_to_session(self, req: Req) -> None: """A cache that keeps records across requests (a streaming session) takes the just-allocated row and the request's tree lock; the request borrows it.""" diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index 78361076ed..5c378616e4 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -1249,12 +1249,7 @@ class BufferModePipeline: ) self._refetch_staged(f) return False - root_id = self._cache.tree_core.empty_match_result.last_device_node - full_indices = self._cache.tree_core.collect_full_device_indices( - node_id, root_id - )[:matched_len] - assert len(full_indices) == matched_len - req.set_prefix_indices(full_indices) + req.prefix_len = matched_len req.last_node = node_id req.kv.cache_protected_len = matched_len full_tokens = max(0, f.matched_len + f.num_tokens - matched_len) @@ -1441,6 +1436,7 @@ class BufferModePipeline: splice_base = plan.device_prefix_len assert req.prefix_len == splice_base + prefix_indices = self._cache.prefix_device_indices(req) trim_tokens = splice_base - f.matched_len assert trim_tokens % cache.page_size == 0, ( f"staged splice trim not page-aligned req={req.rid}: " @@ -1497,7 +1493,7 @@ class BufferModePipeline: anchor_parts = [] host_parts = [] for repair_start, repair_end_ in repair_ranges: - anchor_parts.append(req.prefix_indices[repair_start:repair_end_]) + anchor_parts.append(prefix_indices[repair_start:repair_end_]) host_parts.append( slice(repair_start - window_start, repair_end_ - window_start) ) @@ -1560,16 +1556,14 @@ class BufferModePipeline: key, repair_start, repair_end_, - req.prefix_indices[repair_start:repair_end_], + prefix_indices[repair_start:repair_end_], ): cache._apply_cache_action(action) elif swa_dev is not None: # Register the window's FULL->SWA translation now (attention reads # through it). Keep SWA slots another request may still hold; their # redundant H2D destinations are reclaimed at the transfer ack. - full_window = torch.cat([req.prefix_indices, device_indices])[ - -len(swa_dev) : - ] + full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :] allocator = cache.token_to_kv_pool_allocator old_swa = allocator.translate_swa_indices_for_transfer(full_window) missing = old_swa <= 0 @@ -1619,7 +1613,7 @@ class BufferModePipeline: insert_result = cache.insert( InsertParams( key=key, - value=torch.cat([req.prefix_indices, device_indices]), + value=torch.cat([prefix_indices, device_indices]), prev_prefix_len=splice_base, component_evicted_seqlens={ ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0 diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 22106735bc..b01ae03028 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -209,7 +209,7 @@ def match_kv_cache( match_result = zero_match_result( tree_cache, match_result, extra_key=req.extra_key ) - req.set_prefix_indices(match_result.device_indices) + req.prefix_len = len(match_result.device_indices) ( req.last_node, req.last_host_node, diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 6753427769..e1d25aa7c0 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -624,6 +624,17 @@ class RadixCache(BasePrefixCache): ##### Internal Helper Functions ##### + def prefix_device_indices(self, req: Req) -> torch.Tensor: + values = [] + node = req.last_node + while node is not self.root_node: + values.append(node.value) + node = node.parent + values.reverse() + path = torch.cat(values) + assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len) + return path[: req.prefix_len] + def _match_prefix_helper(self, node: TreeNode, key: RadixKey): access_time = time.monotonic() node.last_access_time = access_time diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index b56ecb57a5..37d817cac9 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -87,6 +87,20 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): # LMCache rebuilds lookup keys from tokens, so any L1 node is valid. return True + def prefix_device_indices(self, req: Req) -> torch.Tensor: + # match_prefix may append loaded slots not yet published to the tree, + # so the path alone can be shorter than the prefix. + root = self.root_node_handle(req.extra_key) + path = self.tree_core.collect_full_device_indices(req.last_node, root) + indices = path[: req.prefix_len] + missing = req.prefix_len - len(indices) + if missing > 0: + load = self._external_flows[req.rid].load + skip = max(len(indices) - load.local_hit_tokens, 0) + indices = torch.cat([indices, load.device_indices[skip : skip + missing]]) + assert len(indices) == req.prefix_len, (req.rid, len(indices), req.prefix_len) + return indices + def match_prefix(self, params: MatchPrefixParams) -> MatchResult: """Report LMCache hits through host fields until GPU slots are ready.""" requested_key_len = len(params.key) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py index a3734c278d..c3e6d213f0 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py @@ -338,7 +338,7 @@ class UnifiedCacheLinkerWrapper: # Insert the newly loaded tail into the tree. prefix_indices = torch.cat( - [req.prefix_indices.to(torch.int64), full_transfer.device_indices] + [cache.prefix_device_indices(req), full_transfer.device_indices] ) mamba_transfer = next( ( diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 0b8ff186d9..6f2b9774ed 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3801,6 +3801,12 @@ class UnifiedRadixCache(BasePrefixCache): # Internal callers (and the session sentinel / None) pass a non-int through. return node_handle + def prefix_device_indices(self, req: Req) -> torch.Tensor: + root = self.root_node_handle(req.extra_key) + path = self.tree_core.collect_full_device_indices(req.last_node, root) + assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len) + return path[: req.prefix_len] + def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId: """The root's NodeId -- URC match results carry NodeIds.""" return self.tree_core.root_node_handle(extra_key) diff --git a/test/manual/chunked_prefill/test_scripted_hybrid_swa.py b/test/manual/chunked_prefill/test_scripted_hybrid_swa.py index 1dd80b8a98..99f79f50cb 100644 --- a/test/manual/chunked_prefill/test_scripted_hybrid_swa.py +++ b/test/manual/chunked_prefill/test_scripted_hybrid_swa.py @@ -96,9 +96,9 @@ class TestSWABasic(ScriptedTestCase): r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2) for _ in range(400): if r.is_chunking: - assert len(r.req.prefix_indices) <= r.req.kv.kv_committed_len, ( + assert r.req.prefix_len <= r.req.kv.kv_committed_len, ( f"prefix_indices must be bounded by kv_committed_len, " - f"got prefix_indices_len={len(r.req.prefix_indices)}, " + f"got prefix_indices_len={r.req.prefix_len}, " f"kv_committed_len={r.req.kv.kv_committed_len}" ) if r.finished: diff --git a/test/manual/chunked_prefill/test_scripted_radix.py b/test/manual/chunked_prefill/test_scripted_radix.py index 430f7f7354..0584ce87a1 100644 --- a/test/manual/chunked_prefill/test_scripted_radix.py +++ b/test/manual/chunked_prefill/test_scripted_radix.py @@ -303,7 +303,7 @@ class TestRadixNoTailChunked(ScriptedTestCase): req = s.chunked_req if req is not None and req.rid == r.rid: observed_mid_chunk = True - prefix_len: int = len(req.prefix_indices) + prefix_len: int = req.prefix_len protected_len: int = req.kv.cache_protected_len assert prefix_len == protected_len, ( f"page_size=1 must take the no-tail else branch: " @@ -459,7 +459,7 @@ class TestRadixPartialPage(ScriptedTestCase): for _ in range(800): req = s.chunked_req if req is not None and req.rid == r.rid: - prefix_len: int = len(req.prefix_indices) + prefix_len: int = req.prefix_len protected_len: int = req.kv.cache_protected_len assert prefix_len >= protected_len, ( f"len(prefix_indices)={prefix_len} dropped below " diff --git a/test/manual/chunked_prefill/test_scripted_regression.py b/test/manual/chunked_prefill/test_scripted_regression.py index 901a0afadc..8988de414c 100644 --- a/test/manual/chunked_prefill/test_scripted_regression.py +++ b/test/manual/chunked_prefill/test_scripted_regression.py @@ -148,7 +148,7 @@ class TestRegressionBasic(ScriptedTestCase): r2 = t.start_req(prompt_len=DEFAULT_CHUNK_SIZE, max_new_tokens=2) yield from run_until(r2, lambda h: h.status == "waiting") - r1_prefix = len(r1.req.prefix_indices) + r1_prefix = r1.req.prefix_len assert r1.is_chunking, "r1 must still be the in-flight chunked req" assert r1_prefix > 0, "r1 must hold a committed prefix as the chunked req" @@ -516,9 +516,9 @@ class TestRegressionGptOss(ScriptedTestCase): committed = r.req.kv.kv_committed_len assert committed > 0 - assert len(r.req.prefix_indices) <= committed, ( + assert r.req.prefix_len <= committed, ( f"checkpoint over-read past kv_committed_len: " - f"prefix_indices_len={len(r.req.prefix_indices)}, " + f"prefix_indices_len={r.req.prefix_len}, " f"kv_committed_len={committed}" ) yield from run_until_finished(r) diff --git a/test/manual/chunked_prefill/test_scripted_special_case.py b/test/manual/chunked_prefill/test_scripted_special_case.py index 60ed31f65d..c7ab81196c 100644 --- a/test/manual/chunked_prefill/test_scripted_special_case.py +++ b/test/manual/chunked_prefill/test_scripted_special_case.py @@ -23,7 +23,7 @@ def _load_inquirer_pending_for_rid(t: ScriptedContext, rid: str) -> int: s = t.scheduler chunked = s.chunked_req if chunked is not None and chunked.rid == rid: - return chunked.seqlen - len(chunked.prefix_indices) + return chunked.seqlen - chunked.prefix_len for req in s.waiting_queue: if req.rid == rid: return req.seqlen @@ -226,9 +226,9 @@ class TestSpecialCaseBasic(ScriptedTestCase): ) for _ in range(DEFAULT_MAX_STEPS): if r.is_chunking: - assert len(r.req.prefix_indices) <= r.req.kv.kv_committed_len, ( + assert r.req.prefix_len <= r.req.kv.kv_committed_len, ( f"streaming-session chunked stash must stay bounded by " - f"kv_committed_len; prefix_indices_len={len(r.req.prefix_indices)}, " + f"kv_committed_len; prefix_indices_len={r.req.prefix_len}, " f"kv_committed_len={r.req.kv.kv_committed_len}" ) if r.finished: @@ -329,7 +329,7 @@ class TestSpecialCaseBasic(ScriptedTestCase): if r.is_chunking and chunked is not None and chunked.rid == r.rid: saw_chunking = True pending = _load_inquirer_pending_for_rid(t, r.rid) - expected = chunked.seqlen - len(chunked.prefix_indices) + expected = chunked.seqlen - chunked.prefix_len assert pending == expected, ( f"load_inquirer chunked contribution must equal the prefix-" f"subtracting formula seqlen - len(prefix_indices) = {expected}; " @@ -339,7 +339,7 @@ class TestSpecialCaseBasic(ScriptedTestCase): f"chunked contribution must never exceed its full seqlen " f"{chunked.seqlen}; got {pending} — dual-queue dedup violated" ) - if len(chunked.prefix_indices) > 0: + if chunked.prefix_len > 0: saw_dedup = pending < chunked.seqlen if r.finished: break @@ -373,7 +373,7 @@ class TestSpecialCaseBasic(ScriptedTestCase): "test requires an empty waiting_queue so the chunked req is " f"the sole pending-token contributor; got {len(s.waiting_queue)}" ) - expected = chunked.seqlen - len(chunked.prefix_indices) + expected = chunked.seqlen - chunked.prefix_len observed = s.load_inquirer._get_num_pending_tokens() assert observed == expected, ( f"chunked contribution must equal remainder " @@ -478,13 +478,12 @@ class TestSpecialCaseBasic(ScriptedTestCase): ): saw_mid_chunk = True assert ( - req.extend_range.end - == len(req.prefix_indices) + req.extend_range.length + req.extend_range.end == req.prefix_len + req.extend_range.length ), ( f"init_next_round_input must rebuild fill_ids to the committed " f"prefix plus the in-flight chunk; " f"fill_ids_len={req.extend_range.end}, " - f"prefix_indices_len={len(req.prefix_indices)}, " + f"prefix_indices_len={req.prefix_len}, " f"extend_input_len={req.extend_range.length}, " f"chunks_done={r.chunks_done}" ) diff --git a/test/manual/test_forward_split_prefill.py b/test/manual/test_forward_split_prefill.py index ef95db318f..6b950e18b0 100644 --- a/test/manual/test_forward_split_prefill.py +++ b/test/manual/test_forward_split_prefill.py @@ -113,9 +113,7 @@ class TestForwardSplitPrefill(CustomTestCase): ) req.full_untruncated_fill_ids = req.origin_input_ids req.logprob_start_len = -1 - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) reqs.append(req) # Create dummy tree_cache for tests (no prefix caching, just allocation) diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 306ff9dc33..7258e87b86 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -665,7 +665,6 @@ class TestDisaggregationWire(unittest.TestCase): def test_prebuilt_skips_unused_prompt_tensor(self): req = SimpleNamespace( kv=ReqKvInfo(req_pool_idx=0), - prefix_indices=[0, 1], prefix_len=2, extend_range=SimpleNamespace(length=3), origin_input_ids=[0, 1, 2, 3, 4], diff --git a/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py b/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py index 832e5aa7b6..0138eaf72e 100644 --- a/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py +++ b/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py @@ -39,7 +39,7 @@ def _make_mock_req( req_pool_idx: int, kv_committed_len: int, kv_allocated_len: int, - prefix_indices_len: int = 0, + prefix_len: int = 0, rid: int = 0, origin_len: int = 0, ): @@ -54,8 +54,7 @@ def _make_mock_req( kv_committed_len=kv_committed_len, kv_allocated_len=kv_allocated_len, ) - req.prefix_indices = list(range(prefix_indices_len)) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = prefix_len req.owned_kv_len = lambda: req.kv.kv_committed_len return req @@ -203,7 +202,7 @@ class TestReleaseFinishedReq(unittest.TestCase): req_pool_idx=0, kv_committed_len=20, kv_allocated_len=20, - prefix_indices_len=5, + prefix_len=5, ) req.kv.cache_protected_len = 5 lock = req.lock diff --git a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py index 0441355b86..66b6574d95 100644 --- a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py +++ b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py @@ -27,8 +27,7 @@ class _Req: self.full_untruncated_fill_ids = self.origin_input_ids + array( "q", [0] * block_size ) - self.prefix_indices = torch.arange(context_len, dtype=torch.int64) - self.prefix_len = len(self.prefix_indices) + self.prefix_len = context_len self.dllm_block_offset = context_len self.dllm_incomplete_ids = array("q") self.dllm_algo_state = None @@ -202,8 +201,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): adder._check_prefill_tile_budget = Mock(return_value=None) req = _Req(context_len=300, block_size=256, prefill=True) req.host_hit_length = req.storage_hit_length = 0 - req.prefix_indices = torch.empty(0, dtype=torch.int64) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 0 admission = adder._select_prefill_admission( req, total_tokens=1024, @@ -228,8 +226,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): adder._update_prefill_budget = Mock() req = _Req(context_len=10, block_size=4, prefill=True) - req.prefix_indices = torch.arange(7) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 7 result = PrefillAdder.add_dllm_staging_req(adder, req) diff --git a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py index 4b5c467325..7a24e2a1de 100644 --- a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py +++ b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py @@ -167,8 +167,7 @@ class _FakeRunner: class _FakeReq: def __init__(self, rid, req_pool_idx=0): self.rid = rid - self.prefix_indices = torch.empty(0, dtype=torch.long) - self.prefix_len = len(self.prefix_indices) + self.prefix_len = 0 self.fill_ids = [0] self.kv = ReqKvInfo(req_pool_idx=req_pool_idx) # Mirrors Req's chunk-finality contract read by diff --git a/test/registered/unit/managers/test_load_inquirer.py b/test/registered/unit/managers/test_load_inquirer.py index 4691f637fc..51363a11bf 100644 --- a/test/registered/unit/managers/test_load_inquirer.py +++ b/test/registered/unit/managers/test_load_inquirer.py @@ -43,9 +43,7 @@ class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase): class TestSchedulerLoadInquirer(unittest.TestCase): def make_inquirer(self, waiting_queue_prefix_matched): waiting_req = SimpleNamespace(seqlen=100, num_matched_prefix_tokens=20) - chunked_req = SimpleNamespace( - seqlen=50, prefix_indices=range(10), prefix_len=10 - ) + chunked_req = SimpleNamespace(seqlen=50, prefix_len=10) return SimpleNamespace( disaggregation_mode=DisaggregationMode.NULL, get_waiting_queue=lambda: [waiting_req], diff --git a/test/registered/unit/managers/test_mamba_checkpoint_depth.py b/test/registered/unit/managers/test_mamba_checkpoint_depth.py index b3250a6917..cf74339972 100644 --- a/test/registered/unit/managers/test_mamba_checkpoint_depth.py +++ b/test/registered/unit/managers/test_mamba_checkpoint_depth.py @@ -53,8 +53,7 @@ def _track_seqlen( sampling_params=sampling_params, vocab_size=128, ) - req.prefix_indices = torch.arange(prefix_len, dtype=torch.int64) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = prefix_len req.set_extend_range(prefix_len, prefix_len + extend_len) req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64) req.kv.mamba_next_track_idx = 0 diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index e7cd0d1c18..7a94fe897c 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -1,4 +1,3 @@ -import functools import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -124,8 +123,7 @@ class TestPrefillAdder(CustomTestCase): req.rid = str(rid) req.cache_request_handle = CacheRequestHandle(req.rid, 0) req.priority = priority - req.set_prefix_indices = functools.partial(Req.set_prefix_indices, req) - req.set_prefix_indices(torch.empty((0,), dtype=torch.int64)) + req.prefix_len = 0 req.full_untruncated_fill_ids = [] req.output_ids = [0] * output_len req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens) @@ -261,8 +259,7 @@ class TestPrefillAdder(CustomTestCase): adder = self.create_shortest_prefill_adder(chunk_tokens=512) req = self.create_shared_req("host-miss") req.full_untruncated_fill_ids = list(range(1024)) - req.prefix_indices = torch.empty(0, dtype=torch.int64) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 0 req.host_hit_length = 768 req.best_match_node = req.last_node req.needs_host_load_back.return_value = True @@ -843,8 +840,7 @@ class TestPrefillAdder(CustomTestCase): def create_sharded_req(self, rid, *, prefix_len=12, extend_len=4): req = self.create_shared_req(rid, max_new_tokens=1) - req.prefix_indices = list(range(prefix_len)) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = prefix_len req.full_untruncated_fill_ids = list(range(prefix_len + extend_len)) return req @@ -1043,8 +1039,7 @@ class TestPrefillAdder(CustomTestCase): ) req = self.create_mock_req("chunked", priority=0, max_new_tokens=128) - req.prefix_indices = [] - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 0 req.full_untruncated_fill_ids = list(range(extend_input_len)) # set_extend_range is the only writer of extend_range; the production # path reads req.extend_range.length right after calling it, so the mock @@ -1156,8 +1151,7 @@ class TestPrefillAdder(CustomTestCase): req = self.create_mock_req( "resume", priority=0, max_new_tokens=40, output_len=10 ) - req.prefix_indices = list(range(PREFIX)) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = PREFIX req.full_untruncated_fill_ids = list(range(PREFIX + EXTEND)) req.host_hit_length = 0 req.swa_host_hit_length = 0 @@ -1207,8 +1201,7 @@ class TestPrefillAdder(CustomTestCase): self.mock_token_allocator, self.mock_tree_cache ) req = self.create_mock_req("dropped-fetch", priority=0, max_new_tokens=8) - req.prefix_indices = torch.empty(0, dtype=torch.int64) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 0 req.full_untruncated_fill_ids = list(range(SPAN)) req.host_hit_length = HOST_HIT req.swa_host_hit_length = WINDOW @@ -1250,7 +1243,7 @@ class TestPrefillAdder(CustomTestCase): # budget. A successful load must not run admission gates again. _, admitted, req = run(HOST_HIT + 4, remaining_after_load=0) self.assertEqual(len(admitted), 1) - self.assertEqual(len(req.prefix_indices), HOST_HIT + 4) + self.assertEqual(req.prefix_len, HOST_HIT + 4) self.assertEqual(req.kv.cache_protected_len, HOST_HIT + 4) req.set_extend_range.assert_called_once_with(HOST_HIT + 4, SPAN) # A partial FULL load stays fatal. @@ -1441,8 +1434,7 @@ class TestPrefillAdder(CustomTestCase): def _create_host_hit_req(self, *, prefix_len=0, host_hit=8192, tail=1024): req = self._create_delayer_req(prefix_len + host_hit + tail) - req.prefix_indices = torch.arange(prefix_len) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = prefix_len req.host_hit_length = host_hit req.needs_host_load_back.return_value = True req.best_match_node = req.last_node diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index ade294a128..da5afca8a1 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -208,7 +208,6 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): req_pool_idx=1, cache_protected_len=2, ), - prefix_indices=torch.tensor([8, 9], dtype=torch.int64), prefix_len=2, priority=3, extra_key=None, diff --git a/test/registered/unit/managers/test_schedule_batch_out_of_place.py b/test/registered/unit/managers/test_schedule_batch_out_of_place.py index 74cadc23ed..5d850bb6a8 100644 --- a/test/registered/unit/managers/test_schedule_batch_out_of_place.py +++ b/test/registered/unit/managers/test_schedule_batch_out_of_place.py @@ -242,7 +242,6 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase): req_with_image = types.SimpleNamespace( rid="img", multimodal_inputs=types.SimpleNamespace(num_image_tokens=2), - prefix_indices=[], prefix_len=0, extend_range=Range(0, 5), logprob_start_len=0, @@ -250,7 +249,6 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase): req_text_only = types.SimpleNamespace( rid="txt", multimodal_inputs=None, - prefix_indices=[], prefix_len=0, extend_range=Range(0, 4), logprob_start_len=0, diff --git a/test/registered/unit/managers/test_schedule_policy.py b/test/registered/unit/managers/test_schedule_policy.py index 80833f051b..3be810ddc8 100644 --- a/test/registered/unit/managers/test_schedule_policy.py +++ b/test/registered/unit/managers/test_schedule_policy.py @@ -139,8 +139,7 @@ class TestShortestPrefillFirst(CustomTestCase): req = _make_req(rid, "", list(range(uncached + cached))) req.full_untruncated_fill_ids = req.origin_input_ids[:] req.num_matched_prefix_tokens = cached - req.prefix_indices = list(range(cached)) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = cached req.time_stats.wait_queue_entry_time = arrived return req diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 97674c5ca9..56211d88c8 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -25,7 +25,7 @@ def _make_req( *, req_pool_idx: int, fill_ids: list, - prefix_indices: torch.Tensor, + prefix_len: int, extend_input_len: int, fill_len: int, ) -> Req: @@ -34,8 +34,7 @@ def _make_req( req.origin_input_ids = array("q", fill_ids) req.output_ids = array("q") req.full_untruncated_fill_ids = array("q", fill_ids) - req.prefix_indices = prefix_indices - req.prefix_len = len(req.prefix_indices) + req.prefix_len = prefix_len req.extend_range = Range(fill_len - extend_input_len, fill_len) req.inflight_middle_chunks = 0 req.host_hit_length = 0 @@ -56,8 +55,6 @@ def _make_req( def _make_req_to_token_pool(num_slots: int, max_context: int) -> SimpleNamespace: - # Slot s contains a recognizable fingerprint [s*1000, s*1000+1, ...] - # so we can tell a corrupted prefix_indices from a healthy one by content. pool = SimpleNamespace() pool.req_to_token = ( torch.arange(max_context, dtype=torch.int32).unsqueeze(0).repeat(num_slots, 1) @@ -119,11 +116,8 @@ def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler: class TestStashGatePreservesPrefix(CustomTestCase): - """Consumer side: real ChunkCache.checkpoint mutates - req.prefix_indices iff stash actually runs, so prefix_indices content - is the bug-detection signal. The stash gate is content-based: - `fill_len > len(prefix_indices)` means there is freshly computed KV to - cache; otherwise the chunk was parked and stashing must be skipped.""" + """The stash gate advances prefix_len iff `fill_len > prefix_len`, i.e. the + chunk computed new KV; a parked chunk must be left untouched.""" POOL_IDX = 4 INITIAL_PREFIX_LEN = 8 # what was really cached last iter @@ -134,35 +128,31 @@ class TestStashGatePreservesPrefix(CustomTestCase): def _build(self, *, fill_len: int): pool = _make_req_to_token_pool(self.NUM_SLOTS, self.MAX_CONTEXT) cache = _make_chunk_cache(pool) - initial_prefix = pool.req_to_token[self.POOL_IDX, : self.INITIAL_PREFIX_LEN].to( - dtype=torch.int64, copy=True - ) req = _make_req( req_pool_idx=self.POOL_IDX, fill_ids=list(range(self.POST_RESET_FILL_LEN)), - prefix_indices=initial_prefix, + prefix_len=self.INITIAL_PREFIX_LEN, extend_input_len=fill_len - self.INITIAL_PREFIX_LEN, fill_len=fill_len, ) s = _scheduler_for_get_next_batch(tree_cache=cache, chunked_req=req) - return s, req, initial_prefix, pool + return s, req, pool def test_parked_chunked_req_keeps_its_prefix(self): # A parked chunk has fill_len == prefix_len: no new KV was computed, # so the gate must skip stash and leave the prefix intact. - s, req, initial_prefix, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN) + s, req, _ = self._build(fill_len=self.INITIAL_PREFIX_LEN) Scheduler.get_next_batch_to_run( s, running_batch=s.running_batch, last_batch=s.last_batch ) self.assertEqual(req.prefix_len, self.INITIAL_PREFIX_LEN) - self.assertTrue(torch.equal(req.prefix_indices, initial_prefix)) def test_scheduled_chunked_req_advances_prefix_via_real_stash(self): # Symmetric guard against over-gating: when fill_len has advanced past # the cached prefix, stash must run and advance prefix_len. - s, req, _, pool = self._build(fill_len=self.POST_RESET_FILL_LEN) + s, req, _ = self._build(fill_len=self.POST_RESET_FILL_LEN) Scheduler.get_next_batch_to_run( s, running_batch=s.running_batch, last_batch=s.last_batch diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index 6dbb9d3af3..9fa80abad7 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -100,8 +100,7 @@ class MockReq: self.lock = lock self.extra_key = None self.cache_salt = None - self.prefix_indices = torch.empty(0, dtype=torch.int64) - self.prefix_len = len(self.prefix_indices) + self.prefix_len = 0 self.priority = 0 self.kv = ReqKvInfo( req_pool_idx=req_pool_idx, diff --git a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py index 0ced064bd9..d600d6b396 100644 --- a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py +++ b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py @@ -61,11 +61,14 @@ class _FakeTreeCache: def maybe_hand_to_session(self, req): pass + def prefix_device_indices(self, req): + return req.tree_prefix + def _make_req(rid, prefix, block_size, *, req_pool_idx=None, reuse=False): return SimpleNamespace( rid=rid, - prefix_indices=torch.tensor(prefix, dtype=torch.int32), + tree_prefix=torch.tensor(prefix, dtype=torch.int32), prefix_len=len(prefix), dllm_incomplete_ids=array("q", range(block_size)) if reuse else array("q"), inflight_middle_chunks=1 if req_pool_idx is not None else 0, @@ -87,10 +90,7 @@ def _remove_allocated_req_slots(pool, *reqs): def _make_batch(pool, allocator, reqs, extend_lens): seq_lens_cpu = torch.tensor( - [ - len(req.prefix_indices) + extend_len - for req, extend_len in zip(reqs, extend_lens) - ], + [req.prefix_len + extend_len for req, extend_len in zip(reqs, extend_lens)], dtype=torch.int64, ) return SimpleNamespace( @@ -99,7 +99,7 @@ def _make_batch(pool, allocator, reqs, extend_lens): req_to_token_pool=pool, token_to_kv_pool_allocator=allocator, tree_cache=_FakeTreeCache(allocator), - prefix_lens=[len(req.prefix_indices) for req in reqs], + prefix_lens=[req.prefix_len for req in reqs], extend_lens=extend_lens, seq_lens=seq_lens_cpu, seq_lens_cpu=seq_lens_cpu, @@ -110,8 +110,8 @@ def _make_batch(pool, allocator, reqs, extend_lens): def _seed_retained_block(pool, req, values): - prefix_len = len(req.prefix_indices) - pool.req_to_token[req.kv.req_pool_idx, :prefix_len] = req.prefix_indices + prefix_len = req.prefix_len + pool.req_to_token[req.kv.req_pool_idx, :prefix_len] = req.tree_prefix pool.req_to_token[req.kv.req_pool_idx, prefix_len : prefix_len + len(values)] = ( torch.tensor(values, dtype=torch.int32) ) diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index 088bf246ce..25e4780d28 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -9,7 +9,6 @@ from unittest import mock import torch from sglang.srt.managers.cache_controller import CacheOperation, HiCacheController -from sglang.srt.managers.schedule_batch import Req from sglang.srt.mem_cache import l2_transfer as transfer_module from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline @@ -35,11 +34,6 @@ from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase - -class _ReqStub(SimpleNamespace): - set_prefix_indices = Req.set_prefix_indices - - register_cpu_ci(est_time=10, suite="base-a-test-cpu") MEMORY_POOL_HOST_MODULE = "sglang.srt.mem_cache.memory_pool_host" @@ -316,10 +310,9 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices( 0, 6 ) - req = _ReqStub( + req = SimpleNamespace( rid="r", cache_request_handle=handle, - prefix_indices=_indices(0, 0), prefix_len=0, kv=SimpleNamespace(cache_protected_len=0), ) @@ -333,10 +326,9 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices( 0, 6 ) - req = _ReqStub( + req = SimpleNamespace( rid="r", cache_request_handle=handle, - prefix_indices=_indices(0, 0), prefix_len=0, kv=SimpleNamespace(cache_protected_len=0), ) diff --git a/test/registered/unit/mem_cache/test_hisparse_allocator.py b/test/registered/unit/mem_cache/test_hisparse_allocator.py index 84510b1b0d..6da4199370 100644 --- a/test/registered/unit/mem_cache/test_hisparse_allocator.py +++ b/test/registered/unit/mem_cache/test_hisparse_allocator.py @@ -247,12 +247,6 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase): req.set_extend_range = set_extend_range - def set_prefix_indices(prefix_indices): - req.prefix_indices = prefix_indices - req.prefix_len = len(prefix_indices) - - req.set_prefix_indices = set_prefix_indices - class ReqToTokenPool: def __init__(self): self.writes = [] diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 8e0f0fd1d5..31bbeca139 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -784,8 +784,7 @@ class _GraftReq: self.kv = ReqKvInfo(req_pool_idx=req_pool_idx) self.extra_key = None self.cache_salt = None - self.prefix_indices = torch.empty(0, dtype=torch.int64) - self.prefix_len = len(self.prefix_indices) + self.prefix_len = 0 self.last_node = None self.priority = 0 self.kv_rotation_base = None diff --git a/test/registered/unit/mem_cache/test_prefill_memory_budget.py b/test/registered/unit/mem_cache/test_prefill_memory_budget.py index f86ff1f152..e94c992f9a 100644 --- a/test/registered/unit/mem_cache/test_prefill_memory_budget.py +++ b/test/registered/unit/mem_cache/test_prefill_memory_budget.py @@ -204,9 +204,7 @@ class TestSharedPrefillAdmission(unittest.TestCase): for page_size in (4, 64): with self.subTest(page_size=page_size): allocator, req, adder = self._new_admission(page_size, pool_pages=6) - req.prefix_indices = allocator.alloc(page_size) - req.prefix_len = len(req.prefix_indices) - self.assertIsNotNone(req.prefix_indices) + req.prefix_len = len(allocator.alloc(page_size)) self.assertTrue(allocator.can_reserve(page_size + 2, page_size + 2)) self.assertIsNone(adder.add_chunked_req(req)) @@ -219,7 +217,7 @@ class TestSharedPrefillAdmission(unittest.TestCase): allocator, req, adder = self._new_admission( page_size, pool_pages=5, ignore_eos=True ) - self.assertEqual(len(req.prefix_indices), 0) + self.assertEqual(req.prefix_len, 0) self.assertTrue(allocator.can_reserve(2 * page_size + 2, 2 * page_size)) adder.add_one_req( diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index 4d3ba3019a..8f2671c9ab 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -16,7 +16,6 @@ from types import SimpleNamespace import torch from sglang.srt.environ import envs -from sglang.srt.managers.schedule_batch import Req from sglang.srt.mem_cache.base_prefix_cache import ( InsertParams, MatchPrefixParams, @@ -33,7 +32,6 @@ class _StubReq: self.output_ids = array("q") self.extra_key = None self.cache_salt = None - self.prefix_indices = None self.prefix_len = 0 self.last_node = None self.last_host_node = None @@ -43,8 +41,6 @@ class _StubReq: self.swa_branching_seqlen = None self.kv = SimpleNamespace(cache_protected_len=None) - set_prefix_indices = Req.set_prefix_indices - def _compute_max_prefix_len(self, input_len): return max(input_len - 1, 0) @@ -117,14 +113,14 @@ class TestMatchKvCacheForceMiss(unittest.TestCase): baseline_req = _StubReq([10, 11, 12, 13, 99, 100]) with envs.SGLANG_RADIX_FORCE_MISS.override(False): match_kv_cache(baseline_req, tree) - self.assertGreater(int(baseline_req.prefix_indices.numel()), 0) + self.assertGreater(baseline_req.prefix_len, 0) self.assertIsNot(baseline_req.last_node, tree.root_node) # With the flag, the same lookup is forced to miss. forced_req = _StubReq([10, 11, 12, 13, 99, 100]) with envs.SGLANG_RADIX_FORCE_MISS.override(True): match_kv_cache(forced_req, tree) - self.assertEqual(int(forced_req.prefix_indices.numel()), 0) + self.assertEqual(forced_req.prefix_len, 0) self.assertIs(forced_req.last_node, tree.root_node) self.assertIs(forced_req.last_host_node, tree.root_node) self.assertEqual(forced_req.host_hit_length, 0) diff --git a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py index 890b7f5867..8c4e1ad11b 100644 --- a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py +++ b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py @@ -11,7 +11,7 @@ from unittest.mock import Mock import torch -from sglang.srt.managers.schedule_batch import Req, split_cached_prefix_by_tier +from sglang.srt.managers.schedule_batch import split_cached_prefix_by_tier from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, InitLoadBackParams, @@ -33,11 +33,6 @@ from sglang.srt.mem_cache.unified_radix_cache import ( from sglang.srt.mem_cache.utils import get_hash_str from sglang.test.ci.ci_register import register_cpu_ci - -class _ReqStub(SimpleNamespace): - set_prefix_indices = Req.set_prefix_indices - - register_cpu_ci(est_time=20, suite="base-a-test-cpu") _REQ = CacheRequestHandle("r", 0) @@ -52,6 +47,7 @@ def _staged_fixture(full_match=2): prefetch_anchor_info=lambda node: (None, None), match_full_device_prefix=Mock(return_value=(full_match, 1, full_match)), collect_full_device_indices=Mock(return_value=torch.arange(8)), + root_node_handle=lambda extra_key=None: 0, inc_full_pin=Mock(), dec_full_pin=Mock(), empty_match_result=SimpleNamespace( @@ -114,10 +110,9 @@ def _staged_fixture(full_match=2): hash_values=["a", "b", "c"], operation_id=1, ) - req = _ReqStub( + req = SimpleNamespace( rid="r", cache_request_handle=_REQ, - prefix_indices=torch.arange(2), prefix_len=2, last_node=1, kv=SimpleNamespace(cache_protected_len=2), @@ -207,8 +202,7 @@ def _two_rank_retry_trace(rank, rendezvous): 1, full_match, ) - req.prefix_indices = torch.arange(full_match) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = full_match if cache.buffer_pipeline.prepare_staged_prefetch(req): assert ( cache.init_load_back( @@ -309,8 +303,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): len(RadixKey(held.key_tokens, is_bigram=bigram)), 10 ) # A joint match counts bigrams, not their extra raw boundary token. - req.prefix_indices = torch.arange(10) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 10 req.kv.cache_protected_len = 10 self.assertTrue(pipeline.prepare_staged_prefetch(req)) self.assertFalse(pipeline.has_staged(req.cache_request_handle)) @@ -352,8 +345,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): # A twin made [0, 6) jointly reusable while the fetch was held. cache.tree_core.match_full_device_prefix.return_value = (8, 1, 8) - req.prefix_indices = torch.arange(6) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 6 self.assertTrue(pipeline.prepare_staged_prefetch(req)) self.assertEqual( split_cached_prefix_by_tier( @@ -390,13 +382,12 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): cache.tree_core.match_full_device_prefix.assert_called_once() # A twin finished first: the next pass's joint match runs past the # staged span and is kept as is (a shrink would strand its recompute). - req.prefix_indices = torch.arange(12) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 12 req.last_node = 9 req.kv.cache_protected_len = 12 self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req)) self.assertIsNone(req.staged_prefetch_plan) - self.assertEqual(len(req.prefix_indices), 12) + self.assertEqual(req.prefix_len, 12) self.assertEqual((req.last_node, req.kv.cache_protected_len), (9, 12)) self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0)) self.assertFalse(pipeline.has_staged(req.cache_request_handle)) @@ -425,8 +416,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): anchor = pipeline.anchor_locks[req.cache_request_handle] cache.tree_core.match_full_device_prefix.reset_mock() for attempt in range(1, 4): - req.prefix_indices = torch.arange(2) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 2 self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req)) self.assertIsNone( cache.init_load_back( @@ -467,16 +457,14 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): # Tree changes occur while queued, before the next preparation pass. cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6) - req.prefix_indices = torch.arange(2) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 2 self.assertTrue(cache.buffer_pipeline.prepare_staged_prefetch(req)) self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4)) self.assertTrue(pipeline.has_staged(req.cache_request_handle)) self.assertEqual(cache.storage_prefetch_retries.pop_ready([req], 0, 8), []) cache.tree_core.match_full_device_prefix.return_value = (0, 0, 0) - req.prefix_indices = torch.arange(0) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 0 self.assertFalse(cache.buffer_pipeline.prepare_staged_prefetch(req)) self.assertFalse(pipeline.has_staged(req.cache_request_handle)) self.assertEqual( @@ -528,8 +516,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): pipeline.max_staged_admission_defers = 3 params = lambda: InitLoadBackParams(None, req.host_hit_length, req=req) for attempt in range(1, 4): - req.prefix_indices = torch.arange(2) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 2 self.assertTrue(pipeline.prepare_staged_prefetch(req)) self.assertIsNone(cache.init_load_back(params())) self.assertEqual(pipeline.has_staged(req.cache_request_handle), attempt < 3) diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index 79fe0795c3..b08a84e779 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -1296,6 +1296,7 @@ def test_linker_load_preserves_swa_boundaries( ) ), resolve_node_handle=lambda node_id: SimpleNamespace(id=0), + prefix_device_indices=lambda req: torch.empty(0, dtype=torch.int64), ) wrapper = UnifiedCacheLinkerWrapper(cache, _FakeLinker()) wrapper.hit_markers["rid"] = ExternalCacheHitMarker( @@ -1315,7 +1316,6 @@ def test_linker_load_preserves_swa_boundaries( req = SimpleNamespace( rid="rid", kv=kv, - prefix_indices=torch.empty(0, dtype=torch.int64), prefix_len=0, last_node=0, priority=0, diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index c52cf12c5f..f7eb92939c 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -634,9 +634,7 @@ def bench_release( req.origin_input_ids = array("q", seq) req.output_ids = array("q") req.full_untruncated_fill_ids = array("q", seq) - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) req.last_node = node req.kv.cache_protected_len = matched_len req.kv.kv_committed_len = len(seq) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index dcefa86a3f..50aa6bc94d 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1,7 +1,6 @@ """Unit tests for UnifiedRadixCache""" import atexit -import functools import json import shutil import sys @@ -122,11 +121,6 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.mem_cache_utils import finish_req from sglang.test.test_utils import CustomTestCase - -class _ReqStub(SimpleNamespace): - set_prefix_indices = Req.set_prefix_indices - - register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small") register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small-amd") @@ -1602,8 +1596,7 @@ class UnifiedRadixCacheSuite: return req def _apply_match_to_req(self, req, match): - req.prefix_indices = match.device_indices - req.prefix_len = len(req.prefix_indices) + req.prefix_len = len(match.device_indices) req.last_node = match.last_device_node req.last_host_node = match.last_host_node req.best_match_node = match.best_match_node @@ -1860,9 +1853,7 @@ class UnifiedRadixCacheSuite: req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", input_ids + output_ids) - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len @@ -1908,9 +1899,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = array("q", prompt_ids) req.output_ids = array("q", output_ids) req.full_untruncated_fill_ids = array("q", prompt_ids + output_ids) - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) kv_len = req.extend_range.end kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -1983,9 +1972,7 @@ class UnifiedRadixCacheSuite: req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) avail_before = allocator.available_size() release_kv_cache(req, cache, checkpoint=False) @@ -2003,9 +1990,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = array("q", tokens) req.output_ids = array("q") req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) kv_len = len(tokens) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -2149,9 +2134,7 @@ class UnifiedRadixCacheSuite: req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", input_ids) - req.set_extend_range( - len(req.prefix_indices), len(req.full_untruncated_fill_ids) - ) + req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len @@ -3452,7 +3435,6 @@ class UnifiedRadixCacheSuite: cache, req_id, prefix_len=None, - prefix_indices=None, timeout: float = 10.0, *, extra_key=None, @@ -3476,20 +3458,7 @@ class UnifiedRadixCacheSuite: req.cache_request_handle = req_id req.extra_key = extra_key req.cache_salt = cache_salt - if prefix_indices is not None: - # Spliceable mid-anchor consumption publishes value=cat(prefix, - # fill) — the real device prefix is required (zeros would insert - # bogus slots into the tree). - assert len(prefix_indices) == prefix_len - req.prefix_indices = prefix_indices - req.prefix_len = len(req.prefix_indices) - else: - req.prefix_indices = torch.zeros( - prefix_len, - dtype=torch.int64, - device=cache.tree_core.empty_match_result.device_indices.device, - ) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = prefix_len req.last_node = cache.root_node_handle() if last_node is None else last_node joint = cache.match_prefix( MatchPrefixParams( @@ -3498,8 +3467,7 @@ class UnifiedRadixCacheSuite: ) ) ) - req.prefix_indices = joint.device_indices - req.prefix_len = len(req.prefix_indices) + req.prefix_len = len(joint.device_indices) joint_covers_span = len(joint.device_indices) >= len(f.key_tokens) adder = PrefillAdder.__new__(PrefillAdder) adder.tree_cache = cache @@ -4517,12 +4485,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None req.cache_salt = None req.last_node = cons.root_node_handle() - req.prefix_indices = torch.zeros( - held.matched_len, - dtype=torch.int64, - device=cons.tree_core.empty_match_result.device_indices.device, - ) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = held.matched_len self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) spliced, _last = cons.init_load_back( InitLoadBackParams( @@ -4707,7 +4670,6 @@ class UnifiedRadixCacheSuite: cons2, anchored_req, prefix_len=len(prefix), - prefix_indices=prefix_match.device_indices, extra_key=extra_key, cache_salt=cache_salt, last_node=anchor, @@ -4886,10 +4848,9 @@ class UnifiedRadixCacheSuite: self.assertIsNotNone(cons_alloc.swa_attn_allocator.alloc(hold // ps * ps)) # The adder's SWA gate for this request (_swa_budget_for_req). - req = _ReqStub( + req = SimpleNamespace( rid=req_id.rid, cache_request_handle=req_id, - prefix_indices=cons.tree_core.empty_match_result.device_indices, prefix_len=len(cons.tree_core.empty_match_result.device_indices), kv=SimpleNamespace(cache_protected_len=0), ) @@ -4911,12 +4872,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None req.cache_salt = None req.last_node = cons.root_node_handle() - req.prefix_indices = torch.zeros( - held.matched_len, - dtype=torch.int64, - device=cons.tree_core.empty_match_result.device_indices.device, - ) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = held.matched_len self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) spliced, last_node = cons.init_load_back( InitLoadBackParams( @@ -5141,7 +5097,7 @@ class UnifiedRadixCacheSuite: return_req=True, ) self.assertEqual(int(spliced.numel()), 0) - self.assertEqual(len(req.prefix_indices), len(seq), "FULL rematch was lost") + self.assertEqual(req.prefix_len, len(seq), "FULL rematch was lost") self.assertEqual(len(dispatched["device"]), window_tokens) self.assertEqual(cons_alloc.full_available_size(), full_avail0) k, v = self._snapshot_full_kv(cons_alloc, masked_full) @@ -5219,7 +5175,7 @@ class UnifiedRadixCacheSuite: spliced, req = self._consume_staged_prefetch( cons, req_id, prefix_len=0, return_req=True ) - self.assertEqual(len(req.prefix_indices), split_at) + self.assertEqual(req.prefix_len, split_at) self.assertEqual(int(spliced.numel()), page_size) self.assertEqual(cons_alloc.full_available_size(), full_avail0 - page_size) k, v = self._snapshot_full_kv(cons_alloc, head_full) @@ -5351,12 +5307,7 @@ class UnifiedRadixCacheSuite: req.cache_request_handle = req_id req.extra_key = None req.cache_salt = None - req.prefix_indices = torch.zeros( - 0, - dtype=torch.int64, - device=cons.tree_core.empty_match_result.device_indices.device, - ) - req.prefix_len = len(req.prefix_indices) + req.prefix_len = 0 req.last_node = cons.root_node_handle() self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) with mock.patch.object(cons.cache_controller, "load", adversarial_load): @@ -5442,10 +5393,9 @@ class UnifiedRadixCacheSuite: self.assertGreater(aux_max, len(seq) - len(head)) # The surfaced host hit is the splice-able tail, not the full span. - req = _ReqStub( + req = SimpleNamespace( rid=req_id.rid, cache_request_handle=req_id, - prefix_indices=sib.device_indices, prefix_len=len(sib.device_indices), kv=SimpleNamespace(cache_protected_len=len(sib.device_indices)), ) @@ -5453,9 +5403,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(req.host_hit_length, len(seq) - len(head)) self.assertTrue(cons.buffer_pipeline.has_staged(req_id)) - spliced = self._consume_staged_prefetch( - cons, req_id, prefix_len=len(head), prefix_indices=sib.device_indices - ) + spliced = self._consume_staged_prefetch(cons, req_id, prefix_len=len(head)) self.assertEqual(int(spliced.numel()), len(seq) - len(head)) m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) @@ -5513,11 +5461,9 @@ class UnifiedRadixCacheSuite: req = mock.Mock( rid=req_id.rid, cache_request_handle=req_id, - prefix_indices=live.device_indices, prefix_len=len(live.device_indices), last_node=live.last_device_node, ) - req.set_prefix_indices = functools.partial(Req.set_prefix_indices, req) self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0)) self.assertFalse(cons.buffer_pipeline.has_staged(req_id)) @@ -5672,7 +5618,6 @@ class UnifiedRadixCacheSuite: spliced = self._consume_staged_prefetch( cons2, CacheRequestHandle("window-req", 0), - prefix_indices=m.device_indices, ) self.assertEqual(int(spliced.numel()), len(seq_ab) - len(seq_a)) self.assertEqual( @@ -7687,7 +7632,7 @@ class UnifiedRadixCacheSuite: ) self.assertEqual(new_node, leaf) - self.assertEqual(len(torch.cat([req.prefix_indices, new_indices])), len(tokens)) + self.assertEqual(req.prefix_len + len(new_indices), len(tokens)) self.assertIsNotNone(_device_value(cache, leaf, ComponentType.MAMBA)) self._finish_pending_loads(cache) self._release_ongoing_load_back_locks(cache) @@ -7729,7 +7674,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(new_node, leaf) self.assertEqual(new_indices.tolist(), leaf_full.tolist()) - self.assertEqual(len(torch.cat([req.prefix_indices, new_indices])), len(tokens)) + self.assertEqual(req.prefix_len + len(new_indices), len(tokens)) self.assertEqual( _device_value(cache, leaf, ComponentType.FULL).tolist(), leaf_full.tolist(), @@ -8017,7 +7962,7 @@ class UnifiedRadixCacheSuite: sampling_params=SamplingParams(temperature=0, max_new_tokens=1), ) req.init_next_round_input(cache) - self.assertEqual((len(req.prefix_indices), req.host_hit_length), (0, 0)) + self.assertEqual((req.prefix_len, req.host_hit_length), (0, 0)) self.assertEqual(req.mamba_host_hit_length, 1) adder = PrefillAdder( page_size=1, @@ -8034,7 +7979,7 @@ class UnifiedRadixCacheSuite: # 18-token demand would reject and scheduler cleanup would free the CoW # destination even though the queued H2D still targets it. self.assertEqual(adder.can_run_list, [req]) - self.assertEqual(len(req.prefix_indices), 12) + self.assertEqual(req.prefix_len, 12) self.assertEqual(req.kv.cache_protected_len, 12) slot = req.kv.mamba_pool_idx.unsqueeze(0) other_slots = pool.mamba_allocator.alloc(pool.mamba_allocator.available_size())