diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py index 18f3bd29a8..e18fc6648a 100644 --- a/python/sglang/srt/arg_groups/serving_hook.py +++ b/python/sglang/srt/arg_groups/serving_hook.py @@ -471,7 +471,7 @@ def handle_other_validations(server_args: Any): elif cfg.enable_hierarchical_cache and not ( ( cfg.hicache_storage_backend is None - and cfg.hicache_write_policy == "write_back" + and cfg.hicache_write_policy in ("write_back", "write_through") ) or ( cfg.hicache_storage_backend is not None @@ -480,8 +480,8 @@ def handle_other_validations(server_args: Any): ) ): logger.warning( - "Optimistic prefill supports L2 write-back or L3 buffer-only " - "write-through hierarchical cache" + "Optimistic prefill supports L2 write-back/write-through or " + "L3 buffer-only write-through hierarchical cache" ) declare_resolution( server_args, diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 0067c98489..a2623adcff 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -80,6 +80,7 @@ from sglang.srt.mem_cache.common import ( ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.observability.req_time_stats import set_schedule_time_batch from sglang.srt.observability.scheduler_stage_metrics import ( SCHEDULER_STAGE_GET_NEXT_BATCH, @@ -117,6 +118,14 @@ def should_force_retry(req: Req) -> bool: return int.from_bytes(digest[:8], "big") < retry_prob * 2**64 +def _uses_write_through_cache(cache: object) -> bool: + return ( + isinstance(cache, UnifiedRadixCache) + and cache.cache_controller is not None + and cache.cache_controller.write_policy == "write_through" + ) + + def _transfer_start_layer(*, pool, hf_text_config) -> int: # Hybrid pools count all layers in start_layer, but peer KV lists contain only # full-attention layers, so translate to a full-attention-relative offset. @@ -500,6 +509,11 @@ class PrefillBootstrapQueue: if ( req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts and not req.is_retracted # engine paused + and not ( + _uses_write_through_cache(self.scheduler.tree_cache) + and req.swa_branching_seqlen is not None + and req.swa_branching_seqlen > req.kv.cache_protected_len + ) ): if not self.ensure_metadata_buffer(req): continue # no more metadata buffer @@ -561,6 +575,26 @@ class SchedulerDisaggregationPrefillMixin: if room is not None and room in kv_mgr.transfer_infos: prefetch(room) + def cache_unfinished_disagg_prefill( + self: Scheduler, req: Req, *, chunked: bool = False + ) -> None: + cache = self.tree_cache + if req.pending_bootstrap and _uses_write_through_cache(cache): + cache.advance_unpublished_req(req, chunked=chunked) + return + + maybe_cache_unfinished_req(req, cache, chunked=chunked) + + def release_aborted_prefill_waiting_req(self: Scheduler, req: Req) -> None: + self.clear_pending_chunk_send(req) + sender = getattr(req, "disagg_kv_sender", None) + if sender is not None and hasattr(sender, "abort"): + sender.abort() + maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator) + if req.kv.holds_kv or req.kv.holds_mamba: + release_kv_cache(req, self.tree_cache, is_insert=False) + req.pending_bootstrap = False + @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE) def resolve_waiting_queue_bootstrap(self: Scheduler) -> None: """Resolve bootstrap status for waiting prefill requests before admission. @@ -851,7 +885,7 @@ class SchedulerDisaggregationPrefillMixin: advance_logprob_pt(i, req) continue - maybe_cache_unfinished_req(req, self.tree_cache) + self.cache_unfinished_disagg_prefill(req) self.disagg_prefill_inflight_queue.append(req) if self.spec_algorithm.is_eagle() and draft_input is not None: req.output_topk_p = draft_input.topk_p[i] @@ -1160,7 +1194,7 @@ class SchedulerDisaggregationPrefillMixin: logger.warning(error_message) req.time_stats.trace_ctx.abort(abort_info={"reason": error_message}) if req.kv.holds_kv or req.kv.holds_mamba: - release_kv_cache(req, self.tree_cache) + release_kv_cache(req, self.tree_cache, is_insert=False) maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator) req.pending_bootstrap = False prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR) @@ -1182,6 +1216,7 @@ class SchedulerDisaggregationPrefillMixin: # Metadata buffer was allocated in pop_bootstrapped before # the request entered the waiting queue, so finalize should not fail. assert self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req) + self.cache_unfinished_disagg_prefill(req) return True else: raise RuntimeError( @@ -1208,7 +1243,7 @@ class SchedulerDisaggregationPrefillMixin: chunked_req_to_exclude = set() if (req := self.chunked_req) is not None: chunked_req_to_exclude.add(req) - maybe_cache_unfinished_req(req, self.tree_cache, chunked=True) + self.cache_unfinished_disagg_prefill(req, chunked=True) if not self.check_bootstrap(req): if is_aborted(req): @@ -1506,7 +1541,9 @@ class SchedulerDisaggregationPrefillMixin: def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None: """Release KV cache and requeue an optimistic prefill request.""" max_attempts = get_disagg().optimistic_prefill_attempts - maybe_cache_unfinished_req(req, self.tree_cache) + uses_write_through_cache = _uses_write_through_cache(self.tree_cache) + if not uses_write_through_cache: + maybe_cache_unfinished_req(req, self.tree_cache) # The cached prefix is evictable once the KV is released. Its length # (capped at what a retry can match) seeds the retry's storage baseline, # so an evicted prefix is looked up in L3 once before it is recomputed. @@ -1521,7 +1558,10 @@ class SchedulerDisaggregationPrefillMixin: self._release_aborted_request(req) # Mamba insertion donates the checkpoint and clears its sequence marker. release_kv_cache( - req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba() + req, + self.tree_cache, + is_insert=not uses_write_through_cache + and not self.tree_cache.supports_mamba(), ) req.reset_for_retract() req.output_ids = array("q") diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 2ec3face15..1bbcc3a360 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3526,7 +3526,10 @@ class Scheduler( self.handle_embedding_request(tokenized_req) def stash_chunked_request(self, req: Req): - maybe_cache_unfinished_req(req, self.tree_cache, chunked=True) + if self.disaggregation_mode == DisaggregationMode.PREFILL: + self.cache_unfinished_disagg_prefill(req, chunked=True) + else: + maybe_cache_unfinished_req(req, self.tree_cache, chunked=True) def process_pending_chunked_abort(self) -> None: """Abort an in-flight chunked-prefill request once it is safe to do so. @@ -5448,24 +5451,13 @@ class Scheduler( # For disaggregation decode mode, the request in the waiting queue has KV cache allocated. if self.disaggregation_mode == DisaggregationMode.DECODE: release_kv_cache(req, self.tree_cache) - # For disaggregation prefill mode, free the metadata buffer index if self.disaggregation_mode == DisaggregationMode.PREFILL: - bootstrap_pending = req.pending_bootstrap - maybe_release_metadata_buffer( - req, self.req_to_metadata_buffer_idx_allocator - ) - if ( - bootstrap_pending - and hasattr(req, "disagg_kv_sender") - and req.disagg_kv_sender is not None - ): - if hasattr(req.disagg_kv_sender, "abort"): - req.disagg_kv_sender.abort() + self.release_aborted_prefill_waiting_req(req) # For mamba radix cache - if ( - req.kv.holds_mamba - and self.disaggregation_mode != DisaggregationMode.DECODE + if req.kv.holds_mamba and self.disaggregation_mode not in ( + DisaggregationMode.PREFILL, + DisaggregationMode.DECODE, ): release_kv_cache(req, self.tree_cache, is_insert=False) logger.debug(f"Abort queued request. {req.rid=}") diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 7803025014..cd45906334 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1111,6 +1111,46 @@ class UnifiedRadixCache(BasePrefixCache): ): self.session_refs.register_session_ref(req) + @rank_consensus(same_params=["req.rid", "chunked"]) + def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None: + assert not self.supports_mamba() + token_ids = req.get_fill_ids() + kv_indices = self.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, : len(token_ids) + ] + insert_params = InsertParams( + prev_prefix_len=req.kv.cache_protected_len, + chunked=chunked, + priority=getattr(req, "priority", 0) or 0, + rotation_base=req.kv_rotation_base, + ) + effective_cache_len = len(token_ids) + for comp in self._components_tuple: + cache_len = comp.prepare_for_caching_req( + req=req, + insert_params=insert_params, + token_ids_len=len(token_ids), + is_finished=False, + ) + if cache_len is not None: + effective_cache_len = min(effective_cache_len, cache_len) + + radix_key = RadixKey( + token_ids[:effective_cache_len], + req.extra_key, + is_bigram=self.tree_core.is_eagle, + cache_salt=req.cache_salt, + ) + if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get(): + for comp in self._components_tuple: + comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params) + + req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) + for comp in self._components_tuple: + comp.cleanup_after_caching_req( + req, is_finished=False, insert_params=insert_params + ) + @rank_consensus(same_params=["req.rid", "chunked"]) def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None: if self.session.try_cache_unfinished_req(req, chunked=chunked, **kwargs): diff --git a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py index 8ae3a54b71..a6290444d0 100644 --- a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py +++ b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py @@ -105,6 +105,9 @@ class TestOptimisticPrefill( "3", "--chunked-prefill-size", "128", + "--enable-hierarchical-cache", + "--hicache-write-policy", + "write_through", "--enable-metrics", "--enable-request-time-stats-logging", ] @@ -187,6 +190,9 @@ class TestOptimisticPrefillFailure(PDDisaggregationServerBase): "3", "--chunked-prefill-size", "128", + "--enable-hierarchical-cache", + "--hicache-write-policy", + "write_through", "--enable-metrics", "--enable-request-time-stats-logging", "--load-format", @@ -345,13 +351,13 @@ class TestOptimisticPrefillMambaAdmission(CustomTestCase): server_args = self._make_args(pp_size=2, **self.BASE) self.assertEqual(self._resolved_attempts(server_args), 0) - def test_hicache_write_policy_restriction_retained(self): + def test_l2_write_through_allowed(self): server_args = self._make_args( enable_hierarchical_cache=True, hicache_write_policy="write_through", **self.BASE, ) - self.assertEqual(self._resolved_attempts(server_args), 0) + self.assertEqual(self._resolved_attempts(server_args), 3) class TestOptimisticPrefillMambaRetryRelease(CustomTestCase): diff --git a/test/registered/unit/managers/test_auxiliary_output.py b/test/registered/unit/managers/test_auxiliary_output.py index 4265ea61ee..4ef9b38921 100644 --- a/test/registered/unit/managers/test_auxiliary_output.py +++ b/test/registered/unit/managers/test_auxiliary_output.py @@ -595,19 +595,20 @@ def test_disaggregated_prefill_consumes_auxiliary_output_after_commit(): spec_algorithm=SimpleNamespace(is_eagle=lambda: False), tree_cache=object(), disagg_prefill_inflight_queue=[], + cache_unfinished_disagg_prefill=Mock(), send_kv_chunk=Mock(), metrics_reporter=SimpleNamespace(report_prefill_stats=Mock()), maybe_send_health_check_signal=Mock(), ) - with patch("sglang.srt.disaggregation.prefill.maybe_cache_unfinished_req"): - SchedulerDisaggregationPrefillMixin.process_batch_result_disagg_prefill( - scheduler, - batch, - result, - ) + SchedulerDisaggregationPrefillMixin.process_batch_result_disagg_prefill( + scheduler, + batch, + result, + ) assert req.output_ids == [7] + scheduler.cache_unfinished_disagg_prefill.assert_called_once_with(req) snapshot_auxiliary_output_starts.assert_called_once_with(batch, result) processor.consume_auxiliary_output.assert_called_once_with( batch, diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index d8a3e4cbc7..fdb90883dc 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -8,12 +8,19 @@ from unittest.mock import MagicMock, patch import numpy as np import torch +from sglang.srt.disaggregation.base import KVPoll # noqa: E402 from sglang.srt.disaggregation.decode import ( # noqa: E402 DecodePreallocQueue, SchedulerDisaggregationDecodeMixin, ) +from sglang.srt.disaggregation.prefill import ( # noqa: E402 + PrefillBootstrapQueue, + SchedulerDisaggregationPrefillMixin, +) from sglang.srt.disaggregation.utils import DisaggregationMode # noqa: E402 +from sglang.srt.environ import envs # noqa: E402 from sglang.srt.lora.lora_manager import LoRAManager # noqa: E402 +from sglang.srt.managers.io_struct import AbortReq # noqa: E402 from sglang.srt.managers.schedule_batch import ( # noqa: E402 FINISH_ABORT, Req, @@ -21,6 +28,7 @@ from sglang.srt.managers.schedule_batch import ( # noqa: E402 ) from sglang.srt.managers.scheduler import Scheduler # noqa: E402 from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache # noqa: E402 from sglang.srt.runtime_context import get_context, publish, reset_context # noqa: E402 from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci @@ -162,6 +170,246 @@ class TestDisaggregationPriorityQueueing(unittest.TestCase): scheduler._prefetch_kvcache.assert_called_once_with(incoming) +class TestOptimisticPrefillCacheOwnership(unittest.TestCase): + def _new_unpublished_swa_state(self): + cache = UnifiedRadixCache.__new__(UnifiedRadixCache) + cache.is_mamba_enabled = False + cache.tree_core = SimpleNamespace(is_eagle=False) + cache.req_to_token_pool = SimpleNamespace( + req_to_token=torch.arange(16, dtype=torch.int64).reshape(2, 8) + ) + component = MagicMock() + component.prepare_for_caching_req.return_value = None + cache._components_tuple = (component,) + cache.insert = MagicMock() + req = SimpleNamespace( + rid="req", + get_fill_ids=lambda: [1, 2, 3, 4], + kv=SimpleNamespace( + req_pool_idx=1, + cache_protected_len=2, + swa_evicted_seqlen=1, + ), + prefix_indices=torch.tensor([8, 9], dtype=torch.int64), + priority=3, + extra_key=None, + cache_salt=None, + kv_rotation_base=None, + last_node=object(), + ) + return cache, component, req + + def _pop_bootstrapping_swa_branch(self, write_policy): + queue = PrefillBootstrapQueue.__new__(PrefillBootstrapQueue) + queue.scheduler_stage_metrics = None + queue.pp_size = 1 + cache = UnifiedRadixCache.__new__(UnifiedRadixCache) + cache.cache_controller = SimpleNamespace(write_policy=write_policy) + queue.scheduler = SimpleNamespace( + attn_cp_cpu_group=None, + attn_tp_cpu_group=None, + tree_cache=cache, + processed_tokens_counter=0, + ) + queue.ensure_metadata_buffer = MagicMock(return_value=True) + req = SimpleNamespace( + rid="req", + disagg_kv_sender=object(), + prefill_attempt_count=0, + is_retracted=False, + swa_branching_seqlen=4, + kv=SimpleNamespace(cache_protected_len=2), + time_stats=MagicMock(), + ) + queue.queue = [req] + + with ( + patch( + "sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group", + return_value=[KVPoll.Bootstrapping], + ), + patch( + "sglang.srt.disaggregation.prefill.get_disagg", + return_value=SimpleNamespace(optimistic_prefill_attempts=2), + ), + ): + result = queue.pop_bootstrapped() + + return queue, req, result + + def test_write_through_pending_swa_branch_waits_for_bootstrap(self): + queue, req, result = self._pop_bootstrapping_swa_branch("write_through") + + self.assertEqual(result, []) + self.assertEqual(queue.queue, [req]) + self.assertEqual(req.prefill_attempt_count, 0) + queue.ensure_metadata_buffer.assert_not_called() + + def test_other_write_policies_stay_optimistic(self): + for write_policy in ("write_back", "write_through_selective"): + with self.subTest(write_policy=write_policy): + queue, req, result = self._pop_bootstrapping_swa_branch(write_policy) + + self.assertEqual(result, [req]) + self.assertEqual(queue.queue, []) + self.assertEqual(req.prefill_attempt_count, 1) + queue.ensure_metadata_buffer.assert_called_once_with(req) + + def test_unpublished_advance_preserves_swa_frontier_and_ownership(self): + cache, component, req = self._new_unpublished_swa_state() + last_node = req.last_node + + with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): + cache.advance_unpublished_req(req, chunked=True) + + self.assertTrue(torch.equal(req.prefix_indices, torch.tensor([8, 9, 10, 11]))) + self.assertEqual(req.kv.cache_protected_len, 2) + self.assertIs(req.last_node, last_node) + prepare_params = component.prepare_for_caching_req.call_args.kwargs[ + "insert_params" + ] + self.assertTrue(prepare_params.chunked) + self.assertEqual(prepare_params.prev_prefix_len, 2) + component.free_out_of_window_slots.assert_called_once_with( + req, 3, prepare_params + ) + component.cleanup_after_caching_req.assert_called_once_with( + req, is_finished=False, insert_params=prepare_params + ) + cache.insert.assert_not_called() + + def test_bootstrap_failure_after_swa_eviction_releases_once(self): + cache, component, req = self._new_unpublished_swa_state() + req.bootstrap_room = 7 + req.disagg_kv_sender = MagicMock() + req.time_stats = SimpleNamespace(trace_ctx=MagicMock()) + req.return_logprob = False + req.pending_bootstrap = True + req.metadata_buffer_index = -1 + req.cache_request_handle = object() + req.kv.holds_kv = True + req.kv.holds_mamba = False + cache.finish = MagicMock() + scheduler = SimpleNamespace( + tree_cache=cache, + clear_pending_chunk_send=MagicMock(), + ps=SimpleNamespace(tp_rank=0), + req_to_metadata_buffer_idx_allocator=MagicMock(), + output_streamer=MagicMock(), + metrics_reporter=SimpleNamespace(enable_metrics=False), + enable_hicache_storage=False, + ) + + with ( + envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True), + patch( + "sglang.srt.disaggregation.prefill.release_kv_cache" + ) as release_kv_cache, + patch("sglang.srt.disaggregation.prefill.prepare_abort"), + patch( + "sglang.srt.disaggregation.prefill.get_parallel", + return_value=SimpleNamespace(tp_rank=0), + ), + ): + cache.advance_unpublished_req(req, chunked=True) + SchedulerDisaggregationPrefillMixin.handle_bootstrap_failure(scheduler, req) + + component.free_out_of_window_slots.assert_called_once() + cache.insert.assert_not_called() + release_kv_cache.assert_called_once_with(req, cache, is_insert=False) + cache.finish.assert_called_once_with( + req.cache_request_handle, CacheRequestOutcome.ABORT + ) + self.assertFalse(req.pending_bootstrap) + + def test_write_through_pending_chunk_does_not_publish(self): + scheduler = SimpleNamespace() + cache = UnifiedRadixCache.__new__(UnifiedRadixCache) + cache.cache_controller = SimpleNamespace(write_policy="write_through") + cache.advance_unpublished_req = MagicMock() + scheduler.tree_cache = cache + req = SimpleNamespace(pending_bootstrap=True) + + SchedulerDisaggregationPrefillMixin.cache_unfinished_disagg_prefill( + scheduler, req, chunked=True + ) + + cache.advance_unpublished_req.assert_called_once_with(req, chunked=True) + + def test_bootstrap_success_publishes_once(self): + scheduler = SimpleNamespace( + disagg_prefill_bootstrap_queue=MagicMock(), + cache_unfinished_disagg_prefill=MagicMock(), + ) + scheduler.disagg_prefill_bootstrap_queue.finalize_bootstrap.return_value = True + req = SimpleNamespace(rid="req") + + result = SchedulerDisaggregationPrefillMixin.handle_pending_bootstrap( + scheduler, req, KVPoll.WaitingForInput + ) + + self.assertTrue(result) + scheduler.disagg_prefill_bootstrap_queue.finalize_bootstrap.assert_called_once_with( + req + ) + scheduler.cache_unfinished_disagg_prefill.assert_called_once_with(req) + + def test_waiting_abort_releases_without_insert(self): + scheduler = SimpleNamespace( + tree_cache=MagicMock(), + req_to_metadata_buffer_idx_allocator=MagicMock(), + clear_pending_chunk_send=MagicMock(), + ) + sender = MagicMock() + req = SimpleNamespace( + disagg_kv_sender=sender, + metadata_buffer_index=3, + kv=SimpleNamespace(holds_kv=True, holds_mamba=False), + pending_bootstrap=True, + ) + + with patch( + "sglang.srt.disaggregation.prefill.release_kv_cache" + ) as release_kv_cache: + SchedulerDisaggregationPrefillMixin.release_aborted_prefill_waiting_req( + scheduler, req + ) + + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, is_insert=False + ) + sender.abort.assert_called_once() + self.assertFalse(req.pending_bootstrap) + + def test_scheduler_waiting_abort_delegates_release_once(self): + scheduler = Scheduler.__new__(Scheduler) + scheduler.chunked_req = None + scheduler.mm_receiver = None + scheduler.waiting_queue = [MagicMock(rid="req")] + scheduler.beam_coordinator = MagicMock() + scheduler.enable_hicache_storage = False + scheduler.ipc_channels = MagicMock() + scheduler.disaggregation_mode = DisaggregationMode.PREFILL + scheduler.release_aborted_prefill_waiting_req = MagicMock() + scheduler.dllm_config = None + scheduler.grammar_manager = MagicMock() + scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[]) + scheduler.disagg_prefill_inflight_queue = [] + scheduler.collect_inflight_reqs = MagicMock(return_value=set()) + scheduler.tree_cache = MagicMock() + + with ( + patch("sglang.srt.managers.scheduler.get_serving") as get_serving, + patch("sglang.srt.managers.scheduler.release_kv_cache") as release, + ): + get_serving.return_value.weight_version = None + scheduler.abort_request(AbortReq(rid="req")) + + scheduler.release_aborted_prefill_waiting_req.assert_called_once() + scheduler.tree_cache.finish.assert_called_once() + release.assert_not_called() + + class TestDecodePreallocQueuePriority(unittest.TestCase): def setUp(self): # The code under test reads its config from the bags. diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 94ac81dd53..200eed26b1 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -13,6 +13,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() +from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.managers.schedule_batch import NextBatchPlan, Req, ReqKvInfo from sglang.srt.managers.scheduler import Scheduler from sglang.srt.mem_cache.chunk_cache import ChunkCache @@ -77,6 +78,7 @@ def _make_chunk_cache(req_to_token_pool) -> ChunkCache: def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler: s = Scheduler.__new__(Scheduler) s.scheduler_stage_metrics = None + s.disaggregation_mode = DisaggregationMode.NULL s.dllm_config = None s.dllm_manager = None s.enable_hisparse = False diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index f8367542c1..6cf4026a24 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -67,6 +67,7 @@ from sglang.srt.arg_groups.serving_hook import ( handle_load_balance_method, handle_missing_default_values, handle_multimodal_feature_transport, + handle_other_validations, handle_ssl_validation, handle_tokenizer_batching, ssl_verify_of, @@ -120,6 +121,25 @@ _mock_device.start() class TestPrepareServerArgs(CustomTestCase): + def test_optimistic_prefill_allows_l2_write_through_only(self): + for policy, expected in ( + ("write_back", 2), + ("write_through", 2), + ("write_through_selective", 0), + ): + with self.subTest(policy=policy): + args = ServerArgs( + model_path="dummy", + disaggregation_mode="prefill", + optimistic_prefill_attempts=2, + enable_hierarchical_cache=True, + hicache_write_policy=policy, + ) + handle_other_validations(args) + self.assertEqual( + resolution_result(args, "optimistic_prefill_attempts"), expected + ) + def test_radix_eviction_policy_explicitness_is_preserved(self): omitted = prepare_server_args(["--model-path", "dummy"]) separated = prepare_server_args( @@ -2040,7 +2060,7 @@ class TestHiCacheArgs(unittest.TestCase): }, 3, ), - ({"hicache_write_policy": "write_through"}, 0), + ({"hicache_write_policy": "write_through"}, 3), ( { "hicache_storage_backend": "file",