Reduce decode bootstrap latency with request-owned speculative KV (#38978)

Signed-off-by: inkcherry <mingzhi.liu@amd.com>
Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
inkcherry
2026-09-23 02:24:53 -07:00
committed by GitHub
co-authored by Duyi-Wang
parent 4cd63da996
commit 58f622ebc4
9 changed files with 382 additions and 33 deletions
+3 -3
View File
@@ -471,7 +471,7 @@ def handle_other_validations(server_args: Any):
elif cfg.enable_hierarchical_cache and not (
(
cfg.hicache_storage_backend is None
and cfg.hicache_write_policy == "write_back"
and cfg.hicache_write_policy in ("write_back", "write_through")
)
or (
cfg.hicache_storage_backend is not None
@@ -480,8 +480,8 @@ def handle_other_validations(server_args: Any):
)
):
logger.warning(
"Optimistic prefill supports L2 write-back or L3 buffer-only "
"write-through hierarchical cache"
"Optimistic prefill supports L2 write-back/write-through or "
"L3 buffer-only write-through hierarchical cache"
)
declare_resolution(
server_args,
+45 -5
View File
@@ -80,6 +80,7 @@ from sglang.srt.mem_cache.common import (
)
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.observability.req_time_stats import set_schedule_time_batch
from sglang.srt.observability.scheduler_stage_metrics import (
SCHEDULER_STAGE_GET_NEXT_BATCH,
@@ -117,6 +118,14 @@ def should_force_retry(req: Req) -> bool:
return int.from_bytes(digest[:8], "big") < retry_prob * 2**64
def _uses_write_through_cache(cache: object) -> bool:
return (
isinstance(cache, UnifiedRadixCache)
and cache.cache_controller is not None
and cache.cache_controller.write_policy == "write_through"
)
def _transfer_start_layer(*, pool, hf_text_config) -> int:
# Hybrid pools count all layers in start_layer, but peer KV lists contain only
# full-attention layers, so translate to a full-attention-relative offset.
@@ -500,6 +509,11 @@ class PrefillBootstrapQueue:
if (
req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts
and not req.is_retracted # engine paused
and not (
_uses_write_through_cache(self.scheduler.tree_cache)
and req.swa_branching_seqlen is not None
and req.swa_branching_seqlen > req.kv.cache_protected_len
)
):
if not self.ensure_metadata_buffer(req):
continue # no more metadata buffer
@@ -561,6 +575,26 @@ class SchedulerDisaggregationPrefillMixin:
if room is not None and room in kv_mgr.transfer_infos:
prefetch(room)
def cache_unfinished_disagg_prefill(
self: Scheduler, req: Req, *, chunked: bool = False
) -> None:
cache = self.tree_cache
if req.pending_bootstrap and _uses_write_through_cache(cache):
cache.advance_unpublished_req(req, chunked=chunked)
return
maybe_cache_unfinished_req(req, cache, chunked=chunked)
def release_aborted_prefill_waiting_req(self: Scheduler, req: Req) -> None:
self.clear_pending_chunk_send(req)
sender = getattr(req, "disagg_kv_sender", None)
if sender is not None and hasattr(sender, "abort"):
sender.abort()
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, is_insert=False)
req.pending_bootstrap = False
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
def resolve_waiting_queue_bootstrap(self: Scheduler) -> None:
"""Resolve bootstrap status for waiting prefill requests before admission.
@@ -851,7 +885,7 @@ class SchedulerDisaggregationPrefillMixin:
advance_logprob_pt(i, req)
continue
maybe_cache_unfinished_req(req, self.tree_cache)
self.cache_unfinished_disagg_prefill(req)
self.disagg_prefill_inflight_queue.append(req)
if self.spec_algorithm.is_eagle() and draft_input is not None:
req.output_topk_p = draft_input.topk_p[i]
@@ -1160,7 +1194,7 @@ class SchedulerDisaggregationPrefillMixin:
logger.warning(error_message)
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache)
release_kv_cache(req, self.tree_cache, is_insert=False)
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
req.pending_bootstrap = False
prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
@@ -1182,6 +1216,7 @@ class SchedulerDisaggregationPrefillMixin:
# Metadata buffer was allocated in pop_bootstrapped before
# the request entered the waiting queue, so finalize should not fail.
assert self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req)
self.cache_unfinished_disagg_prefill(req)
return True
else:
raise RuntimeError(
@@ -1208,7 +1243,7 @@ class SchedulerDisaggregationPrefillMixin:
chunked_req_to_exclude = set()
if (req := self.chunked_req) is not None:
chunked_req_to_exclude.add(req)
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
self.cache_unfinished_disagg_prefill(req, chunked=True)
if not self.check_bootstrap(req):
if is_aborted(req):
@@ -1506,7 +1541,9 @@ class SchedulerDisaggregationPrefillMixin:
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
"""Release KV cache and requeue an optimistic prefill request."""
max_attempts = get_disagg().optimistic_prefill_attempts
maybe_cache_unfinished_req(req, self.tree_cache)
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
if not uses_write_through_cache:
maybe_cache_unfinished_req(req, self.tree_cache)
# The cached prefix is evictable once the KV is released. Its length
# (capped at what a retry can match) seeds the retry's storage baseline,
# so an evicted prefix is looked up in L3 once before it is recomputed.
@@ -1521,7 +1558,10 @@ class SchedulerDisaggregationPrefillMixin:
self._release_aborted_request(req)
# Mamba insertion donates the checkpoint and clears its sequence marker.
release_kv_cache(
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
req,
self.tree_cache,
is_insert=not uses_write_through_cache
and not self.tree_cache.supports_mamba(),
)
req.reset_for_retract()
req.output_ids = array("q")
+8 -16
View File
@@ -3526,7 +3526,10 @@ class Scheduler(
self.handle_embedding_request(tokenized_req)
def stash_chunked_request(self, req: Req):
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.cache_unfinished_disagg_prefill(req, chunked=True)
else:
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
def process_pending_chunked_abort(self) -> None:
"""Abort an in-flight chunked-prefill request once it is safe to do so.
@@ -5448,24 +5451,13 @@ class Scheduler(
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
if self.disaggregation_mode == DisaggregationMode.DECODE:
release_kv_cache(req, self.tree_cache)
# For disaggregation prefill mode, free the metadata buffer index
if self.disaggregation_mode == DisaggregationMode.PREFILL:
bootstrap_pending = req.pending_bootstrap
maybe_release_metadata_buffer(
req, self.req_to_metadata_buffer_idx_allocator
)
if (
bootstrap_pending
and hasattr(req, "disagg_kv_sender")
and req.disagg_kv_sender is not None
):
if hasattr(req.disagg_kv_sender, "abort"):
req.disagg_kv_sender.abort()
self.release_aborted_prefill_waiting_req(req)
# For mamba radix cache
if (
req.kv.holds_mamba
and self.disaggregation_mode != DisaggregationMode.DECODE
if req.kv.holds_mamba and self.disaggregation_mode not in (
DisaggregationMode.PREFILL,
DisaggregationMode.DECODE,
):
release_kv_cache(req, self.tree_cache, is_insert=False)
logger.debug(f"Abort queued request. {req.rid=}")
@@ -1111,6 +1111,46 @@ class UnifiedRadixCache(BasePrefixCache):
):
self.session_refs.register_session_ref(req)
@rank_consensus(same_params=["req.rid", "chunked"])
def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None:
assert not self.supports_mamba()
token_ids = req.get_fill_ids()
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : len(token_ids)
]
insert_params = InsertParams(
prev_prefix_len=req.kv.cache_protected_len,
chunked=chunked,
priority=getattr(req, "priority", 0) or 0,
rotation_base=req.kv_rotation_base,
)
effective_cache_len = len(token_ids)
for comp in self._components_tuple:
cache_len = comp.prepare_for_caching_req(
req=req,
insert_params=insert_params,
token_ids_len=len(token_ids),
is_finished=False,
)
if cache_len is not None:
effective_cache_len = min(effective_cache_len, cache_len)
radix_key = RadixKey(
token_ids[:effective_cache_len],
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
)
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
for comp in self._components_tuple:
comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params)
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req, is_finished=False, insert_params=insert_params
)
@rank_consensus(same_params=["req.rid", "chunked"])
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
if self.session.try_cache_unfinished_req(req, chunked=chunked, **kwargs):