Reduce decode bootstrap latency with request-owned speculative KV (#38978)
Signed-off-by: inkcherry <mingzhi.liu@amd.com> Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
@@ -471,7 +471,7 @@ def handle_other_validations(server_args: Any):
|
||||
elif cfg.enable_hierarchical_cache and not (
|
||||
(
|
||||
cfg.hicache_storage_backend is None
|
||||
and cfg.hicache_write_policy == "write_back"
|
||||
and cfg.hicache_write_policy in ("write_back", "write_through")
|
||||
)
|
||||
or (
|
||||
cfg.hicache_storage_backend is not None
|
||||
@@ -480,8 +480,8 @@ def handle_other_validations(server_args: Any):
|
||||
)
|
||||
):
|
||||
logger.warning(
|
||||
"Optimistic prefill supports L2 write-back or L3 buffer-only "
|
||||
"write-through hierarchical cache"
|
||||
"Optimistic prefill supports L2 write-back/write-through or "
|
||||
"L3 buffer-only write-through hierarchical cache"
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
|
||||
@@ -80,6 +80,7 @@ from sglang.srt.mem_cache.common import (
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.observability.req_time_stats import set_schedule_time_batch
|
||||
from sglang.srt.observability.scheduler_stage_metrics import (
|
||||
SCHEDULER_STAGE_GET_NEXT_BATCH,
|
||||
@@ -117,6 +118,14 @@ def should_force_retry(req: Req) -> bool:
|
||||
return int.from_bytes(digest[:8], "big") < retry_prob * 2**64
|
||||
|
||||
|
||||
def _uses_write_through_cache(cache: object) -> bool:
|
||||
return (
|
||||
isinstance(cache, UnifiedRadixCache)
|
||||
and cache.cache_controller is not None
|
||||
and cache.cache_controller.write_policy == "write_through"
|
||||
)
|
||||
|
||||
|
||||
def _transfer_start_layer(*, pool, hf_text_config) -> int:
|
||||
# Hybrid pools count all layers in start_layer, but peer KV lists contain only
|
||||
# full-attention layers, so translate to a full-attention-relative offset.
|
||||
@@ -500,6 +509,11 @@ class PrefillBootstrapQueue:
|
||||
if (
|
||||
req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts
|
||||
and not req.is_retracted # engine paused
|
||||
and not (
|
||||
_uses_write_through_cache(self.scheduler.tree_cache)
|
||||
and req.swa_branching_seqlen is not None
|
||||
and req.swa_branching_seqlen > req.kv.cache_protected_len
|
||||
)
|
||||
):
|
||||
if not self.ensure_metadata_buffer(req):
|
||||
continue # no more metadata buffer
|
||||
@@ -561,6 +575,26 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if room is not None and room in kv_mgr.transfer_infos:
|
||||
prefetch(room)
|
||||
|
||||
def cache_unfinished_disagg_prefill(
|
||||
self: Scheduler, req: Req, *, chunked: bool = False
|
||||
) -> None:
|
||||
cache = self.tree_cache
|
||||
if req.pending_bootstrap and _uses_write_through_cache(cache):
|
||||
cache.advance_unpublished_req(req, chunked=chunked)
|
||||
return
|
||||
|
||||
maybe_cache_unfinished_req(req, cache, chunked=chunked)
|
||||
|
||||
def release_aborted_prefill_waiting_req(self: Scheduler, req: Req) -> None:
|
||||
self.clear_pending_chunk_send(req)
|
||||
sender = getattr(req, "disagg_kv_sender", None)
|
||||
if sender is not None and hasattr(sender, "abort"):
|
||||
sender.abort()
|
||||
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
req.pending_bootstrap = False
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
|
||||
def resolve_waiting_queue_bootstrap(self: Scheduler) -> None:
|
||||
"""Resolve bootstrap status for waiting prefill requests before admission.
|
||||
@@ -851,7 +885,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
advance_logprob_pt(i, req)
|
||||
continue
|
||||
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.cache_unfinished_disagg_prefill(req)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
@@ -1160,7 +1194,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
logger.warning(error_message)
|
||||
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
|
||||
req.pending_bootstrap = False
|
||||
prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
@@ -1182,6 +1216,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# Metadata buffer was allocated in pop_bootstrapped before
|
||||
# the request entered the waiting queue, so finalize should not fail.
|
||||
assert self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req)
|
||||
self.cache_unfinished_disagg_prefill(req)
|
||||
return True
|
||||
else:
|
||||
raise RuntimeError(
|
||||
@@ -1208,7 +1243,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
chunked_req_to_exclude = set()
|
||||
if (req := self.chunked_req) is not None:
|
||||
chunked_req_to_exclude.add(req)
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
self.cache_unfinished_disagg_prefill(req, chunked=True)
|
||||
|
||||
if not self.check_bootstrap(req):
|
||||
if is_aborted(req):
|
||||
@@ -1506,7 +1541,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
|
||||
"""Release KV cache and requeue an optimistic prefill request."""
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
|
||||
if not uses_write_through_cache:
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
# The cached prefix is evictable once the KV is released. Its length
|
||||
# (capped at what a retry can match) seeds the retry's storage baseline,
|
||||
# so an evicted prefix is looked up in L3 once before it is recomputed.
|
||||
@@ -1521,7 +1558,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self._release_aborted_request(req)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
release_kv_cache(
|
||||
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
|
||||
req,
|
||||
self.tree_cache,
|
||||
is_insert=not uses_write_through_cache
|
||||
and not self.tree_cache.supports_mamba(),
|
||||
)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
|
||||
@@ -3526,7 +3526,10 @@ class Scheduler(
|
||||
self.handle_embedding_request(tokenized_req)
|
||||
|
||||
def stash_chunked_request(self, req: Req):
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self.cache_unfinished_disagg_prefill(req, chunked=True)
|
||||
else:
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
|
||||
def process_pending_chunked_abort(self) -> None:
|
||||
"""Abort an in-flight chunked-prefill request once it is safe to do so.
|
||||
@@ -5448,24 +5451,13 @@ class Scheduler(
|
||||
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
# For disaggregation prefill mode, free the metadata buffer index
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
bootstrap_pending = req.pending_bootstrap
|
||||
maybe_release_metadata_buffer(
|
||||
req, self.req_to_metadata_buffer_idx_allocator
|
||||
)
|
||||
if (
|
||||
bootstrap_pending
|
||||
and hasattr(req, "disagg_kv_sender")
|
||||
and req.disagg_kv_sender is not None
|
||||
):
|
||||
if hasattr(req.disagg_kv_sender, "abort"):
|
||||
req.disagg_kv_sender.abort()
|
||||
self.release_aborted_prefill_waiting_req(req)
|
||||
|
||||
# For mamba radix cache
|
||||
if (
|
||||
req.kv.holds_mamba
|
||||
and self.disaggregation_mode != DisaggregationMode.DECODE
|
||||
if req.kv.holds_mamba and self.disaggregation_mode not in (
|
||||
DisaggregationMode.PREFILL,
|
||||
DisaggregationMode.DECODE,
|
||||
):
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
logger.debug(f"Abort queued request. {req.rid=}")
|
||||
|
||||
@@ -1111,6 +1111,46 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
):
|
||||
self.session_refs.register_session_ref(req)
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "chunked"])
|
||||
def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None:
|
||||
assert not self.supports_mamba()
|
||||
token_ids = req.get_fill_ids()
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : len(token_ids)
|
||||
]
|
||||
insert_params = InsertParams(
|
||||
prev_prefix_len=req.kv.cache_protected_len,
|
||||
chunked=chunked,
|
||||
priority=getattr(req, "priority", 0) or 0,
|
||||
rotation_base=req.kv_rotation_base,
|
||||
)
|
||||
effective_cache_len = len(token_ids)
|
||||
for comp in self._components_tuple:
|
||||
cache_len = comp.prepare_for_caching_req(
|
||||
req=req,
|
||||
insert_params=insert_params,
|
||||
token_ids_len=len(token_ids),
|
||||
is_finished=False,
|
||||
)
|
||||
if cache_len is not None:
|
||||
effective_cache_len = min(effective_cache_len, cache_len)
|
||||
|
||||
radix_key = RadixKey(
|
||||
token_ids[:effective_cache_len],
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
)
|
||||
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
|
||||
for comp in self._components_tuple:
|
||||
comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params)
|
||||
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req, is_finished=False, insert_params=insert_params
|
||||
)
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "chunked"])
|
||||
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
|
||||
if self.session.try_cache_unfinished_req(req, chunked=chunked, **kwargs):
|
||||
|
||||
Reference in New Issue
Block a user