Enable optimistic prefill for Mamba radix-cache models (#40184)
This commit is contained in:
@@ -13,7 +13,6 @@ from typing import Any
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
declare_resolution,
|
||||
model_config_of,
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
@@ -486,15 +485,6 @@ def handle_other_validations(server_args: Any):
|
||||
"_handle_other_validations",
|
||||
optimistic_prefill_attempts=0,
|
||||
)
|
||||
elif resolved_view(server_args).uses_mamba_radix_cache:
|
||||
logger.warning(
|
||||
"Optimistic prefill does not support models that use mamba radix cache."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_other_validations",
|
||||
optimistic_prefill_attempts=0,
|
||||
)
|
||||
|
||||
# Handle model inference tensor dump.
|
||||
if cfg.debug_tensor_dump_output_folder is not None:
|
||||
|
||||
@@ -1510,7 +1510,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
)
|
||||
self._release_aborted_request(req)
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
release_kv_cache(
|
||||
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
|
||||
)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
Reference in New Issue
Block a user