Enable optimistic prefill for Mamba radix-cache models (#40184)

This commit is contained in:
Tri Dao
2026-09-19 19:28:11 -07:00
committed by GitHub
parent e9300f643e
commit d82d653f96
3 changed files with 209 additions and 13 deletions
@@ -13,7 +13,6 @@ from typing import Any
from sglang.srt.arg_groups.overrides import (
declare_resolution,
model_config_of,
resolved_view,
resolving_view,
)
from sglang.srt.environ import envs
@@ -486,15 +485,6 @@ def handle_other_validations(server_args: Any):
"_handle_other_validations",
optimistic_prefill_attempts=0,
)
elif resolved_view(server_args).uses_mamba_radix_cache:
logger.warning(
"Optimistic prefill does not support models that use mamba radix cache."
)
declare_resolution(
server_args,
"_handle_other_validations",
optimistic_prefill_attempts=0,
)
# Handle model inference tensor dump.
if cfg.debug_tensor_dump_output_folder is not None:
+4 -1
View File
@@ -1510,7 +1510,10 @@ class SchedulerDisaggregationPrefillMixin:
)
)
self._release_aborted_request(req)
release_kv_cache(req, self.tree_cache)
# Mamba insertion donates the checkpoint and clears its sequence marker.
release_kv_cache(
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
)
req.reset_for_retract()
req.output_ids = array("q")
req.start_send_idx = 0