[PD] Enable deferred decode-side KV release by default (#41023)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
4142235c2b
commit
cd11037009
@@ -141,6 +141,10 @@ class BaseKVManager(ABC):
|
||||
# role switch (release transfer resources; the scheduler owns the KV pool).
|
||||
supports_role_switch: bool = False
|
||||
|
||||
# Opt-in per backend: set True once the prefill acks a drained abort
|
||||
# (ABORT_ACK carrying the sender rank). Without it a hold only ends on timeout.
|
||||
supports_deferred_decode_kv_release: bool = False
|
||||
|
||||
def teardown(self) -> None:
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support PD role switch teardown"
|
||||
|
||||
@@ -183,6 +183,7 @@ class CommonKVManager(BaseKVManager):
|
||||
self.server_args = server_args
|
||||
self.enable_deferred_decode_kv_release = (
|
||||
envs.SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE.get()
|
||||
and self.supports_deferred_decode_kv_release
|
||||
)
|
||||
self._dcp_pack_buffers = None
|
||||
self._dcp_pack_max_tokens: Optional[int] = None
|
||||
|
||||
@@ -217,6 +217,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
AUX_DATA_HEADER = b"AUX_DATA"
|
||||
# Implements teardown() below, so runtime PD role switching is supported.
|
||||
supports_role_switch = True
|
||||
# Bootstrap thread defers the ABORT ack until the transfer worker drains.
|
||||
supports_deferred_decode_kv_release = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -412,6 +412,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# message is tagged too. It is new to NIXL, hence free to carry the reason.
|
||||
kv_status_msg_tag = b"KV_STATUS"
|
||||
kv_status_msg_carries_reason = True
|
||||
# ABORT handler defers the ack until the transfer worker drains.
|
||||
supports_deferred_decode_kv_release = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -698,8 +698,9 @@ class Envs:
|
||||
SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL = EnvInt(120)
|
||||
# Deferred decode-side KV release: on abort, hold an in-flight request's KV
|
||||
# pages/slot until the prefill acks the transfer drained, or the timeout
|
||||
# below fires. Off by default (no behavior/perf impact when disabled).
|
||||
SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE = EnvBool(False)
|
||||
# below fires. Only applies to backends that ack the drain
|
||||
# (supports_deferred_decode_kv_release).
|
||||
SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE = EnvBool(True)
|
||||
SGLANG_DISAGGREGATION_DEFERRED_DECODE_KV_RELEASE_TIMEOUT = EnvFloat(30.0)
|
||||
|
||||
# ===================================================================
|
||||
|
||||
Reference in New Issue
Block a user