[Feat] mooncake: Enable the failed-session probe by default with backoff and idle pruning (#45338)

This commit is contained in:
Shangming Cai
2026-10-21 17:16:19 +08:00
committed by GitHub
parent 1278087336
commit be65391abd
6 changed files with 293 additions and 32 deletions
+100 -28
View File
@@ -87,6 +87,15 @@ FAILED_SESSION_RECOVERIES = Counter(
)
@dataclasses.dataclass
class FailedSessionProbe:
"""Probe schedule of one blacklisted session, in time.monotonic()."""
last_requested: float
next_probe_at: float
interval: float
# decode
@dataclasses.dataclass
class TransferInfo:
@@ -269,9 +278,37 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# Set by teardown() to make worker threads exit (P<->D role switch).
self._stopped = False
self._worker_threads: List[threading.Thread] = []
# Checked in every role, so a bad value fails startup rather than a
# later decode -> prefill role switch.
self.enable_failed_session_probe = envs.SGLANG_ENABLE_FAILED_SESSION_PROBE.get()
if self.enable_failed_session_probe:
self.failed_session_probe_interval = (
envs.SGLANG_FAILED_SESSION_PROBE_INTERVAL_S.get()
)
self.failed_session_probe_max_interval = (
envs.SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S.get()
)
self.failed_session_idle_timeout = (
envs.SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S.get()
)
if not (
0
< self.failed_session_probe_interval
<= self.failed_session_probe_max_interval
):
raise ValueError(
"Need 0 < SGLANG_FAILED_SESSION_PROBE_INTERVAL_S <= "
"SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S"
)
if self.failed_session_idle_timeout <= 0:
raise ValueError(
"SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S must be positive"
)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.session_failures = defaultdict(int)
self.failed_sessions = set()
# Only the probe loop fills this; empty when the probe is off.
self.failed_session_probes: Dict[str, FailedSessionProbe] = {}
# Separate from failed_sessions, which _run_one_probe_pass clears
# on a successful probe: an incompatible decode is still reachable,
# so only a re-registration may lift this.
@@ -329,13 +366,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
t.start()
self._worker_threads.append(t)
self.enable_failed_session_probe = (
envs.SGLANG_ENABLE_FAILED_SESSION_PROBE.get()
)
if self.enable_failed_session_probe:
self.failed_session_probe_interval = (
envs.SGLANG_FAILED_SESSION_PROBE_INTERVAL_S.get()
)
self._failed_session_probe_shutdown = threading.Event()
t = threading.Thread(
target=self._failed_session_probe_loop,
@@ -897,8 +928,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
"""
fully_checked, mismatch = self._validate_peer_state_layout(decode_kv_args)
with self.session_lock:
self.failed_sessions.discard(mooncake_session_id)
self.session_failures.pop(mooncake_session_id, None)
self._unblacklist_session(mooncake_session_id)
self.state_layout_rejections.pop(mooncake_session_id, None)
self.state_strides_validated.discard(mooncake_session_id)
if mismatch is not None:
@@ -2523,6 +2553,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
layout_rejection is not None
or req.mooncake_session_id in self.failed_sessions
):
probe = self.failed_session_probes.get(
req.mooncake_session_id
)
if probe is not None:
probe.last_requested = time.monotonic()
self.conclude_failure(
bootstrap_room=kv_chunk.room,
failure_reason=layout_rejection
@@ -3081,31 +3116,68 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
def get_session_id(self):
return self.engine.get_session_id()
def _unblacklist_session(self, session_id: str) -> None:
"""Caller holds session_lock."""
self.failed_sessions.discard(session_id)
self.session_failures.pop(session_id, None)
self.failed_session_probes.pop(session_id, None)
def _send_probe(self, session_id: str) -> int:
try:
return self.engine.send_probe(session_id)
except Exception as e:
logger.warning("send_probe(%s) raised: %s", session_id, e)
return -1
def _run_one_probe_pass(self) -> None:
"""Probe due sessions with per-session exponential backoff, and drop
sessions no request has targeted within the idle timeout: a restarted
decode comes back under a new session id, so its old one never
re-registers."""
now = time.monotonic()
due = []
with self.session_lock:
snapshot = list(self.failed_sessions)
for session_id in snapshot:
send_probe = getattr(self.engine, "send_probe", None)
if send_probe is None:
rc = -1
else:
try:
rc = send_probe(session_id)
except Exception as e:
logger.warning("send_probe(%s) raised: %s", session_id, e)
continue
if rc == 0:
with self.session_lock:
was_blacklisted = session_id in self.failed_sessions
self.failed_sessions.discard(session_id)
self.session_failures.pop(session_id, None)
if was_blacklisted:
for session_id in list(self.failed_session_probes):
if session_id not in self.failed_sessions:
del self.failed_session_probes[session_id]
for session_id in list(self.failed_sessions):
probe = self.failed_session_probes.setdefault(
session_id,
FailedSessionProbe(
last_requested=now,
next_probe_at=now,
interval=self.failed_session_probe_interval,
),
)
if now - probe.last_requested >= self.failed_session_idle_timeout:
self._unblacklist_session(session_id)
logger.info(
"Session %s recovered via probe; un-blacklisted",
"Dropped blacklisted session %s: no request for %.0fs",
session_id,
now - probe.last_requested,
)
FAILED_SESSION_RECOVERIES.inc()
else:
elif now >= probe.next_probe_at:
due.append(session_id)
for session_id in due:
rc = self._send_probe(session_id)
with self.session_lock:
if rc == 0:
was_blacklisted = session_id in self.failed_sessions
self._unblacklist_session(session_id)
else:
was_blacklisted = False
probe = self.failed_session_probes.get(session_id)
if probe is not None:
probe.interval = min(
2 * probe.interval, self.failed_session_probe_max_interval
)
probe.next_probe_at = time.monotonic() + probe.interval
if was_blacklisted:
logger.info(
"Session %s recovered via probe; un-blacklisted", session_id
)
FAILED_SESSION_RECOVERIES.inc()
elif rc != 0:
logger.debug("Probe still failing for %s (rc=%d)", session_id, rc)
def _failed_session_probe_loop(self) -> None:
+3 -1
View File
@@ -832,8 +832,10 @@ class Envs:
ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False)
ASCEND_NPU_PHY_ID = EnvInt(-1)
SGLANG_MOONCAKE_SEND_AUX_TCP = EnvBool(False)
SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(False)
SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(True)
SGLANG_FAILED_SESSION_PROBE_INTERVAL_S = EnvFloat(30.0)
SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S = EnvFloat(300.0)
SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S = EnvFloat(600.0)
# ===================================================================
# Mooncake store