diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx
index c6ef4258b5..8629c8c4e5 100644
--- a/docs/docs/references/environment_variables.mdx
+++ b/docs/docs/references/environment_variables.mdx
@@ -1564,14 +1564,24 @@ SGLang supports various environment variables that can be used to configure its
SGLANG_ENABLE_FAILED_SESSION_PROBE |
- Probe failed Mooncake sessions for recovery. |
- false |
+ Periodically probe blacklisted Mooncake sessions and un-blacklist the ones that answer. Probes run on their own thread, off the KV transfer path. |
+ true |
SGLANG_FAILED_SESSION_PROBE_INTERVAL_S |
- Interval (seconds) between failed-session probes. |
+ How often (seconds) the probe loop runs, and the first retry interval of a failing session. |
30.0 |
+
+ SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S |
+ Cap (seconds) on a session's probe interval, which doubles after each failed probe. |
+ 300.0 |
+
+
+ SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S |
+ Drop a blacklisted Mooncake session that no request has targeted for this many seconds, e.g. the old session id of a restarted decode. |
+ 600.0 |
+
MOONCAKE_MASTER |
Address of the Mooncake master. |
diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py
index 69b8ddf0ce..351937c24d 100644
--- a/python/sglang/srt/disaggregation/mooncake/conn.py
+++ b/python/sglang/srt/disaggregation/mooncake/conn.py
@@ -87,6 +87,15 @@ FAILED_SESSION_RECOVERIES = Counter(
)
+@dataclasses.dataclass
+class FailedSessionProbe:
+ """Probe schedule of one blacklisted session, in time.monotonic()."""
+
+ last_requested: float
+ next_probe_at: float
+ interval: float
+
+
# decode
@dataclasses.dataclass
class TransferInfo:
@@ -269,9 +278,37 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# Set by teardown() to make worker threads exit (P<->D role switch).
self._stopped = False
self._worker_threads: List[threading.Thread] = []
+ # Checked in every role, so a bad value fails startup rather than a
+ # later decode -> prefill role switch.
+ self.enable_failed_session_probe = envs.SGLANG_ENABLE_FAILED_SESSION_PROBE.get()
+ if self.enable_failed_session_probe:
+ self.failed_session_probe_interval = (
+ envs.SGLANG_FAILED_SESSION_PROBE_INTERVAL_S.get()
+ )
+ self.failed_session_probe_max_interval = (
+ envs.SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S.get()
+ )
+ self.failed_session_idle_timeout = (
+ envs.SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S.get()
+ )
+ if not (
+ 0
+ < self.failed_session_probe_interval
+ <= self.failed_session_probe_max_interval
+ ):
+ raise ValueError(
+ "Need 0 < SGLANG_FAILED_SESSION_PROBE_INTERVAL_S <= "
+ "SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S"
+ )
+ if self.failed_session_idle_timeout <= 0:
+ raise ValueError(
+ "SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S must be positive"
+ )
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.session_failures = defaultdict(int)
self.failed_sessions = set()
+ # Only the probe loop fills this; empty when the probe is off.
+ self.failed_session_probes: Dict[str, FailedSessionProbe] = {}
# Separate from failed_sessions, which _run_one_probe_pass clears
# on a successful probe: an incompatible decode is still reachable,
# so only a re-registration may lift this.
@@ -329,13 +366,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
t.start()
self._worker_threads.append(t)
- self.enable_failed_session_probe = (
- envs.SGLANG_ENABLE_FAILED_SESSION_PROBE.get()
- )
if self.enable_failed_session_probe:
- self.failed_session_probe_interval = (
- envs.SGLANG_FAILED_SESSION_PROBE_INTERVAL_S.get()
- )
self._failed_session_probe_shutdown = threading.Event()
t = threading.Thread(
target=self._failed_session_probe_loop,
@@ -897,8 +928,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
"""
fully_checked, mismatch = self._validate_peer_state_layout(decode_kv_args)
with self.session_lock:
- self.failed_sessions.discard(mooncake_session_id)
- self.session_failures.pop(mooncake_session_id, None)
+ self._unblacklist_session(mooncake_session_id)
self.state_layout_rejections.pop(mooncake_session_id, None)
self.state_strides_validated.discard(mooncake_session_id)
if mismatch is not None:
@@ -2523,6 +2553,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
layout_rejection is not None
or req.mooncake_session_id in self.failed_sessions
):
+ probe = self.failed_session_probes.get(
+ req.mooncake_session_id
+ )
+ if probe is not None:
+ probe.last_requested = time.monotonic()
self.conclude_failure(
bootstrap_room=kv_chunk.room,
failure_reason=layout_rejection
@@ -3081,31 +3116,68 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
def get_session_id(self):
return self.engine.get_session_id()
+ def _unblacklist_session(self, session_id: str) -> None:
+ """Caller holds session_lock."""
+ self.failed_sessions.discard(session_id)
+ self.session_failures.pop(session_id, None)
+ self.failed_session_probes.pop(session_id, None)
+
+ def _send_probe(self, session_id: str) -> int:
+ try:
+ return self.engine.send_probe(session_id)
+ except Exception as e:
+ logger.warning("send_probe(%s) raised: %s", session_id, e)
+ return -1
+
def _run_one_probe_pass(self) -> None:
+ """Probe due sessions with per-session exponential backoff, and drop
+ sessions no request has targeted within the idle timeout: a restarted
+ decode comes back under a new session id, so its old one never
+ re-registers."""
+ now = time.monotonic()
+ due = []
with self.session_lock:
- snapshot = list(self.failed_sessions)
- for session_id in snapshot:
- send_probe = getattr(self.engine, "send_probe", None)
- if send_probe is None:
- rc = -1
- else:
- try:
- rc = send_probe(session_id)
- except Exception as e:
- logger.warning("send_probe(%s) raised: %s", session_id, e)
- continue
- if rc == 0:
- with self.session_lock:
- was_blacklisted = session_id in self.failed_sessions
- self.failed_sessions.discard(session_id)
- self.session_failures.pop(session_id, None)
- if was_blacklisted:
+ for session_id in list(self.failed_session_probes):
+ if session_id not in self.failed_sessions:
+ del self.failed_session_probes[session_id]
+ for session_id in list(self.failed_sessions):
+ probe = self.failed_session_probes.setdefault(
+ session_id,
+ FailedSessionProbe(
+ last_requested=now,
+ next_probe_at=now,
+ interval=self.failed_session_probe_interval,
+ ),
+ )
+ if now - probe.last_requested >= self.failed_session_idle_timeout:
+ self._unblacklist_session(session_id)
logger.info(
- "Session %s recovered via probe; un-blacklisted",
+ "Dropped blacklisted session %s: no request for %.0fs",
session_id,
+ now - probe.last_requested,
)
- FAILED_SESSION_RECOVERIES.inc()
- else:
+ elif now >= probe.next_probe_at:
+ due.append(session_id)
+ for session_id in due:
+ rc = self._send_probe(session_id)
+ with self.session_lock:
+ if rc == 0:
+ was_blacklisted = session_id in self.failed_sessions
+ self._unblacklist_session(session_id)
+ else:
+ was_blacklisted = False
+ probe = self.failed_session_probes.get(session_id)
+ if probe is not None:
+ probe.interval = min(
+ 2 * probe.interval, self.failed_session_probe_max_interval
+ )
+ probe.next_probe_at = time.monotonic() + probe.interval
+ if was_blacklisted:
+ logger.info(
+ "Session %s recovered via probe; un-blacklisted", session_id
+ )
+ FAILED_SESSION_RECOVERIES.inc()
+ elif rc != 0:
logger.debug("Probe still failing for %s (rc=%d)", session_id, rc)
def _failed_session_probe_loop(self) -> None:
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 8b6209e428..5afc6aacf5 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -832,8 +832,10 @@ class Envs:
ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False)
ASCEND_NPU_PHY_ID = EnvInt(-1)
SGLANG_MOONCAKE_SEND_AUX_TCP = EnvBool(False)
- SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(False)
+ SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(True)
SGLANG_FAILED_SESSION_PROBE_INTERVAL_S = EnvFloat(30.0)
+ SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S = EnvFloat(300.0)
+ SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S = EnvFloat(600.0)
# ===================================================================
# Mooncake store
diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py
index 9fb145fea7..2766f613f4 100644
--- a/test/registered/unit/disaggregation/test_disaggregation_wire.py
+++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py
@@ -33,6 +33,7 @@ from sglang.srt.disaggregation.decode_schedule_batch_mixin import (
ScheduleBatchDisaggregationDecodeMixin,
)
from sglang.srt.disaggregation.mooncake.conn import (
+ FailedSessionProbe,
KVArgsRegisterInfo,
MooncakeKVManager,
MooncakeKVSender,
@@ -160,6 +161,7 @@ class TestDisaggregationWire(unittest.TestCase):
manager.failure_lock = threading.Lock()
manager.failed_sessions = set()
manager.session_failures = defaultdict(int)
+ manager.failed_session_probes = {}
manager.state_layout_rejections = {}
manager.state_strides_validated = set()
manager.failure_records = {}
@@ -369,6 +371,11 @@ class TestDisaggregationWire(unittest.TestCase):
manager.session_lock = threading.Lock()
manager.failed_sessions = {"decode"}
manager.session_failures = defaultdict(int, {"decode": 2})
+ manager.failed_session_probes = {
+ "decode": FailedSessionProbe(
+ last_requested=0.0, next_probe_at=0.0, interval=300.0
+ )
+ }
manager.state_layout_rejections = {}
manager.state_strides_validated = set()
manager.decode_kv_args_table = {}
@@ -391,6 +398,7 @@ class TestDisaggregationWire(unittest.TestCase):
# A transport blacklist is cleared by registering, as before.
self.assertFalse(manager.failed_sessions)
self.assertFalse(manager.session_failures)
+ self.assertFalse(manager.failed_session_probes)
bad, mismatch = register(64)
self.assertIn("item length mismatch", mismatch)
@@ -476,6 +484,7 @@ class TestDisaggregationWire(unittest.TestCase):
manager.failure_lock = threading.Lock()
manager.failed_sessions = set()
manager.session_failures = defaultdict(int)
+ manager.failed_session_probes = {}
manager.failure_records = {}
manager._staging_outstanding = defaultdict(int)
manager.request_status = {42: KVPoll.Transferring}
@@ -526,8 +535,12 @@ class TestDisaggregationWire(unittest.TestCase):
manager.session_lock = threading.Lock()
manager.failed_sessions = {"decode"}
manager.session_failures = defaultdict(int, {"decode": 1})
+ manager.failed_session_probes = {}
reason = f"{StateType.SWA} item length mismatch for positional entry 0"
manager.state_layout_rejections = {"decode": reason}
+ manager.failed_session_probe_interval = 30.0
+ manager.failed_session_probe_max_interval = 300.0
+ manager.failed_session_idle_timeout = 600.0
manager.engine = SimpleNamespace(send_probe=Mock(return_value=0))
manager._run_one_probe_pass()
diff --git a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py
index e39324a45d..6bc138cc4d 100644
--- a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py
+++ b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py
@@ -159,6 +159,7 @@ class TestMooncakeAbortNotification(
manager._staging_outstanding = defaultdict(int)
manager.session_lock = threading.Lock()
manager.session_failures = defaultdict(int)
+ manager.failed_session_probes = {}
manager.failed_sessions = set()
manager.state_layout_rejections = {}
manager.req_to_decode_prefix_len = {}
diff --git a/test/registered/unit/disaggregation/test_mooncake_failed_session_probe.py b/test/registered/unit/disaggregation/test_mooncake_failed_session_probe.py
new file mode 100644
index 0000000000..477e2683a8
--- /dev/null
+++ b/test/registered/unit/disaggregation/test_mooncake_failed_session_probe.py
@@ -0,0 +1,163 @@
+"""Backoff and idle pruning in the Mooncake failed-session probe loop."""
+
+import threading
+import unittest
+from collections import defaultdict
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+import numpy as np
+
+from sglang.srt.disaggregation.base.conn import KVPoll
+from sglang.srt.disaggregation.common.utils import FastQueue, TransferKVChunk
+from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager, TransferInfo
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=2, suite="base-a-test-cpu")
+
+CLOCK = "sglang.srt.disaggregation.mooncake.conn.time.monotonic"
+SESSION = "10.0.0.2:15000"
+ROOM = 7
+
+
+def _manager(probe_rc: int = -1, idle_timeout: float = 600.0) -> MooncakeKVManager:
+ manager = MooncakeKVManager.__new__(MooncakeKVManager)
+ manager.session_lock = threading.Lock()
+ manager.failed_sessions = {SESSION}
+ manager.session_failures = defaultdict(int, {SESSION: 1})
+ manager.failed_session_probes = {}
+ manager.failed_session_probe_interval = 30.0
+ manager.failed_session_probe_max_interval = 300.0
+ manager.failed_session_idle_timeout = idle_timeout
+ manager.engine = SimpleNamespace(send_probe=Mock(return_value=probe_rc))
+ return manager
+
+
+def _run_passes(manager, until: float, step: float = 30.0) -> list:
+ """Run the probe loop every `step` seconds; return when probes were sent."""
+ sent = []
+ manager.engine.send_probe.side_effect = lambda session_id: (
+ sent.append(now) or manager.engine.send_probe.return_value
+ )
+ now = 0.0
+ while now <= until:
+ with patch(CLOCK, return_value=now):
+ manager._run_one_probe_pass()
+ now += step
+ return sent
+
+
+def _request_blacklisted_session(manager, now: float) -> None:
+ """Send one chunk for the blacklisted session through transfer_worker."""
+ manager.kv_args = SimpleNamespace(kv_data_ptrs=[], state_types=[])
+ manager.is_mla_backend = True
+ manager.is_hybrid_mla_backend = False
+ manager.attn_tp_size = manager.attn_cp_size = manager.pp_size = 1
+ manager.attn_tp_rank = manager.attn_cp_rank = manager.pp_rank = 0
+ manager.enable_custom_mem_pool = False
+ manager.max_transfer_batch_indices = 0
+ manager.enable_trace = manager.enable_staging = False
+ manager.bootstrap_port = 1234
+ manager.failure_lock = threading.Lock()
+ manager.failure_records = {}
+ manager._staging_outstanding = defaultdict(int)
+ manager.state_layout_rejections = {}
+ manager.state_strides_validated = set()
+ manager.decode_kv_args_table = {}
+ manager.req_to_decode_prefix_len = {ROOM: 0}
+ manager.request_status = {ROOM: KVPoll.Transferring}
+ manager.transfer_infos = {
+ ROOM: {
+ SESSION: TransferInfo(
+ room=ROOM,
+ endpoint="10.0.0.2",
+ dst_port=1234,
+ mooncake_session_id=SESSION,
+ dst_kv_indices=np.array([], dtype=np.int32),
+ dst_aux_index=0,
+ dst_state_indices=None,
+ required_dst_info_num=1,
+ is_dummy=False,
+ )
+ }
+ }
+ manager._send_multipart_locked = Mock()
+ manager._transfer_data = Mock(return_value=0)
+ queue = FastQueue()
+ queue.put(
+ TransferKVChunk(
+ room=ROOM,
+ prefill_kv_indices=np.array([], dtype=np.int32),
+ index_slice=slice(0, 0),
+ is_last_chunk=False,
+ prefill_aux_index=None,
+ state_indices=None,
+ )
+ )
+ queue.put(None)
+ with patch(CLOCK, return_value=now):
+ manager.transfer_worker(queue, executor=None)
+ assert manager.request_status[ROOM] == KVPoll.Failed
+ manager._transfer_data.assert_not_called()
+
+
+class TestMooncakeFailedSessionProbe(CustomTestCase):
+ def test_failing_probe_backs_off_to_the_cap(self):
+ manager = _manager(idle_timeout=1e9)
+
+ sent = _run_passes(manager, until=1200)
+
+ self.assertEqual(sent, [0, 60, 180, 420, 720, 1020])
+ self.assertIn(SESSION, manager.failed_sessions)
+
+ def test_recovery_unblacklists_and_resets_the_schedule(self):
+ manager = _manager()
+ _run_passes(manager, until=60)
+ self.assertEqual(manager.failed_session_probes[SESSION].interval, 120)
+
+ manager.engine.send_probe.return_value = 0
+ with patch(CLOCK, return_value=180):
+ manager._run_one_probe_pass()
+ self.assertFalse(manager.failed_sessions)
+ self.assertFalse(manager.session_failures)
+ self.assertFalse(manager.failed_session_probes)
+
+ # Blacklisted again later: probed on the next pass, from the base interval.
+ manager.failed_sessions.add(SESSION)
+ manager.engine.send_probe.return_value = -1
+ manager.engine.send_probe.reset_mock()
+ with patch(CLOCK, return_value=500):
+ manager._run_one_probe_pass()
+ manager.engine.send_probe.assert_called_once_with(SESSION)
+ self.assertEqual(manager.failed_session_probes[SESSION].interval, 60)
+
+ def test_unrequested_session_is_dropped(self):
+ # E.g. the old session id of a restarted decode.
+ manager = _manager()
+
+ sent = _run_passes(manager, until=1200)
+
+ self.assertEqual(sent, [0, 60, 180, 420])
+ self.assertFalse(manager.failed_sessions)
+ self.assertFalse(manager.session_failures)
+ self.assertFalse(manager.failed_session_probes)
+
+ def test_requested_session_stays_blacklisted_and_probed(self):
+ manager = _manager()
+ with patch(CLOCK, return_value=0):
+ manager._run_one_probe_pass()
+
+ _request_blacklisted_session(manager, now=590)
+ with patch(CLOCK, return_value=600):
+ manager._run_one_probe_pass()
+ self.assertIn(SESSION, manager.failed_sessions)
+ self.assertEqual(manager.failed_session_probes[SESSION].last_requested, 590)
+
+ with patch(CLOCK, return_value=1190):
+ manager._run_one_probe_pass()
+ self.assertFalse(manager.failed_sessions)
+
+
+if __name__ == "__main__":
+ unittest.main()