diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index c6ef4258b5..8629c8c4e5 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -1564,14 +1564,24 @@ SGLang supports various environment variables that can be used to configure its SGLANG_ENABLE_FAILED_SESSION_PROBE - Probe failed Mooncake sessions for recovery. - false + Periodically probe blacklisted Mooncake sessions and un-blacklist the ones that answer. Probes run on their own thread, off the KV transfer path. + true SGLANG_FAILED_SESSION_PROBE_INTERVAL_S - Interval (seconds) between failed-session probes. + How often (seconds) the probe loop runs, and the first retry interval of a failing session. 30.0 + + SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S + Cap (seconds) on a session's probe interval, which doubles after each failed probe. + 300.0 + + + SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S + Drop a blacklisted Mooncake session that no request has targeted for this many seconds, e.g. the old session id of a restarted decode. + 600.0 + MOONCAKE_MASTER Address of the Mooncake master. diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 69b8ddf0ce..351937c24d 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -87,6 +87,15 @@ FAILED_SESSION_RECOVERIES = Counter( ) +@dataclasses.dataclass +class FailedSessionProbe: + """Probe schedule of one blacklisted session, in time.monotonic().""" + + last_requested: float + next_probe_at: float + interval: float + + # decode @dataclasses.dataclass class TransferInfo: @@ -269,9 +278,37 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): # Set by teardown() to make worker threads exit (P<->D role switch). self._stopped = False self._worker_threads: List[threading.Thread] = [] + # Checked in every role, so a bad value fails startup rather than a + # later decode -> prefill role switch. + self.enable_failed_session_probe = envs.SGLANG_ENABLE_FAILED_SESSION_PROBE.get() + if self.enable_failed_session_probe: + self.failed_session_probe_interval = ( + envs.SGLANG_FAILED_SESSION_PROBE_INTERVAL_S.get() + ) + self.failed_session_probe_max_interval = ( + envs.SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S.get() + ) + self.failed_session_idle_timeout = ( + envs.SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S.get() + ) + if not ( + 0 + < self.failed_session_probe_interval + <= self.failed_session_probe_max_interval + ): + raise ValueError( + "Need 0 < SGLANG_FAILED_SESSION_PROBE_INTERVAL_S <= " + "SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S" + ) + if self.failed_session_idle_timeout <= 0: + raise ValueError( + "SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S must be positive" + ) if self.disaggregation_mode == DisaggregationMode.PREFILL: self.session_failures = defaultdict(int) self.failed_sessions = set() + # Only the probe loop fills this; empty when the probe is off. + self.failed_session_probes: Dict[str, FailedSessionProbe] = {} # Separate from failed_sessions, which _run_one_probe_pass clears # on a successful probe: an incompatible decode is still reachable, # so only a re-registration may lift this. @@ -329,13 +366,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ) t.start() self._worker_threads.append(t) - self.enable_failed_session_probe = ( - envs.SGLANG_ENABLE_FAILED_SESSION_PROBE.get() - ) if self.enable_failed_session_probe: - self.failed_session_probe_interval = ( - envs.SGLANG_FAILED_SESSION_PROBE_INTERVAL_S.get() - ) self._failed_session_probe_shutdown = threading.Event() t = threading.Thread( target=self._failed_session_probe_loop, @@ -897,8 +928,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): """ fully_checked, mismatch = self._validate_peer_state_layout(decode_kv_args) with self.session_lock: - self.failed_sessions.discard(mooncake_session_id) - self.session_failures.pop(mooncake_session_id, None) + self._unblacklist_session(mooncake_session_id) self.state_layout_rejections.pop(mooncake_session_id, None) self.state_strides_validated.discard(mooncake_session_id) if mismatch is not None: @@ -2523,6 +2553,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): layout_rejection is not None or req.mooncake_session_id in self.failed_sessions ): + probe = self.failed_session_probes.get( + req.mooncake_session_id + ) + if probe is not None: + probe.last_requested = time.monotonic() self.conclude_failure( bootstrap_room=kv_chunk.room, failure_reason=layout_rejection @@ -3081,31 +3116,68 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): def get_session_id(self): return self.engine.get_session_id() + def _unblacklist_session(self, session_id: str) -> None: + """Caller holds session_lock.""" + self.failed_sessions.discard(session_id) + self.session_failures.pop(session_id, None) + self.failed_session_probes.pop(session_id, None) + + def _send_probe(self, session_id: str) -> int: + try: + return self.engine.send_probe(session_id) + except Exception as e: + logger.warning("send_probe(%s) raised: %s", session_id, e) + return -1 + def _run_one_probe_pass(self) -> None: + """Probe due sessions with per-session exponential backoff, and drop + sessions no request has targeted within the idle timeout: a restarted + decode comes back under a new session id, so its old one never + re-registers.""" + now = time.monotonic() + due = [] with self.session_lock: - snapshot = list(self.failed_sessions) - for session_id in snapshot: - send_probe = getattr(self.engine, "send_probe", None) - if send_probe is None: - rc = -1 - else: - try: - rc = send_probe(session_id) - except Exception as e: - logger.warning("send_probe(%s) raised: %s", session_id, e) - continue - if rc == 0: - with self.session_lock: - was_blacklisted = session_id in self.failed_sessions - self.failed_sessions.discard(session_id) - self.session_failures.pop(session_id, None) - if was_blacklisted: + for session_id in list(self.failed_session_probes): + if session_id not in self.failed_sessions: + del self.failed_session_probes[session_id] + for session_id in list(self.failed_sessions): + probe = self.failed_session_probes.setdefault( + session_id, + FailedSessionProbe( + last_requested=now, + next_probe_at=now, + interval=self.failed_session_probe_interval, + ), + ) + if now - probe.last_requested >= self.failed_session_idle_timeout: + self._unblacklist_session(session_id) logger.info( - "Session %s recovered via probe; un-blacklisted", + "Dropped blacklisted session %s: no request for %.0fs", session_id, + now - probe.last_requested, ) - FAILED_SESSION_RECOVERIES.inc() - else: + elif now >= probe.next_probe_at: + due.append(session_id) + for session_id in due: + rc = self._send_probe(session_id) + with self.session_lock: + if rc == 0: + was_blacklisted = session_id in self.failed_sessions + self._unblacklist_session(session_id) + else: + was_blacklisted = False + probe = self.failed_session_probes.get(session_id) + if probe is not None: + probe.interval = min( + 2 * probe.interval, self.failed_session_probe_max_interval + ) + probe.next_probe_at = time.monotonic() + probe.interval + if was_blacklisted: + logger.info( + "Session %s recovered via probe; un-blacklisted", session_id + ) + FAILED_SESSION_RECOVERIES.inc() + elif rc != 0: logger.debug("Probe still failing for %s (rc=%d)", session_id, rc) def _failed_session_probe_loop(self) -> None: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 8b6209e428..5afc6aacf5 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -832,8 +832,10 @@ class Envs: ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False) ASCEND_NPU_PHY_ID = EnvInt(-1) SGLANG_MOONCAKE_SEND_AUX_TCP = EnvBool(False) - SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(False) + SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(True) SGLANG_FAILED_SESSION_PROBE_INTERVAL_S = EnvFloat(30.0) + SGLANG_FAILED_SESSION_PROBE_MAX_INTERVAL_S = EnvFloat(300.0) + SGLANG_FAILED_SESSION_IDLE_TIMEOUT_S = EnvFloat(600.0) # =================================================================== # Mooncake store diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 9fb145fea7..2766f613f4 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -33,6 +33,7 @@ from sglang.srt.disaggregation.decode_schedule_batch_mixin import ( ScheduleBatchDisaggregationDecodeMixin, ) from sglang.srt.disaggregation.mooncake.conn import ( + FailedSessionProbe, KVArgsRegisterInfo, MooncakeKVManager, MooncakeKVSender, @@ -160,6 +161,7 @@ class TestDisaggregationWire(unittest.TestCase): manager.failure_lock = threading.Lock() manager.failed_sessions = set() manager.session_failures = defaultdict(int) + manager.failed_session_probes = {} manager.state_layout_rejections = {} manager.state_strides_validated = set() manager.failure_records = {} @@ -369,6 +371,11 @@ class TestDisaggregationWire(unittest.TestCase): manager.session_lock = threading.Lock() manager.failed_sessions = {"decode"} manager.session_failures = defaultdict(int, {"decode": 2}) + manager.failed_session_probes = { + "decode": FailedSessionProbe( + last_requested=0.0, next_probe_at=0.0, interval=300.0 + ) + } manager.state_layout_rejections = {} manager.state_strides_validated = set() manager.decode_kv_args_table = {} @@ -391,6 +398,7 @@ class TestDisaggregationWire(unittest.TestCase): # A transport blacklist is cleared by registering, as before. self.assertFalse(manager.failed_sessions) self.assertFalse(manager.session_failures) + self.assertFalse(manager.failed_session_probes) bad, mismatch = register(64) self.assertIn("item length mismatch", mismatch) @@ -476,6 +484,7 @@ class TestDisaggregationWire(unittest.TestCase): manager.failure_lock = threading.Lock() manager.failed_sessions = set() manager.session_failures = defaultdict(int) + manager.failed_session_probes = {} manager.failure_records = {} manager._staging_outstanding = defaultdict(int) manager.request_status = {42: KVPoll.Transferring} @@ -526,8 +535,12 @@ class TestDisaggregationWire(unittest.TestCase): manager.session_lock = threading.Lock() manager.failed_sessions = {"decode"} manager.session_failures = defaultdict(int, {"decode": 1}) + manager.failed_session_probes = {} reason = f"{StateType.SWA} item length mismatch for positional entry 0" manager.state_layout_rejections = {"decode": reason} + manager.failed_session_probe_interval = 30.0 + manager.failed_session_probe_max_interval = 300.0 + manager.failed_session_idle_timeout = 600.0 manager.engine = SimpleNamespace(send_probe=Mock(return_value=0)) manager._run_one_probe_pass() diff --git a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py index e39324a45d..6bc138cc4d 100644 --- a/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py +++ b/test/registered/unit/disaggregation/test_mooncake_deferred_kv_release.py @@ -159,6 +159,7 @@ class TestMooncakeAbortNotification( manager._staging_outstanding = defaultdict(int) manager.session_lock = threading.Lock() manager.session_failures = defaultdict(int) + manager.failed_session_probes = {} manager.failed_sessions = set() manager.state_layout_rejections = {} manager.req_to_decode_prefix_len = {} diff --git a/test/registered/unit/disaggregation/test_mooncake_failed_session_probe.py b/test/registered/unit/disaggregation/test_mooncake_failed_session_probe.py new file mode 100644 index 0000000000..477e2683a8 --- /dev/null +++ b/test/registered/unit/disaggregation/test_mooncake_failed_session_probe.py @@ -0,0 +1,163 @@ +"""Backoff and idle pruning in the Mooncake failed-session probe loop.""" + +import threading +import unittest +from collections import defaultdict +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import numpy as np + +from sglang.srt.disaggregation.base.conn import KVPoll +from sglang.srt.disaggregation.common.utils import FastQueue, TransferKVChunk +from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager, TransferInfo +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +CLOCK = "sglang.srt.disaggregation.mooncake.conn.time.monotonic" +SESSION = "10.0.0.2:15000" +ROOM = 7 + + +def _manager(probe_rc: int = -1, idle_timeout: float = 600.0) -> MooncakeKVManager: + manager = MooncakeKVManager.__new__(MooncakeKVManager) + manager.session_lock = threading.Lock() + manager.failed_sessions = {SESSION} + manager.session_failures = defaultdict(int, {SESSION: 1}) + manager.failed_session_probes = {} + manager.failed_session_probe_interval = 30.0 + manager.failed_session_probe_max_interval = 300.0 + manager.failed_session_idle_timeout = idle_timeout + manager.engine = SimpleNamespace(send_probe=Mock(return_value=probe_rc)) + return manager + + +def _run_passes(manager, until: float, step: float = 30.0) -> list: + """Run the probe loop every `step` seconds; return when probes were sent.""" + sent = [] + manager.engine.send_probe.side_effect = lambda session_id: ( + sent.append(now) or manager.engine.send_probe.return_value + ) + now = 0.0 + while now <= until: + with patch(CLOCK, return_value=now): + manager._run_one_probe_pass() + now += step + return sent + + +def _request_blacklisted_session(manager, now: float) -> None: + """Send one chunk for the blacklisted session through transfer_worker.""" + manager.kv_args = SimpleNamespace(kv_data_ptrs=[], state_types=[]) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.attn_tp_size = manager.attn_cp_size = manager.pp_size = 1 + manager.attn_tp_rank = manager.attn_cp_rank = manager.pp_rank = 0 + manager.enable_custom_mem_pool = False + manager.max_transfer_batch_indices = 0 + manager.enable_trace = manager.enable_staging = False + manager.bootstrap_port = 1234 + manager.failure_lock = threading.Lock() + manager.failure_records = {} + manager._staging_outstanding = defaultdict(int) + manager.state_layout_rejections = {} + manager.state_strides_validated = set() + manager.decode_kv_args_table = {} + manager.req_to_decode_prefix_len = {ROOM: 0} + manager.request_status = {ROOM: KVPoll.Transferring} + manager.transfer_infos = { + ROOM: { + SESSION: TransferInfo( + room=ROOM, + endpoint="10.0.0.2", + dst_port=1234, + mooncake_session_id=SESSION, + dst_kv_indices=np.array([], dtype=np.int32), + dst_aux_index=0, + dst_state_indices=None, + required_dst_info_num=1, + is_dummy=False, + ) + } + } + manager._send_multipart_locked = Mock() + manager._transfer_data = Mock(return_value=0) + queue = FastQueue() + queue.put( + TransferKVChunk( + room=ROOM, + prefill_kv_indices=np.array([], dtype=np.int32), + index_slice=slice(0, 0), + is_last_chunk=False, + prefill_aux_index=None, + state_indices=None, + ) + ) + queue.put(None) + with patch(CLOCK, return_value=now): + manager.transfer_worker(queue, executor=None) + assert manager.request_status[ROOM] == KVPoll.Failed + manager._transfer_data.assert_not_called() + + +class TestMooncakeFailedSessionProbe(CustomTestCase): + def test_failing_probe_backs_off_to_the_cap(self): + manager = _manager(idle_timeout=1e9) + + sent = _run_passes(manager, until=1200) + + self.assertEqual(sent, [0, 60, 180, 420, 720, 1020]) + self.assertIn(SESSION, manager.failed_sessions) + + def test_recovery_unblacklists_and_resets_the_schedule(self): + manager = _manager() + _run_passes(manager, until=60) + self.assertEqual(manager.failed_session_probes[SESSION].interval, 120) + + manager.engine.send_probe.return_value = 0 + with patch(CLOCK, return_value=180): + manager._run_one_probe_pass() + self.assertFalse(manager.failed_sessions) + self.assertFalse(manager.session_failures) + self.assertFalse(manager.failed_session_probes) + + # Blacklisted again later: probed on the next pass, from the base interval. + manager.failed_sessions.add(SESSION) + manager.engine.send_probe.return_value = -1 + manager.engine.send_probe.reset_mock() + with patch(CLOCK, return_value=500): + manager._run_one_probe_pass() + manager.engine.send_probe.assert_called_once_with(SESSION) + self.assertEqual(manager.failed_session_probes[SESSION].interval, 60) + + def test_unrequested_session_is_dropped(self): + # E.g. the old session id of a restarted decode. + manager = _manager() + + sent = _run_passes(manager, until=1200) + + self.assertEqual(sent, [0, 60, 180, 420]) + self.assertFalse(manager.failed_sessions) + self.assertFalse(manager.session_failures) + self.assertFalse(manager.failed_session_probes) + + def test_requested_session_stays_blacklisted_and_probed(self): + manager = _manager() + with patch(CLOCK, return_value=0): + manager._run_one_probe_pass() + + _request_blacklisted_session(manager, now=590) + with patch(CLOCK, return_value=600): + manager._run_one_probe_pass() + self.assertIn(SESSION, manager.failed_sessions) + self.assertEqual(manager.failed_session_probes[SESSION].last_requested, 590) + + with patch(CLOCK, return_value=1190): + manager._run_one_probe_pass() + self.assertFalse(manager.failed_sessions) + + +if __name__ == "__main__": + unittest.main()