[Feat] disaggregation: Let a KV transfer backend skip the checksum for an incomparable peer (#44605)
Co-authored-by: jiayisuse <jiayisuse@users.noreply.github.com>
This commit is contained in:
co-authored by
jiayisuse
parent
63a9b7a713
commit
989ccf4863
@@ -129,6 +129,13 @@ class BaseKVManager(ABC):
|
||||
enable_deferred_decode_kv_release: bool = False
|
||||
supports_host_destination: bool = False
|
||||
|
||||
def should_verify_kv_checksum(self, bootstrap_addr: str) -> bool:
|
||||
"""Whether the checksum sent by the prefill at bootstrap_addr is comparable
|
||||
with one over this rank's local KV. A backend returns False for a peer
|
||||
whose digest covers a different byte range, e.g. a TP width mismatch where
|
||||
each decode rank receives only a slice of what the sender digested."""
|
||||
return True
|
||||
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -3201,11 +3201,15 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
if computer is None:
|
||||
return
|
||||
|
||||
kv_manager = self.disagg_decode_prealloc_queue.kv_manager
|
||||
verified: List[int] = []
|
||||
for i, req in enumerate(batch.reqs):
|
||||
if is_health_check_req(req):
|
||||
verified.append(i)
|
||||
continue
|
||||
if not kv_manager.should_verify_kv_checksum(_bootstrap_addr(req)):
|
||||
verified.append(i)
|
||||
continue
|
||||
expected = req.expected_kv_checksum
|
||||
if expected == 0:
|
||||
verified.append(i)
|
||||
|
||||
@@ -207,6 +207,11 @@ class _FakeScheduler(SchedulerDisaggregationDecodeMixin):
|
||||
self.ngram_embedding_manager = Mock()
|
||||
self.chunked_req = None
|
||||
self.future_map = None
|
||||
self.disagg_decode_prealloc_queue = SimpleNamespace(
|
||||
kv_manager=SimpleNamespace(
|
||||
should_verify_kv_checksum=lambda bootstrap_addr: True
|
||||
)
|
||||
)
|
||||
|
||||
def stream_output(self, reqs, return_logprob):
|
||||
self.streamed_aborts.extend(reqs)
|
||||
@@ -218,9 +223,11 @@ class _FakePrefillScheduler(SchedulerDisaggregationPrefillMixin):
|
||||
self.disagg_metadata_buffers = SimpleNamespace(set_kv_checksum=Mock())
|
||||
|
||||
|
||||
def _make_req(expected_chksum, num_input_tokens, rid="r0"):
|
||||
def _make_req(expected_chksum, num_input_tokens, rid="r0", bootstrap_host="127.0.0.1"):
|
||||
return SimpleNamespace(
|
||||
rid=rid,
|
||||
bootstrap_host=bootstrap_host,
|
||||
bootstrap_port=8998,
|
||||
bootstrap_room=12345,
|
||||
kv=SimpleNamespace(req_pool_idx=0),
|
||||
origin_input_ids=list(range(num_input_tokens)),
|
||||
@@ -356,6 +363,29 @@ class TestGetNewPrebuiltBatchChecksum(unittest.TestCase):
|
||||
sched.waiting_queue = [_make_req(0, self.num_pages)]
|
||||
self.assertEqual(len(self._run_once(sched).reqs), 1)
|
||||
|
||||
def test_backend_can_skip_incomparable_checksum(self):
|
||||
"""A request whose sender the backend declares incomparable is admitted
|
||||
without a digest, decided per bootstrap address, and skipping it must not
|
||||
disturb the index-based filtering of a mismatching sibling."""
|
||||
sched = self._make_sched(_SENTINEL)
|
||||
sched.disagg_decode_prealloc_queue.kv_manager.should_verify_kv_checksum = (
|
||||
lambda bootstrap_addr: not bootstrap_addr.startswith("10.0.0.1:")
|
||||
)
|
||||
aborted = _make_req(0xDEADBEEF, self.num_pages, rid="aborted")
|
||||
skipped = _make_req(
|
||||
0xDEADBEEF, self.num_pages, rid="skipped", bootstrap_host="10.0.0.1"
|
||||
)
|
||||
sched.waiting_queue = [aborted, skipped]
|
||||
with (
|
||||
envs.SGLANG_IS_IN_CI.override(False),
|
||||
patch("sglang.srt.disaggregation.decode.prepare_abort") as mock_abort,
|
||||
patch("sglang.srt.disaggregation.decode.release_kv_cache"),
|
||||
):
|
||||
batch = self._run_once(sched)
|
||||
self.assertEqual(batch.reqs, [skipped])
|
||||
self.assertEqual(sched.streamed_aborts, [aborted])
|
||||
mock_abort.assert_called_once()
|
||||
|
||||
|
||||
_SENTINEL = object()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user