[Feat] disaggregation: Let a KV transfer backend skip the checksum for an incomparable peer (#44605)

Co-authored-by: jiayisuse <jiayisuse@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 10:36:46 -07:00
committed by GitHub
co-authored by jiayisuse
parent 63a9b7a713
commit 989ccf4863
3 changed files with 42 additions and 1 deletions
@@ -129,6 +129,13 @@ class BaseKVManager(ABC):
enable_deferred_decode_kv_release: bool = False
supports_host_destination: bool = False
def should_verify_kv_checksum(self, bootstrap_addr: str) -> bool:
"""Whether the checksum sent by the prefill at bootstrap_addr is comparable
with one over this rank's local KV. A backend returns False for a peer
whose digest covers a different byte range, e.g. a TP width mismatch where
each decode rank receives only a slice of what the sender digested."""
return True
@abstractmethod
def __init__(
self,
@@ -3201,11 +3201,15 @@ class SchedulerDisaggregationDecodeMixin:
if computer is None:
return
kv_manager = self.disagg_decode_prealloc_queue.kv_manager
verified: List[int] = []
for i, req in enumerate(batch.reqs):
if is_health_check_req(req):
verified.append(i)
continue
if not kv_manager.should_verify_kv_checksum(_bootstrap_addr(req)):
verified.append(i)
continue
expected = req.expected_kv_checksum
if expected == 0:
verified.append(i)
@@ -207,6 +207,11 @@ class _FakeScheduler(SchedulerDisaggregationDecodeMixin):
self.ngram_embedding_manager = Mock()
self.chunked_req = None
self.future_map = None
self.disagg_decode_prealloc_queue = SimpleNamespace(
kv_manager=SimpleNamespace(
should_verify_kv_checksum=lambda bootstrap_addr: True
)
)
def stream_output(self, reqs, return_logprob):
self.streamed_aborts.extend(reqs)
@@ -218,9 +223,11 @@ class _FakePrefillScheduler(SchedulerDisaggregationPrefillMixin):
self.disagg_metadata_buffers = SimpleNamespace(set_kv_checksum=Mock())
def _make_req(expected_chksum, num_input_tokens, rid="r0"):
def _make_req(expected_chksum, num_input_tokens, rid="r0", bootstrap_host="127.0.0.1"):
return SimpleNamespace(
rid=rid,
bootstrap_host=bootstrap_host,
bootstrap_port=8998,
bootstrap_room=12345,
kv=SimpleNamespace(req_pool_idx=0),
origin_input_ids=list(range(num_input_tokens)),
@@ -356,6 +363,29 @@ class TestGetNewPrebuiltBatchChecksum(unittest.TestCase):
sched.waiting_queue = [_make_req(0, self.num_pages)]
self.assertEqual(len(self._run_once(sched).reqs), 1)
def test_backend_can_skip_incomparable_checksum(self):
"""A request whose sender the backend declares incomparable is admitted
without a digest, decided per bootstrap address, and skipping it must not
disturb the index-based filtering of a mismatching sibling."""
sched = self._make_sched(_SENTINEL)
sched.disagg_decode_prealloc_queue.kv_manager.should_verify_kv_checksum = (
lambda bootstrap_addr: not bootstrap_addr.startswith("10.0.0.1:")
)
aborted = _make_req(0xDEADBEEF, self.num_pages, rid="aborted")
skipped = _make_req(
0xDEADBEEF, self.num_pages, rid="skipped", bootstrap_host="10.0.0.1"
)
sched.waiting_queue = [aborted, skipped]
with (
envs.SGLANG_IS_IN_CI.override(False),
patch("sglang.srt.disaggregation.decode.prepare_abort") as mock_abort,
patch("sglang.srt.disaggregation.decode.release_kv_cache"),
):
batch = self._run_once(sched)
self.assertEqual(batch.reqs, [skipped])
self.assertEqual(sched.streamed_aborts, [aborted])
mock_abort.assert_called_once()
_SENTINEL = object()