From 98b950366d32e890a66ac2ffc65ab980040cb597 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 12 Oct 2026 23:41:23 +0800 Subject: [PATCH] mooncake PD transfer: mix bootstrap_room into lane shard hash (fixes 1/16 lane collapse with single decode instance) --- .../sglang/srt/disaggregation/mooncake/conn.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 3833b5738c..df6059e44e 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2998,12 +2998,22 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): # add further chunks into the transfer queue. return - # NOTE(shangming): sharding according to the dst_infos to make sure - # requests with the same dst_sessions will be added into the same - # queue, which enables early abort with failed sessions. + # Shard by dst session ports AND bootstrap_room. The port term alone + # degenerates to a constant when all requests target a single decode + # instance (its session ports are fixed), collapsing every transfer + # onto 1/16 of the lanes and throttling P->D intake (measured at + # client concurrency 300: #transfer-req piled at 100+ while decode + # itself stayed healthy). Mixing in bootstrap_room keeps all chunks + # of one request on the same lane (per-request FIFO order preserved) + # while spreading distinct rooms across all lanes. Abort/cleanup is + # unaffected: chunks are dropped per-room at dequeue via + # request_status (see the worker loop) and per-request via + # failed_sessions before send — no lane-level purge exists, and the + # deferred abort ack is room-scoped (_staging_outstanding). Same + # trade-off as mori/conn.py which shards by bootstrap_room alone. dst_infos = self.transfer_infos[bootstrap_room].keys() session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos) - shard_idx = session_port_sum % len(self.transfer_queues) + shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues) if trace_ctx is None: trace_ctx = TraceNullContext()