[EPD] Fix HTTP dispatch lock blocking cross-request encoder batching (#31275)

Co-authored-by: 晟海 <huangtingwei.htw@antgroup.com>
This commit is contained in:
Yuang Chen
2026-07-25 14:47:58 +08:00
committed by GitHub
co-authored by 晟海
parent 9eb2dccbb7
commit f5155d9602
@@ -3880,9 +3880,6 @@ async def handle_encode_request(request: dict):
encoder.background_tasks.add(task)
task.add_done_callback(encoder.background_tasks.discard)
# broadcast request, lock together with rank0 await so NCCL
# launch order matches the ZMQ dispatch order rank>0 sees.
async with encoder.encode_dispatch_lock:
request.update({"enter_time": time.time()})
modality = Modality.from_str(request["modality"])
if time_stats_json:
@@ -3912,6 +3909,9 @@ async def handle_encode_request(request: dict):
},
)
else:
# Lock direct dispatch together with rank0 await so its NCCL launch
# order matches the ZMQ dispatch order rank>0 sees.
async with encoder.encode_dispatch_lock:
for socket in send_sockets:
sock_send(socket, wrap_as_pickle(request))
nbytes, embedding_len, embedding_dim, error_msg, error_code = (