[EPD] Fix HTTP dispatch lock blocking cross-request encoder batching (#31275)
Co-authored-by: 晟海 <huangtingwei.htw@antgroup.com>
This commit is contained in:
@@ -3880,9 +3880,6 @@ async def handle_encode_request(request: dict):
|
||||
encoder.background_tasks.add(task)
|
||||
task.add_done_callback(encoder.background_tasks.discard)
|
||||
|
||||
# broadcast request, lock together with rank0 await so NCCL
|
||||
# launch order matches the ZMQ dispatch order rank>0 sees.
|
||||
async with encoder.encode_dispatch_lock:
|
||||
request.update({"enter_time": time.time()})
|
||||
modality = Modality.from_str(request["modality"])
|
||||
if time_stats_json:
|
||||
@@ -3912,6 +3909,9 @@ async def handle_encode_request(request: dict):
|
||||
},
|
||||
)
|
||||
else:
|
||||
# Lock direct dispatch together with rank0 await so its NCCL launch
|
||||
# order matches the ZMQ dispatch order rank>0 sees.
|
||||
async with encoder.encode_dispatch_lock:
|
||||
for socket in send_sockets:
|
||||
sock_send(socket, wrap_as_pickle(request))
|
||||
nbytes, embedding_len, embedding_dim, error_msg, error_code = (
|
||||
|
||||
Reference in New Issue
Block a user