minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[EPD] Fix HTTP dispatch lock blocking cross-request encoder batching (#31275)

Browse Source 
Co-authored-by: 晟海 <huangtingwei.htw@antgroup.com>
This commit is contained in:
Yuang Chen
2026-07-25 14:47:58 +08:00
committed by GitHub
co-authored by 晟海
parent 9eb2dccbb7
commit f5155d9602
1 changed files with 31 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/disaggregation/encode_server.py
+31 -31
View File
@@ -3880,38 +3880,38 @@ async def handle_encode_request(request: dict):
encoder.background_tasks.add(task) encoder.background_tasks.add(task)
task.add_done_callback(encoder.background_tasks.discard) task.add_done_callback(encoder.background_tasks.discard)
# broadcast request, lock together with rank0 await so NCCL request.update({"enter_time": time.time()})
# launch order matches the ZMQ dispatch order rank>0 sees. modality = Modality.from_str(request["modality"])
async with encoder.encode_dispatch_lock: if time_stats_json:
request.update({"enter_time": time.time()}) time_stats.decode_json(time_stats_json)
modality = Modality.from_str(request["modality"])
if time_stats_json:
time_stats.decode_json(time_stats_json)
modality_str = modality.name.lower() modality_str = modality.name.lower()
time_stats.modality = modality_str time_stats.modality = modality_str
time_stats.set_metrics_collector(encoder_metrics_collector) time_stats.set_metrics_collector(encoder_metrics_collector)
time_stats.set_mm_encode_start_time() time_stats.set_mm_encode_start_time()
if encoder_metrics_collector is not None: if encoder_metrics_collector is not None:
encoder_metrics_collector.inc_requests_received(modality=modality_str) encoder_metrics_collector.inc_requests_received(modality=modality_str)
if encoder_scheduler is not None and modality in _BATCHABLE_MODALITIES: if encoder_scheduler is not None and modality in _BATCHABLE_MODALITIES:
try: try:
nbytes, embedding_len, embedding_dim, error_msg, error_code = ( nbytes, embedding_len, embedding_dim, error_msg, error_code = (
await encoder_scheduler.submit(request) await encoder_scheduler.submit(request)
) )
except asyncio.TimeoutError: except asyncio.TimeoutError:
time_stats.trace_ctx.abort( time_stats.trace_ctx.abort(
abort_info={"reason": "encoder batch timed out"} abort_info={"reason": "encoder batch timed out"}
) )
return ORJSONResponse( return ORJSONResponse(
status_code=HTTPStatus.GATEWAY_TIMEOUT, status_code=HTTPStatus.GATEWAY_TIMEOUT,
content={ content={
"status": "error", "status": "error",
"message": "encoder batch timed out", "message": "encoder batch timed out",
"req_id": req_id, "req_id": req_id,
}, },
) )
else: else:
# Lock direct dispatch together with rank0 await so its NCCL launch
# order matches the ZMQ dispatch order rank>0 sees.
async with encoder.encode_dispatch_lock:
for socket in send_sockets: for socket in send_sockets:
sock_send(socket, wrap_as_pickle(request)) sock_send(socket, wrap_as_pickle(request))
nbytes, embedding_len, embedding_dim, error_msg, error_code = ( nbytes, embedding_len, embedding_dim, error_msg, error_code = (