[Bug fix] Add /model_info endpoint to mini_lb (#14535)
This commit is contained in:
@@ -341,8 +341,7 @@ async def get_server_info():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/get_model_info")
|
async def _get_model_info_impl():
|
||||||
async def get_model_info():
|
|
||||||
if not lb or not lb.prefill_urls:
|
if not lb or not lb.prefill_urls:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
||||||
@@ -350,7 +349,7 @@ async def get_model_info():
|
|||||||
)
|
)
|
||||||
|
|
||||||
target_server_url = lb.prefill_urls[0]
|
target_server_url = lb.prefill_urls[0]
|
||||||
endpoint_url = f"{target_server_url}/get_model_info"
|
endpoint_url = f"{target_server_url}/model_info"
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
try:
|
try:
|
||||||
@@ -375,6 +374,16 @@ async def get_model_info():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/model_info")
|
||||||
|
async def model_info():
|
||||||
|
return await _get_model_info_impl()
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/get_model_info")
|
||||||
|
async def get_model_info():
|
||||||
|
return await _get_model_info_impl()
|
||||||
|
|
||||||
|
|
||||||
@app.post("/generate")
|
@app.post("/generate")
|
||||||
async def handle_generate_request(request_data: dict):
|
async def handle_generate_request(request_data: dict):
|
||||||
prefill_server, bootstrap_port, decode_server = lb.select_pair()
|
prefill_server, bootstrap_port, decode_server = lb.select_pair()
|
||||||
|
|||||||
Reference in New Issue
Block a user