minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

add check for none status code in FinishAbort (#22535)

Browse Source 
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
pdasgup
2026-04-16 16:21:07 -07:00
committed by GitHub
co-authored by Xinyuan Tong hnyls2002
parent 2211b4d9c6
commit f639425ff0
2 changed files with 21 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/serving_chat.py
+11 -7
View File
@@ -729,11 +729,16 @@ class OpenAIServingChat(OpenAIServingBase):
# Track finish_reason for each index # Track finish_reason for each index
if finish_reason_type: if finish_reason_type:
# If the abort is from scheduler. # Abort with an explicit error status_code is a system error
if finish_reason_type == "abort": # (timeout, OOM, validation): emit a streaming error chunk.
code = finish_reason.get( # A graceful abort (no status_code, e.g. user-initiated via
"status_code", HTTPStatus.INTERNAL_SERVER_ERROR # /abort_request or session lifecycle cleanup) falls through
) # to the normal chunk path, matching the non-stream behavior
# in tokenizer_manager._handle_abort_finish_reason.
if finish_reason_type == "abort" and isinstance(
finish_reason.get("status_code"), HTTPStatus
):
code = finish_reason["status_code"]
error = self.create_streaming_error_response( error = self.create_streaming_error_response(
finish_reason.get("message", "Generation aborted."), finish_reason.get("message", "Generation aborted."),
code.name, code.name,
@@ -741,8 +746,7 @@ class OpenAIServingChat(OpenAIServingBase):
) )
yield f"data: {error}\n\n" yield f"data: {error}\n\n"
break break
else: finish_reasons[index] = finish_reason
finish_reasons[index] = finish_reason
# First chunk with role # First chunk with role
if is_firsts.get(index, True): if is_firsts.get(index, True):
python/sglang/srt/entrypoints/openai/serving_completions.py
+10 -5
View File
@@ -307,11 +307,16 @@ class OpenAIServingCompletion(OpenAIServingBase):
finish_reason = content["meta_info"].get("finish_reason", None) finish_reason = content["meta_info"].get("finish_reason", None)
finish_reason_type = finish_reason["type"] if finish_reason else None finish_reason_type = finish_reason["type"] if finish_reason else None
# If the abort is from scheduler. # Abort with an explicit error status_code is a system error
if finish_reason_type == "abort": # (timeout, OOM, validation): emit a streaming error chunk.
code = finish_reason.get( # A graceful abort (no status_code, e.g. user-initiated via
"status_code", HTTPStatus.INTERNAL_SERVER_ERROR # /abort_request or session lifecycle cleanup) falls through
) # to the normal chunk path, matching the non-stream behavior
# in tokenizer_manager._handle_abort_finish_reason.
if finish_reason_type == "abort" and isinstance(
finish_reason.get("status_code"), HTTPStatus
):
code = finish_reason["status_code"]
error = self.create_streaming_error_response( error = self.create_streaming_error_response(
finish_reason.get("message", "Generation aborted."), finish_reason.get("message", "Generation aborted."),
code.name, code.name,