fix: fix image benchmark backend parity (#30867)
This commit is contained in:
@@ -199,22 +199,23 @@ def create_mm_data_row(
|
||||
# Vision tokens = total tokens - text tokens
|
||||
vision_prompt_len = prompt_len - text_prompt_len
|
||||
|
||||
supported_backends = [
|
||||
supported_backends = (
|
||||
"sglang",
|
||||
"sglang-native",
|
||||
"sglang-oai-chat",
|
||||
"vllm-chat",
|
||||
]
|
||||
"lmdeploy-chat",
|
||||
)
|
||||
if backend not in supported_backends:
|
||||
raise ValueError(
|
||||
f"Image dataset only supports backends: {supported_backends}, "
|
||||
f"got '{backend}'."
|
||||
)
|
||||
|
||||
# OpenAI chat handlers apply the chat template and receive images separately, so
|
||||
# send the raw text. /generate does not apply a chat template, so it needs
|
||||
# prompt_str, which contains the multimodal processor's image placeholders.
|
||||
use_raw_prompt = backend in ("sglang-oai-chat", "vllm-chat")
|
||||
# Chat-completions backends apply their own chat template, so send raw text.
|
||||
# Native SGLang /generate does not apply a template and needs the image
|
||||
# placeholder-bearing prompt generated by the processor.
|
||||
use_raw_prompt = backend in ("sglang-oai-chat", "vllm-chat", "lmdeploy-chat")
|
||||
|
||||
return DatasetRow(
|
||||
prompt=text_prompt if use_raw_prompt else prompt_str,
|
||||
|
||||
Reference in New Issue
Block a user