[diffusion] feat: add opt-in SRT prompt enhancement to image and video APIs (#41095)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-09-25 10:00:04 +08:00
committed by GitHub
co-authored by Mick Qian
parent 9dc4c5d891
commit 6103a8c743
11 changed files with 886 additions and 69 deletions
@@ -17,6 +17,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.entrypoints.action import api as action_api
from sglang.multimodal_gen.runtime.entrypoints.action import openpi
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
PromptEnhancer,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
VertexGenerateReqInput,
)
@@ -136,6 +139,11 @@ async def lifespan(app: FastAPI):
warmup_done.set()
try:
app.state.prompt_enhancer = (
PromptEnhancer.from_file(server_args.prompt_enhancer_config)
if server_args.prompt_enhancer_config is not None
else None
)
yield
finally:
if warmup_task is not None and not warmup_task.done():
@@ -146,6 +154,8 @@ async def lifespan(app: FastAPI):
# On shutdown
logger.info("FastAPI app is shutting down...")
await shutdown_video_jobs()
if app.state.prompt_enhancer is not None:
await app.state.prompt_enhancer.close()
broker_task.cancel()
with suppress(asyncio.CancelledError):
await broker_task
@@ -444,4 +454,5 @@ def create_app(server_args: ServerArgs):
app.include_router(rollout_api.router)
app.state.server_args = server_args
app.state.prompt_enhancer = None
return app
@@ -25,6 +25,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
SamplingParams,
generate_request_id,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
maybe_enhance_prompt,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
ImageGenerationsRequest,
ImageResponse,
@@ -274,11 +277,14 @@ async def generations(
else sampling_params_cls.default_image_output_format()
)
ext = choose_output_image_ext(output_format, request.background)
prompt = await maybe_enhance_prompt(
raw_request, request.prompt, enabled=request.enhance_prompt, task="image"
)
with temp_dir_if_disabled(server_args.output_path) as output_dir:
sampling = build_sampling_params(
request_id,
prompt=request.prompt,
prompt=prompt,
size=request.size,
width=request.width,
height=request.height,
@@ -383,7 +389,7 @@ async def generations(
response_kwargs = _build_image_response_kwargs(
save_file_path_list,
resp_format,
request.prompt,
prompt,
request_id,
result,
b64_list=b64_list,
@@ -404,6 +410,7 @@ async def edits(
url: Optional[List[str]] = Form(None),
url_array: Optional[List[str]] = Form(None, alias="url[]"),
prompt: str = Form(...),
enhance_prompt: bool = Form(False),
mask: Optional[UploadFile] = File(None),
model: Optional[str] = Form(None),
n: Optional[int] = Form(1),
@@ -464,6 +471,13 @@ async def edits(
)
ext = choose_output_image_ext(output_format, background)
prompt = await maybe_enhance_prompt(
raw_request,
prompt,
enabled=enhance_prompt,
task="image_edit",
image_paths=input_paths,
)
sampling = build_sampling_params(
request_id,
prompt=prompt,
@@ -0,0 +1,179 @@
import asyncio
import base64
import json
import os
from pathlib import Path
from typing import Any, Literal
import httpx
from fastapi import HTTPException, Request
from PIL import Image
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
class PromptEnhancerConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
base_url: HttpUrl
model: str = Field(min_length=1)
system_prompt: str = Field(
default=(
"You expand prompts for image and video generation. The user supplies "
"JSON with task and prompt fields. Expand the prompt into a concise, "
"visually detailed description, clarifying composition, lighting and "
"appearance where appropriate instead of simply repeating it. Preserve "
"the requested subjects, count, quoted text, actions, style and language. "
"Preserve special tokens and reference labels exactly. "
"Clarify visual details without inventing new subjects or changing intent. "
"For image editing, describe only the requested edits and preserve "
"everything else. For video, preserve the requested motion and timing. "
"Do not invent facts about reference media you cannot see. "
"Return only the rewritten prompt, without surrounding quotes, "
"commentary or markdown."
),
min_length=1,
)
include_images: bool = False
timeout: float = Field(default=60.0, gt=0, allow_inf_nan=False)
api_key_env: str | None = None
generation_kwargs: dict[str, Any] = Field(default_factory=dict)
@field_validator("base_url")
@classmethod
def validate_base_url(cls, value: HttpUrl) -> HttpUrl:
if value.username or value.password or value.query or value.fragment:
raise ValueError(
"base_url must not contain credentials, a query or fragment"
)
return value
@field_validator("generation_kwargs")
@classmethod
def validate_generation_kwargs(cls, value: dict[str, Any]) -> dict[str, Any]:
# one non-streaming completion per diffusion request
reserved = value.keys() & {
"model",
"messages",
"stream",
"n",
"tools",
"tool_choice",
}
if reserved:
raise ValueError(f"generation_kwargs cannot override {sorted(reserved)}")
return value
class PromptEnhancer:
"""HTTP frontend client; owns its connection pool, never runs in GPU workers."""
def __init__(self, config: PromptEnhancerConfig):
self.config = config
headers = {}
if config.api_key_env is not None:
api_key = os.environ.get(config.api_key_env)
if not api_key:
raise ValueError(
f"Prompt enhancer API key env {config.api_key_env} is unset"
)
headers["Authorization"] = f"Bearer {api_key}"
self.client = httpx.AsyncClient(
base_url=str(config.base_url).rstrip("/") + "/",
headers=headers,
timeout=config.timeout,
trust_env=False,
)
@classmethod
def from_file(cls, path: str) -> "PromptEnhancer":
return cls(PromptEnhancerConfig.model_validate_json(Path(path).read_text()))
async def close(self) -> None:
await self.client.aclose()
async def enhance(
self,
prompt: str,
*,
task: Literal["image", "image_edit", "video"],
image_paths: list[str],
) -> str:
content: str | list[dict[str, Any]] = json.dumps(
{"task": task, "prompt": prompt}, ensure_ascii=False
)
if self.config.include_images and image_paths:
content = [{"type": "text", "text": content}]
for path in image_paths:
if path.startswith(("https://", "http://", "data:image/")):
url = path
else:
url = await asyncio.to_thread(_local_image_url, path)
content.append({"type": "image_url", "image_url": {"url": url}})
payload = {
"max_tokens": 512,
"temperature": 0,
**self.config.generation_kwargs,
"model": self.config.model,
"messages": [
{"role": "system", "content": self.config.system_prompt},
{"role": "user", "content": content},
],
"stream": False,
"n": 1,
}
try:
response = await self.client.post("chat/completions", json=payload)
response.raise_for_status()
except httpx.TimeoutException as e:
raise HTTPException(504, "Prompt enhancer timed out") from e
except httpx.HTTPError as e:
# do not expose upstream response bodies, prompts or auth headers
raise HTTPException(502, "Prompt enhancer request failed") from e
try:
choice = response.json()["choices"][0]
if choice["finish_reason"] != "stop":
raise ValueError("Incomplete completion")
rewritten = choice["message"]["content"]
if not isinstance(rewritten, str) or not rewritten.strip():
raise ValueError("Empty completion")
except (ValueError, KeyError, IndexError, TypeError) as e:
raise HTTPException(
502,
"Prompt enhancer did not return a complete, nonempty text response; "
"check its template and token limit",
) from e
return rewritten.strip()
def _local_image_url(path: str) -> str:
# uploads live on the diffusion host, not necessarily on the SRT host
try:
with Image.open(path) as image:
mime = image.get_format_mimetype()
image.verify()
if mime is None:
raise ValueError("Unknown image MIME type")
encoded = base64.b64encode(Path(path).read_bytes()).decode("ascii")
except (OSError, ValueError, SyntaxError) as e:
raise HTTPException(
400, "Cannot read reference image for prompt enhancement"
) from e
return f"data:{mime};base64,{encoded}"
async def maybe_enhance_prompt(
request: Request,
prompt: str,
*,
enabled: bool,
task: Literal["image", "image_edit", "video"],
image_paths: list[str] | None = None,
) -> str:
if not enabled:
return prompt
enhancer: PromptEnhancer | None = request.app.state.prompt_enhancer
if enhancer is None:
raise HTTPException(400, "Prompt enhancement requires --prompt-enhancer-config")
return await enhancer.enhance(prompt, task=task, image_paths=image_paths or [])
@@ -46,6 +46,7 @@ class ImageGenerationsRequest(BaseModel):
model_config = ConfigDict(extra="allow")
prompt: str
enhance_prompt: bool = False
model: Optional[str] = None
n: Optional[int] = 1
quality: Optional[str] = "auto"
@@ -105,6 +106,7 @@ class VideoResponse(BaseModel):
num_outputs: Optional[int] = None
peak_memory_mb: Optional[float] = None
inference_time_s: Optional[float] = None
revised_prompt: Optional[str] = None
action: Optional[Dict[str, Any]] = None
@@ -112,6 +114,7 @@ class VideoGenerationsRequest(BaseModel):
model_config = ConfigDict(extra="allow")
prompt: str
enhance_prompt: bool = False
input_reference: Optional[str] = None
reference_url: Optional[str] = None
video_path: Optional[str] = None
@@ -168,6 +171,7 @@ class VideoRemixRequest(BaseModel):
class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
enhance_prompt: Literal[False] = False
type: Literal["init"]
# WebSocket does not support multipart/form-data image uploads
first_frame: Optional[bytes | str] = None
@@ -27,6 +27,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
SamplingParams,
generate_request_id,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
maybe_enhance_prompt,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
VideoGenerationsRequest,
VideoListResponse,
@@ -465,6 +468,7 @@ async def create_video(
request: Request,
# multipart/form-data fields (optional; used only when content-type is multipart)
prompt: Optional[str] = Form(None),
enhance_prompt: Optional[bool] = Form(None),
input_reference: Optional[UploadFile] = File(None),
reference_url: Optional[str] = Form(None),
video_reference: Optional[UploadFile] = File(None),
@@ -607,6 +611,7 @@ async def create_video(
req = VideoGenerationsRequest(
prompt=prompt,
enhance_prompt=form_value("enhance_prompt", enhance_prompt) or False,
input_reference=input_path,
video_path=form_value("video_path", video_input_path),
video_url=form_value("video_url", video_url),
@@ -730,10 +735,20 @@ async def create_video(
logger.debug(f"Server received from create_video endpoint: req={req}")
try:
image_path = _resolve_image_path(req, _resolve_video_path(req))
req.prompt = await maybe_enhance_prompt(
request,
req.prompt,
enabled=req.enhance_prompt,
task="video",
image_paths=[image_path] if image_path else None,
)
sampling_params = _build_video_sampling_params(request_id, req)
except (ValueError, TypeError) as e:
except (asyncio.CancelledError, HTTPException, ValueError, TypeError) as e:
for td in temp_dirs:
shutil.rmtree(td, ignore_errors=True)
if isinstance(e, (asyncio.CancelledError, HTTPException)):
raise
raise HTTPException(status_code=400, detail=str(e))
batch: Req | None = None
@@ -766,6 +781,8 @@ async def create_video(
sampling_params,
server_args.served_model_name,
)
if req.enhance_prompt:
job["revised_prompt"] = req.prompt
job.update(sampling_params.project_video_queued_job_fields(batch))
await VIDEO_STORE.upsert(request_id, job)
except Exception as e:
@@ -613,6 +613,7 @@ class ServerArgs(DisaggServerArgsMixin):
# SGLang server for PE model inference
pe_server_url: str | None = None
prompt_enhancer_config: str | None = None
@property
def broker_port(self) -> int:
@@ -3054,6 +3055,12 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.pe_server_url,
help="URL of SGLang server for PE model",
)
parser.add_argument(
"--prompt-enhancer-config",
type=str,
default=ServerArgs.prompt_enhancer_config,
help="JSON config file for an external SRT prompt enhancer, used by HTTP requests with enhance_prompt=true.",
)
return parser
@@ -0,0 +1,460 @@
import asyncio
import io
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import httpx
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
from PIL import Image
from pydantic import ValidationError
from sglang.multimodal_gen import registry
from sglang.multimodal_gen.configs.pipeline_configs.ernie_image import (
ErnieImagePipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.ideogram import (
Ideogram4PipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import (
LongCatImagePipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
MiniMaxH3PipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
QwenImageEditPlusPipelineConfig,
QwenImagePipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
from sglang.multimodal_gen.configs.sample.ernie_image import ErnieImageSamplingParams
from sglang.multimodal_gen.configs.sample.flux import FluxSamplingParams
from sglang.multimodal_gen.configs.sample.ideogram import Ideogram4SamplingParams
from sglang.multimodal_gen.configs.sample.longcat_image import (
LongCatImageSamplingParams,
)
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
from sglang.multimodal_gen.configs.sample.qwenimage import (
QwenImageEditPlusSamplingParams,
QwenImageSamplingParams,
)
from sglang.multimodal_gen.configs.sample.wan import WanT2V_1_3B_SamplingParams
from sglang.multimodal_gen.configs.sample.zimage import ZImageTurboSamplingParams
from sglang.multimodal_gen.runtime.entrypoints import http_server
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
PromptEnhancer,
PromptEnhancerConfig,
maybe_enhance_prompt,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
RealtimeVideoGenerationsRequest,
VideoGenerationsRequest,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.server_args import server_args as server_args_module
def completion(text="Rewritten prompt", finish_reason="stop"):
return {"choices": [{"message": {"content": text}, "finish_reason": finish_reason}]}
@pytest.mark.parametrize(
"override",
[
{"timeout": 0},
{"timeout": float("inf")},
{"model": ""},
{"base_url": "file:///tmp/server"},
{"base_url": "https://user:secret@example.org/v1"},
{"base_url": "https://example.org/v1?key=secret"},
{"generation_kwargs": {"stream": True}},
{"generation_kwargs": {"messages": []}},
{"generation_kwargs": {"n": 2}},
{"typo": True},
],
)
def test_config_rejects_invalid_or_conflicting_fields(override):
with pytest.raises(ValidationError):
PromptEnhancerConfig.model_validate(
{"base_url": "http://localhost:30001/v1", "model": "enhancer", **override}
)
def test_missing_auth_env_fails_at_startup(monkeypatch):
monkeypatch.delenv("TEST_ENHANCER_KEY", raising=False)
with pytest.raises(ValueError, match="TEST_ENHANCER_KEY is unset"):
PromptEnhancer(
PromptEnhancerConfig(
base_url="http://localhost:30001/v1",
model="enhancer",
api_key_env="TEST_ENHANCER_KEY",
)
)
def test_disabled_path_does_not_need_a_client():
assert (
asyncio.run(maybe_enhance_prompt(None, "original", enabled=False, task="image"))
== "original"
)
@pytest.mark.parametrize(
"upstream, status",
[
(completion(" "), 502),
(completion(None), 502),
(completion("partial", "length"), 502),
(completion("refusal", "content_filter"), 502),
({"choices": []}, 502),
({"error": "sensitive upstream detail"}, 502),
(httpx.Response(401, text="secret"), 502),
(httpx.ReadTimeout("secret"), 504),
(httpx.ConnectError("secret"), 502),
],
)
def test_upstream_failures_are_not_silent(upstream, status):
async def run():
enhancer = PromptEnhancer(
PromptEnhancerConfig(base_url="http://localhost/v1", model="enhancer")
)
def respond(request):
if isinstance(upstream, Exception):
raise upstream
if isinstance(upstream, httpx.Response):
return upstream
return httpx.Response(200, json=upstream)
await enhancer.client.aclose()
enhancer.client = httpx.AsyncClient(
base_url="http://localhost/v1/", transport=httpx.MockTransport(respond)
)
try:
with pytest.raises(HTTPException) as error:
await enhancer.enhance("original", task="image", image_paths=[])
assert error.value.status_code == status
assert "secret" not in error.value.detail
assert "sensitive" not in error.value.detail
finally:
await enhancer.close()
asyncio.run(run())
@pytest.fixture
def server(monkeypatch, tmp_path):
calls = []
batches = []
rewritten = '{"high_level_description": "A red teapot"}'
config_path = tmp_path / "enhancer.json"
config_path.write_text(
json.dumps(
{
"base_url": "http://enhancer.local/v1",
"model": "test-vlm",
"include_images": True,
"api_key_env": "TEST_ENHANCER_KEY",
"system_prompt": "Return the target model's JSON caption.",
"generation_kwargs": {
"max_tokens": 128,
"response_format": {"type": "json_object"},
},
}
)
)
monkeypatch.setenv("TEST_ENHANCER_KEY", "test-key")
def respond(request):
assert request.url == "http://enhancer.local/v1/chat/completions"
assert request.headers["authorization"] == "Bearer test-key"
calls.append(json.loads(request.content))
return httpx.Response(200, json=completion(rewritten))
# keep the real HTTP client and lifespan; replace only the external transport
original_init = httpx.AsyncClient.__init__
def client_init(self, *args, **kwargs):
kwargs.setdefault("transport", httpx.MockTransport(respond))
original_init(self, *args, **kwargs)
monkeypatch.setattr(httpx.AsyncClient, "__init__", client_init)
monkeypatch.setattr(ServerArgs, "__post_init__", lambda self: None)
args = ServerArgs(
model_path="test-model",
served_model_name="test-model",
pipeline_config=ZImagePipelineConfig(),
num_gpus=1,
prompt_enhancer_config=str(config_path),
warmup_mode="off",
output_path=str(tmp_path / "outputs"),
input_save_path=str(tmp_path / "inputs"),
)
monkeypatch.setattr(server_args_module, "_global_server_args", args)
model_info = SimpleNamespace(sampling_param_cls=ZImageTurboSamplingParams)
monkeypatch.setattr(registry, "get_model_info", lambda *args, **kwargs: model_info)
monkeypatch.setattr(async_scheduler_client, "initialize", lambda args: None)
monkeypatch.setattr(async_scheduler_client, "close", lambda: None)
monkeypatch.setattr(http_server, "_wait_until_http_live", AsyncMock())
monkeypatch.setattr(
"sglang.multimodal_gen.runtime.scheduler_client.run_zeromq_broker", AsyncMock()
)
async def forward(requests):
batches.extend(requests)
batch = requests[0]
paths = []
for idx in range(batch.num_outputs_per_prompt):
path = tmp_path / f"{batch.request_id}_{idx}.png"
Image.new("RGB", (16, 16), "red").save(path)
paths.append(str(path))
return OutputBatch(output_file_paths=paths)
monkeypatch.setattr(async_scheduler_client, "forward", forward)
app = http_server.create_app(args)
with TestClient(app) as client:
yield SimpleNamespace(
client=client,
app=app,
args=args,
model_info=model_info,
calls=calls,
batches=batches,
rewritten=rewritten,
)
assert app.state.prompt_enhancer.client.is_closed
@pytest.mark.parametrize("enabled", [False, True])
@pytest.mark.parametrize("endpoint", ["image", "edit", "video_json", "video_form"])
def test_http_routes_enhance_once_before_sampling_and_preserve_options(
server, endpoint, enabled
):
original = "A red teapot"
if endpoint == "image":
response = server.client.post(
"/v1/images/generations",
json={
"prompt": original,
"enhance_prompt": enabled,
"seed": 123,
"negative_prompt": "blur",
"n": 2,
"response_format": "b64_json",
},
)
elif endpoint == "edit":
server.args.pipeline_config = QwenImageEditPlusPipelineConfig()
server.model_info.sampling_param_cls = QwenImageEditPlusSamplingParams
buffer = io.BytesIO()
Image.new("RGB", (16, 16), "blue").save(buffer, format="PNG")
response = server.client.post(
"/v1/images/edits",
data={
"prompt": original,
"enhance_prompt": str(enabled).lower(),
"seed": "123",
"negative_prompt": "blur",
"n": "2",
},
files={"image": ("reference.png", buffer.getvalue(), "image/png")},
)
else:
server.args.pipeline_config = WanT2V480PConfig()
server.model_info.sampling_param_cls = WanT2V_1_3B_SamplingParams
payload = {
"prompt": original,
"seed": 123,
"negative_prompt": "blur",
"n": 2,
"num_frames": 5,
}
if endpoint == "video_json":
response = server.client.post(
"/v1/videos", json={**payload, "enhance_prompt": enabled}
)
else:
response = server.client.post(
"/v1/videos",
data={**payload, "enhance_prompt": str(enabled).lower()},
files={"unused": ("empty", b"")},
)
assert response.status_code == 200, response.text
expected = server.rewritten if enabled else original
assert len(server.calls) == int(enabled)
if endpoint.startswith("video"):
assert response.json()["revised_prompt"] == (expected if enabled else None)
# wait for the real background job, not a mocked dispatch
async def wait_jobs():
if video_api._VIDEO_JOB_TASKS:
await asyncio.gather(*list(video_api._VIDEO_JOB_TASKS.values()))
server.client.portal.call(wait_jobs)
else:
assert len(response.json()["data"]) == 2
assert all(
item["revised_prompt"] == expected for item in response.json()["data"]
)
assert len(server.batches) == 1
batch = server.batches[0]
assert batch.prompt == expected
assert batch.seed == 123
assert batch.negative_prompt == "blur"
assert batch.num_outputs_per_prompt == 2
if enabled:
payload = server.calls[0]
assert payload["model"] == "test-vlm"
assert payload["max_tokens"] == 128
assert payload["response_format"] == {"type": "json_object"}
if endpoint == "edit":
assert payload["messages"][1]["content"][1]["image_url"]["url"].startswith(
"data:image/png;base64,"
)
else:
assert json.loads(payload["messages"][1]["content"])["prompt"] == original
def test_unconfigured_enhancer_rejects_opt_in_without_generation(server):
enhancer = server.app.state.prompt_enhancer
server.app.state.prompt_enhancer = None
try:
response = server.client.post(
"/v1/images/generations",
json={"prompt": "a teapot", "enhance_prompt": True},
)
assert response.status_code == 400
assert not server.batches
finally:
server.app.state.prompt_enhancer = enhancer
def test_missing_switch_preserves_original_prompt(server):
response = server.client.post(
"/v1/images/generations",
json={"prompt": "keep exactly this", "response_format": "b64_json"},
)
assert response.status_code == 200, response.text
assert not server.calls
assert server.batches[0].prompt == "keep exactly this"
@pytest.mark.parametrize(
"pipeline_cls, sampling_cls, extras",
[
(FluxPipelineConfig, FluxSamplingParams, {}),
(QwenImagePipelineConfig, QwenImageSamplingParams, {}),
(Ideogram4PipelineConfig, Ideogram4SamplingParams, {}),
(ErnieImagePipelineConfig, ErnieImageSamplingParams, {"use_pe": False}),
(
LongCatImagePipelineConfig,
LongCatImageSamplingParams,
{"enable_prompt_rewrite": False},
),
],
)
def test_model_sampling_contracts_remain_owned_by_the_model(
server, pipeline_cls, sampling_cls, extras
):
server.args.pipeline_config = pipeline_cls()
server.model_info.sampling_param_cls = sampling_cls
response = server.client.post(
"/v1/images/generations",
json={
"prompt": "a teapot",
"enhance_prompt": True,
"response_format": "b64_json",
**extras,
},
)
assert response.status_code == 200, response.text
sampling = server.batches[0].sampling_params
assert type(sampling) is sampling_cls
assert sampling.prompt == server.rewritten
for name, value in extras.items():
assert vars(sampling)[name] == value
def test_h3_native_request_contract_survives_enhancement(server):
server.args.pipeline_config = MiniMaxH3PipelineConfig()
server.model_info.sampling_param_cls = MiniMaxH3SamplingParams
target = {"short_edge": 768, "aspect_ratio": "16:9", "duration_seconds": 5.0}
request = VideoGenerationsRequest(
prompt="A red teapot", task="t2va", conditions=[], target=target, seed=123
)
async def enhance_and_lower():
request.prompt = await server.app.state.prompt_enhancer.enhance(
request.prompt, task="video", image_paths=[]
)
return video_api._build_video_sampling_params("h3-enhanced", request)
sampling = server.client.portal.call(enhance_and_lower)
assert type(sampling) is MiniMaxH3SamplingParams
assert sampling.prompt == server.rewritten
assert sampling.seed == 123
assert sampling.task == "t2va"
assert sampling.conditions == []
assert sampling.target == target
@pytest.mark.parametrize(
"source", ["https://example.com/reference.png", "data:image/png;base64,AAAA"]
)
@pytest.mark.parametrize("include_images", [True, False])
def test_reference_transport_is_explicit(server, source, include_images):
enhancer = server.app.state.prompt_enhancer
enhancer.config.include_images = include_images
async def run():
return await enhancer.enhance(
"keep the subject", task="image_edit", image_paths=[source]
)
assert server.client.portal.call(run) == server.rewritten
content = server.calls[0]["messages"][1]["content"]
if include_images:
assert content[1] == {"type": "image_url", "image_url": {"url": source}}
else:
assert isinstance(content, str)
assert source not in content
def test_enhancer_failure_does_not_queue_video_or_leak_uploads(
server, monkeypatch, tmp_path
):
server.args.pipeline_config = WanT2V480PConfig()
server.args.input_save_path = None
server.args.output_path = None
monkeypatch.setattr(video_api.tempfile, "tempdir", str(tmp_path))
async def fail(request):
return httpx.Response(503, text="upstream secret")
enhancer = server.app.state.prompt_enhancer
server.client.portal.call(enhancer.client.aclose)
enhancer.client = httpx.AsyncClient(
base_url="http://enhancer.local/v1/", transport=httpx.MockTransport(fail)
)
response = server.client.post(
"/v1/videos", json={"prompt": "a teapot", "enhance_prompt": True}
)
assert response.status_code == 502
assert "secret" not in response.text
assert not server.batches
assert not list(tmp_path.glob("sglang_*"))
def test_realtime_does_not_silently_accept_prompt_enhancement():
with pytest.raises(ValidationError):
RealtimeVideoGenerationsRequest(
type="init", prompt="a teapot", enhance_prompt=True
)