[diffusion] feat: add opt-in SRT prompt enhancement to image and video APIs (#41095)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -17,6 +17,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.entrypoints.action import api as action_api
|
||||
from sglang.multimodal_gen.runtime.entrypoints.action import openpi
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
|
||||
PromptEnhancer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
VertexGenerateReqInput,
|
||||
)
|
||||
@@ -136,6 +139,11 @@ async def lifespan(app: FastAPI):
|
||||
warmup_done.set()
|
||||
|
||||
try:
|
||||
app.state.prompt_enhancer = (
|
||||
PromptEnhancer.from_file(server_args.prompt_enhancer_config)
|
||||
if server_args.prompt_enhancer_config is not None
|
||||
else None
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
if warmup_task is not None and not warmup_task.done():
|
||||
@@ -146,6 +154,8 @@ async def lifespan(app: FastAPI):
|
||||
# On shutdown
|
||||
logger.info("FastAPI app is shutting down...")
|
||||
await shutdown_video_jobs()
|
||||
if app.state.prompt_enhancer is not None:
|
||||
await app.state.prompt_enhancer.close()
|
||||
broker_task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await broker_task
|
||||
@@ -444,4 +454,5 @@ def create_app(server_args: ServerArgs):
|
||||
app.include_router(rollout_api.router)
|
||||
|
||||
app.state.server_args = server_args
|
||||
app.state.prompt_enhancer = None
|
||||
return app
|
||||
|
||||
@@ -25,6 +25,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
SamplingParams,
|
||||
generate_request_id,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
|
||||
maybe_enhance_prompt,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
ImageGenerationsRequest,
|
||||
ImageResponse,
|
||||
@@ -274,11 +277,14 @@ async def generations(
|
||||
else sampling_params_cls.default_image_output_format()
|
||||
)
|
||||
ext = choose_output_image_ext(output_format, request.background)
|
||||
prompt = await maybe_enhance_prompt(
|
||||
raw_request, request.prompt, enabled=request.enhance_prompt, task="image"
|
||||
)
|
||||
|
||||
with temp_dir_if_disabled(server_args.output_path) as output_dir:
|
||||
sampling = build_sampling_params(
|
||||
request_id,
|
||||
prompt=request.prompt,
|
||||
prompt=prompt,
|
||||
size=request.size,
|
||||
width=request.width,
|
||||
height=request.height,
|
||||
@@ -383,7 +389,7 @@ async def generations(
|
||||
response_kwargs = _build_image_response_kwargs(
|
||||
save_file_path_list,
|
||||
resp_format,
|
||||
request.prompt,
|
||||
prompt,
|
||||
request_id,
|
||||
result,
|
||||
b64_list=b64_list,
|
||||
@@ -404,6 +410,7 @@ async def edits(
|
||||
url: Optional[List[str]] = Form(None),
|
||||
url_array: Optional[List[str]] = Form(None, alias="url[]"),
|
||||
prompt: str = Form(...),
|
||||
enhance_prompt: bool = Form(False),
|
||||
mask: Optional[UploadFile] = File(None),
|
||||
model: Optional[str] = Form(None),
|
||||
n: Optional[int] = Form(1),
|
||||
@@ -464,6 +471,13 @@ async def edits(
|
||||
)
|
||||
|
||||
ext = choose_output_image_ext(output_format, background)
|
||||
prompt = await maybe_enhance_prompt(
|
||||
raw_request,
|
||||
prompt,
|
||||
enabled=enhance_prompt,
|
||||
task="image_edit",
|
||||
image_paths=input_paths,
|
||||
)
|
||||
sampling = build_sampling_params(
|
||||
request_id,
|
||||
prompt=prompt,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, Request
|
||||
from PIL import Image
|
||||
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
|
||||
|
||||
|
||||
class PromptEnhancerConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
base_url: HttpUrl
|
||||
model: str = Field(min_length=1)
|
||||
system_prompt: str = Field(
|
||||
default=(
|
||||
"You expand prompts for image and video generation. The user supplies "
|
||||
"JSON with task and prompt fields. Expand the prompt into a concise, "
|
||||
"visually detailed description, clarifying composition, lighting and "
|
||||
"appearance where appropriate instead of simply repeating it. Preserve "
|
||||
"the requested subjects, count, quoted text, actions, style and language. "
|
||||
"Preserve special tokens and reference labels exactly. "
|
||||
"Clarify visual details without inventing new subjects or changing intent. "
|
||||
"For image editing, describe only the requested edits and preserve "
|
||||
"everything else. For video, preserve the requested motion and timing. "
|
||||
"Do not invent facts about reference media you cannot see. "
|
||||
"Return only the rewritten prompt, without surrounding quotes, "
|
||||
"commentary or markdown."
|
||||
),
|
||||
min_length=1,
|
||||
)
|
||||
include_images: bool = False
|
||||
timeout: float = Field(default=60.0, gt=0, allow_inf_nan=False)
|
||||
api_key_env: str | None = None
|
||||
generation_kwargs: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def validate_base_url(cls, value: HttpUrl) -> HttpUrl:
|
||||
if value.username or value.password or value.query or value.fragment:
|
||||
raise ValueError(
|
||||
"base_url must not contain credentials, a query or fragment"
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("generation_kwargs")
|
||||
@classmethod
|
||||
def validate_generation_kwargs(cls, value: dict[str, Any]) -> dict[str, Any]:
|
||||
# one non-streaming completion per diffusion request
|
||||
reserved = value.keys() & {
|
||||
"model",
|
||||
"messages",
|
||||
"stream",
|
||||
"n",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
}
|
||||
if reserved:
|
||||
raise ValueError(f"generation_kwargs cannot override {sorted(reserved)}")
|
||||
return value
|
||||
|
||||
|
||||
class PromptEnhancer:
|
||||
"""HTTP frontend client; owns its connection pool, never runs in GPU workers."""
|
||||
|
||||
def __init__(self, config: PromptEnhancerConfig):
|
||||
self.config = config
|
||||
headers = {}
|
||||
if config.api_key_env is not None:
|
||||
api_key = os.environ.get(config.api_key_env)
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
f"Prompt enhancer API key env {config.api_key_env} is unset"
|
||||
)
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
self.client = httpx.AsyncClient(
|
||||
base_url=str(config.base_url).rstrip("/") + "/",
|
||||
headers=headers,
|
||||
timeout=config.timeout,
|
||||
trust_env=False,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: str) -> "PromptEnhancer":
|
||||
return cls(PromptEnhancerConfig.model_validate_json(Path(path).read_text()))
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.client.aclose()
|
||||
|
||||
async def enhance(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
task: Literal["image", "image_edit", "video"],
|
||||
image_paths: list[str],
|
||||
) -> str:
|
||||
content: str | list[dict[str, Any]] = json.dumps(
|
||||
{"task": task, "prompt": prompt}, ensure_ascii=False
|
||||
)
|
||||
if self.config.include_images and image_paths:
|
||||
content = [{"type": "text", "text": content}]
|
||||
for path in image_paths:
|
||||
if path.startswith(("https://", "http://", "data:image/")):
|
||||
url = path
|
||||
else:
|
||||
url = await asyncio.to_thread(_local_image_url, path)
|
||||
content.append({"type": "image_url", "image_url": {"url": url}})
|
||||
|
||||
payload = {
|
||||
"max_tokens": 512,
|
||||
"temperature": 0,
|
||||
**self.config.generation_kwargs,
|
||||
"model": self.config.model,
|
||||
"messages": [
|
||||
{"role": "system", "content": self.config.system_prompt},
|
||||
{"role": "user", "content": content},
|
||||
],
|
||||
"stream": False,
|
||||
"n": 1,
|
||||
}
|
||||
try:
|
||||
response = await self.client.post("chat/completions", json=payload)
|
||||
response.raise_for_status()
|
||||
except httpx.TimeoutException as e:
|
||||
raise HTTPException(504, "Prompt enhancer timed out") from e
|
||||
except httpx.HTTPError as e:
|
||||
# do not expose upstream response bodies, prompts or auth headers
|
||||
raise HTTPException(502, "Prompt enhancer request failed") from e
|
||||
|
||||
try:
|
||||
choice = response.json()["choices"][0]
|
||||
if choice["finish_reason"] != "stop":
|
||||
raise ValueError("Incomplete completion")
|
||||
rewritten = choice["message"]["content"]
|
||||
if not isinstance(rewritten, str) or not rewritten.strip():
|
||||
raise ValueError("Empty completion")
|
||||
except (ValueError, KeyError, IndexError, TypeError) as e:
|
||||
raise HTTPException(
|
||||
502,
|
||||
"Prompt enhancer did not return a complete, nonempty text response; "
|
||||
"check its template and token limit",
|
||||
) from e
|
||||
return rewritten.strip()
|
||||
|
||||
|
||||
def _local_image_url(path: str) -> str:
|
||||
# uploads live on the diffusion host, not necessarily on the SRT host
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
mime = image.get_format_mimetype()
|
||||
image.verify()
|
||||
if mime is None:
|
||||
raise ValueError("Unknown image MIME type")
|
||||
encoded = base64.b64encode(Path(path).read_bytes()).decode("ascii")
|
||||
except (OSError, ValueError, SyntaxError) as e:
|
||||
raise HTTPException(
|
||||
400, "Cannot read reference image for prompt enhancement"
|
||||
) from e
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
async def maybe_enhance_prompt(
|
||||
request: Request,
|
||||
prompt: str,
|
||||
*,
|
||||
enabled: bool,
|
||||
task: Literal["image", "image_edit", "video"],
|
||||
image_paths: list[str] | None = None,
|
||||
) -> str:
|
||||
if not enabled:
|
||||
return prompt
|
||||
enhancer: PromptEnhancer | None = request.app.state.prompt_enhancer
|
||||
if enhancer is None:
|
||||
raise HTTPException(400, "Prompt enhancement requires --prompt-enhancer-config")
|
||||
return await enhancer.enhance(prompt, task=task, image_paths=image_paths or [])
|
||||
@@ -46,6 +46,7 @@ class ImageGenerationsRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
prompt: str
|
||||
enhance_prompt: bool = False
|
||||
model: Optional[str] = None
|
||||
n: Optional[int] = 1
|
||||
quality: Optional[str] = "auto"
|
||||
@@ -105,6 +106,7 @@ class VideoResponse(BaseModel):
|
||||
num_outputs: Optional[int] = None
|
||||
peak_memory_mb: Optional[float] = None
|
||||
inference_time_s: Optional[float] = None
|
||||
revised_prompt: Optional[str] = None
|
||||
action: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
@@ -112,6 +114,7 @@ class VideoGenerationsRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
prompt: str
|
||||
enhance_prompt: bool = False
|
||||
input_reference: Optional[str] = None
|
||||
reference_url: Optional[str] = None
|
||||
video_path: Optional[str] = None
|
||||
@@ -168,6 +171,7 @@ class VideoRemixRequest(BaseModel):
|
||||
|
||||
|
||||
class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
|
||||
enhance_prompt: Literal[False] = False
|
||||
type: Literal["init"]
|
||||
# WebSocket does not support multipart/form-data image uploads
|
||||
first_frame: Optional[bytes | str] = None
|
||||
|
||||
@@ -27,6 +27,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
SamplingParams,
|
||||
generate_request_id,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
|
||||
maybe_enhance_prompt,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
VideoGenerationsRequest,
|
||||
VideoListResponse,
|
||||
@@ -465,6 +468,7 @@ async def create_video(
|
||||
request: Request,
|
||||
# multipart/form-data fields (optional; used only when content-type is multipart)
|
||||
prompt: Optional[str] = Form(None),
|
||||
enhance_prompt: Optional[bool] = Form(None),
|
||||
input_reference: Optional[UploadFile] = File(None),
|
||||
reference_url: Optional[str] = Form(None),
|
||||
video_reference: Optional[UploadFile] = File(None),
|
||||
@@ -607,6 +611,7 @@ async def create_video(
|
||||
|
||||
req = VideoGenerationsRequest(
|
||||
prompt=prompt,
|
||||
enhance_prompt=form_value("enhance_prompt", enhance_prompt) or False,
|
||||
input_reference=input_path,
|
||||
video_path=form_value("video_path", video_input_path),
|
||||
video_url=form_value("video_url", video_url),
|
||||
@@ -730,10 +735,20 @@ async def create_video(
|
||||
logger.debug(f"Server received from create_video endpoint: req={req}")
|
||||
|
||||
try:
|
||||
image_path = _resolve_image_path(req, _resolve_video_path(req))
|
||||
req.prompt = await maybe_enhance_prompt(
|
||||
request,
|
||||
req.prompt,
|
||||
enabled=req.enhance_prompt,
|
||||
task="video",
|
||||
image_paths=[image_path] if image_path else None,
|
||||
)
|
||||
sampling_params = _build_video_sampling_params(request_id, req)
|
||||
except (ValueError, TypeError) as e:
|
||||
except (asyncio.CancelledError, HTTPException, ValueError, TypeError) as e:
|
||||
for td in temp_dirs:
|
||||
shutil.rmtree(td, ignore_errors=True)
|
||||
if isinstance(e, (asyncio.CancelledError, HTTPException)):
|
||||
raise
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
batch: Req | None = None
|
||||
@@ -766,6 +781,8 @@ async def create_video(
|
||||
sampling_params,
|
||||
server_args.served_model_name,
|
||||
)
|
||||
if req.enhance_prompt:
|
||||
job["revised_prompt"] = req.prompt
|
||||
job.update(sampling_params.project_video_queued_job_fields(batch))
|
||||
await VIDEO_STORE.upsert(request_id, job)
|
||||
except Exception as e:
|
||||
|
||||
@@ -613,6 +613,7 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
|
||||
# SGLang server for PE model inference
|
||||
pe_server_url: str | None = None
|
||||
prompt_enhancer_config: str | None = None
|
||||
|
||||
@property
|
||||
def broker_port(self) -> int:
|
||||
@@ -3054,6 +3055,12 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
default=ServerArgs.pe_server_url,
|
||||
help="URL of SGLang server for PE model",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-enhancer-config",
|
||||
type=str,
|
||||
default=ServerArgs.prompt_enhancer_config,
|
||||
help="JSON config file for an external SRT prompt enhancer, used by HTTP requests with enhance_prompt=true.",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@@ -0,0 +1,460 @@
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
from pydantic import ValidationError
|
||||
|
||||
from sglang.multimodal_gen import registry
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ernie_image import (
|
||||
ErnieImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ideogram import (
|
||||
Ideogram4PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import (
|
||||
LongCatImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
|
||||
MiniMaxH3PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
||||
QwenImageEditPlusPipelineConfig,
|
||||
QwenImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
|
||||
from sglang.multimodal_gen.configs.sample.ernie_image import ErnieImageSamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.flux import FluxSamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.ideogram import Ideogram4SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.longcat_image import (
|
||||
LongCatImageSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.qwenimage import (
|
||||
QwenImageEditPlusSamplingParams,
|
||||
QwenImageSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.wan import WanT2V_1_3B_SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.zimage import ZImageTurboSamplingParams
|
||||
from sglang.multimodal_gen.runtime.entrypoints import http_server
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
|
||||
PromptEnhancer,
|
||||
PromptEnhancerConfig,
|
||||
maybe_enhance_prompt,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
RealtimeVideoGenerationsRequest,
|
||||
VideoGenerationsRequest,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
|
||||
from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.server_args import server_args as server_args_module
|
||||
|
||||
|
||||
def completion(text="Rewritten prompt", finish_reason="stop"):
|
||||
return {"choices": [{"message": {"content": text}, "finish_reason": finish_reason}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
{"timeout": 0},
|
||||
{"timeout": float("inf")},
|
||||
{"model": ""},
|
||||
{"base_url": "file:///tmp/server"},
|
||||
{"base_url": "https://user:secret@example.org/v1"},
|
||||
{"base_url": "https://example.org/v1?key=secret"},
|
||||
{"generation_kwargs": {"stream": True}},
|
||||
{"generation_kwargs": {"messages": []}},
|
||||
{"generation_kwargs": {"n": 2}},
|
||||
{"typo": True},
|
||||
],
|
||||
)
|
||||
def test_config_rejects_invalid_or_conflicting_fields(override):
|
||||
with pytest.raises(ValidationError):
|
||||
PromptEnhancerConfig.model_validate(
|
||||
{"base_url": "http://localhost:30001/v1", "model": "enhancer", **override}
|
||||
)
|
||||
|
||||
|
||||
def test_missing_auth_env_fails_at_startup(monkeypatch):
|
||||
monkeypatch.delenv("TEST_ENHANCER_KEY", raising=False)
|
||||
with pytest.raises(ValueError, match="TEST_ENHANCER_KEY is unset"):
|
||||
PromptEnhancer(
|
||||
PromptEnhancerConfig(
|
||||
base_url="http://localhost:30001/v1",
|
||||
model="enhancer",
|
||||
api_key_env="TEST_ENHANCER_KEY",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_path_does_not_need_a_client():
|
||||
assert (
|
||||
asyncio.run(maybe_enhance_prompt(None, "original", enabled=False, task="image"))
|
||||
== "original"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"upstream, status",
|
||||
[
|
||||
(completion(" "), 502),
|
||||
(completion(None), 502),
|
||||
(completion("partial", "length"), 502),
|
||||
(completion("refusal", "content_filter"), 502),
|
||||
({"choices": []}, 502),
|
||||
({"error": "sensitive upstream detail"}, 502),
|
||||
(httpx.Response(401, text="secret"), 502),
|
||||
(httpx.ReadTimeout("secret"), 504),
|
||||
(httpx.ConnectError("secret"), 502),
|
||||
],
|
||||
)
|
||||
def test_upstream_failures_are_not_silent(upstream, status):
|
||||
async def run():
|
||||
enhancer = PromptEnhancer(
|
||||
PromptEnhancerConfig(base_url="http://localhost/v1", model="enhancer")
|
||||
)
|
||||
|
||||
def respond(request):
|
||||
if isinstance(upstream, Exception):
|
||||
raise upstream
|
||||
if isinstance(upstream, httpx.Response):
|
||||
return upstream
|
||||
return httpx.Response(200, json=upstream)
|
||||
|
||||
await enhancer.client.aclose()
|
||||
enhancer.client = httpx.AsyncClient(
|
||||
base_url="http://localhost/v1/", transport=httpx.MockTransport(respond)
|
||||
)
|
||||
try:
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await enhancer.enhance("original", task="image", image_paths=[])
|
||||
assert error.value.status_code == status
|
||||
assert "secret" not in error.value.detail
|
||||
assert "sensitive" not in error.value.detail
|
||||
finally:
|
||||
await enhancer.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
batches = []
|
||||
rewritten = '{"high_level_description": "A red teapot"}'
|
||||
config_path = tmp_path / "enhancer.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"base_url": "http://enhancer.local/v1",
|
||||
"model": "test-vlm",
|
||||
"include_images": True,
|
||||
"api_key_env": "TEST_ENHANCER_KEY",
|
||||
"system_prompt": "Return the target model's JSON caption.",
|
||||
"generation_kwargs": {
|
||||
"max_tokens": 128,
|
||||
"response_format": {"type": "json_object"},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
monkeypatch.setenv("TEST_ENHANCER_KEY", "test-key")
|
||||
|
||||
def respond(request):
|
||||
assert request.url == "http://enhancer.local/v1/chat/completions"
|
||||
assert request.headers["authorization"] == "Bearer test-key"
|
||||
calls.append(json.loads(request.content))
|
||||
return httpx.Response(200, json=completion(rewritten))
|
||||
|
||||
# keep the real HTTP client and lifespan; replace only the external transport
|
||||
original_init = httpx.AsyncClient.__init__
|
||||
|
||||
def client_init(self, *args, **kwargs):
|
||||
kwargs.setdefault("transport", httpx.MockTransport(respond))
|
||||
original_init(self, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "__init__", client_init)
|
||||
monkeypatch.setattr(ServerArgs, "__post_init__", lambda self: None)
|
||||
args = ServerArgs(
|
||||
model_path="test-model",
|
||||
served_model_name="test-model",
|
||||
pipeline_config=ZImagePipelineConfig(),
|
||||
num_gpus=1,
|
||||
prompt_enhancer_config=str(config_path),
|
||||
warmup_mode="off",
|
||||
output_path=str(tmp_path / "outputs"),
|
||||
input_save_path=str(tmp_path / "inputs"),
|
||||
)
|
||||
monkeypatch.setattr(server_args_module, "_global_server_args", args)
|
||||
|
||||
model_info = SimpleNamespace(sampling_param_cls=ZImageTurboSamplingParams)
|
||||
monkeypatch.setattr(registry, "get_model_info", lambda *args, **kwargs: model_info)
|
||||
monkeypatch.setattr(async_scheduler_client, "initialize", lambda args: None)
|
||||
monkeypatch.setattr(async_scheduler_client, "close", lambda: None)
|
||||
monkeypatch.setattr(http_server, "_wait_until_http_live", AsyncMock())
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.runtime.scheduler_client.run_zeromq_broker", AsyncMock()
|
||||
)
|
||||
|
||||
async def forward(requests):
|
||||
batches.extend(requests)
|
||||
batch = requests[0]
|
||||
paths = []
|
||||
for idx in range(batch.num_outputs_per_prompt):
|
||||
path = tmp_path / f"{batch.request_id}_{idx}.png"
|
||||
Image.new("RGB", (16, 16), "red").save(path)
|
||||
paths.append(str(path))
|
||||
return OutputBatch(output_file_paths=paths)
|
||||
|
||||
monkeypatch.setattr(async_scheduler_client, "forward", forward)
|
||||
app = http_server.create_app(args)
|
||||
with TestClient(app) as client:
|
||||
yield SimpleNamespace(
|
||||
client=client,
|
||||
app=app,
|
||||
args=args,
|
||||
model_info=model_info,
|
||||
calls=calls,
|
||||
batches=batches,
|
||||
rewritten=rewritten,
|
||||
)
|
||||
assert app.state.prompt_enhancer.client.is_closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
@pytest.mark.parametrize("endpoint", ["image", "edit", "video_json", "video_form"])
|
||||
def test_http_routes_enhance_once_before_sampling_and_preserve_options(
|
||||
server, endpoint, enabled
|
||||
):
|
||||
original = "A red teapot"
|
||||
if endpoint == "image":
|
||||
response = server.client.post(
|
||||
"/v1/images/generations",
|
||||
json={
|
||||
"prompt": original,
|
||||
"enhance_prompt": enabled,
|
||||
"seed": 123,
|
||||
"negative_prompt": "blur",
|
||||
"n": 2,
|
||||
"response_format": "b64_json",
|
||||
},
|
||||
)
|
||||
elif endpoint == "edit":
|
||||
server.args.pipeline_config = QwenImageEditPlusPipelineConfig()
|
||||
server.model_info.sampling_param_cls = QwenImageEditPlusSamplingParams
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (16, 16), "blue").save(buffer, format="PNG")
|
||||
response = server.client.post(
|
||||
"/v1/images/edits",
|
||||
data={
|
||||
"prompt": original,
|
||||
"enhance_prompt": str(enabled).lower(),
|
||||
"seed": "123",
|
||||
"negative_prompt": "blur",
|
||||
"n": "2",
|
||||
},
|
||||
files={"image": ("reference.png", buffer.getvalue(), "image/png")},
|
||||
)
|
||||
else:
|
||||
server.args.pipeline_config = WanT2V480PConfig()
|
||||
server.model_info.sampling_param_cls = WanT2V_1_3B_SamplingParams
|
||||
payload = {
|
||||
"prompt": original,
|
||||
"seed": 123,
|
||||
"negative_prompt": "blur",
|
||||
"n": 2,
|
||||
"num_frames": 5,
|
||||
}
|
||||
if endpoint == "video_json":
|
||||
response = server.client.post(
|
||||
"/v1/videos", json={**payload, "enhance_prompt": enabled}
|
||||
)
|
||||
else:
|
||||
response = server.client.post(
|
||||
"/v1/videos",
|
||||
data={**payload, "enhance_prompt": str(enabled).lower()},
|
||||
files={"unused": ("empty", b"")},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
expected = server.rewritten if enabled else original
|
||||
assert len(server.calls) == int(enabled)
|
||||
if endpoint.startswith("video"):
|
||||
assert response.json()["revised_prompt"] == (expected if enabled else None)
|
||||
|
||||
# wait for the real background job, not a mocked dispatch
|
||||
async def wait_jobs():
|
||||
if video_api._VIDEO_JOB_TASKS:
|
||||
await asyncio.gather(*list(video_api._VIDEO_JOB_TASKS.values()))
|
||||
|
||||
server.client.portal.call(wait_jobs)
|
||||
else:
|
||||
assert len(response.json()["data"]) == 2
|
||||
assert all(
|
||||
item["revised_prompt"] == expected for item in response.json()["data"]
|
||||
)
|
||||
assert len(server.batches) == 1
|
||||
batch = server.batches[0]
|
||||
assert batch.prompt == expected
|
||||
assert batch.seed == 123
|
||||
assert batch.negative_prompt == "blur"
|
||||
assert batch.num_outputs_per_prompt == 2
|
||||
if enabled:
|
||||
payload = server.calls[0]
|
||||
assert payload["model"] == "test-vlm"
|
||||
assert payload["max_tokens"] == 128
|
||||
assert payload["response_format"] == {"type": "json_object"}
|
||||
if endpoint == "edit":
|
||||
assert payload["messages"][1]["content"][1]["image_url"]["url"].startswith(
|
||||
"data:image/png;base64,"
|
||||
)
|
||||
else:
|
||||
assert json.loads(payload["messages"][1]["content"])["prompt"] == original
|
||||
|
||||
|
||||
def test_unconfigured_enhancer_rejects_opt_in_without_generation(server):
|
||||
enhancer = server.app.state.prompt_enhancer
|
||||
server.app.state.prompt_enhancer = None
|
||||
try:
|
||||
response = server.client.post(
|
||||
"/v1/images/generations",
|
||||
json={"prompt": "a teapot", "enhance_prompt": True},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert not server.batches
|
||||
finally:
|
||||
server.app.state.prompt_enhancer = enhancer
|
||||
|
||||
|
||||
def test_missing_switch_preserves_original_prompt(server):
|
||||
response = server.client.post(
|
||||
"/v1/images/generations",
|
||||
json={"prompt": "keep exactly this", "response_format": "b64_json"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert not server.calls
|
||||
assert server.batches[0].prompt == "keep exactly this"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pipeline_cls, sampling_cls, extras",
|
||||
[
|
||||
(FluxPipelineConfig, FluxSamplingParams, {}),
|
||||
(QwenImagePipelineConfig, QwenImageSamplingParams, {}),
|
||||
(Ideogram4PipelineConfig, Ideogram4SamplingParams, {}),
|
||||
(ErnieImagePipelineConfig, ErnieImageSamplingParams, {"use_pe": False}),
|
||||
(
|
||||
LongCatImagePipelineConfig,
|
||||
LongCatImageSamplingParams,
|
||||
{"enable_prompt_rewrite": False},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_model_sampling_contracts_remain_owned_by_the_model(
|
||||
server, pipeline_cls, sampling_cls, extras
|
||||
):
|
||||
server.args.pipeline_config = pipeline_cls()
|
||||
server.model_info.sampling_param_cls = sampling_cls
|
||||
response = server.client.post(
|
||||
"/v1/images/generations",
|
||||
json={
|
||||
"prompt": "a teapot",
|
||||
"enhance_prompt": True,
|
||||
"response_format": "b64_json",
|
||||
**extras,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
sampling = server.batches[0].sampling_params
|
||||
assert type(sampling) is sampling_cls
|
||||
assert sampling.prompt == server.rewritten
|
||||
for name, value in extras.items():
|
||||
assert vars(sampling)[name] == value
|
||||
|
||||
|
||||
def test_h3_native_request_contract_survives_enhancement(server):
|
||||
server.args.pipeline_config = MiniMaxH3PipelineConfig()
|
||||
server.model_info.sampling_param_cls = MiniMaxH3SamplingParams
|
||||
target = {"short_edge": 768, "aspect_ratio": "16:9", "duration_seconds": 5.0}
|
||||
request = VideoGenerationsRequest(
|
||||
prompt="A red teapot", task="t2va", conditions=[], target=target, seed=123
|
||||
)
|
||||
|
||||
async def enhance_and_lower():
|
||||
request.prompt = await server.app.state.prompt_enhancer.enhance(
|
||||
request.prompt, task="video", image_paths=[]
|
||||
)
|
||||
return video_api._build_video_sampling_params("h3-enhanced", request)
|
||||
|
||||
sampling = server.client.portal.call(enhance_and_lower)
|
||||
assert type(sampling) is MiniMaxH3SamplingParams
|
||||
assert sampling.prompt == server.rewritten
|
||||
assert sampling.seed == 123
|
||||
assert sampling.task == "t2va"
|
||||
assert sampling.conditions == []
|
||||
assert sampling.target == target
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source", ["https://example.com/reference.png", "data:image/png;base64,AAAA"]
|
||||
)
|
||||
@pytest.mark.parametrize("include_images", [True, False])
|
||||
def test_reference_transport_is_explicit(server, source, include_images):
|
||||
enhancer = server.app.state.prompt_enhancer
|
||||
enhancer.config.include_images = include_images
|
||||
|
||||
async def run():
|
||||
return await enhancer.enhance(
|
||||
"keep the subject", task="image_edit", image_paths=[source]
|
||||
)
|
||||
|
||||
assert server.client.portal.call(run) == server.rewritten
|
||||
content = server.calls[0]["messages"][1]["content"]
|
||||
if include_images:
|
||||
assert content[1] == {"type": "image_url", "image_url": {"url": source}}
|
||||
else:
|
||||
assert isinstance(content, str)
|
||||
assert source not in content
|
||||
|
||||
|
||||
def test_enhancer_failure_does_not_queue_video_or_leak_uploads(
|
||||
server, monkeypatch, tmp_path
|
||||
):
|
||||
server.args.pipeline_config = WanT2V480PConfig()
|
||||
server.args.input_save_path = None
|
||||
server.args.output_path = None
|
||||
monkeypatch.setattr(video_api.tempfile, "tempdir", str(tmp_path))
|
||||
|
||||
async def fail(request):
|
||||
return httpx.Response(503, text="upstream secret")
|
||||
|
||||
enhancer = server.app.state.prompt_enhancer
|
||||
server.client.portal.call(enhancer.client.aclose)
|
||||
enhancer.client = httpx.AsyncClient(
|
||||
base_url="http://enhancer.local/v1/", transport=httpx.MockTransport(fail)
|
||||
)
|
||||
response = server.client.post(
|
||||
"/v1/videos", json={"prompt": "a teapot", "enhance_prompt": True}
|
||||
)
|
||||
assert response.status_code == 502
|
||||
assert "secret" not in response.text
|
||||
assert not server.batches
|
||||
assert not list(tmp_path.glob("sglang_*"))
|
||||
|
||||
|
||||
def test_realtime_does_not_silently_accept_prompt_enhancement():
|
||||
with pytest.raises(ValidationError):
|
||||
RealtimeVideoGenerationsRequest(
|
||||
type="init", prompt="a teapot", enhance_prompt=True
|
||||
)
|
||||
Reference in New Issue
Block a user