diff --git a/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx b/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx index 9e17b3b271..7b46738e25 100644 --- a/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx +++ b/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx @@ -188,6 +188,12 @@ response = client.images.generate( Base Ideogram 4 presets are `V4_DEFAULT_20`, `V4_QUALITY_48`, and `V4_TURBO_12`. The fal variants automatically select `V4_FAST_20` and `V4_INSTANT_8`, respectively. A preset controls both `num_inference_steps` and guidance, so do not set those fields directly. +You can also produce captions with a separate SRT LLM using +[prompt enhancement](/docs/sglang-diffusion/models_with_pe). Configure an +Ideogram-specific JSON caption template and schema; the generic prose template +is not appropriate for these checkpoints. This does not reproduce fal's +checkpoint-specific prompt expander or establish equivalent image quality. + ## 5. Run in ComfyUI import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index ee7ec2cc1a..faf961c866 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -115,6 +115,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis - `--otlp-service-name {NAME}`: `service.name` attached to exported traces. Resolution order is this flag, `OTEL_SERVICE_NAME`, then `sglang-diffusion`. - `--enable-metrics`: expose Prometheus metrics at `/metrics` (default: disabled). Includes request counts, queue time, host-side stage timing and LoRA state, separated by role and DP replica. See [Production metrics](/docs/sglang-diffusion/production_metrics) for metric semantics and disaggregated scraping. - `--pe-server-url {HTTPADDRESS}`: url of SGLang server hosting the PE model (e.g., for ERNIE-Image). See [Models with Prompt Enhancement](/docs/sglang-diffusion/models_with_pe). +- `--prompt-enhancer-config {JSON_FILE}`: configure an independent SRT LLM/VLM for opt-in HTTP prompt rewriting. Requests use `enhance_prompt=true`; no enhancer runs by default. See [Prompt enhancement](/docs/sglang-diffusion/models_with_pe). ### Sampling and output diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx index 79fc30b994..54aa18a9ea 100644 --- a/docs/docs/sglang-diffusion/api/openai_api.mdx +++ b/docs/docs/sglang-diffusion/api/openai_api.mdx @@ -96,6 +96,14 @@ curl -sS "http://localhost:30010/v1/models/wan-t2v" ## Endpoints +### Prompt enhancement + +Image generation, image edits and video creation accept `enhance_prompt=true` +when the server is configured with `--prompt-enhancer-config`. An independent +SRT service rewrites the prompt before diffusion runs. The default is `false`; +image responses return `data[].revised_prompt`, and video responses return +`revised_prompt`. See [deployment and model compatibility](/docs/sglang-diffusion/models_with_pe). + ### Model-specific request fields The image and video request schemas contain only OpenAI fields and stable diff --git a/docs/docs/sglang-diffusion/models_with_pe.mdx b/docs/docs/sglang-diffusion/models_with_pe.mdx index 777228f833..ba27a3618f 100644 --- a/docs/docs/sglang-diffusion/models_with_pe.mdx +++ b/docs/docs/sglang-diffusion/models_with_pe.mdx @@ -1,76 +1,186 @@ --- -title: "Diffusion Models with Prompt Enhancement (PE)" -description: "Run ERNIE-Image with built-in prompt enhancement or a separate SGLang-served PE model." +title: "Prompt enhancement" +description: "Connect an independently deployed SGLang LLM or VLM to diffusion image and video APIs." --- - -## Quick Start - -By default, the diffusion server loads the PE model in-process with SGLang's native Ministral3 implementation. Deploy the PE model as a separate SGLang server when it needs independent resources or scaling. This document uses `baidu/ERNIE-Image` as an example. - -Run the model with the built-in native PE implementation (default): - -```bash -# Terminal 1: launch server -sglang serve --model-path baidu/ERNIE-Image --port ${PORT} -``` -```bash -# Terminal 2: launch client -curl -X POST http://${HOST}:${PORT}/v1/images/generations \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "This is a photograph depicting an urban street scene. Shot at eye level, it shows a covered pedestrian or commercial street. Slightly below the center of the frame, a cyclist rides away from the camera toward the background, appearing as a dark silhouette against backlighting with indistinct details. The ground is paved with regular square tiles, bisected by a prominent tactile paving strip running through the scene, whose raised textures are clearly visible under the light. Light streams in diagonally from the right side of the frame, creating a strong backlight effect with a distinct Tyndall effect—visible light beams illuminating dust or vapor in the air and casting long shadows across the street. Several pedestrians appear on the left side and in the distance, some with their backs to the camera and others walking sideways, all rendered as silhouettes or semi-silhouettes. The overall color palette is warm, dominated by golden yellows and dark browns, evoking the atmosphere of dusk or early morning.", - "height": 1264, - "width": 848, - "num_inference_steps": 50, - "guidance_scale": 4.0, - "use_pe": true - }' -``` - -Run the model with an SGLang-served PE model (high performance): - -```bash -# Terminal 1: launch SGLang PE model server -sglang serve --model-path /path/to/baidu/ERNIE-Image/pe/ --port ${PE_PORT} -``` -```bash -# Terminal 2: launch diffusion model server with PE server -sglang serve --model-path /path/to/baidu/ERNIE-Image/ \ - --pe-server-url "http://${HOST}:${PE_PORT}" \ - --port ${PORT} -``` -```bash -# Terminal 3: launch client -curl -X POST http://${HOST}:${PORT}/v1/images/generations \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "This is a photograph depicting an urban street scene. Shot at eye level, it shows a covered pedestrian or commercial street. Slightly below the center of the frame, a cyclist rides away from the camera toward the background, appearing as a dark silhouette against backlighting with indistinct details. The ground is paved with regular square tiles, bisected by a prominent tactile paving strip running through the scene, whose raised textures are clearly visible under the light. Light streams in diagonally from the right side of the frame, creating a strong backlight effect with a distinct Tyndall effect—visible light beams illuminating dust or vapor in the air and casting long shadows across the street. Several pedestrians appear on the left side and in the distance, some with their backs to the camera and others walking sideways, all rendered as silhouettes or semi-silhouettes. The overall color palette is warm, dominated by golden yellows and dark browns, evoking the atmosphere of dusk or early morning.", - "height": 1264, - "width": 848, - "num_inference_steps": 50, - "guidance_scale": 4.0, - "use_pe": true - }' + +Run an SRT server for prompt expansion and a diffusion server for generation. +The diffusion HTTP frontend calls SRT's `/v1/chat/completions` once before +constructing the generation request. The enhancer does not replace the +diffusion model's text encoder or run inside its GPU workers. + +```text +Application -> diffusion HTTP server -> SRT enhancer server + <- rewritten prompt + <- image/video <- diffusion pipeline ``` -For a memory-constrained in-process deployment, the native PE decoder supports -layerwise offload: +Prompt enhancement is opt-in per request. It changes conditioning and can +change intent or quality; it is not a lossless acceleration feature. Evaluate +the chosen enhancer and template on your workload before enabling it. + +## Deploy two services + +This example uses Linux with NVIDIA CUDA and two GPUs, one for each server. +Install [SGLang](/docs/get-started/install) for SRT and +[SGLang Diffusion](/docs/sglang-diffusion/installation) for diffusion. Separate +environments and hosts are supported; the diffusion host needs HTTP access to SRT. + +Start a text-only enhancer: ```bash -sglang serve --model-path baidu/ERNIE-Image \ - --layerwise-offload-components pe \ - --port ${PORT} +CUDA_VISIBLE_DEVICES=0 sglang serve \ + --model-path Qwen/Qwen3-4B-Instruct-2507 \ + --port 31000 ``` -This option streams PE decoder layers from CPU and can increase prompt-enhancement -latency. It does not apply when `--pe-server-url` selects an external PE server. +Create `enhancer.json` on the diffusion host: -## Support matrix - -| Model | Built-in PE | SGLang PE Server | -|-------|-------------|------------------| -| ERNIE-Image | ✅ | ✅ | - -## Ascend NPU Environment +```json +{ + "base_url": "http://127.0.0.1:31000/v1", + "model": "Qwen/Qwen3-4B-Instruct-2507" +} +``` -See [Diffusion models with AR stage like GLM-Image](/docs/sglang-diffusion/models_with_ar#ascend-npu-env). +Start diffusion in another terminal: + +```bash +CUDA_VISIBLE_DEVICES=1 sglang serve \ + --model-path Tongyi-MAI/Z-Image-Turbo \ + --prompt-enhancer-config enhancer.json \ + --port 30000 +``` + +Keep the SRT port separate from diffusion's HTTP and internal ports; diffusion +reserves `--port + 1` for its broker. For separate hosts, bind SRT to its service +interface with `--host` and update +`base_url`. Restrict network access and configure SRT authentication. Do not +expose an unauthenticated enhancer publicly. A shared enhancer can serve +multiple diffusion servers; size its concurrency and memory independently. + +Send a generation request: + +```bash +curl --fail-with-body http://127.0.0.1:30000/v1/images/generations \ + -H 'Content-Type: application/json' \ + -d '{ + "prompt": "A red teapot on a windowsill, morning light", + "enhance_prompt": true, + "size": "512x512", + "seed": 42, + "response_format": "b64_json" + }' +``` + +Omit `enhance_prompt` or set it to `false` to use the original prompt without +calling SRT. With the OpenAI Python SDK, pass `extra_body={"enhance_prompt": True}` +to `images.generate()`. + +The same switch works with `/v1/images/edits` and `/v1/videos`, including +multipart requests (`-F 'enhance_prompt=true'`). Image responses expose the +rewritten text as `data[].revised_prompt`; video responses use `revised_prompt`. +Video creation waits for enhancement before returning the queued job. All `n` +outputs share one rewritten prompt. + +Include enhancement in client-observed end-to-end latency. The response's +`inference_time_s` measures diffusion execution, not the preceding SRT call. + +## Configure the enhancer + +The configuration file is read at server startup. Its supported fields are: + +| Field | Meaning | +| --- | --- | +| `base_url` | SRT OpenAI API base URL, including `/v1` | +| `model` | SRT model ID or its configured served model name | +| `system_prompt` | Optional replacement for the built-in, task-aware rewriting instruction | +| `include_images` | Forward reference images to a VLM; defaults to `false` | +| `timeout` | HTTP operation timeout in seconds; defaults to `60` | +| `api_key_env` | Name of an environment variable on the diffusion server containing the SRT API key | +| `generation_kwargs` | SRT chat-completion options, such as `max_tokens`, `temperature`, `chat_template_kwargs`, or `response_format` | + +Default generation options are `max_tokens=512` and `temperature=0`. +`generation_kwargs` cannot replace the model, messages, single-output or +non-streaming settings, or enable tools. Choose a non-thinking model or configure +its supported chat template to suppress reasoning; only message `content` is +used as the diffusion prompt. + +The enhancer receives a JSON text message containing `task` (`image`, +`image_edit`, or `video`) and `prompt`. It returns the replacement prompt as +text. Seed, negative prompt, references and diffusion sampling options are not +rewritten. The diffusion frontend reuses HTTP connections and makes no enhancer +calls during synthetic warmup. + +Timeouts return HTTP 504. Upstream failures, empty responses and truncated +completions return HTTP 502 before diffusion is queued. There is no automatic +retry or fallback to the original prompt. An unconfigured server rejects +`enhance_prompt=true` with HTTP 400. + +## Choose a model combination + +There is no enhancer or diffusion model allowlist in this integration. Protocol +compatibility does not establish quality: use a template appropriate for the +target checkpoint and verify prompt adherence. + +| Diffusion input | Enhancer requirements | +| --- | --- | +| Natural-language image/video prompts, such as Z-Image, FLUX, Qwen-Image, Wan, LTX or MiniMax-H3 | An SRT-served instruction model with chat completions; the default template is a starting point | +| Image editing or image-to-video | Text-only instruction rewriting, or an SRT VLM with `include_images=true` for image-aware rewriting | +| Structured captions, such as Ideogram 4 or LingBot-Video-MoE | A model-specific `system_prompt` and preferably schema-constrained `response_format`; the default prose template is unsuitable | +| ERNIE-Image or LongCat with built-in rewriting | Disable the built-in rewriter for the request when using this generic enhancer, to avoid rewriting twice | + +For a VLM, deploy an image-capable SRT model and set `include_images=true` in +its configuration. Uploaded images are sent as data URLs, so the SRT server +does not need access to the diffusion host's filesystem. HTTP image URLs must +be reachable from SRT. Text-only models should leave this option disabled. +Audio and video references are not forwarded to the enhancer; it cannot inspect +their content. Image edits forward the supplied images; video creation forwards +the primary image reference, not model-specific reference bundles. + +For [Ideogram 4](/cookbook/diffusion/Ideogram/Ideogram4), instruct the enhancer +to produce the checkpoint's JSON caption schema, including +`high_level_description` and `compositional_deconstruction`. Set the matching +JSON schema through `generation_kwargs.response_format`. JSON text is passed +through unchanged, not converted to prose. Valid JSON alone does not guarantee +the correct caption schema or equivalent output to a hosted service's expander. + +This integration applies to the three HTTP image/video endpoints above, +including their normal parallelism, caching and disaggregated diffusion paths. +It does not add enhancement to offline `sglang generate`, realtime streaming, +mesh or action endpoints. For those workflows, call SRT in your application +first and pass its returned text as the diffusion prompt. For reproducibility, +store the rewritten prompt and replay it with `enhance_prompt=false`; a fixed +diffusion seed does not fix the enhancer's output. + +## ERNIE-Image's checkpoint-provided enhancer + +ERNIE-Image also has a separate, existing PE integration. By default, diffusion +loads its native Ministral3 enhancer in-process: + +```bash +sglang serve --model-path baidu/ERNIE-Image --port 30000 +``` + +Use `use_pe=false` to skip it for a request. For a memory-constrained deployment, +`--layerwise-offload-components pe` streams the native PE decoder from CPU. + +To serve the checkpoint-provided PE through SRT, use a local checkpoint's PE +directory with its tokenizer and retain the ERNIE-specific protocol: + +```bash +CUDA_VISIBLE_DEVICES=0 sglang serve \ + --model-path /models/ERNIE-Image/pe --port 31000 + +CUDA_VISIBLE_DEVICES=1 sglang serve \ + --model-path /models/ERNIE-Image \ + --pe-server-url http://127.0.0.1:31000 --port 30000 +``` + +`--pe-server-url` uses the checkpoint's PE tokenizer/template and SRT's +`/generate`; it is not the generic chat-completion integration. Keep using it +when you need ERNIE's trained PE behavior. If you instead use +`--prompt-enhancer-config`, send both `enhance_prompt=true` and `use_pe=false`. +For LongCat, use `enable_prompt_rewrite=false` with the generic enhancer. +These request flags skip native rewriting, not loading its model components. + +For Ascend setup, see [models with AR stages](/docs/sglang-diffusion/models_with_ar#ascend-npu-env). diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py index c990c908b8..a5953feeb1 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py @@ -17,6 +17,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.entrypoints.action import api as action_api from sglang.multimodal_gen.runtime.entrypoints.action import openpi from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api +from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import ( + PromptEnhancer, +) from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( VertexGenerateReqInput, ) @@ -136,6 +139,11 @@ async def lifespan(app: FastAPI): warmup_done.set() try: + app.state.prompt_enhancer = ( + PromptEnhancer.from_file(server_args.prompt_enhancer_config) + if server_args.prompt_enhancer_config is not None + else None + ) yield finally: if warmup_task is not None and not warmup_task.done(): @@ -146,6 +154,8 @@ async def lifespan(app: FastAPI): # On shutdown logger.info("FastAPI app is shutting down...") await shutdown_video_jobs() + if app.state.prompt_enhancer is not None: + await app.state.prompt_enhancer.close() broker_task.cancel() with suppress(asyncio.CancelledError): await broker_task @@ -444,4 +454,5 @@ def create_app(server_args: ServerArgs): app.include_router(rollout_api.router) app.state.server_args = server_args + app.state.prompt_enhancer = None return app diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py index 59dfd4f83d..c65576af77 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py @@ -25,6 +25,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( SamplingParams, generate_request_id, ) +from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import ( + maybe_enhance_prompt, +) from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( ImageGenerationsRequest, ImageResponse, @@ -274,11 +277,14 @@ async def generations( else sampling_params_cls.default_image_output_format() ) ext = choose_output_image_ext(output_format, request.background) + prompt = await maybe_enhance_prompt( + raw_request, request.prompt, enabled=request.enhance_prompt, task="image" + ) with temp_dir_if_disabled(server_args.output_path) as output_dir: sampling = build_sampling_params( request_id, - prompt=request.prompt, + prompt=prompt, size=request.size, width=request.width, height=request.height, @@ -383,7 +389,7 @@ async def generations( response_kwargs = _build_image_response_kwargs( save_file_path_list, resp_format, - request.prompt, + prompt, request_id, result, b64_list=b64_list, @@ -404,6 +410,7 @@ async def edits( url: Optional[List[str]] = Form(None), url_array: Optional[List[str]] = Form(None, alias="url[]"), prompt: str = Form(...), + enhance_prompt: bool = Form(False), mask: Optional[UploadFile] = File(None), model: Optional[str] = Form(None), n: Optional[int] = Form(1), @@ -464,6 +471,13 @@ async def edits( ) ext = choose_output_image_ext(output_format, background) + prompt = await maybe_enhance_prompt( + raw_request, + prompt, + enabled=enhance_prompt, + task="image_edit", + image_paths=input_paths, + ) sampling = build_sampling_params( request_id, prompt=prompt, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/prompt_enhancement.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/prompt_enhancement.py new file mode 100644 index 0000000000..1651dde9ed --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/prompt_enhancement.py @@ -0,0 +1,179 @@ +import asyncio +import base64 +import json +import os +from pathlib import Path +from typing import Any, Literal + +import httpx +from fastapi import HTTPException, Request +from PIL import Image +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator + + +class PromptEnhancerConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + base_url: HttpUrl + model: str = Field(min_length=1) + system_prompt: str = Field( + default=( + "You expand prompts for image and video generation. The user supplies " + "JSON with task and prompt fields. Expand the prompt into a concise, " + "visually detailed description, clarifying composition, lighting and " + "appearance where appropriate instead of simply repeating it. Preserve " + "the requested subjects, count, quoted text, actions, style and language. " + "Preserve special tokens and reference labels exactly. " + "Clarify visual details without inventing new subjects or changing intent. " + "For image editing, describe only the requested edits and preserve " + "everything else. For video, preserve the requested motion and timing. " + "Do not invent facts about reference media you cannot see. " + "Return only the rewritten prompt, without surrounding quotes, " + "commentary or markdown." + ), + min_length=1, + ) + include_images: bool = False + timeout: float = Field(default=60.0, gt=0, allow_inf_nan=False) + api_key_env: str | None = None + generation_kwargs: dict[str, Any] = Field(default_factory=dict) + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, value: HttpUrl) -> HttpUrl: + if value.username or value.password or value.query or value.fragment: + raise ValueError( + "base_url must not contain credentials, a query or fragment" + ) + return value + + @field_validator("generation_kwargs") + @classmethod + def validate_generation_kwargs(cls, value: dict[str, Any]) -> dict[str, Any]: + # one non-streaming completion per diffusion request + reserved = value.keys() & { + "model", + "messages", + "stream", + "n", + "tools", + "tool_choice", + } + if reserved: + raise ValueError(f"generation_kwargs cannot override {sorted(reserved)}") + return value + + +class PromptEnhancer: + """HTTP frontend client; owns its connection pool, never runs in GPU workers.""" + + def __init__(self, config: PromptEnhancerConfig): + self.config = config + headers = {} + if config.api_key_env is not None: + api_key = os.environ.get(config.api_key_env) + if not api_key: + raise ValueError( + f"Prompt enhancer API key env {config.api_key_env} is unset" + ) + headers["Authorization"] = f"Bearer {api_key}" + self.client = httpx.AsyncClient( + base_url=str(config.base_url).rstrip("/") + "/", + headers=headers, + timeout=config.timeout, + trust_env=False, + ) + + @classmethod + def from_file(cls, path: str) -> "PromptEnhancer": + return cls(PromptEnhancerConfig.model_validate_json(Path(path).read_text())) + + async def close(self) -> None: + await self.client.aclose() + + async def enhance( + self, + prompt: str, + *, + task: Literal["image", "image_edit", "video"], + image_paths: list[str], + ) -> str: + content: str | list[dict[str, Any]] = json.dumps( + {"task": task, "prompt": prompt}, ensure_ascii=False + ) + if self.config.include_images and image_paths: + content = [{"type": "text", "text": content}] + for path in image_paths: + if path.startswith(("https://", "http://", "data:image/")): + url = path + else: + url = await asyncio.to_thread(_local_image_url, path) + content.append({"type": "image_url", "image_url": {"url": url}}) + + payload = { + "max_tokens": 512, + "temperature": 0, + **self.config.generation_kwargs, + "model": self.config.model, + "messages": [ + {"role": "system", "content": self.config.system_prompt}, + {"role": "user", "content": content}, + ], + "stream": False, + "n": 1, + } + try: + response = await self.client.post("chat/completions", json=payload) + response.raise_for_status() + except httpx.TimeoutException as e: + raise HTTPException(504, "Prompt enhancer timed out") from e + except httpx.HTTPError as e: + # do not expose upstream response bodies, prompts or auth headers + raise HTTPException(502, "Prompt enhancer request failed") from e + + try: + choice = response.json()["choices"][0] + if choice["finish_reason"] != "stop": + raise ValueError("Incomplete completion") + rewritten = choice["message"]["content"] + if not isinstance(rewritten, str) or not rewritten.strip(): + raise ValueError("Empty completion") + except (ValueError, KeyError, IndexError, TypeError) as e: + raise HTTPException( + 502, + "Prompt enhancer did not return a complete, nonempty text response; " + "check its template and token limit", + ) from e + return rewritten.strip() + + +def _local_image_url(path: str) -> str: + # uploads live on the diffusion host, not necessarily on the SRT host + try: + with Image.open(path) as image: + mime = image.get_format_mimetype() + image.verify() + if mime is None: + raise ValueError("Unknown image MIME type") + encoded = base64.b64encode(Path(path).read_bytes()).decode("ascii") + except (OSError, ValueError, SyntaxError) as e: + raise HTTPException( + 400, "Cannot read reference image for prompt enhancement" + ) from e + return f"data:{mime};base64,{encoded}" + + +async def maybe_enhance_prompt( + request: Request, + prompt: str, + *, + enabled: bool, + task: Literal["image", "image_edit", "video"], + image_paths: list[str] | None = None, +) -> str: + if not enabled: + return prompt + enhancer: PromptEnhancer | None = request.app.state.prompt_enhancer + if enhancer is None: + raise HTTPException(400, "Prompt enhancement requires --prompt-enhancer-config") + return await enhancer.enhance(prompt, task=task, image_paths=image_paths or []) diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py index 6db73b3169..bec145de03 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py @@ -46,6 +46,7 @@ class ImageGenerationsRequest(BaseModel): model_config = ConfigDict(extra="allow") prompt: str + enhance_prompt: bool = False model: Optional[str] = None n: Optional[int] = 1 quality: Optional[str] = "auto" @@ -105,6 +106,7 @@ class VideoResponse(BaseModel): num_outputs: Optional[int] = None peak_memory_mb: Optional[float] = None inference_time_s: Optional[float] = None + revised_prompt: Optional[str] = None action: Optional[Dict[str, Any]] = None @@ -112,6 +114,7 @@ class VideoGenerationsRequest(BaseModel): model_config = ConfigDict(extra="allow") prompt: str + enhance_prompt: bool = False input_reference: Optional[str] = None reference_url: Optional[str] = None video_path: Optional[str] = None @@ -168,6 +171,7 @@ class VideoRemixRequest(BaseModel): class RealtimeVideoGenerationsRequest(VideoGenerationsRequest): + enhance_prompt: Literal[False] = False type: Literal["init"] # WebSocket does not support multipart/form-data image uploads first_frame: Optional[bytes | str] = None diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index 2fe2599329..ace350df26 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -27,6 +27,9 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( SamplingParams, generate_request_id, ) +from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import ( + maybe_enhance_prompt, +) from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( VideoGenerationsRequest, VideoListResponse, @@ -465,6 +468,7 @@ async def create_video( request: Request, # multipart/form-data fields (optional; used only when content-type is multipart) prompt: Optional[str] = Form(None), + enhance_prompt: Optional[bool] = Form(None), input_reference: Optional[UploadFile] = File(None), reference_url: Optional[str] = Form(None), video_reference: Optional[UploadFile] = File(None), @@ -607,6 +611,7 @@ async def create_video( req = VideoGenerationsRequest( prompt=prompt, + enhance_prompt=form_value("enhance_prompt", enhance_prompt) or False, input_reference=input_path, video_path=form_value("video_path", video_input_path), video_url=form_value("video_url", video_url), @@ -730,10 +735,20 @@ async def create_video( logger.debug(f"Server received from create_video endpoint: req={req}") try: + image_path = _resolve_image_path(req, _resolve_video_path(req)) + req.prompt = await maybe_enhance_prompt( + request, + req.prompt, + enabled=req.enhance_prompt, + task="video", + image_paths=[image_path] if image_path else None, + ) sampling_params = _build_video_sampling_params(request_id, req) - except (ValueError, TypeError) as e: + except (asyncio.CancelledError, HTTPException, ValueError, TypeError) as e: for td in temp_dirs: shutil.rmtree(td, ignore_errors=True) + if isinstance(e, (asyncio.CancelledError, HTTPException)): + raise raise HTTPException(status_code=400, detail=str(e)) batch: Req | None = None @@ -766,6 +781,8 @@ async def create_video( sampling_params, server_args.served_model_name, ) + if req.enhance_prompt: + job["revised_prompt"] = req.prompt job.update(sampling_params.project_video_queued_job_fields(batch)) await VIDEO_STORE.upsert(request_id, job) except Exception as e: diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 9424ee44bb..6dce4718fd 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -613,6 +613,7 @@ class ServerArgs(DisaggServerArgsMixin): # SGLang server for PE model inference pe_server_url: str | None = None + prompt_enhancer_config: str | None = None @property def broker_port(self) -> int: @@ -3054,6 +3055,12 @@ class ServerArgs(DisaggServerArgsMixin): default=ServerArgs.pe_server_url, help="URL of SGLang server for PE model", ) + parser.add_argument( + "--prompt-enhancer-config", + type=str, + default=ServerArgs.prompt_enhancer_config, + help="JSON config file for an external SRT prompt enhancer, used by HTTP requests with enhance_prompt=true.", + ) return parser diff --git a/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py b/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py new file mode 100644 index 0000000000..a882f0dc5f --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py @@ -0,0 +1,460 @@ +import asyncio +import io +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +from fastapi import HTTPException +from fastapi.testclient import TestClient +from PIL import Image +from pydantic import ValidationError + +from sglang.multimodal_gen import registry +from sglang.multimodal_gen.configs.pipeline_configs.ernie_image import ( + ErnieImagePipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.ideogram import ( + Ideogram4PipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import ( + LongCatImagePipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import ( + MiniMaxH3PipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( + QwenImageEditPlusPipelineConfig, + QwenImagePipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig +from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig +from sglang.multimodal_gen.configs.sample.ernie_image import ErnieImageSamplingParams +from sglang.multimodal_gen.configs.sample.flux import FluxSamplingParams +from sglang.multimodal_gen.configs.sample.ideogram import Ideogram4SamplingParams +from sglang.multimodal_gen.configs.sample.longcat_image import ( + LongCatImageSamplingParams, +) +from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams +from sglang.multimodal_gen.configs.sample.qwenimage import ( + QwenImageEditPlusSamplingParams, + QwenImageSamplingParams, +) +from sglang.multimodal_gen.configs.sample.wan import WanT2V_1_3B_SamplingParams +from sglang.multimodal_gen.configs.sample.zimage import ZImageTurboSamplingParams +from sglang.multimodal_gen.runtime.entrypoints import http_server +from sglang.multimodal_gen.runtime.entrypoints.openai import video_api +from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import ( + PromptEnhancer, + PromptEnhancerConfig, + maybe_enhance_prompt, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + RealtimeVideoGenerationsRequest, + VideoGenerationsRequest, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.server_args import server_args as server_args_module + + +def completion(text="Rewritten prompt", finish_reason="stop"): + return {"choices": [{"message": {"content": text}, "finish_reason": finish_reason}]} + + +@pytest.mark.parametrize( + "override", + [ + {"timeout": 0}, + {"timeout": float("inf")}, + {"model": ""}, + {"base_url": "file:///tmp/server"}, + {"base_url": "https://user:secret@example.org/v1"}, + {"base_url": "https://example.org/v1?key=secret"}, + {"generation_kwargs": {"stream": True}}, + {"generation_kwargs": {"messages": []}}, + {"generation_kwargs": {"n": 2}}, + {"typo": True}, + ], +) +def test_config_rejects_invalid_or_conflicting_fields(override): + with pytest.raises(ValidationError): + PromptEnhancerConfig.model_validate( + {"base_url": "http://localhost:30001/v1", "model": "enhancer", **override} + ) + + +def test_missing_auth_env_fails_at_startup(monkeypatch): + monkeypatch.delenv("TEST_ENHANCER_KEY", raising=False) + with pytest.raises(ValueError, match="TEST_ENHANCER_KEY is unset"): + PromptEnhancer( + PromptEnhancerConfig( + base_url="http://localhost:30001/v1", + model="enhancer", + api_key_env="TEST_ENHANCER_KEY", + ) + ) + + +def test_disabled_path_does_not_need_a_client(): + assert ( + asyncio.run(maybe_enhance_prompt(None, "original", enabled=False, task="image")) + == "original" + ) + + +@pytest.mark.parametrize( + "upstream, status", + [ + (completion(" "), 502), + (completion(None), 502), + (completion("partial", "length"), 502), + (completion("refusal", "content_filter"), 502), + ({"choices": []}, 502), + ({"error": "sensitive upstream detail"}, 502), + (httpx.Response(401, text="secret"), 502), + (httpx.ReadTimeout("secret"), 504), + (httpx.ConnectError("secret"), 502), + ], +) +def test_upstream_failures_are_not_silent(upstream, status): + async def run(): + enhancer = PromptEnhancer( + PromptEnhancerConfig(base_url="http://localhost/v1", model="enhancer") + ) + + def respond(request): + if isinstance(upstream, Exception): + raise upstream + if isinstance(upstream, httpx.Response): + return upstream + return httpx.Response(200, json=upstream) + + await enhancer.client.aclose() + enhancer.client = httpx.AsyncClient( + base_url="http://localhost/v1/", transport=httpx.MockTransport(respond) + ) + try: + with pytest.raises(HTTPException) as error: + await enhancer.enhance("original", task="image", image_paths=[]) + assert error.value.status_code == status + assert "secret" not in error.value.detail + assert "sensitive" not in error.value.detail + finally: + await enhancer.close() + + asyncio.run(run()) + + +@pytest.fixture +def server(monkeypatch, tmp_path): + calls = [] + batches = [] + rewritten = '{"high_level_description": "A red teapot"}' + config_path = tmp_path / "enhancer.json" + config_path.write_text( + json.dumps( + { + "base_url": "http://enhancer.local/v1", + "model": "test-vlm", + "include_images": True, + "api_key_env": "TEST_ENHANCER_KEY", + "system_prompt": "Return the target model's JSON caption.", + "generation_kwargs": { + "max_tokens": 128, + "response_format": {"type": "json_object"}, + }, + } + ) + ) + monkeypatch.setenv("TEST_ENHANCER_KEY", "test-key") + + def respond(request): + assert request.url == "http://enhancer.local/v1/chat/completions" + assert request.headers["authorization"] == "Bearer test-key" + calls.append(json.loads(request.content)) + return httpx.Response(200, json=completion(rewritten)) + + # keep the real HTTP client and lifespan; replace only the external transport + original_init = httpx.AsyncClient.__init__ + + def client_init(self, *args, **kwargs): + kwargs.setdefault("transport", httpx.MockTransport(respond)) + original_init(self, *args, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "__init__", client_init) + monkeypatch.setattr(ServerArgs, "__post_init__", lambda self: None) + args = ServerArgs( + model_path="test-model", + served_model_name="test-model", + pipeline_config=ZImagePipelineConfig(), + num_gpus=1, + prompt_enhancer_config=str(config_path), + warmup_mode="off", + output_path=str(tmp_path / "outputs"), + input_save_path=str(tmp_path / "inputs"), + ) + monkeypatch.setattr(server_args_module, "_global_server_args", args) + + model_info = SimpleNamespace(sampling_param_cls=ZImageTurboSamplingParams) + monkeypatch.setattr(registry, "get_model_info", lambda *args, **kwargs: model_info) + monkeypatch.setattr(async_scheduler_client, "initialize", lambda args: None) + monkeypatch.setattr(async_scheduler_client, "close", lambda: None) + monkeypatch.setattr(http_server, "_wait_until_http_live", AsyncMock()) + monkeypatch.setattr( + "sglang.multimodal_gen.runtime.scheduler_client.run_zeromq_broker", AsyncMock() + ) + + async def forward(requests): + batches.extend(requests) + batch = requests[0] + paths = [] + for idx in range(batch.num_outputs_per_prompt): + path = tmp_path / f"{batch.request_id}_{idx}.png" + Image.new("RGB", (16, 16), "red").save(path) + paths.append(str(path)) + return OutputBatch(output_file_paths=paths) + + monkeypatch.setattr(async_scheduler_client, "forward", forward) + app = http_server.create_app(args) + with TestClient(app) as client: + yield SimpleNamespace( + client=client, + app=app, + args=args, + model_info=model_info, + calls=calls, + batches=batches, + rewritten=rewritten, + ) + assert app.state.prompt_enhancer.client.is_closed + + +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize("endpoint", ["image", "edit", "video_json", "video_form"]) +def test_http_routes_enhance_once_before_sampling_and_preserve_options( + server, endpoint, enabled +): + original = "A red teapot" + if endpoint == "image": + response = server.client.post( + "/v1/images/generations", + json={ + "prompt": original, + "enhance_prompt": enabled, + "seed": 123, + "negative_prompt": "blur", + "n": 2, + "response_format": "b64_json", + }, + ) + elif endpoint == "edit": + server.args.pipeline_config = QwenImageEditPlusPipelineConfig() + server.model_info.sampling_param_cls = QwenImageEditPlusSamplingParams + buffer = io.BytesIO() + Image.new("RGB", (16, 16), "blue").save(buffer, format="PNG") + response = server.client.post( + "/v1/images/edits", + data={ + "prompt": original, + "enhance_prompt": str(enabled).lower(), + "seed": "123", + "negative_prompt": "blur", + "n": "2", + }, + files={"image": ("reference.png", buffer.getvalue(), "image/png")}, + ) + else: + server.args.pipeline_config = WanT2V480PConfig() + server.model_info.sampling_param_cls = WanT2V_1_3B_SamplingParams + payload = { + "prompt": original, + "seed": 123, + "negative_prompt": "blur", + "n": 2, + "num_frames": 5, + } + if endpoint == "video_json": + response = server.client.post( + "/v1/videos", json={**payload, "enhance_prompt": enabled} + ) + else: + response = server.client.post( + "/v1/videos", + data={**payload, "enhance_prompt": str(enabled).lower()}, + files={"unused": ("empty", b"")}, + ) + assert response.status_code == 200, response.text + expected = server.rewritten if enabled else original + assert len(server.calls) == int(enabled) + if endpoint.startswith("video"): + assert response.json()["revised_prompt"] == (expected if enabled else None) + + # wait for the real background job, not a mocked dispatch + async def wait_jobs(): + if video_api._VIDEO_JOB_TASKS: + await asyncio.gather(*list(video_api._VIDEO_JOB_TASKS.values())) + + server.client.portal.call(wait_jobs) + else: + assert len(response.json()["data"]) == 2 + assert all( + item["revised_prompt"] == expected for item in response.json()["data"] + ) + assert len(server.batches) == 1 + batch = server.batches[0] + assert batch.prompt == expected + assert batch.seed == 123 + assert batch.negative_prompt == "blur" + assert batch.num_outputs_per_prompt == 2 + if enabled: + payload = server.calls[0] + assert payload["model"] == "test-vlm" + assert payload["max_tokens"] == 128 + assert payload["response_format"] == {"type": "json_object"} + if endpoint == "edit": + assert payload["messages"][1]["content"][1]["image_url"]["url"].startswith( + "data:image/png;base64," + ) + else: + assert json.loads(payload["messages"][1]["content"])["prompt"] == original + + +def test_unconfigured_enhancer_rejects_opt_in_without_generation(server): + enhancer = server.app.state.prompt_enhancer + server.app.state.prompt_enhancer = None + try: + response = server.client.post( + "/v1/images/generations", + json={"prompt": "a teapot", "enhance_prompt": True}, + ) + assert response.status_code == 400 + assert not server.batches + finally: + server.app.state.prompt_enhancer = enhancer + + +def test_missing_switch_preserves_original_prompt(server): + response = server.client.post( + "/v1/images/generations", + json={"prompt": "keep exactly this", "response_format": "b64_json"}, + ) + assert response.status_code == 200, response.text + assert not server.calls + assert server.batches[0].prompt == "keep exactly this" + + +@pytest.mark.parametrize( + "pipeline_cls, sampling_cls, extras", + [ + (FluxPipelineConfig, FluxSamplingParams, {}), + (QwenImagePipelineConfig, QwenImageSamplingParams, {}), + (Ideogram4PipelineConfig, Ideogram4SamplingParams, {}), + (ErnieImagePipelineConfig, ErnieImageSamplingParams, {"use_pe": False}), + ( + LongCatImagePipelineConfig, + LongCatImageSamplingParams, + {"enable_prompt_rewrite": False}, + ), + ], +) +def test_model_sampling_contracts_remain_owned_by_the_model( + server, pipeline_cls, sampling_cls, extras +): + server.args.pipeline_config = pipeline_cls() + server.model_info.sampling_param_cls = sampling_cls + response = server.client.post( + "/v1/images/generations", + json={ + "prompt": "a teapot", + "enhance_prompt": True, + "response_format": "b64_json", + **extras, + }, + ) + assert response.status_code == 200, response.text + sampling = server.batches[0].sampling_params + assert type(sampling) is sampling_cls + assert sampling.prompt == server.rewritten + for name, value in extras.items(): + assert vars(sampling)[name] == value + + +def test_h3_native_request_contract_survives_enhancement(server): + server.args.pipeline_config = MiniMaxH3PipelineConfig() + server.model_info.sampling_param_cls = MiniMaxH3SamplingParams + target = {"short_edge": 768, "aspect_ratio": "16:9", "duration_seconds": 5.0} + request = VideoGenerationsRequest( + prompt="A red teapot", task="t2va", conditions=[], target=target, seed=123 + ) + + async def enhance_and_lower(): + request.prompt = await server.app.state.prompt_enhancer.enhance( + request.prompt, task="video", image_paths=[] + ) + return video_api._build_video_sampling_params("h3-enhanced", request) + + sampling = server.client.portal.call(enhance_and_lower) + assert type(sampling) is MiniMaxH3SamplingParams + assert sampling.prompt == server.rewritten + assert sampling.seed == 123 + assert sampling.task == "t2va" + assert sampling.conditions == [] + assert sampling.target == target + + +@pytest.mark.parametrize( + "source", ["https://example.com/reference.png", "data:image/png;base64,AAAA"] +) +@pytest.mark.parametrize("include_images", [True, False]) +def test_reference_transport_is_explicit(server, source, include_images): + enhancer = server.app.state.prompt_enhancer + enhancer.config.include_images = include_images + + async def run(): + return await enhancer.enhance( + "keep the subject", task="image_edit", image_paths=[source] + ) + + assert server.client.portal.call(run) == server.rewritten + content = server.calls[0]["messages"][1]["content"] + if include_images: + assert content[1] == {"type": "image_url", "image_url": {"url": source}} + else: + assert isinstance(content, str) + assert source not in content + + +def test_enhancer_failure_does_not_queue_video_or_leak_uploads( + server, monkeypatch, tmp_path +): + server.args.pipeline_config = WanT2V480PConfig() + server.args.input_save_path = None + server.args.output_path = None + monkeypatch.setattr(video_api.tempfile, "tempdir", str(tmp_path)) + + async def fail(request): + return httpx.Response(503, text="upstream secret") + + enhancer = server.app.state.prompt_enhancer + server.client.portal.call(enhancer.client.aclose) + enhancer.client = httpx.AsyncClient( + base_url="http://enhancer.local/v1/", transport=httpx.MockTransport(fail) + ) + response = server.client.post( + "/v1/videos", json={"prompt": "a teapot", "enhance_prompt": True} + ) + assert response.status_code == 502 + assert "secret" not in response.text + assert not server.batches + assert not list(tmp_path.glob("sglang_*")) + + +def test_realtime_does_not_silently_accept_prompt_enhancement(): + with pytest.raises(ValidationError): + RealtimeVideoGenerationsRequest( + type="init", prompt="a teapot", enhance_prompt=True + )