[diffusion] feat: support multiple task types for pipelines (#38762)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Yihao Wang
2026-09-28 10:00:45 +08:00
committed by GitHub
co-authored by Mick Qian
parent 6cbdfec453
commit 8b2ca8ecc2
25 changed files with 1036 additions and 155 deletions
@@ -6,7 +6,7 @@ import math
import os
from collections.abc import Callable
from dataclasses import asdict, dataclass, field, fields
from enum import Enum, auto
from enum import Enum
from operator import attrgetter
from typing import Any, ClassVar
@@ -26,7 +26,7 @@ from sglang.multimodal_gen.configs.models.encoders.t5 import T5Config
from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import (
ModelDeploymentConfig,
)
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
from sglang.multimodal_gen.configs.task_type import DataType, ModelTaskType
from sglang.multimodal_gen.configs.utils import update_config_from_args
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy
from sglang.multimodal_gen.runtime.distributed.communication_op import (
@@ -46,71 +46,6 @@ from sglang.multimodal_gen.runtime.utils.vision import get_default_height_width
logger = init_logger(__name__)
# NOTE: possible duplication with DataType
# this may focus on the model's original ability
class ModelTaskType(Enum):
# TODO: check if I2V/TI2V models can work w/wo text
I2V = auto() # Image to Video
T2V = auto() # Text to Video
TI2V = auto() # Text and Image to Video
T2I = auto() # Text to Image
I2I = auto() # Image to Image
TI2I = auto() # Image to Image or Text-Image to Image
I2M = auto() # Image to Mesh
VLA_ACTION = auto() # Vision-language-action policy output
def is_image_gen(self) -> bool:
return (
self == ModelTaskType.T2I
or self == ModelTaskType.I2I
or self == ModelTaskType.TI2I
)
def is_action_gen(self) -> bool:
return self == ModelTaskType.VLA_ACTION
def is_mesh_gen(self) -> bool:
return self == ModelTaskType.I2M
def is_video_gen(self) -> bool:
return (
self == ModelTaskType.I2V
or self == ModelTaskType.T2V
or self == ModelTaskType.TI2V
)
def is_visual_gen(self) -> bool:
return self.is_image_gen() or self.is_video_gen()
def requires_image_input(self) -> bool:
return (
self == ModelTaskType.I2V
or self == ModelTaskType.I2I
or self == ModelTaskType.I2M
)
def accepts_image_input(self) -> bool:
return (
self == ModelTaskType.I2V
or self == ModelTaskType.I2I
or self == ModelTaskType.TI2I
or self == ModelTaskType.TI2V
or self == ModelTaskType.I2M
or self == ModelTaskType.VLA_ACTION
)
def data_type(self) -> DataType:
if self.is_action_gen():
return DataType.ACTION
if self.is_mesh_gen():
return DataType.MESH
if self.is_image_gen():
return DataType.IMAGE
return DataType.VIDEO
class STA_Mode(str, Enum):
"""STA (Sliding Tile Attention) modes."""
@@ -199,7 +134,9 @@ class PipelineConfig:
"""The base configuration class for a generation pipeline."""
native_only_components: ClassVar[tuple[str, ...]] = ()
# Default task; supported_task_types is a model capability, not a user knob.
task_type: ModelTaskType = ModelTaskType.I2I
supported_task_types: ClassVar[tuple[ModelTaskType, ...] | None] = None
skip_input_image_preprocess: bool = False
# False when changing component placement after a calibration request is
# known to alter the pipeline's numerical path.
@@ -284,6 +221,91 @@ class PipelineConfig:
# return the model-specific config for optimal deployment setting
return ModelDeploymentConfig()
def get_supported_task_types(self) -> tuple[ModelTaskType, ...]:
"""Return the tasks implemented by this configured pipeline.
Existing pipelines keep their single default task. Multi-task models
declare a tuple, or override this method for configuration-dependent
capabilities. Declaration alone does not add model execution paths.
"""
default = ModelTaskType.parse(self.task_type)
declared = self.supported_task_types
if declared is None:
return (default,)
tasks = tuple(ModelTaskType.parse(task) for task in declared)
if not tasks or len(set(tasks)) != len(tasks) or default not in tasks:
raise ValueError(
"supported_task_types must be nonempty, unique, and include "
f"the default task_type {default.name}"
)
return tasks
def resolve_task_type(
self,
requested: ModelTaskType | str | None = None,
*,
data_type: DataType | None = None,
has_image: bool = False,
has_video: bool = False,
) -> ModelTaskType:
"""Resolve a request without changing shared pipeline configuration.
HTTP endpoints constrain the output type. Otherwise the default task
wins when its input contract fits. An ambiguous alternative requires
an explicit task_type; declaration order never selects a task.
"""
supported = self.get_supported_task_types()
if requested is not None:
task = ModelTaskType.parse(requested)
if task not in supported:
raise ValueError(
f"Unsupported task_type {task.name}; this pipeline supports "
f"{[task.name for task in supported]}"
)
if data_type is not None and task.data_type() != data_type:
raise ValueError(
f"task_type {task.name} produces {task.data_type().name}, "
f"but this endpoint produces {data_type.name}"
)
return task
candidates = tuple(
task
for task in supported
if data_type is None or task.data_type() == data_type
)
if not candidates:
raise ValueError(
f"This pipeline does not support {data_type.name} output; "
f"supported task types: {[task.name for task in supported]}"
)
if len(candidates) == 1:
return candidates[0]
compatible = tuple(
task
for task in candidates
if (not has_image or task.accepts_image_input())
and (not task.requires_image_input() or has_image)
and (not has_video or task.accepts_video_input())
and (not task.requires_video_input() or has_video)
)
default = ModelTaskType.parse(self.task_type)
if default in compatible:
return default
if data_type is None:
same_output = tuple(
task for task in compatible if task.data_type() == default.data_type()
)
compatible = same_output or compatible
if len(compatible) == 1:
return compatible[0]
raise ValueError(
"Specify task_type for this request; compatible tasks: "
f"{[task.name for task in compatible]}, "
f"supported tasks: {[task.name for task in candidates]}"
)
def validate_server_args(self, server_args: Any) -> None:
"""Validate model-owned constraints after server args are normalized."""
@@ -292,7 +314,7 @@ class PipelineConfig:
def supports_action_endpoint(self) -> bool:
"""Whether this pipeline exposes the generic action generation API."""
return self.task_type.is_action_gen()
return any(task.is_action_gen() for task in self.get_supported_task_types())
def supports_openpi_endpoint(self) -> bool:
"""Whether this pipeline implements the OpenPI policy websocket."""
@@ -424,7 +446,10 @@ class PipelineConfig:
The scheduler still checks each request before merging it into a batch.
"""
return self.task_type in (ModelTaskType.T2I, ModelTaskType.T2V)
return all(
task in (ModelTaskType.T2I, ModelTaskType.T2V)
for task in self.get_supported_task_types()
)
def supports_dynamic_batching_for_request(self, batch) -> bool:
"""Return whether one request may participate in dynamic batching."""
@@ -960,7 +985,15 @@ class PipelineConfig:
def update_config_from_dict(self, args: dict[str, Any], prefix: str = "") -> None:
prefix_with_dot = f"{prefix}." if (prefix.strip() != "") else ""
update_config_from_args(self, args, prefix, pop_args=True)
# Flat task_type is a request field; configure a different default in
# the nested pipeline_config. Capabilities belong to the model class.
update_config_from_args(
self,
args,
prefix,
pop_args=True,
exclude=("task_type", "supported_task_types"),
)
update_config_from_args(
self.vae_config, args, f"{prefix_with_dot}vae_config", pop_args=True
)
@@ -1137,6 +1170,8 @@ class PipelineConfig:
return pipeline_config
def check_pipeline_config(self) -> None:
self.task_type = ModelTaskType.parse(self.task_type)
self.get_supported_task_types()
if self.vae_sp and not self.vae_tiling:
raise ValueError(
"Currently enabling vae_sp requires enabling vae_tiling, please set --vae-tiling to True."
@@ -11,6 +11,7 @@ in the stages from ``num_frames`` and ``image_path``; T2I overrides
import functools
import os
from dataclasses import dataclass, field
from typing import ClassVar
from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
from sglang.multimodal_gen.configs.models.dits.cosmos3video import Cosmos3VideoConfig
@@ -109,6 +110,11 @@ class Cosmos3Config(PipelineConfig):
# TI2V (text + image → video) so the request validator accepts ``image_path``
# without requiring it. T2V ignores it; I2V uses it; T2I disregards it.
task_type: ModelTaskType = ModelTaskType.TI2V
supported_task_types: ClassVar[tuple[ModelTaskType, ...]] = (
ModelTaskType.TI2V,
ModelTaskType.T2I,
ModelTaskType.V2V,
)
dit_config: DiTConfig = field(default_factory=Cosmos3VideoConfig)
@@ -10,6 +10,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
_sanitize_filename,
)
from sglang.multimodal_gen.configs.task_type import ModelTaskType
from sglang.multimodal_gen.configs.utils import expand_path_fields
from sglang.multimodal_gen.runtime.utils.argparse import StoreBoolean
@@ -22,6 +23,7 @@ class ActionSamplingParams:
"""Sampling parameters for policies that generate continuous actions."""
data_type: DataType = DataType.ACTION
task_type: ModelTaskType = ModelTaskType.VLA_ACTION
request_id: str | None = field(default=None, metadata={"batch_sig_exclude": True})
prompt: str | list[str] | None = field(
default="", metadata={"batch_sig_exclude": True}
@@ -119,13 +121,15 @@ class ActionSamplingParams:
)
def _validate_with_pipeline_config(self, pipeline_config):
if not pipeline_config.task_type.is_action_gen():
task_type = pipeline_config.resolve_task_type(self.task_type)
if not task_type.is_action_gen():
raise ValueError(
f"ActionSamplingParams requires an ACTION pipeline, got {pipeline_config.task_type.name}"
f"ActionSamplingParams requires an ACTION pipeline, got {task_type.name}"
)
def _adjust(self, server_args: "ServerArgs"):
expand_path_fields(self)
self._validate_with_pipeline_config(server_args.pipeline_config)
self.data_type = DataType.ACTION
self.return_file_paths_only = False
if self.output_path is None and server_args.output_path is not None:
@@ -17,6 +17,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
)
from sglang.multimodal_gen.configs.task_type import ModelTaskType
COSMOS3_DEFAULT_GUIDANCE_SCALE = 4.0
COSMOS3_EDGE_T2I_GUIDANCE_SCALE = 7.0
@@ -405,6 +406,15 @@ class Cosmos3SamplingParams(SamplingParams):
return kwargs
def _adjust(self, server_args) -> None:
if (
self.task_type is None
and self.num_frames == 1
and self.image_path is None
and self.video_path is None
and self.action_mode is None
):
self.task_type = ModelTaskType.T2I
# adjust distil and edge args — read from the pre-computed config fields
# so no checkpoint download happens at request time.
pipeline_config = server_args.pipeline_config
@@ -13,10 +13,11 @@ import time
import unicodedata
import uuid
from dataclasses import dataclass, field
from enum import Enum, auto
from enum import Enum
from typing import TYPE_CHECKING, Any, ClassVar
from sglang.multimodal_gen.configs.post_training import RLRolloutArgs
from sglang.multimodal_gen.configs.task_type import DataType, ModelTaskType
from sglang.multimodal_gen.configs.utils import expand_path_fields
from sglang.multimodal_gen.runtime.utils.argparse import StoreBoolean
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -189,22 +190,6 @@ def align_num_frames_for_num_gpus(
return new_latent_num_frames
class DataType(Enum):
IMAGE = auto()
VIDEO = auto()
MESH = auto()
ACTION = auto()
def get_default_extension(self) -> str:
if self == DataType.IMAGE:
return "png"
if self == DataType.VIDEO:
return "mp4"
if self == DataType.ACTION:
return "json"
return "glb"
@dataclass
class SamplingParams:
"""
@@ -222,6 +207,8 @@ class SamplingParams:
"""
data_type: DataType = DataType.VIDEO
# Included in the dynamic batching signature and retained by replace/pickle.
task_type: ModelTaskType | str | None = None
request_id: str | None = field(default=None, metadata={"batch_sig_exclude": True})
@@ -732,13 +719,13 @@ class SamplingParams:
"""
check if the sampling params is compatible and valid with server_args
"""
task_type = pipeline_config.task_type
task_type = self.resolve_task_type(pipeline_config)
if task_type.is_action_gen():
return
if task_type.requires_image_input():
# requires image input
if self.image_path is None:
if not self.image_path:
raise ValueError(
f"Served model with task type '{task_type.name}' requires an 'image_path' input, but none was provided"
)
@@ -750,6 +737,34 @@ class SamplingParams:
f"input_reference is not supported for {task_type.name} models."
)
if task_type.requires_video_input() and not self.video_path:
raise ValueError(f"Task {task_type.name} requires a 'video_path' input")
# Legacy pipelines retain their model-owned video conditioning checks.
# Explicit multi-task declarations use the standard per-task contract.
if (
self.video_path
and (
getattr(pipeline_config, "supported_task_types", None) is not None
or (
hasattr(pipeline_config, "get_supported_task_types")
and len(pipeline_config.get_supported_task_types()) > 1
)
)
and not task_type.accepts_video_input()
):
raise ValueError(f"video_path is not supported for task {task_type.name}")
def resolve_task_type(self, pipeline_config) -> ModelTaskType:
resolver = getattr(pipeline_config, "resolve_task_type", None)
if resolver is None:
# Compatibility with integrations that supply a lightweight config.
return ModelTaskType.parse(self.task_type or pipeline_config.task_type)
return resolver(
self.task_type,
has_image=bool(self.image_path),
has_video=bool(self.video_path),
)
def _adjust(
self,
server_args,
@@ -761,7 +776,8 @@ class SamplingParams:
# TODO: SamplingParams should not rely on ServerArgs
pipeline_config = server_args.pipeline_config
task_type = pipeline_config.task_type
task_type = self.resolve_task_type(pipeline_config)
self.task_type = task_type
self.data_type = task_type.data_type()
self._adjust_output_path(server_args)
@@ -864,7 +880,7 @@ class SamplingParams:
"Sequence dimension shard is enabled, disabling frame adjustment for better performance"
)
if pipeline_config.task_type.is_image_gen():
if self.resolve_task_type(pipeline_config).is_image_gen():
# settle num_frames
if not server_args.pipeline_config.allow_set_num_frames():
logger.debug("Setting `num_frames` to 1 for image generation model")
@@ -1007,6 +1023,11 @@ class SamplingParams:
return parser.add_argument(*name_or_flags, **kwargs)
add_argument("--data-type", type=str, nargs="+")
add_argument(
"--task-type",
type=str,
help="Request task (for example t2i, i2i, t2v, i2v, v2v, f2v); must be supported by the pipeline.",
)
# Predict the shot length from the caption, overriding `--num-frames`.
add_argument("--use-diffusion-decoder", action="store_true")
add_argument("--auto-duration", action="store_true")
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
"""Shared task and output types for pipeline capabilities and requests."""
from enum import Enum, auto
class DataType(Enum):
IMAGE = auto()
VIDEO = auto()
MESH = auto()
ACTION = auto()
def get_default_extension(self) -> str:
if self == DataType.IMAGE:
return "png"
if self == DataType.VIDEO:
return "mp4"
if self == DataType.ACTION:
return "json"
return "glb"
class ModelTaskType(Enum):
# TODO: check if I2V/TI2V models can work w/wo text
I2V = auto() # Image to Video
T2V = auto() # Text to Video
TI2V = auto() # Text and Image to Video
T2I = auto() # Text to Image
I2I = auto() # Image to Image
TI2I = auto() # Image to Image or Text-Image to Image
I2M = auto() # Image to Mesh
VLA_ACTION = auto() # Vision-language-action policy output
V2V = auto() # Video to Video
F2V = auto() # Video-prefix-conditioned Video
@classmethod
def parse(cls, value: "ModelTaskType | str") -> "ModelTaskType":
if isinstance(value, cls):
return value
if isinstance(value, str):
try:
return cls[value.strip().upper()]
except KeyError:
pass
raise ValueError(
f"Unknown task_type {value!r}; choose from {[task.name for task in cls]}"
)
def requires_video_input(self) -> bool:
return self in (ModelTaskType.V2V, ModelTaskType.F2V)
def accepts_video_input(self) -> bool:
return self.requires_video_input()
def is_image_gen(self) -> bool:
return (
self == ModelTaskType.T2I
or self == ModelTaskType.I2I
or self == ModelTaskType.TI2I
)
def is_action_gen(self) -> bool:
return self == ModelTaskType.VLA_ACTION
def is_mesh_gen(self) -> bool:
return self == ModelTaskType.I2M
def is_video_gen(self) -> bool:
return (
self == ModelTaskType.I2V
or self == ModelTaskType.T2V
or self == ModelTaskType.TI2V
or self == ModelTaskType.V2V
or self == ModelTaskType.F2V
)
def is_visual_gen(self) -> bool:
return self.is_image_gen() or self.is_video_gen()
def requires_image_input(self) -> bool:
return (
self == ModelTaskType.I2V
or self == ModelTaskType.I2I
or self == ModelTaskType.I2M
)
def accepts_image_input(self) -> bool:
return (
self == ModelTaskType.I2V
or self == ModelTaskType.I2I
or self == ModelTaskType.TI2I
or self == ModelTaskType.TI2V
or self == ModelTaskType.I2M
or self == ModelTaskType.VLA_ACTION
)
def data_type(self) -> DataType:
if self.is_action_gen():
return DataType.ACTION
if self.is_mesh_gen():
return DataType.MESH
if self.is_image_gen():
return DataType.IMAGE
return DataType.VIDEO
def get_request_task_type(batch, pipeline_config) -> ModelTaskType:
"""Read the admitted task, falling back for legacy direct stage callers."""
return ModelTaskType.parse(
getattr(batch, "task_type", None) or pipeline_config.task_type
)
@@ -31,7 +31,12 @@ def expand_path_fields(obj) -> None:
def update_config_from_args(
config: Any, args_dict: dict[str, Any], prefix: str = "", pop_args: bool = False
config: Any,
args_dict: dict[str, Any],
prefix: str = "",
pop_args: bool = False,
*,
exclude: tuple[str, ...] = (),
) -> bool:
"""
Update configuration object from arguments dictionary.
@@ -50,7 +55,7 @@ def update_config_from_args(
args_to_remove = []
if prefix.strip() == "":
for key, value in args_dict.items():
if hasattr(config, key) and value is not None:
if key not in exclude and hasattr(config, key) and value is not None:
if key == "text_encoder_precisions" and isinstance(value, list):
setattr(config, key, tuple(value))
else:
@@ -63,6 +68,8 @@ def update_config_from_args(
for key, value in args_dict.items():
if key.startswith(prefix_with_dot) and value is not None:
attr_name = key[len(prefix_with_dot) :]
if attr_name in exclude:
continue
if hasattr(config, attr_name):
setattr(config, attr_name, value)
if pop_args:
@@ -189,23 +189,28 @@ async def get_models(request: Request):
Use /v1/models instead for OpenAI-compatible model discovery.
This endpoint will be removed in a future version.
"""
from sglang.multimodal_gen.registry import get_model_info
from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import (
get_served_pipeline_class,
)
server_args: ServerArgs = request.app.state.server_args
model_info = get_model_info(server_args.model_path, model_id=server_args.model_id)
pipeline_cls = get_served_pipeline_class(server_args)
response = {
"model_path": server_args.model_path,
"num_gpus": server_args.num_gpus,
"task_type": server_args.pipeline_config.task_type.name,
"supported_task_types": [
task.name for task in server_args.pipeline_config.get_supported_task_types()
],
"dit_precision": server_args.pipeline_config.dit_precision,
"vae_precision": server_args.pipeline_config.vae_precision,
"vae_decode_precision": server_args.pipeline_config.vae_decode_precision,
}
if model_info:
response["pipeline_name"] = model_info.pipeline_cls.pipeline_name
response["pipeline_class"] = model_info.pipeline_cls.__name__
if pipeline_cls:
response["pipeline_name"] = pipeline_cls.pipeline_name
response["pipeline_class"] = pipeline_cls.__name__
return response
@@ -235,34 +240,35 @@ async def model_info_endpoint(request: Request):
Returns fields compatible with the LLM engine's /model_info so that
the model gateway can detect capabilities for diffusion workers.
"""
from sglang.multimodal_gen.registry import get_model_info
from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import (
get_served_pipeline_class,
)
server_args: ServerArgs = request.app.state.server_args
task_type = server_args.pipeline_config.task_type
supported_tasks = server_args.pipeline_config.get_supported_task_types()
try:
registry_info = get_model_info(
server_args.model_path,
backend=server_args.backend,
model_id=server_args.model_id,
)
pipeline_cls = get_served_pipeline_class(server_args)
except Exception:
logger.warning("Failed to resolve model info from registry", exc_info=True)
registry_info = None
pipeline_cls = None
return {
# Fields consumed by the model gateway for worker discovery
"model_path": server_args.model_path,
"is_generation": True,
"model_type": "diffusion",
"architectures": (
[registry_info.pipeline_cls.__name__] if registry_info else None
),
"architectures": [pipeline_cls.__name__] if pipeline_cls else None,
# Fields matching the LLM engine's /model_info shape
"has_image_understanding": task_type.accepts_image_input(),
"has_image_understanding": any(
task.accepts_image_input() for task in supported_tasks
),
"has_audio_understanding": False,
# Diffusion-specific fields
"task_type": task_type.name,
"supported_task_types": [task.name for task in supported_tasks],
"output_types": sorted({task.data_type().name for task in supported_tasks}),
"is_image_gen": task_type.is_image_gen(),
}
@@ -4,7 +4,7 @@ from typing import Any, List, Optional, Union
from fastapi import APIRouter, Body, HTTPException
from pydantic import BaseModel, Field
from sglang.multimodal_gen.registry import get_model_info
from sglang.multimodal_gen.registry import get_model_info, get_pipeline_class
from sglang.multimodal_gen.runtime.entrypoints.control_requests import (
ListLorasReq,
MergeLoraWeightsReq,
@@ -41,31 +41,43 @@ class DiffusionModelCard(ModelCard):
num_gpus: Optional[int] = None
task_type: Optional[str] = None
supported_task_types: list[str] | None = None
dit_precision: Optional[str] = None
vae_precision: Optional[str] = None
pipeline_name: Optional[str] = None
pipeline_class: Optional[str] = None
def _build_model_card(
server_args: ServerArgs, served_model_name: str
) -> DiffusionModelCard:
def get_served_pipeline_class(server_args: ServerArgs):
"""Resolve discovery metadata using the same explicit pipeline as serving."""
if server_args.pipeline_class_name:
return get_pipeline_class(server_args.pipeline_class_name)
model_info = get_model_info(
server_args.model_path,
backend=server_args.backend,
model_id=server_args.model_id,
)
return model_info.pipeline_cls if model_info else None
def _build_model_card(
server_args: ServerArgs, served_model_name: str
) -> DiffusionModelCard:
pipeline_cls = get_served_pipeline_class(server_args)
card_kwargs: dict[str, Any] = {
"id": served_model_name,
"root": served_model_name,
"num_gpus": server_args.num_gpus,
"task_type": server_args.pipeline_config.task_type.name,
"supported_task_types": [
task.name for task in server_args.pipeline_config.get_supported_task_types()
],
"dit_precision": server_args.pipeline_config.dit_precision,
"vae_precision": server_args.pipeline_config.vae_precision,
}
if model_info:
card_kwargs["pipeline_name"] = model_info.pipeline_cls.pipeline_name
card_kwargs["pipeline_class"] = model_info.pipeline_cls.__name__
if pipeline_cls:
card_kwargs["pipeline_name"] = pipeline_cls.pipeline_name
card_kwargs["pipeline_class"] = pipeline_cls.__name__
return DiffusionModelCard(**card_kwargs)
@@ -25,6 +25,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
SamplingParams,
generate_request_id,
)
from sglang.multimodal_gen.configs.task_type import DataType
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
maybe_enhance_prompt,
)
@@ -291,6 +292,8 @@ async def generations(
sampling = build_sampling_params(
request_id,
prompt=prompt,
task_type=request.task_type,
request_data_type=DataType.IMAGE,
size=request.size,
width=request.width,
height=request.height,
@@ -419,6 +422,7 @@ async def edits(
enhance_prompt: bool = Form(False),
mask: Optional[UploadFile] = File(None),
model: Optional[str] = Form(None),
task_type: Optional[str] = Form(None),
n: Optional[int] = Form(1),
response_format: Optional[str] = Form(None),
size: Optional[str] = Form(None),
@@ -489,6 +493,8 @@ async def edits(
sampling = build_sampling_params(
request_id,
prompt=prompt,
task_type=task_type,
request_data_type=DataType.IMAGE,
size=size,
num_outputs_per_prompt=max(1, min(int(n or 1), 10)),
output_file_name=f"{request_id}.{ext}",
@@ -19,6 +19,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
SamplingParams,
generate_request_id,
)
from sglang.multimodal_gen.configs.task_type import ModelTaskType
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
MeshGenerationsRequest,
MeshListResponse,
@@ -54,6 +55,7 @@ def _build_sampling_params_from_request(
server_args = get_global_server_args()
sampling_kwargs: Dict[str, Any] = {
"request_id": request_id,
"task_type": ModelTaskType.I2M,
"prompt": req.prompt,
"num_frames": 1,
"image_path": [image_path] if image_path else None,
@@ -46,6 +46,9 @@ class ImageGenerationsRequest(BaseModel):
model_config = ConfigDict(extra="allow")
prompt: str
task_type: Optional[str] = Field(
default=None, description="Requested task from the server supported_task_types."
)
enhance_prompt: bool = False
model: Optional[str] = None
n: Optional[int] = 1
@@ -119,6 +122,9 @@ class VideoGenerationsRequest(BaseModel):
reference_url: Optional[str] = None
video_path: Optional[str] = None
video_url: Optional[str] = None
task_type: Optional[str] = Field(
default=None, description="Requested task from the server supported_task_types."
)
model: Optional[str] = None
n: Optional[int] = 1
num_outputs_per_prompt: Optional[int] = None
@@ -214,6 +214,17 @@ def build_sampling_params(request_id: str, **kwargs) -> SamplingParams:
# pop HTTP-layer params that aren't SamplingParams fields
output_quality = kwargs.pop("output_quality", None)
request_data_type = kwargs.pop("request_data_type", None)
if request_data_type is not None:
try:
kwargs["task_type"] = server_args.pipeline_config.resolve_task_type(
kwargs.get("task_type"),
data_type=request_data_type,
has_image=bool(kwargs.get("image_path")),
has_video=bool(kwargs.get("video_path")),
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
has_explicit_compression = kwargs.get("output_compression") is not None
@@ -240,12 +251,15 @@ def build_sampling_params(request_id: str, **kwargs) -> SamplingParams:
kwargs = {k: v for k, v in kwargs.items() if v is not None}
kwargs.setdefault("save_output", True)
sampling_params = SamplingParams.from_user_sampling_params_args(
model_path=server_args.model_path,
server_args=server_args,
request_id=request_id,
**kwargs,
)
try:
sampling_params = SamplingParams.from_user_sampling_params_args(
model_path=server_args.model_path,
server_args=server_args,
request_id=request_id,
**kwargs,
)
except (ValueError, TypeError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
# resolve output_quality → output_compression with the correct data_type.
# SamplingParams.__post_init__ may have resolved with the wrong data_type
@@ -270,6 +270,8 @@ def _build_video_sampling_params(request_id: str, request: VideoGenerationsReque
kwargs = {
"prompt": request.prompt,
"task_type": request.task_type,
"request_data_type": DataType.VIDEO,
"num_outputs_per_prompt": max(1, min(int(num_outputs), 10)),
"size": request.size,
"width": request.width,
@@ -475,6 +477,7 @@ async def create_video(
video_url: Optional[str] = Form(None),
video_path: Optional[str] = Form(None),
model: Optional[str] = Form(None),
task_type: Optional[str] = Form(None),
n: Optional[int] = Form(1),
num_outputs_per_prompt: Optional[int] = Form(None),
seconds: Optional[int] = Form(None),
@@ -509,7 +512,6 @@ async def create_video(
request_id = generate_request_id()
server_args = get_global_server_args()
task_type = server_args.pipeline_config.task_type
is_multipart = "multipart/form-data" in content_type
raw_form: Any = None
extra_from_form: Dict[str, Any] = {}
@@ -567,12 +569,6 @@ async def create_video(
video_input_path = reference_url
image_sources = merge_image_input_list(input_reference)
# Validate image input based on model task type
if task_type.requires_image_input() and not image_sources:
raise HTTPException(
status_code=400,
detail="input_reference or reference_url is required for image-to-video generation",
)
input_path = None
if image_sources:
try:
@@ -616,6 +612,7 @@ async def create_video(
video_path=form_value("video_path", video_input_path),
video_url=form_value("video_url", video_url),
model=form_value("model", model),
task_type=form_text_value("task_type", task_type),
n=form_value("n", n),
num_outputs_per_prompt=form_value(
"num_outputs_per_prompt", num_outputs_per_prompt
@@ -688,18 +685,6 @@ async def create_video(
if _is_probably_video_source(payload.get("input_reference")):
payload.setdefault("video_path", payload.get("input_reference"))
has_image_input = (
payload.get("reference_url")
and not _is_probably_video_source(payload.get("reference_url"))
) or (
payload.get("input_reference")
and not _is_probably_video_source(payload.get("input_reference"))
)
if task_type.requires_image_input() and not has_image_input:
raise HTTPException(
status_code=400,
detail="input_reference or reference_url is required for image-to-video generation",
)
# for non-multipart/form-data type
if payload.get("reference_url") and not _is_probably_video_source(
payload.get("reference_url")
@@ -59,6 +59,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
quality_allows_kernel_fusions,
resolve_skip_softmax_params,
)
from sglang.multimodal_gen.configs.task_type import get_request_task_type
from sglang.multimodal_gen.runtime.breakable_cuda_graph import (
prompt_padding as bcg_utils,
)
@@ -1642,9 +1643,10 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
# 1. Prepare latent inputs in the model's compute dtype.
latent_model_input = ctx.latents.to(ctx.target_dtype)
if batch.image_latent is not None:
assert not server_args.pipeline_config.task_type == ModelTaskType.TI2V, (
"image latents should not be provided for TI2V task"
)
assert (
get_request_task_type(batch, server_args.pipeline_config)
!= ModelTaskType.TI2V
), "image latents should not be provided for TI2V task"
latent_model_input = torch.cat(
[latent_model_input, batch.image_latent], dim=1
).to(ctx.target_dtype)
@@ -15,6 +15,7 @@ from PIL import Image
from sglang.multimodal_gen.configs.pipeline_configs import WanI2V480PConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
from sglang.multimodal_gen.configs.pipeline_configs.mova import MOVAPipelineConfig
from sglang.multimodal_gen.configs.task_type import get_request_task_type
from sglang.multimodal_gen.runtime.pipelines_core.request_utils import (
expand_request_outputs,
)
@@ -192,8 +193,10 @@ class InputValidationStage(PipelineStage):
NOTE: condition image resizing is only allowed in InputValidationStage
"""
if batch.condition_image is not None and (
server_args.pipeline_config.task_type == ModelTaskType.I2I
or server_args.pipeline_config.task_type == ModelTaskType.TI2I
get_request_task_type(batch, server_args.pipeline_config)
== ModelTaskType.I2I
or get_request_task_type(batch, server_args.pipeline_config)
== ModelTaskType.TI2I
):
# calculate new condition image size
if not isinstance(batch.condition_image, list):
@@ -235,7 +238,10 @@ class InputValidationStage(PipelineStage):
batch.width = width
batch.height = height
elif server_args.pipeline_config.task_type == ModelTaskType.TI2V:
elif (
get_request_task_type(batch, server_args.pipeline_config)
== ModelTaskType.TI2V
):
if server_args.pipeline_config.skip_input_image_preprocess:
return
# duplicate with vae_image_processor
@@ -360,7 +366,8 @@ class InputValidationStage(PipelineStage):
self._generate_seeds(batch, server_args)
if (
server_args.pipeline_config.task_type == ModelTaskType.I2M
get_request_task_type(batch, server_args.pipeline_config)
== ModelTaskType.I2M
and batch.num_inference_steps is None
and hasattr(server_args.pipeline_config, "shape_num_inference_steps")
):
@@ -370,7 +377,8 @@ class InputValidationStage(PipelineStage):
# Ensure prompt is properly formatted (I2M can be image-only)
if (
server_args.pipeline_config.task_type != ModelTaskType.I2M
get_request_task_type(batch, server_args.pipeline_config)
!= ModelTaskType.I2M
and batch.prompt is None
and batch.prompt_embeds is None
):
@@ -456,7 +464,10 @@ class InputValidationStage(PipelineStage):
)
batch.original_condition_image_size = image.size
if server_args.pipeline_config.task_type != ModelTaskType.I2M:
if (
get_request_task_type(batch, server_args.pipeline_config)
!= ModelTaskType.I2M
):
self.preprocess_condition_image(
batch, server_args, condition_image_width, condition_image_height
)
@@ -492,7 +503,10 @@ class InputValidationStage(PipelineStage):
result.add_check(
"num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int
)
if server_args.pipeline_config.task_type != ModelTaskType.I2M:
if (
get_request_task_type(batch, server_args.pipeline_config)
!= ModelTaskType.I2M
):
result.add_check(
"prompt_or_embeds",
None,
@@ -502,7 +516,10 @@ class InputValidationStage(PipelineStage):
),
)
if server_args.pipeline_config.task_type != ModelTaskType.I2M:
if (
get_request_task_type(batch, server_args.pipeline_config)
!= ModelTaskType.I2M
):
result.add_check(
"num_inference_steps", batch.num_inference_steps, V.positive_int
)
@@ -614,6 +614,7 @@ def build_warmup_reqs(
include_warmup_image = should_include_warmup_image(server_args, server_based_warmup)
for width, height, num_frames, is_probe in shapes:
req_kwargs = dict(
task_type=task_type,
data_type=task_type.data_type(),
width=width,
height=height,
@@ -871,6 +871,8 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
server_based_expected = {
ModelTaskType.T2I: False,
ModelTaskType.T2V: False,
ModelTaskType.V2V: False,
ModelTaskType.F2V: False,
ModelTaskType.TI2I: True,
ModelTaskType.TI2V: True,
ModelTaskType.I2I: True,
@@ -264,5 +264,20 @@ class TestFlux2TI2ISizeResolution(unittest.TestCase):
self.assertEqual((batch.width, batch.height), (768, 512))
class TestPerRequestTask(unittest.TestCase):
def test_image_preprocessing_uses_request_on_video_default(self):
config = _DummyTI2IConfig()
config.task_type = ModelTaskType.T2V
image = Image.new("RGB", (1255, 833))
batch = _make_batch(image)
batch.task_type = ModelTaskType.I2I
InputValidationStage().preprocess_condition_image(
batch, _make_server_args(config), image.width, image.height
)
self.assertEqual((batch.width, batch.height), (1248, 832))
self.assertIsInstance(batch.condition_image, list)
self.assertEqual(config.task_type, ModelTaskType.T2V)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
from types import SimpleNamespace
from fastapi import FastAPI
from fastapi.testclient import TestClient
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.task_type import ModelTaskType
from sglang.multimodal_gen.runtime.entrypoints import http_server
from sglang.multimodal_gen.runtime.entrypoints.openai import common_api
def test_explicit_pipeline_discovery_does_not_require_diffusers_index(monkeypatch):
class CustomPipeline:
pipeline_name = "CustomPipeline"
class MultiConfig(PipelineConfig):
supported_task_types = (ModelTaskType.T2V, ModelTaskType.I2I)
args = SimpleNamespace(
model_path="/models/custom/dit.safetensors",
pipeline_class_name="CustomPipeline",
pipeline_config=MultiConfig(task_type=ModelTaskType.T2V),
num_gpus=1,
served_model_name="custom",
backend="sglang",
model_id=None,
)
def unexpected_lookup(*args, **kwargs):
raise AssertionError(
"Explicit pipeline must bypass Diffusers model_index lookup"
)
monkeypatch.setattr(common_api, "get_model_info", unexpected_lookup)
monkeypatch.setattr(common_api, "get_pipeline_class", lambda name: CustomPipeline)
monkeypatch.setattr(common_api, "get_global_server_args", lambda: args)
app = FastAPI()
app.state.server_args = args
app.include_router(http_server.health_router)
app.include_router(common_api.router)
with TestClient(app) as client:
for endpoint in ("/models", "/v1/models", "/model_info"):
response = client.get(endpoint)
assert response.status_code == 200, response.text
data = response.json()
if endpoint == "/v1/models":
data = data["data"][0]
assert data["task_type"] == "T2V"
assert data["supported_task_types"] == ["T2V", "I2I"]
info = client.get("/model_info").json()
assert info["architectures"] == ["CustomPipeline"]
assert info["output_types"] == ["IMAGE", "VIDEO"]
assert info["has_image_understanding"] is True
@@ -0,0 +1,403 @@
# SPDX-License-Identifier: Apache-2.0
"""Capability selection and real HTTP parsing, with model execution stubbed."""
import argparse
import io
import pickle
from dataclasses import dataclass, replace
from types import SimpleNamespace
from typing import ClassVar
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from PIL import Image
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config
from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.task_type import DataType
from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, utils, video_api
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
@dataclass
class MultiConfig(PipelineConfig):
task_type: Task = Task.T2V
supported_task_types: ClassVar[tuple[Task, ...]] = (
Task.T2I,
Task.I2I,
Task.T2V,
Task.I2V,
Task.V2V,
Task.F2V,
)
def make_args(config, output=None):
return SimpleNamespace(
pipeline_config=config,
model_path="test-model",
backend="sglang",
model_id=None,
pipeline_class_name=None,
output_path=output,
input_save_path=None,
served_model_name="test-model",
num_gpus=1,
comfyui_mode=False,
attention_backend_config={},
enable_trace=False,
enable_torch_compile=False,
enable_breakable_cuda_graph=False,
enable_cfg_parallel=False,
warmup_steps=1,
)
def test_legacy_enums_and_capabilities():
from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType as OldDataType,
)
assert ModelTaskType is Task
assert OldDataType is DataType
for task in Task:
config = PipelineConfig(task_type=task)
assert config.get_supported_task_types() == (task,)
assert config.resolve_task_type() == task
assert config.resolve_task_type(task.name.lower()) == task
@pytest.mark.parametrize("declared", [(), (Task.T2I,), (Task.T2V, Task.T2V)])
def test_invalid_capability_declarations(declared):
class Invalid(MultiConfig):
supported_task_types = declared
with pytest.raises(ValueError, match="supported_task_types"):
Invalid().get_supported_task_types()
def test_selection_and_ambiguity():
config = MultiConfig()
assert config.resolve_task_type() == Task.T2V
assert config.resolve_task_type(data_type=DataType.IMAGE) == Task.T2I
assert (
config.resolve_task_type(data_type=DataType.IMAGE, has_image=True) == Task.I2I
)
assert config.resolve_task_type(has_image=True) == Task.I2V
with pytest.raises(ValueError, match="Specify task_type"):
config.resolve_task_type(has_video=True)
assert config.resolve_task_type("f2v", has_video=True) == Task.F2V
assert config.task_type == Task.T2V
with pytest.raises(ValueError, match="endpoint produces IMAGE"):
config.resolve_task_type("t2v", data_type=DataType.IMAGE)
with pytest.raises(ValueError, match="Unsupported task_type"):
config.resolve_task_type("i2m")
with pytest.raises(ValueError, match="Unknown task_type"):
config.resolve_task_type("imaginary")
@pytest.mark.parametrize(
"task,image,video,output",
[
("t2i", None, None, DataType.IMAGE),
("i2i", "reference.png", None, DataType.IMAGE),
("t2v", None, None, DataType.VIDEO),
("i2v", "reference.png", None, DataType.VIDEO),
("v2v", None, "reference.mp4", DataType.VIDEO),
("f2v", None, "reference.mp4", DataType.VIDEO),
],
)
def test_preparation_copy_transport_and_extension(task, image, video, output, tmp_path):
config = MultiConfig()
sp = SamplingParams(
prompt="test",
task_type=task,
image_path=image,
video_path=video,
num_frames=25,
output_file_name="result",
adjust_frames=False,
)
sp._adjust(make_args(config, str(tmp_path)))
sp._validate_with_pipeline_config(config)
sp = pickle.loads(pickle.dumps(replace(sp, prompt="expanded prompt")))
assert sp.task_type == Task.parse(task)
assert sp.data_type == output
assert sp.output_file_path().endswith(output.get_default_extension())
assert sp.num_frames == (1 if output == DataType.IMAGE else 25)
assert config.task_type == Task.T2V
@pytest.mark.parametrize(
"task,image,video",
[
("i2i", None, None),
("i2v", None, None),
("v2v", None, None),
("f2v", None, None),
("t2i", "a.png", None),
("t2v", None, "a.mp4"),
("v2v", "a.png", "a.mp4"),
],
)
def test_task_input_contracts(task, image, video):
sp = SamplingParams(task_type=task, image_path=image, video_path=video)
with pytest.raises(ValueError):
sp._validate_with_pipeline_config(MultiConfig())
def test_cli_accepts_standard_task_type():
parser = argparse.ArgumentParser()
SamplingParams.add_cli_args(parser)
assert parser.parse_args(["--task-type", "i2v"]).task_type == "i2v"
def test_batch_signature_separates_tasks():
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
scheduler = Scheduler.__new__(Scheduler)
scheduler.server_args = make_args(MultiConfig())
a = Req(sampling_params=SamplingParams(task_type=Task.V2V, video_path="same.mp4"))
b = Req(sampling_params=SamplingParams(task_type=Task.F2V, video_path="same.mp4"))
assert scheduler._build_dynamic_batch_signature(
a
) != scheduler._build_dynamic_batch_signature(b)
def test_multi_task_warmup_uses_default_without_optional_image(monkeypatch):
from sglang.multimodal_gen.runtime import warmup_request_builder as warmup
monkeypatch.setattr(
warmup, "get_model_sampling_defaults", lambda _: SamplingParams()
)
reqs = warmup.build_warmup_reqs(
make_args(MultiConfig()),
warmup_resolutions=["512x512"],
server_based_warmup=True,
)
assert len(reqs) == 1
assert reqs[0].task_type == Task.T2V
assert reqs[0].data_type == DataType.VIDEO
assert reqs[0].image_path is None
@pytest.fixture(params=[(MultiConfig, SamplingParams)])
def http_client(monkeypatch, tmp_path, request):
config_cls, sampling_cls = request.param
config = config_cls()
args = make_args(config, str(tmp_path))
for module in (image_api, video_api, utils):
monkeypatch.setattr(module, "get_global_server_args", lambda: args)
for module in (image_api, video_api):
monkeypatch.setattr(
module, "resolve_sampling_params_cls", lambda _: sampling_cls
)
monkeypatch.setattr(
SamplingParams, "from_pretrained", classmethod(lambda cls, *a, **kw: cls())
)
admitted = []
async def generate(client, req, **kwargs):
admitted.append(req)
path = req.output_file_path()
Image.new("RGB", (64, 64)).save(path)
return [path], OutputBatch()
async def upload(paths):
return [None] * len(paths)
async def complete_dispatch():
pass
def dispatch(job_id, req, **kwargs):
# capture admission before the background coroutine can race the response
admitted.append(req)
return complete_dispatch()
monkeypatch.setattr(image_api, "process_generation_batch", generate)
monkeypatch.setattr(image_api, "_upload_and_cleanup_images", upload)
monkeypatch.setattr(video_api, "_dispatch_job_async", dispatch)
app = FastAPI()
app.include_router(image_api.router)
app.include_router(video_api.router)
with TestClient(app) as client:
yield client, admitted, config
def png_bytes():
buf = io.BytesIO()
Image.new("RGB", (64, 64)).save(buf, format="PNG")
return buf.getvalue()
@pytest.mark.parametrize(
"http_client", [(Cosmos3Config, Cosmos3SamplingParams)], indirect=True
)
@pytest.mark.parametrize("task", [None, "t2i"])
def test_cosmos3_image_endpoint_selects_image_task(http_client, task):
client, admitted, config = http_client
response = client.post(
"/v1/images/generations",
json={"prompt": "text", "size": "64x64", "task_type": task},
)
assert response.status_code == 200, response.text
assert admitted[-1].task_type == Task.T2I
assert admitted[-1].data_type == DataType.IMAGE
assert admitted[-1].num_frames == 1
assert config.task_type == Task.TI2V
@pytest.mark.parametrize(
"http_client", [(Cosmos3Config, Cosmos3SamplingParams)], indirect=True
)
@pytest.mark.parametrize(
"conditioning,task",
[
({}, Task.TI2V),
({"input_reference": "/tmp/image.png"}, Task.TI2V),
({"video_path": "/tmp/clip.mp4"}, Task.V2V),
],
)
def test_cosmos3_video_endpoint_preserves_conditioning(http_client, conditioning, task):
client, admitted, config = http_client
response = client.post(
"/v1/videos", json={"prompt": "video", "size": "64x64", **conditioning}
)
assert response.status_code == 200, response.text
assert admitted[-1].task_type == task
assert admitted[-1].data_type == DataType.VIDEO
assert admitted[-1].image_path == conditioning.get("input_reference")
assert admitted[-1].video_path == conditioning.get("video_path")
assert config.task_type == Task.TI2V
@pytest.mark.parametrize(
"frames,conditioning,task",
[
(1, {}, Task.T2I),
(81, {}, Task.TI2V),
(81, {"image_path": "/tmp/image.png"}, Task.TI2V),
(81, {"video_path": "/tmp/clip.mp4"}, Task.V2V),
],
)
def test_cosmos3_sampling_resolves_existing_modes(frames, conditioning, task, tmp_path):
config = Cosmos3Config()
sp = Cosmos3SamplingParams(prompt="test", num_frames=frames, **conditioning)
sp._adjust(make_args(config, str(tmp_path)))
sp._validate_with_pipeline_config(config)
assert sp.task_type == task
assert sp.data_type == task.data_type()
assert sp.output_file_path().endswith(sp.data_type.get_default_extension())
assert config.task_type == Task.TI2V
def test_image_json_and_multipart_choose_tasks_on_video_default(http_client):
client, admitted, config = http_client
response = client.post(
"/v1/images/generations",
json={
"prompt": "text",
"response_format": "b64_json",
"size": "64x64",
"task_type": "t2i",
},
)
assert response.status_code == 200, response.text
assert admitted[-1].task_type == Task.T2I
for mode in ({}, {"task_type": "i2i"}):
response = client.post(
"/v1/images/edits",
data={
"prompt": "edit",
"size": "64x64",
"response_format": "b64_json",
**mode,
},
files={"image": ("image.png", png_bytes(), "image/png")},
)
assert response.status_code == 200, response.text
assert admitted[-1].task_type == Task.I2I
assert config.task_type == Task.T2V
@pytest.mark.parametrize("multipart", [False, True])
def test_video_transports_keep_selected_task(http_client, multipart):
client, admitted, config = http_client
payload = {
"prompt": "video",
"task_type": "f2v",
"size": "64x64",
"video_path": "/tmp/clip.mp4",
}
if multipart:
response = client.post(
"/v1/videos", files={k: (None, v) for k, v in payload.items()}
)
else:
response = client.post("/v1/videos", json=payload)
assert response.status_code == 200, response.text
assert admitted[-1].task_type == Task.F2V
assert admitted[-1].data_type == DataType.VIDEO
assert config.task_type == Task.T2V
def test_http_rejects_invalid_tasks_before_model_execution(http_client):
client, admitted, _ = http_client
for endpoint, payload in [
("images/generations", {"task_type": "t2v"}),
("images/generations", {"task_type": "i2i"}),
("videos", {"task_type": "t2i"}),
("videos", {"task_type": "v2v"}),
("videos", {"task_type": "t2v", "video_path": "/tmp/clip.mp4"}),
("videos", {"task_type": "unknown"}),
]:
response = client.post("/v1/" + endpoint, json={"prompt": "invalid", **payload})
assert response.status_code == 400, response.text
assert admitted == []
@pytest.mark.parametrize("multipart", [False, True])
def test_unconditioned_request_on_image_conditioned_default(http_client, multipart):
client, admitted, config = http_client
config.task_type = Task.I2V
payload = {"prompt": "video", "task_type": "t2v", "size": "64x64"}
if multipart:
response = client.post(
"/v1/videos", files={k: (None, v) for k, v in payload.items()}
)
else:
response = client.post("/v1/videos", json=payload)
assert response.status_code == 200, response.text
assert admitted[-1].task_type == Task.T2V
assert config.task_type == Task.I2V
def test_capability_discovery(monkeypatch):
from sglang.multimodal_gen import registry
from sglang.multimodal_gen.runtime.entrypoints import http_server
from sglang.multimodal_gen.runtime.entrypoints.openai import common_api
monkeypatch.setattr(registry, "get_model_info", lambda *a, **kw: None)
monkeypatch.setattr(common_api, "get_model_info", lambda *a, **kw: None)
args = make_args(MultiConfig())
expected = [task.name for task in MultiConfig.supported_task_types]
card = common_api._build_model_card(args, "test-model")
assert card.task_type == "T2V"
assert card.supported_task_types == expected
app = FastAPI()
app.state.server_args = args
app.include_router(http_server.health_router)
with TestClient(app) as client:
for endpoint in ("/models", "/model_info"):
response = client.get(endpoint)
assert response.status_code == 200, response.text
assert response.json()["supported_task_types"] == expected
info = client.get("/model_info").json()
assert info["task_type"] == "T2V"
assert info["output_types"] == ["IMAGE", "VIDEO"]
assert info["has_image_understanding"] is True
@@ -233,16 +233,20 @@ def server(monkeypatch, tmp_path):
assert app.state.prompt_enhancer.client.is_closed
@pytest.mark.parametrize("explicit_task", [False, True])
@pytest.mark.parametrize("enabled", [False, True])
@pytest.mark.parametrize("endpoint", ["image", "edit", "video_json", "video_form"])
def test_http_routes_enhance_once_before_sampling_and_preserve_options(
server, endpoint, enabled
server, endpoint, enabled, explicit_task
):
original = "A red teapot"
task = {"image": "t2i", "edit": "i2i"}.get(endpoint, "t2v")
task_fields = {"task_type": task} if explicit_task else {}
if endpoint == "image":
response = server.client.post(
"/v1/images/generations",
json={
**task_fields,
"prompt": original,
"enhance_prompt": enabled,
"seed": 123,
@@ -259,6 +263,7 @@ def test_http_routes_enhance_once_before_sampling_and_preserve_options(
response = server.client.post(
"/v1/images/edits",
data={
**task_fields,
"prompt": original,
"enhance_prompt": str(enabled).lower(),
"seed": "123",
@@ -271,6 +276,7 @@ def test_http_routes_enhance_once_before_sampling_and_preserve_options(
server.args.pipeline_config = WanT2V480PConfig()
server.model_info.sampling_param_cls = WanT2V_1_3B_SamplingParams
payload = {
**task_fields,
"prompt": original,
"seed": 123,
"negative_prompt": "blur",
@@ -307,6 +313,8 @@ def test_http_routes_enhance_once_before_sampling_and_preserve_options(
assert len(server.batches) == 1
batch = server.batches[0]
assert batch.prompt == expected
assert batch.task_type.name.lower() == task
assert server.args.pipeline_config.task_type.name.lower() == task
assert batch.seed == 123
assert batch.negative_prompt == "blur"
assert batch.num_outputs_per_prompt == 2
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
from types import SimpleNamespace
import pytest
from sglang.multimodal_gen import registry
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task
class MultiConfig(PipelineConfig):
supported_task_types = (Task.T2V, Task.I2V)
@pytest.mark.parametrize("prefix", ["", "custom"])
def test_request_task_and_capabilities_are_not_pipeline_overrides(prefix):
config = MultiConfig(task_type=Task.T2V)
stem = prefix + "." if prefix else ""
args = {
stem + "task_type": "i2v",
stem + "supported_task_types": (Task.T2I,),
stem + "dit_precision": "fp32",
}
config.update_config_from_dict(args, prefix)
assert config.task_type == Task.T2V
assert config.get_supported_task_types() == (Task.T2V, Task.I2V)
assert config.dit_precision == "fp32"
assert args[stem + "task_type"] == "i2v"
def test_config_construction_preserves_nested_default_and_request_task(monkeypatch):
monkeypatch.setattr(
registry,
"get_model_info",
lambda *a, **kw: SimpleNamespace(pipeline_config_cls=MultiConfig),
)
kwargs = {
"model_path": "test-model",
"task_type": "i2v",
"pipeline_config": {"task_type": Task.T2V, "supported_task_types": (Task.I2V,)},
}
config = PipelineConfig.from_kwargs(kwargs)
assert config.task_type == Task.T2V
assert config.get_supported_task_types() == (Task.T2V, Task.I2V)
assert kwargs["task_type"] == "i2v"
def test_request_cannot_expand_legacy_capabilities():
config = PipelineConfig(task_type=Task.T2V)
config.update_config_from_dict({"task_type": "i2v"})
assert config.get_supported_task_types() == (Task.T2V,)
with pytest.raises(ValueError, match="Unsupported task_type"):
config.resolve_task_type("i2v")