From 8b2ca8ecc2410f3ab170adff23fcd2db4cd47d2b Mon Sep 17 00:00:00 2001 From: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Date: Sun, 27 Sep 2026 19:00:45 -0700 Subject: [PATCH] [diffusion] feat: support multiple task types for pipelines (#38762) Co-authored-by: Mick Qian --- docs/docs/sglang-diffusion/api/openai_api.mdx | 32 ++ .../sglang-diffusion/support_new_models.mdx | 66 +++ .../configs/pipeline_configs/base.py | 175 +++++--- .../configs/pipeline_configs/cosmos3.py | 6 + .../multimodal_gen/configs/sample/action.py | 8 +- .../multimodal_gen/configs/sample/cosmos3.py | 10 + .../configs/sample/sampling_params.py | 63 ++- .../multimodal_gen/configs/task_type.py | 114 +++++ python/sglang/multimodal_gen/configs/utils.py | 11 +- .../runtime/entrypoints/http_server.py | 38 +- .../runtime/entrypoints/openai/common_api.py | 26 +- .../runtime/entrypoints/openai/image_api.py | 6 + .../runtime/entrypoints/openai/mesh_api.py | 2 + .../runtime/entrypoints/openai/protocol.py | 6 + .../runtime/entrypoints/openai/utils.py | 26 +- .../runtime/entrypoints/openai/video_api.py | 23 +- .../pipelines_core/stages/denoising.py | 8 +- .../pipelines_core/stages/input_validation.py | 33 +- .../runtime/warmup_request_builder.py | 1 + .../test/unit/test_cfg_parallel_warmup.py | 2 + .../test/unit/test_input_validation.py | 15 + .../unit/test_model_discovery_task_types.py | 54 +++ .../test/unit/test_multiple_task_types.py | 403 ++++++++++++++++++ .../test/unit/test_prompt_enhancement.py | 10 +- .../test/unit/test_task_type_config_args.py | 53 +++ 25 files changed, 1036 insertions(+), 155 deletions(-) create mode 100644 python/sglang/multimodal_gen/configs/task_type.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_model_discovery_task_types.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_task_type_config_args.py diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx index 54aa18a9ea..ba6d70d11c 100644 --- a/docs/docs/sglang-diffusion/api/openai_api.mdx +++ b/docs/docs/sglang-diffusion/api/openai_api.mdx @@ -75,6 +75,7 @@ curl -sS "http://localhost:30010/v1/models" "max_model_len": null, "num_gpus": 4, "task_type": "T2V", + "supported_task_types": ["T2V"], "dit_precision": "bf16", "vae_precision": "fp16", "pipeline_name": "WanPipeline", @@ -96,6 +97,37 @@ curl -sS "http://localhost:30010/v1/models/wan-t2v" ## Endpoints +### Select a request task + +`GET /v1/models` reports `task_type` (the configured default) and +`supported_task_types` (the tasks implemented by that configured pipeline). +`GET /model_info` also reports the supported `output_types`. A pipeline may +support image and video output on one server when its implementation and +loaded components provide both paths. + +Send the optional SGLang extension `task_type` as a case-insensitive enum name: + +| Interface | Task field | Output constraint | +| --- | --- | --- | +| `POST /v1/images/generations` JSON | `"task_type": "t2i"` | Image | +| `POST /v1/images/edits` multipart | `-F task_type=i2i` | Image | +| `POST /v1/videos` JSON or multipart | `"task_type": "i2v"` or `-F task_type=i2v` | Video | +| OpenAI Python client | `extra_body={"task_type": "t2i"}` | Selected endpoint | +| Native Python `generate` sampling parameters | `"task_type": "t2i"` | Selected task | +| `sglang generate` | `--task-type t2i` | Selected task | + +Choose only a task in the server's advertised set. An unsupported task, a task +with the wrong output type for the endpoint, or missing required conditioning +returns HTTP 400. For example, `i2v` needs image input, while `v2v` and `f2v` +need video input. Model-specific size, frame count, and reference constraints +still apply. + +If you omit `task_type`, the server uses the endpoint and supplied inputs to +choose a compatible task, preferring its default when compatible. It asks for +an explicit task if several alternatives remain (for example `v2v` versus +`f2v`). Pipelines with one task keep that task as their default. Sending a +request never changes the server default or subsequent requests. + ### Prompt enhancement Image generation, image edits and video creation accept `enhance_prompt=true` diff --git a/docs/docs/sglang-diffusion/support_new_models.mdx b/docs/docs/sglang-diffusion/support_new_models.mdx index 5733f9cd78..5528211f64 100644 --- a/docs/docs/sglang-diffusion/support_new_models.mdx +++ b/docs/docs/sglang-diffusion/support_new_models.mdx @@ -559,3 +559,69 @@ Common failure points: - [ ] Verified image or video quality against a reference output. - [ ] Completed the distributed and memory integration checks above. - [ ] Tested CFG parallelism and distributed serving paths when they apply. + + +## Declare multiple task types + +Keep `PipelineConfig.task_type` as the default task. To let one configured +pipeline serve several tasks, declare its capabilities as a class variable: + +```python +from dataclasses import dataclass +from typing import ClassVar + +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.task_type import ModelTaskType + + +@dataclass +class MultiTaskPipelineConfig(PipelineConfig): + task_type: ModelTaskType = ModelTaskType.T2V + supported_task_types: ClassVar[tuple[ModelTaskType, ...]] = ( + ModelTaskType.T2I, + ModelTaskType.I2I, + ModelTaskType.T2V, + ModelTaskType.I2V, + ) +``` + +The declaration must be nonempty, contain no duplicates, and include the +configured default. Existing pipelines that omit it retain the singleton +`(task_type,)`. Override `get_supported_task_types()` if capabilities depend +on the loaded checkpoint or configuration, preserving these invariants. +Flat CLI/Python `task_type` arguments select the request. To change the +configured default, use `pipeline_config={"task_type": ModelTaskType.I2V}` +(or its serialized configuration), choosing a supported task. + +Capabilities describe implemented execution paths; declaring a task does not +load additional components or implement conditioning and decoding for it. + +Requests carry `SamplingParams.task_type`, accepting an enum or its name +(case-insensitive). Admission resolves it to an enum and derives the output +`data_type`. Explicit tasks must belong to the supported set and match the +HTTP endpoint's output type. Without an explicit task, admission filters by +output type and conditioning inputs, prefers a compatible default, then +selects a unique compatible alternative. Python/CLI requests also prefer the +default's output type among alternatives. Ambiguous requests must specify +`task_type`. No request changes the shared configuration. + +Use the admitted `batch.task_type` in model stages when behavior depends on +the request. Shared stages use `get_request_task_type(batch, pipeline_config)` +to preserve compatibility with direct callers that have no resolved task. +Audit model-specific frame adjustment, reference loading, latent construction, +and decoding for assumptions about the server default. Declare the union of +components used by all supported tasks in stage residency metadata. Synthetic +warmup uses the default task; provide the pipeline's warmup hook when that task +needs model-specific reference inputs. Dynamic batching compares the admitted +task along with the other sampling fields. + +`I2I` and `I2V` require image input; `T2I` and `T2V` reject it. Existing `TI2I` +and `TI2V` retain optional image conditioning. `V2V` and `F2V` require video +input; `F2V` denotes video-prefix conditioning. Models own the details of +prefix length and temporal alignment. Pipelines with explicit capabilities +reject video input for tasks that do not accept it; legacy singleton pipelines +retain their model-owned video checks. + +See [request task selection](/docs/sglang-diffusion/api/openai_api#select-a-request-task) +for the HTTP interface. Test task selection, invalid inputs, discovery, +serialization, batching, and alternating tasks on one running server. diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py index 24e5fed0c5..a481249a7b 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py @@ -6,7 +6,7 @@ import math import os from collections.abc import Callable from dataclasses import asdict, dataclass, field, fields -from enum import Enum, auto +from enum import Enum from operator import attrgetter from typing import Any, ClassVar @@ -26,7 +26,7 @@ from sglang.multimodal_gen.configs.models.encoders.t5 import T5Config from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import ( ModelDeploymentConfig, ) -from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.configs.task_type import DataType, ModelTaskType from sglang.multimodal_gen.configs.utils import update_config_from_args from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy from sglang.multimodal_gen.runtime.distributed.communication_op import ( @@ -46,71 +46,6 @@ from sglang.multimodal_gen.runtime.utils.vision import get_default_height_width logger = init_logger(__name__) -# NOTE: possible duplication with DataType -# this may focus on the model's original ability -class ModelTaskType(Enum): - # TODO: check if I2V/TI2V models can work w/wo text - - I2V = auto() # Image to Video - T2V = auto() # Text to Video - TI2V = auto() # Text and Image to Video - - T2I = auto() # Text to Image - I2I = auto() # Image to Image - TI2I = auto() # Image to Image or Text-Image to Image - I2M = auto() # Image to Mesh - VLA_ACTION = auto() # Vision-language-action policy output - - def is_image_gen(self) -> bool: - return ( - self == ModelTaskType.T2I - or self == ModelTaskType.I2I - or self == ModelTaskType.TI2I - ) - - def is_action_gen(self) -> bool: - return self == ModelTaskType.VLA_ACTION - - def is_mesh_gen(self) -> bool: - return self == ModelTaskType.I2M - - def is_video_gen(self) -> bool: - return ( - self == ModelTaskType.I2V - or self == ModelTaskType.T2V - or self == ModelTaskType.TI2V - ) - - def is_visual_gen(self) -> bool: - return self.is_image_gen() or self.is_video_gen() - - def requires_image_input(self) -> bool: - return ( - self == ModelTaskType.I2V - or self == ModelTaskType.I2I - or self == ModelTaskType.I2M - ) - - def accepts_image_input(self) -> bool: - return ( - self == ModelTaskType.I2V - or self == ModelTaskType.I2I - or self == ModelTaskType.TI2I - or self == ModelTaskType.TI2V - or self == ModelTaskType.I2M - or self == ModelTaskType.VLA_ACTION - ) - - def data_type(self) -> DataType: - if self.is_action_gen(): - return DataType.ACTION - if self.is_mesh_gen(): - return DataType.MESH - if self.is_image_gen(): - return DataType.IMAGE - return DataType.VIDEO - - class STA_Mode(str, Enum): """STA (Sliding Tile Attention) modes.""" @@ -199,7 +134,9 @@ class PipelineConfig: """The base configuration class for a generation pipeline.""" native_only_components: ClassVar[tuple[str, ...]] = () + # Default task; supported_task_types is a model capability, not a user knob. task_type: ModelTaskType = ModelTaskType.I2I + supported_task_types: ClassVar[tuple[ModelTaskType, ...] | None] = None skip_input_image_preprocess: bool = False # False when changing component placement after a calibration request is # known to alter the pipeline's numerical path. @@ -284,6 +221,91 @@ class PipelineConfig: # return the model-specific config for optimal deployment setting return ModelDeploymentConfig() + def get_supported_task_types(self) -> tuple[ModelTaskType, ...]: + """Return the tasks implemented by this configured pipeline. + + Existing pipelines keep their single default task. Multi-task models + declare a tuple, or override this method for configuration-dependent + capabilities. Declaration alone does not add model execution paths. + """ + default = ModelTaskType.parse(self.task_type) + declared = self.supported_task_types + if declared is None: + return (default,) + tasks = tuple(ModelTaskType.parse(task) for task in declared) + if not tasks or len(set(tasks)) != len(tasks) or default not in tasks: + raise ValueError( + "supported_task_types must be nonempty, unique, and include " + f"the default task_type {default.name}" + ) + return tasks + + def resolve_task_type( + self, + requested: ModelTaskType | str | None = None, + *, + data_type: DataType | None = None, + has_image: bool = False, + has_video: bool = False, + ) -> ModelTaskType: + """Resolve a request without changing shared pipeline configuration. + + HTTP endpoints constrain the output type. Otherwise the default task + wins when its input contract fits. An ambiguous alternative requires + an explicit task_type; declaration order never selects a task. + """ + supported = self.get_supported_task_types() + if requested is not None: + task = ModelTaskType.parse(requested) + if task not in supported: + raise ValueError( + f"Unsupported task_type {task.name}; this pipeline supports " + f"{[task.name for task in supported]}" + ) + if data_type is not None and task.data_type() != data_type: + raise ValueError( + f"task_type {task.name} produces {task.data_type().name}, " + f"but this endpoint produces {data_type.name}" + ) + return task + + candidates = tuple( + task + for task in supported + if data_type is None or task.data_type() == data_type + ) + if not candidates: + raise ValueError( + f"This pipeline does not support {data_type.name} output; " + f"supported task types: {[task.name for task in supported]}" + ) + if len(candidates) == 1: + return candidates[0] + + compatible = tuple( + task + for task in candidates + if (not has_image or task.accepts_image_input()) + and (not task.requires_image_input() or has_image) + and (not has_video or task.accepts_video_input()) + and (not task.requires_video_input() or has_video) + ) + default = ModelTaskType.parse(self.task_type) + if default in compatible: + return default + if data_type is None: + same_output = tuple( + task for task in compatible if task.data_type() == default.data_type() + ) + compatible = same_output or compatible + if len(compatible) == 1: + return compatible[0] + raise ValueError( + "Specify task_type for this request; compatible tasks: " + f"{[task.name for task in compatible]}, " + f"supported tasks: {[task.name for task in candidates]}" + ) + def validate_server_args(self, server_args: Any) -> None: """Validate model-owned constraints after server args are normalized.""" @@ -292,7 +314,7 @@ class PipelineConfig: def supports_action_endpoint(self) -> bool: """Whether this pipeline exposes the generic action generation API.""" - return self.task_type.is_action_gen() + return any(task.is_action_gen() for task in self.get_supported_task_types()) def supports_openpi_endpoint(self) -> bool: """Whether this pipeline implements the OpenPI policy websocket.""" @@ -424,7 +446,10 @@ class PipelineConfig: The scheduler still checks each request before merging it into a batch. """ - return self.task_type in (ModelTaskType.T2I, ModelTaskType.T2V) + return all( + task in (ModelTaskType.T2I, ModelTaskType.T2V) + for task in self.get_supported_task_types() + ) def supports_dynamic_batching_for_request(self, batch) -> bool: """Return whether one request may participate in dynamic batching.""" @@ -960,7 +985,15 @@ class PipelineConfig: def update_config_from_dict(self, args: dict[str, Any], prefix: str = "") -> None: prefix_with_dot = f"{prefix}." if (prefix.strip() != "") else "" - update_config_from_args(self, args, prefix, pop_args=True) + # Flat task_type is a request field; configure a different default in + # the nested pipeline_config. Capabilities belong to the model class. + update_config_from_args( + self, + args, + prefix, + pop_args=True, + exclude=("task_type", "supported_task_types"), + ) update_config_from_args( self.vae_config, args, f"{prefix_with_dot}vae_config", pop_args=True ) @@ -1137,6 +1170,8 @@ class PipelineConfig: return pipeline_config def check_pipeline_config(self) -> None: + self.task_type = ModelTaskType.parse(self.task_type) + self.get_supported_task_types() if self.vae_sp and not self.vae_tiling: raise ValueError( "Currently enabling vae_sp requires enabling vae_tiling, please set --vae-tiling to True." diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py index fc692380a2..748ccfc26e 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py @@ -11,6 +11,7 @@ in the stages from ``num_frames`` and ``image_path``; T2I overrides import functools import os from dataclasses import dataclass, field +from typing import ClassVar from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig from sglang.multimodal_gen.configs.models.dits.cosmos3video import Cosmos3VideoConfig @@ -109,6 +110,11 @@ class Cosmos3Config(PipelineConfig): # TI2V (text + image → video) so the request validator accepts ``image_path`` # without requiring it. T2V ignores it; I2V uses it; T2I disregards it. task_type: ModelTaskType = ModelTaskType.TI2V + supported_task_types: ClassVar[tuple[ModelTaskType, ...]] = ( + ModelTaskType.TI2V, + ModelTaskType.T2I, + ModelTaskType.V2V, + ) dit_config: DiTConfig = field(default_factory=Cosmos3VideoConfig) diff --git a/python/sglang/multimodal_gen/configs/sample/action.py b/python/sglang/multimodal_gen/configs/sample/action.py index 24fb8914c0..c3f341516e 100644 --- a/python/sglang/multimodal_gen/configs/sample/action.py +++ b/python/sglang/multimodal_gen/configs/sample/action.py @@ -10,6 +10,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( DataType, _sanitize_filename, ) +from sglang.multimodal_gen.configs.task_type import ModelTaskType from sglang.multimodal_gen.configs.utils import expand_path_fields from sglang.multimodal_gen.runtime.utils.argparse import StoreBoolean @@ -22,6 +23,7 @@ class ActionSamplingParams: """Sampling parameters for policies that generate continuous actions.""" data_type: DataType = DataType.ACTION + task_type: ModelTaskType = ModelTaskType.VLA_ACTION request_id: str | None = field(default=None, metadata={"batch_sig_exclude": True}) prompt: str | list[str] | None = field( default="", metadata={"batch_sig_exclude": True} @@ -119,13 +121,15 @@ class ActionSamplingParams: ) def _validate_with_pipeline_config(self, pipeline_config): - if not pipeline_config.task_type.is_action_gen(): + task_type = pipeline_config.resolve_task_type(self.task_type) + if not task_type.is_action_gen(): raise ValueError( - f"ActionSamplingParams requires an ACTION pipeline, got {pipeline_config.task_type.name}" + f"ActionSamplingParams requires an ACTION pipeline, got {task_type.name}" ) def _adjust(self, server_args: "ServerArgs"): expand_path_fields(self) + self._validate_with_pipeline_config(server_args.pipeline_config) self.data_type = DataType.ACTION self.return_file_paths_only = False if self.output_path is None and server_args.output_path is not None: diff --git a/python/sglang/multimodal_gen/configs/sample/cosmos3.py b/python/sglang/multimodal_gen/configs/sample/cosmos3.py index 75512d53fd..a55b45044f 100644 --- a/python/sglang/multimodal_gen/configs/sample/cosmos3.py +++ b/python/sglang/multimodal_gen/configs/sample/cosmos3.py @@ -17,6 +17,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( DataType, SamplingParams, ) +from sglang.multimodal_gen.configs.task_type import ModelTaskType COSMOS3_DEFAULT_GUIDANCE_SCALE = 4.0 COSMOS3_EDGE_T2I_GUIDANCE_SCALE = 7.0 @@ -405,6 +406,15 @@ class Cosmos3SamplingParams(SamplingParams): return kwargs def _adjust(self, server_args) -> None: + if ( + self.task_type is None + and self.num_frames == 1 + and self.image_path is None + and self.video_path is None + and self.action_mode is None + ): + self.task_type = ModelTaskType.T2I + # adjust distil and edge args — read from the pre-computed config fields # so no checkpoint download happens at request time. pipeline_config = server_args.pipeline_config diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index f3e079bfd6..01dc5a9c99 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -13,10 +13,11 @@ import time import unicodedata import uuid from dataclasses import dataclass, field -from enum import Enum, auto +from enum import Enum from typing import TYPE_CHECKING, Any, ClassVar from sglang.multimodal_gen.configs.post_training import RLRolloutArgs +from sglang.multimodal_gen.configs.task_type import DataType, ModelTaskType from sglang.multimodal_gen.configs.utils import expand_path_fields from sglang.multimodal_gen.runtime.utils.argparse import StoreBoolean from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -189,22 +190,6 @@ def align_num_frames_for_num_gpus( return new_latent_num_frames -class DataType(Enum): - IMAGE = auto() - VIDEO = auto() - MESH = auto() - ACTION = auto() - - def get_default_extension(self) -> str: - if self == DataType.IMAGE: - return "png" - if self == DataType.VIDEO: - return "mp4" - if self == DataType.ACTION: - return "json" - return "glb" - - @dataclass class SamplingParams: """ @@ -222,6 +207,8 @@ class SamplingParams: """ data_type: DataType = DataType.VIDEO + # Included in the dynamic batching signature and retained by replace/pickle. + task_type: ModelTaskType | str | None = None request_id: str | None = field(default=None, metadata={"batch_sig_exclude": True}) @@ -732,13 +719,13 @@ class SamplingParams: """ check if the sampling params is compatible and valid with server_args """ - task_type = pipeline_config.task_type + task_type = self.resolve_task_type(pipeline_config) if task_type.is_action_gen(): return if task_type.requires_image_input(): # requires image input - if self.image_path is None: + if not self.image_path: raise ValueError( f"Served model with task type '{task_type.name}' requires an 'image_path' input, but none was provided" ) @@ -750,6 +737,34 @@ class SamplingParams: f"input_reference is not supported for {task_type.name} models." ) + if task_type.requires_video_input() and not self.video_path: + raise ValueError(f"Task {task_type.name} requires a 'video_path' input") + # Legacy pipelines retain their model-owned video conditioning checks. + # Explicit multi-task declarations use the standard per-task contract. + if ( + self.video_path + and ( + getattr(pipeline_config, "supported_task_types", None) is not None + or ( + hasattr(pipeline_config, "get_supported_task_types") + and len(pipeline_config.get_supported_task_types()) > 1 + ) + ) + and not task_type.accepts_video_input() + ): + raise ValueError(f"video_path is not supported for task {task_type.name}") + + def resolve_task_type(self, pipeline_config) -> ModelTaskType: + resolver = getattr(pipeline_config, "resolve_task_type", None) + if resolver is None: + # Compatibility with integrations that supply a lightweight config. + return ModelTaskType.parse(self.task_type or pipeline_config.task_type) + return resolver( + self.task_type, + has_image=bool(self.image_path), + has_video=bool(self.video_path), + ) + def _adjust( self, server_args, @@ -761,7 +776,8 @@ class SamplingParams: # TODO: SamplingParams should not rely on ServerArgs pipeline_config = server_args.pipeline_config - task_type = pipeline_config.task_type + task_type = self.resolve_task_type(pipeline_config) + self.task_type = task_type self.data_type = task_type.data_type() self._adjust_output_path(server_args) @@ -864,7 +880,7 @@ class SamplingParams: "Sequence dimension shard is enabled, disabling frame adjustment for better performance" ) - if pipeline_config.task_type.is_image_gen(): + if self.resolve_task_type(pipeline_config).is_image_gen(): # settle num_frames if not server_args.pipeline_config.allow_set_num_frames(): logger.debug("Setting `num_frames` to 1 for image generation model") @@ -1007,6 +1023,11 @@ class SamplingParams: return parser.add_argument(*name_or_flags, **kwargs) add_argument("--data-type", type=str, nargs="+") + add_argument( + "--task-type", + type=str, + help="Request task (for example t2i, i2i, t2v, i2v, v2v, f2v); must be supported by the pipeline.", + ) # Predict the shot length from the caption, overriding `--num-frames`. add_argument("--use-diffusion-decoder", action="store_true") add_argument("--auto-duration", action="store_true") diff --git a/python/sglang/multimodal_gen/configs/task_type.py b/python/sglang/multimodal_gen/configs/task_type.py new file mode 100644 index 0000000000..0c8c915fc8 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/task_type.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared task and output types for pipeline capabilities and requests.""" + +from enum import Enum, auto + + +class DataType(Enum): + IMAGE = auto() + VIDEO = auto() + MESH = auto() + ACTION = auto() + + def get_default_extension(self) -> str: + if self == DataType.IMAGE: + return "png" + if self == DataType.VIDEO: + return "mp4" + if self == DataType.ACTION: + return "json" + return "glb" + + +class ModelTaskType(Enum): + # TODO: check if I2V/TI2V models can work w/wo text + + I2V = auto() # Image to Video + T2V = auto() # Text to Video + TI2V = auto() # Text and Image to Video + + T2I = auto() # Text to Image + I2I = auto() # Image to Image + TI2I = auto() # Image to Image or Text-Image to Image + I2M = auto() # Image to Mesh + VLA_ACTION = auto() # Vision-language-action policy output + + V2V = auto() # Video to Video + F2V = auto() # Video-prefix-conditioned Video + + @classmethod + def parse(cls, value: "ModelTaskType | str") -> "ModelTaskType": + if isinstance(value, cls): + return value + if isinstance(value, str): + try: + return cls[value.strip().upper()] + except KeyError: + pass + raise ValueError( + f"Unknown task_type {value!r}; choose from {[task.name for task in cls]}" + ) + + def requires_video_input(self) -> bool: + return self in (ModelTaskType.V2V, ModelTaskType.F2V) + + def accepts_video_input(self) -> bool: + return self.requires_video_input() + + def is_image_gen(self) -> bool: + return ( + self == ModelTaskType.T2I + or self == ModelTaskType.I2I + or self == ModelTaskType.TI2I + ) + + def is_action_gen(self) -> bool: + return self == ModelTaskType.VLA_ACTION + + def is_mesh_gen(self) -> bool: + return self == ModelTaskType.I2M + + def is_video_gen(self) -> bool: + return ( + self == ModelTaskType.I2V + or self == ModelTaskType.T2V + or self == ModelTaskType.TI2V + or self == ModelTaskType.V2V + or self == ModelTaskType.F2V + ) + + def is_visual_gen(self) -> bool: + return self.is_image_gen() or self.is_video_gen() + + def requires_image_input(self) -> bool: + return ( + self == ModelTaskType.I2V + or self == ModelTaskType.I2I + or self == ModelTaskType.I2M + ) + + def accepts_image_input(self) -> bool: + return ( + self == ModelTaskType.I2V + or self == ModelTaskType.I2I + or self == ModelTaskType.TI2I + or self == ModelTaskType.TI2V + or self == ModelTaskType.I2M + or self == ModelTaskType.VLA_ACTION + ) + + def data_type(self) -> DataType: + if self.is_action_gen(): + return DataType.ACTION + if self.is_mesh_gen(): + return DataType.MESH + if self.is_image_gen(): + return DataType.IMAGE + return DataType.VIDEO + + +def get_request_task_type(batch, pipeline_config) -> ModelTaskType: + """Read the admitted task, falling back for legacy direct stage callers.""" + return ModelTaskType.parse( + getattr(batch, "task_type", None) or pipeline_config.task_type + ) diff --git a/python/sglang/multimodal_gen/configs/utils.py b/python/sglang/multimodal_gen/configs/utils.py index 53ecc5bc58..4388dccb7c 100644 --- a/python/sglang/multimodal_gen/configs/utils.py +++ b/python/sglang/multimodal_gen/configs/utils.py @@ -31,7 +31,12 @@ def expand_path_fields(obj) -> None: def update_config_from_args( - config: Any, args_dict: dict[str, Any], prefix: str = "", pop_args: bool = False + config: Any, + args_dict: dict[str, Any], + prefix: str = "", + pop_args: bool = False, + *, + exclude: tuple[str, ...] = (), ) -> bool: """ Update configuration object from arguments dictionary. @@ -50,7 +55,7 @@ def update_config_from_args( args_to_remove = [] if prefix.strip() == "": for key, value in args_dict.items(): - if hasattr(config, key) and value is not None: + if key not in exclude and hasattr(config, key) and value is not None: if key == "text_encoder_precisions" and isinstance(value, list): setattr(config, key, tuple(value)) else: @@ -63,6 +68,8 @@ def update_config_from_args( for key, value in args_dict.items(): if key.startswith(prefix_with_dot) and value is not None: attr_name = key[len(prefix_with_dot) :] + if attr_name in exclude: + continue if hasattr(config, attr_name): setattr(config, attr_name, value) if pop_args: diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py index a5953feeb1..d7ea394227 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py @@ -189,23 +189,28 @@ async def get_models(request: Request): Use /v1/models instead for OpenAI-compatible model discovery. This endpoint will be removed in a future version. """ - from sglang.multimodal_gen.registry import get_model_info + from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import ( + get_served_pipeline_class, + ) server_args: ServerArgs = request.app.state.server_args - model_info = get_model_info(server_args.model_path, model_id=server_args.model_id) + pipeline_cls = get_served_pipeline_class(server_args) response = { "model_path": server_args.model_path, "num_gpus": server_args.num_gpus, "task_type": server_args.pipeline_config.task_type.name, + "supported_task_types": [ + task.name for task in server_args.pipeline_config.get_supported_task_types() + ], "dit_precision": server_args.pipeline_config.dit_precision, "vae_precision": server_args.pipeline_config.vae_precision, "vae_decode_precision": server_args.pipeline_config.vae_decode_precision, } - if model_info: - response["pipeline_name"] = model_info.pipeline_cls.pipeline_name - response["pipeline_class"] = model_info.pipeline_cls.__name__ + if pipeline_cls: + response["pipeline_name"] = pipeline_cls.pipeline_name + response["pipeline_class"] = pipeline_cls.__name__ return response @@ -235,34 +240,35 @@ async def model_info_endpoint(request: Request): Returns fields compatible with the LLM engine's /model_info so that the model gateway can detect capabilities for diffusion workers. """ - from sglang.multimodal_gen.registry import get_model_info + from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import ( + get_served_pipeline_class, + ) server_args: ServerArgs = request.app.state.server_args task_type = server_args.pipeline_config.task_type + supported_tasks = server_args.pipeline_config.get_supported_task_types() try: - registry_info = get_model_info( - server_args.model_path, - backend=server_args.backend, - model_id=server_args.model_id, - ) + pipeline_cls = get_served_pipeline_class(server_args) except Exception: logger.warning("Failed to resolve model info from registry", exc_info=True) - registry_info = None + pipeline_cls = None return { # Fields consumed by the model gateway for worker discovery "model_path": server_args.model_path, "is_generation": True, "model_type": "diffusion", - "architectures": ( - [registry_info.pipeline_cls.__name__] if registry_info else None - ), + "architectures": [pipeline_cls.__name__] if pipeline_cls else None, # Fields matching the LLM engine's /model_info shape - "has_image_understanding": task_type.accepts_image_input(), + "has_image_understanding": any( + task.accepts_image_input() for task in supported_tasks + ), "has_audio_understanding": False, # Diffusion-specific fields "task_type": task_type.name, + "supported_task_types": [task.name for task in supported_tasks], + "output_types": sorted({task.data_type().name for task in supported_tasks}), "is_image_gen": task_type.is_image_gen(), } diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py index d3c9234a16..5a099de3f1 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py @@ -4,7 +4,7 @@ from typing import Any, List, Optional, Union from fastapi import APIRouter, Body, HTTPException from pydantic import BaseModel, Field -from sglang.multimodal_gen.registry import get_model_info +from sglang.multimodal_gen.registry import get_model_info, get_pipeline_class from sglang.multimodal_gen.runtime.entrypoints.control_requests import ( ListLorasReq, MergeLoraWeightsReq, @@ -41,31 +41,43 @@ class DiffusionModelCard(ModelCard): num_gpus: Optional[int] = None task_type: Optional[str] = None + supported_task_types: list[str] | None = None dit_precision: Optional[str] = None vae_precision: Optional[str] = None pipeline_name: Optional[str] = None pipeline_class: Optional[str] = None -def _build_model_card( - server_args: ServerArgs, served_model_name: str -) -> DiffusionModelCard: +def get_served_pipeline_class(server_args: ServerArgs): + """Resolve discovery metadata using the same explicit pipeline as serving.""" + if server_args.pipeline_class_name: + return get_pipeline_class(server_args.pipeline_class_name) model_info = get_model_info( server_args.model_path, backend=server_args.backend, model_id=server_args.model_id, ) + return model_info.pipeline_cls if model_info else None + + +def _build_model_card( + server_args: ServerArgs, served_model_name: str +) -> DiffusionModelCard: + pipeline_cls = get_served_pipeline_class(server_args) card_kwargs: dict[str, Any] = { "id": served_model_name, "root": served_model_name, "num_gpus": server_args.num_gpus, "task_type": server_args.pipeline_config.task_type.name, + "supported_task_types": [ + task.name for task in server_args.pipeline_config.get_supported_task_types() + ], "dit_precision": server_args.pipeline_config.dit_precision, "vae_precision": server_args.pipeline_config.vae_precision, } - if model_info: - card_kwargs["pipeline_name"] = model_info.pipeline_cls.pipeline_name - card_kwargs["pipeline_class"] = model_info.pipeline_cls.__name__ + if pipeline_cls: + card_kwargs["pipeline_name"] = pipeline_cls.pipeline_name + card_kwargs["pipeline_class"] = pipeline_cls.__name__ return DiffusionModelCard(**card_kwargs) diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py index 8f8673b406..c483f0104b 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py @@ -25,6 +25,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( SamplingParams, generate_request_id, ) +from sglang.multimodal_gen.configs.task_type import DataType from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import ( maybe_enhance_prompt, ) @@ -291,6 +292,8 @@ async def generations( sampling = build_sampling_params( request_id, prompt=prompt, + task_type=request.task_type, + request_data_type=DataType.IMAGE, size=request.size, width=request.width, height=request.height, @@ -419,6 +422,7 @@ async def edits( enhance_prompt: bool = Form(False), mask: Optional[UploadFile] = File(None), model: Optional[str] = Form(None), + task_type: Optional[str] = Form(None), n: Optional[int] = Form(1), response_format: Optional[str] = Form(None), size: Optional[str] = Form(None), @@ -489,6 +493,8 @@ async def edits( sampling = build_sampling_params( request_id, prompt=prompt, + task_type=task_type, + request_data_type=DataType.IMAGE, size=size, num_outputs_per_prompt=max(1, min(int(n or 1), 10)), output_file_name=f"{request_id}.{ext}", diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/mesh_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/mesh_api.py index 81284878cf..be917951d7 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/mesh_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/mesh_api.py @@ -19,6 +19,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( SamplingParams, generate_request_id, ) +from sglang.multimodal_gen.configs.task_type import ModelTaskType from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( MeshGenerationsRequest, MeshListResponse, @@ -54,6 +55,7 @@ def _build_sampling_params_from_request( server_args = get_global_server_args() sampling_kwargs: Dict[str, Any] = { "request_id": request_id, + "task_type": ModelTaskType.I2M, "prompt": req.prompt, "num_frames": 1, "image_path": [image_path] if image_path else None, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py index bec145de03..f421a3790f 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py @@ -46,6 +46,9 @@ class ImageGenerationsRequest(BaseModel): model_config = ConfigDict(extra="allow") prompt: str + task_type: Optional[str] = Field( + default=None, description="Requested task from the server supported_task_types." + ) enhance_prompt: bool = False model: Optional[str] = None n: Optional[int] = 1 @@ -119,6 +122,9 @@ class VideoGenerationsRequest(BaseModel): reference_url: Optional[str] = None video_path: Optional[str] = None video_url: Optional[str] = None + task_type: Optional[str] = Field( + default=None, description="Requested task from the server supported_task_types." + ) model: Optional[str] = None n: Optional[int] = 1 num_outputs_per_prompt: Optional[int] = None diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py index f5c966d804..fb75a79ec1 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py @@ -214,6 +214,17 @@ def build_sampling_params(request_id: str, **kwargs) -> SamplingParams: # pop HTTP-layer params that aren't SamplingParams fields output_quality = kwargs.pop("output_quality", None) + request_data_type = kwargs.pop("request_data_type", None) + if request_data_type is not None: + try: + kwargs["task_type"] = server_args.pipeline_config.resolve_task_type( + kwargs.get("task_type"), + data_type=request_data_type, + has_image=bool(kwargs.get("image_path")), + has_video=bool(kwargs.get("video_path")), + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc has_explicit_compression = kwargs.get("output_compression") is not None @@ -240,12 +251,15 @@ def build_sampling_params(request_id: str, **kwargs) -> SamplingParams: kwargs = {k: v for k, v in kwargs.items() if v is not None} kwargs.setdefault("save_output", True) - sampling_params = SamplingParams.from_user_sampling_params_args( - model_path=server_args.model_path, - server_args=server_args, - request_id=request_id, - **kwargs, - ) + try: + sampling_params = SamplingParams.from_user_sampling_params_args( + model_path=server_args.model_path, + server_args=server_args, + request_id=request_id, + **kwargs, + ) + except (ValueError, TypeError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc # resolve output_quality → output_compression with the correct data_type. # SamplingParams.__post_init__ may have resolved with the wrong data_type diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index ace350df26..a4380e1606 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -270,6 +270,8 @@ def _build_video_sampling_params(request_id: str, request: VideoGenerationsReque kwargs = { "prompt": request.prompt, + "task_type": request.task_type, + "request_data_type": DataType.VIDEO, "num_outputs_per_prompt": max(1, min(int(num_outputs), 10)), "size": request.size, "width": request.width, @@ -475,6 +477,7 @@ async def create_video( video_url: Optional[str] = Form(None), video_path: Optional[str] = Form(None), model: Optional[str] = Form(None), + task_type: Optional[str] = Form(None), n: Optional[int] = Form(1), num_outputs_per_prompt: Optional[int] = Form(None), seconds: Optional[int] = Form(None), @@ -509,7 +512,6 @@ async def create_video( request_id = generate_request_id() server_args = get_global_server_args() - task_type = server_args.pipeline_config.task_type is_multipart = "multipart/form-data" in content_type raw_form: Any = None extra_from_form: Dict[str, Any] = {} @@ -567,12 +569,6 @@ async def create_video( video_input_path = reference_url image_sources = merge_image_input_list(input_reference) - # Validate image input based on model task type - if task_type.requires_image_input() and not image_sources: - raise HTTPException( - status_code=400, - detail="input_reference or reference_url is required for image-to-video generation", - ) input_path = None if image_sources: try: @@ -616,6 +612,7 @@ async def create_video( video_path=form_value("video_path", video_input_path), video_url=form_value("video_url", video_url), model=form_value("model", model), + task_type=form_text_value("task_type", task_type), n=form_value("n", n), num_outputs_per_prompt=form_value( "num_outputs_per_prompt", num_outputs_per_prompt @@ -688,18 +685,6 @@ async def create_video( if _is_probably_video_source(payload.get("input_reference")): payload.setdefault("video_path", payload.get("input_reference")) - has_image_input = ( - payload.get("reference_url") - and not _is_probably_video_source(payload.get("reference_url")) - ) or ( - payload.get("input_reference") - and not _is_probably_video_source(payload.get("input_reference")) - ) - if task_type.requires_image_input() and not has_image_input: - raise HTTPException( - status_code=400, - detail="input_reference or reference_url is required for image-to-video generation", - ) # for non-multipart/form-data type if payload.get("reference_url") and not _is_probably_video_source( payload.get("reference_url") diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index c48fd8875f..bd6a0886ce 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -59,6 +59,7 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( quality_allows_kernel_fusions, resolve_skip_softmax_params, ) +from sglang.multimodal_gen.configs.task_type import get_request_task_type from sglang.multimodal_gen.runtime.breakable_cuda_graph import ( prompt_padding as bcg_utils, ) @@ -1642,9 +1643,10 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): # 1. Prepare latent inputs in the model's compute dtype. latent_model_input = ctx.latents.to(ctx.target_dtype) if batch.image_latent is not None: - assert not server_args.pipeline_config.task_type == ModelTaskType.TI2V, ( - "image latents should not be provided for TI2V task" - ) + assert ( + get_request_task_type(batch, server_args.pipeline_config) + != ModelTaskType.TI2V + ), "image latents should not be provided for TI2V task" latent_model_input = torch.cat( [latent_model_input, batch.image_latent], dim=1 ).to(ctx.target_dtype) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py index 955cdeded9..0fc6ddc383 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py @@ -15,6 +15,7 @@ from PIL import Image from sglang.multimodal_gen.configs.pipeline_configs import WanI2V480PConfig from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType from sglang.multimodal_gen.configs.pipeline_configs.mova import MOVAPipelineConfig +from sglang.multimodal_gen.configs.task_type import get_request_task_type from sglang.multimodal_gen.runtime.pipelines_core.request_utils import ( expand_request_outputs, ) @@ -192,8 +193,10 @@ class InputValidationStage(PipelineStage): NOTE: condition image resizing is only allowed in InputValidationStage """ if batch.condition_image is not None and ( - server_args.pipeline_config.task_type == ModelTaskType.I2I - or server_args.pipeline_config.task_type == ModelTaskType.TI2I + get_request_task_type(batch, server_args.pipeline_config) + == ModelTaskType.I2I + or get_request_task_type(batch, server_args.pipeline_config) + == ModelTaskType.TI2I ): # calculate new condition image size if not isinstance(batch.condition_image, list): @@ -235,7 +238,10 @@ class InputValidationStage(PipelineStage): batch.width = width batch.height = height - elif server_args.pipeline_config.task_type == ModelTaskType.TI2V: + elif ( + get_request_task_type(batch, server_args.pipeline_config) + == ModelTaskType.TI2V + ): if server_args.pipeline_config.skip_input_image_preprocess: return # duplicate with vae_image_processor @@ -360,7 +366,8 @@ class InputValidationStage(PipelineStage): self._generate_seeds(batch, server_args) if ( - server_args.pipeline_config.task_type == ModelTaskType.I2M + get_request_task_type(batch, server_args.pipeline_config) + == ModelTaskType.I2M and batch.num_inference_steps is None and hasattr(server_args.pipeline_config, "shape_num_inference_steps") ): @@ -370,7 +377,8 @@ class InputValidationStage(PipelineStage): # Ensure prompt is properly formatted (I2M can be image-only) if ( - server_args.pipeline_config.task_type != ModelTaskType.I2M + get_request_task_type(batch, server_args.pipeline_config) + != ModelTaskType.I2M and batch.prompt is None and batch.prompt_embeds is None ): @@ -456,7 +464,10 @@ class InputValidationStage(PipelineStage): ) batch.original_condition_image_size = image.size - if server_args.pipeline_config.task_type != ModelTaskType.I2M: + if ( + get_request_task_type(batch, server_args.pipeline_config) + != ModelTaskType.I2M + ): self.preprocess_condition_image( batch, server_args, condition_image_width, condition_image_height ) @@ -492,7 +503,10 @@ class InputValidationStage(PipelineStage): result.add_check( "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int ) - if server_args.pipeline_config.task_type != ModelTaskType.I2M: + if ( + get_request_task_type(batch, server_args.pipeline_config) + != ModelTaskType.I2M + ): result.add_check( "prompt_or_embeds", None, @@ -502,7 +516,10 @@ class InputValidationStage(PipelineStage): ), ) - if server_args.pipeline_config.task_type != ModelTaskType.I2M: + if ( + get_request_task_type(batch, server_args.pipeline_config) + != ModelTaskType.I2M + ): result.add_check( "num_inference_steps", batch.num_inference_steps, V.positive_int ) diff --git a/python/sglang/multimodal_gen/runtime/warmup_request_builder.py b/python/sglang/multimodal_gen/runtime/warmup_request_builder.py index c060a9493e..4076f4755a 100644 --- a/python/sglang/multimodal_gen/runtime/warmup_request_builder.py +++ b/python/sglang/multimodal_gen/runtime/warmup_request_builder.py @@ -614,6 +614,7 @@ def build_warmup_reqs( include_warmup_image = should_include_warmup_image(server_args, server_based_warmup) for width, height, num_frames, is_probe in shapes: req_kwargs = dict( + task_type=task_type, data_type=task_type.data_type(), width=width, height=height, diff --git a/python/sglang/multimodal_gen/test/unit/test_cfg_parallel_warmup.py b/python/sglang/multimodal_gen/test/unit/test_cfg_parallel_warmup.py index 717176a907..d5d59e87e7 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cfg_parallel_warmup.py +++ b/python/sglang/multimodal_gen/test/unit/test_cfg_parallel_warmup.py @@ -871,6 +871,8 @@ class TestWarmupReqCfgParallel(unittest.TestCase): server_based_expected = { ModelTaskType.T2I: False, ModelTaskType.T2V: False, + ModelTaskType.V2V: False, + ModelTaskType.F2V: False, ModelTaskType.TI2I: True, ModelTaskType.TI2V: True, ModelTaskType.I2I: True, diff --git a/python/sglang/multimodal_gen/test/unit/test_input_validation.py b/python/sglang/multimodal_gen/test/unit/test_input_validation.py index b97cc06dd0..04a749cf16 100644 --- a/python/sglang/multimodal_gen/test/unit/test_input_validation.py +++ b/python/sglang/multimodal_gen/test/unit/test_input_validation.py @@ -264,5 +264,20 @@ class TestFlux2TI2ISizeResolution(unittest.TestCase): self.assertEqual((batch.width, batch.height), (768, 512)) +class TestPerRequestTask(unittest.TestCase): + def test_image_preprocessing_uses_request_on_video_default(self): + config = _DummyTI2IConfig() + config.task_type = ModelTaskType.T2V + image = Image.new("RGB", (1255, 833)) + batch = _make_batch(image) + batch.task_type = ModelTaskType.I2I + InputValidationStage().preprocess_condition_image( + batch, _make_server_args(config), image.width, image.height + ) + self.assertEqual((batch.width, batch.height), (1248, 832)) + self.assertIsInstance(batch.condition_image, list) + self.assertEqual(config.task_type, ModelTaskType.T2V) + + if __name__ == "__main__": unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_model_discovery_task_types.py b/python/sglang/multimodal_gen/test/unit/test_model_discovery_task_types.py new file mode 100644 index 0000000000..e5590a1630 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_model_discovery_task_types.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +from types import SimpleNamespace + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.task_type import ModelTaskType +from sglang.multimodal_gen.runtime.entrypoints import http_server +from sglang.multimodal_gen.runtime.entrypoints.openai import common_api + + +def test_explicit_pipeline_discovery_does_not_require_diffusers_index(monkeypatch): + class CustomPipeline: + pipeline_name = "CustomPipeline" + + class MultiConfig(PipelineConfig): + supported_task_types = (ModelTaskType.T2V, ModelTaskType.I2I) + + args = SimpleNamespace( + model_path="/models/custom/dit.safetensors", + pipeline_class_name="CustomPipeline", + pipeline_config=MultiConfig(task_type=ModelTaskType.T2V), + num_gpus=1, + served_model_name="custom", + backend="sglang", + model_id=None, + ) + + def unexpected_lookup(*args, **kwargs): + raise AssertionError( + "Explicit pipeline must bypass Diffusers model_index lookup" + ) + + monkeypatch.setattr(common_api, "get_model_info", unexpected_lookup) + monkeypatch.setattr(common_api, "get_pipeline_class", lambda name: CustomPipeline) + monkeypatch.setattr(common_api, "get_global_server_args", lambda: args) + app = FastAPI() + app.state.server_args = args + app.include_router(http_server.health_router) + app.include_router(common_api.router) + with TestClient(app) as client: + for endpoint in ("/models", "/v1/models", "/model_info"): + response = client.get(endpoint) + assert response.status_code == 200, response.text + data = response.json() + if endpoint == "/v1/models": + data = data["data"][0] + assert data["task_type"] == "T2V" + assert data["supported_task_types"] == ["T2V", "I2I"] + info = client.get("/model_info").json() + assert info["architectures"] == ["CustomPipeline"] + assert info["output_types"] == ["IMAGE", "VIDEO"] + assert info["has_image_understanding"] is True diff --git a/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py b/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py new file mode 100644 index 0000000000..e2ab75baca --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py @@ -0,0 +1,403 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Capability selection and real HTTP parsing, with model execution stubbed.""" + +import argparse +import io +import pickle +from dataclasses import dataclass, replace +from types import SimpleNamespace +from typing import ClassVar + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from PIL import Image + +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config +from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.task_type import DataType +from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task +from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, utils, video_api +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req + + +@dataclass +class MultiConfig(PipelineConfig): + task_type: Task = Task.T2V + supported_task_types: ClassVar[tuple[Task, ...]] = ( + Task.T2I, + Task.I2I, + Task.T2V, + Task.I2V, + Task.V2V, + Task.F2V, + ) + + +def make_args(config, output=None): + return SimpleNamespace( + pipeline_config=config, + model_path="test-model", + backend="sglang", + model_id=None, + pipeline_class_name=None, + output_path=output, + input_save_path=None, + served_model_name="test-model", + num_gpus=1, + comfyui_mode=False, + attention_backend_config={}, + enable_trace=False, + enable_torch_compile=False, + enable_breakable_cuda_graph=False, + enable_cfg_parallel=False, + warmup_steps=1, + ) + + +def test_legacy_enums_and_capabilities(): + from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType + from sglang.multimodal_gen.configs.sample.sampling_params import ( + DataType as OldDataType, + ) + + assert ModelTaskType is Task + assert OldDataType is DataType + for task in Task: + config = PipelineConfig(task_type=task) + assert config.get_supported_task_types() == (task,) + assert config.resolve_task_type() == task + assert config.resolve_task_type(task.name.lower()) == task + + +@pytest.mark.parametrize("declared", [(), (Task.T2I,), (Task.T2V, Task.T2V)]) +def test_invalid_capability_declarations(declared): + class Invalid(MultiConfig): + supported_task_types = declared + + with pytest.raises(ValueError, match="supported_task_types"): + Invalid().get_supported_task_types() + + +def test_selection_and_ambiguity(): + config = MultiConfig() + assert config.resolve_task_type() == Task.T2V + assert config.resolve_task_type(data_type=DataType.IMAGE) == Task.T2I + assert ( + config.resolve_task_type(data_type=DataType.IMAGE, has_image=True) == Task.I2I + ) + assert config.resolve_task_type(has_image=True) == Task.I2V + with pytest.raises(ValueError, match="Specify task_type"): + config.resolve_task_type(has_video=True) + assert config.resolve_task_type("f2v", has_video=True) == Task.F2V + assert config.task_type == Task.T2V + with pytest.raises(ValueError, match="endpoint produces IMAGE"): + config.resolve_task_type("t2v", data_type=DataType.IMAGE) + with pytest.raises(ValueError, match="Unsupported task_type"): + config.resolve_task_type("i2m") + with pytest.raises(ValueError, match="Unknown task_type"): + config.resolve_task_type("imaginary") + + +@pytest.mark.parametrize( + "task,image,video,output", + [ + ("t2i", None, None, DataType.IMAGE), + ("i2i", "reference.png", None, DataType.IMAGE), + ("t2v", None, None, DataType.VIDEO), + ("i2v", "reference.png", None, DataType.VIDEO), + ("v2v", None, "reference.mp4", DataType.VIDEO), + ("f2v", None, "reference.mp4", DataType.VIDEO), + ], +) +def test_preparation_copy_transport_and_extension(task, image, video, output, tmp_path): + config = MultiConfig() + sp = SamplingParams( + prompt="test", + task_type=task, + image_path=image, + video_path=video, + num_frames=25, + output_file_name="result", + adjust_frames=False, + ) + sp._adjust(make_args(config, str(tmp_path))) + sp._validate_with_pipeline_config(config) + sp = pickle.loads(pickle.dumps(replace(sp, prompt="expanded prompt"))) + assert sp.task_type == Task.parse(task) + assert sp.data_type == output + assert sp.output_file_path().endswith(output.get_default_extension()) + assert sp.num_frames == (1 if output == DataType.IMAGE else 25) + assert config.task_type == Task.T2V + + +@pytest.mark.parametrize( + "task,image,video", + [ + ("i2i", None, None), + ("i2v", None, None), + ("v2v", None, None), + ("f2v", None, None), + ("t2i", "a.png", None), + ("t2v", None, "a.mp4"), + ("v2v", "a.png", "a.mp4"), + ], +) +def test_task_input_contracts(task, image, video): + sp = SamplingParams(task_type=task, image_path=image, video_path=video) + with pytest.raises(ValueError): + sp._validate_with_pipeline_config(MultiConfig()) + + +def test_cli_accepts_standard_task_type(): + parser = argparse.ArgumentParser() + SamplingParams.add_cli_args(parser) + assert parser.parse_args(["--task-type", "i2v"]).task_type == "i2v" + + +def test_batch_signature_separates_tasks(): + from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler + + scheduler = Scheduler.__new__(Scheduler) + scheduler.server_args = make_args(MultiConfig()) + a = Req(sampling_params=SamplingParams(task_type=Task.V2V, video_path="same.mp4")) + b = Req(sampling_params=SamplingParams(task_type=Task.F2V, video_path="same.mp4")) + assert scheduler._build_dynamic_batch_signature( + a + ) != scheduler._build_dynamic_batch_signature(b) + + +def test_multi_task_warmup_uses_default_without_optional_image(monkeypatch): + from sglang.multimodal_gen.runtime import warmup_request_builder as warmup + + monkeypatch.setattr( + warmup, "get_model_sampling_defaults", lambda _: SamplingParams() + ) + reqs = warmup.build_warmup_reqs( + make_args(MultiConfig()), + warmup_resolutions=["512x512"], + server_based_warmup=True, + ) + assert len(reqs) == 1 + assert reqs[0].task_type == Task.T2V + assert reqs[0].data_type == DataType.VIDEO + assert reqs[0].image_path is None + + +@pytest.fixture(params=[(MultiConfig, SamplingParams)]) +def http_client(monkeypatch, tmp_path, request): + config_cls, sampling_cls = request.param + config = config_cls() + args = make_args(config, str(tmp_path)) + for module in (image_api, video_api, utils): + monkeypatch.setattr(module, "get_global_server_args", lambda: args) + for module in (image_api, video_api): + monkeypatch.setattr( + module, "resolve_sampling_params_cls", lambda _: sampling_cls + ) + monkeypatch.setattr( + SamplingParams, "from_pretrained", classmethod(lambda cls, *a, **kw: cls()) + ) + admitted = [] + + async def generate(client, req, **kwargs): + admitted.append(req) + path = req.output_file_path() + Image.new("RGB", (64, 64)).save(path) + return [path], OutputBatch() + + async def upload(paths): + return [None] * len(paths) + + async def complete_dispatch(): + pass + + def dispatch(job_id, req, **kwargs): + # capture admission before the background coroutine can race the response + admitted.append(req) + return complete_dispatch() + + monkeypatch.setattr(image_api, "process_generation_batch", generate) + monkeypatch.setattr(image_api, "_upload_and_cleanup_images", upload) + monkeypatch.setattr(video_api, "_dispatch_job_async", dispatch) + app = FastAPI() + app.include_router(image_api.router) + app.include_router(video_api.router) + with TestClient(app) as client: + yield client, admitted, config + + +def png_bytes(): + buf = io.BytesIO() + Image.new("RGB", (64, 64)).save(buf, format="PNG") + return buf.getvalue() + + +@pytest.mark.parametrize( + "http_client", [(Cosmos3Config, Cosmos3SamplingParams)], indirect=True +) +@pytest.mark.parametrize("task", [None, "t2i"]) +def test_cosmos3_image_endpoint_selects_image_task(http_client, task): + client, admitted, config = http_client + response = client.post( + "/v1/images/generations", + json={"prompt": "text", "size": "64x64", "task_type": task}, + ) + assert response.status_code == 200, response.text + assert admitted[-1].task_type == Task.T2I + assert admitted[-1].data_type == DataType.IMAGE + assert admitted[-1].num_frames == 1 + assert config.task_type == Task.TI2V + + +@pytest.mark.parametrize( + "http_client", [(Cosmos3Config, Cosmos3SamplingParams)], indirect=True +) +@pytest.mark.parametrize( + "conditioning,task", + [ + ({}, Task.TI2V), + ({"input_reference": "/tmp/image.png"}, Task.TI2V), + ({"video_path": "/tmp/clip.mp4"}, Task.V2V), + ], +) +def test_cosmos3_video_endpoint_preserves_conditioning(http_client, conditioning, task): + client, admitted, config = http_client + response = client.post( + "/v1/videos", json={"prompt": "video", "size": "64x64", **conditioning} + ) + assert response.status_code == 200, response.text + assert admitted[-1].task_type == task + assert admitted[-1].data_type == DataType.VIDEO + assert admitted[-1].image_path == conditioning.get("input_reference") + assert admitted[-1].video_path == conditioning.get("video_path") + assert config.task_type == Task.TI2V + + +@pytest.mark.parametrize( + "frames,conditioning,task", + [ + (1, {}, Task.T2I), + (81, {}, Task.TI2V), + (81, {"image_path": "/tmp/image.png"}, Task.TI2V), + (81, {"video_path": "/tmp/clip.mp4"}, Task.V2V), + ], +) +def test_cosmos3_sampling_resolves_existing_modes(frames, conditioning, task, tmp_path): + config = Cosmos3Config() + sp = Cosmos3SamplingParams(prompt="test", num_frames=frames, **conditioning) + sp._adjust(make_args(config, str(tmp_path))) + sp._validate_with_pipeline_config(config) + assert sp.task_type == task + assert sp.data_type == task.data_type() + assert sp.output_file_path().endswith(sp.data_type.get_default_extension()) + assert config.task_type == Task.TI2V + + +def test_image_json_and_multipart_choose_tasks_on_video_default(http_client): + client, admitted, config = http_client + response = client.post( + "/v1/images/generations", + json={ + "prompt": "text", + "response_format": "b64_json", + "size": "64x64", + "task_type": "t2i", + }, + ) + assert response.status_code == 200, response.text + assert admitted[-1].task_type == Task.T2I + for mode in ({}, {"task_type": "i2i"}): + response = client.post( + "/v1/images/edits", + data={ + "prompt": "edit", + "size": "64x64", + "response_format": "b64_json", + **mode, + }, + files={"image": ("image.png", png_bytes(), "image/png")}, + ) + assert response.status_code == 200, response.text + assert admitted[-1].task_type == Task.I2I + assert config.task_type == Task.T2V + + +@pytest.mark.parametrize("multipart", [False, True]) +def test_video_transports_keep_selected_task(http_client, multipart): + client, admitted, config = http_client + payload = { + "prompt": "video", + "task_type": "f2v", + "size": "64x64", + "video_path": "/tmp/clip.mp4", + } + if multipart: + response = client.post( + "/v1/videos", files={k: (None, v) for k, v in payload.items()} + ) + else: + response = client.post("/v1/videos", json=payload) + assert response.status_code == 200, response.text + assert admitted[-1].task_type == Task.F2V + assert admitted[-1].data_type == DataType.VIDEO + assert config.task_type == Task.T2V + + +def test_http_rejects_invalid_tasks_before_model_execution(http_client): + client, admitted, _ = http_client + for endpoint, payload in [ + ("images/generations", {"task_type": "t2v"}), + ("images/generations", {"task_type": "i2i"}), + ("videos", {"task_type": "t2i"}), + ("videos", {"task_type": "v2v"}), + ("videos", {"task_type": "t2v", "video_path": "/tmp/clip.mp4"}), + ("videos", {"task_type": "unknown"}), + ]: + response = client.post("/v1/" + endpoint, json={"prompt": "invalid", **payload}) + assert response.status_code == 400, response.text + assert admitted == [] + + +@pytest.mark.parametrize("multipart", [False, True]) +def test_unconditioned_request_on_image_conditioned_default(http_client, multipart): + client, admitted, config = http_client + config.task_type = Task.I2V + payload = {"prompt": "video", "task_type": "t2v", "size": "64x64"} + if multipart: + response = client.post( + "/v1/videos", files={k: (None, v) for k, v in payload.items()} + ) + else: + response = client.post("/v1/videos", json=payload) + assert response.status_code == 200, response.text + assert admitted[-1].task_type == Task.T2V + assert config.task_type == Task.I2V + + +def test_capability_discovery(monkeypatch): + from sglang.multimodal_gen import registry + from sglang.multimodal_gen.runtime.entrypoints import http_server + from sglang.multimodal_gen.runtime.entrypoints.openai import common_api + + monkeypatch.setattr(registry, "get_model_info", lambda *a, **kw: None) + monkeypatch.setattr(common_api, "get_model_info", lambda *a, **kw: None) + args = make_args(MultiConfig()) + expected = [task.name for task in MultiConfig.supported_task_types] + card = common_api._build_model_card(args, "test-model") + assert card.task_type == "T2V" + assert card.supported_task_types == expected + app = FastAPI() + app.state.server_args = args + app.include_router(http_server.health_router) + with TestClient(app) as client: + for endpoint in ("/models", "/model_info"): + response = client.get(endpoint) + assert response.status_code == 200, response.text + assert response.json()["supported_task_types"] == expected + info = client.get("/model_info").json() + assert info["task_type"] == "T2V" + assert info["output_types"] == ["IMAGE", "VIDEO"] + assert info["has_image_understanding"] is True diff --git a/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py b/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py index a882f0dc5f..b9ad360d61 100644 --- a/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py +++ b/python/sglang/multimodal_gen/test/unit/test_prompt_enhancement.py @@ -233,16 +233,20 @@ def server(monkeypatch, tmp_path): assert app.state.prompt_enhancer.client.is_closed +@pytest.mark.parametrize("explicit_task", [False, True]) @pytest.mark.parametrize("enabled", [False, True]) @pytest.mark.parametrize("endpoint", ["image", "edit", "video_json", "video_form"]) def test_http_routes_enhance_once_before_sampling_and_preserve_options( - server, endpoint, enabled + server, endpoint, enabled, explicit_task ): original = "A red teapot" + task = {"image": "t2i", "edit": "i2i"}.get(endpoint, "t2v") + task_fields = {"task_type": task} if explicit_task else {} if endpoint == "image": response = server.client.post( "/v1/images/generations", json={ + **task_fields, "prompt": original, "enhance_prompt": enabled, "seed": 123, @@ -259,6 +263,7 @@ def test_http_routes_enhance_once_before_sampling_and_preserve_options( response = server.client.post( "/v1/images/edits", data={ + **task_fields, "prompt": original, "enhance_prompt": str(enabled).lower(), "seed": "123", @@ -271,6 +276,7 @@ def test_http_routes_enhance_once_before_sampling_and_preserve_options( server.args.pipeline_config = WanT2V480PConfig() server.model_info.sampling_param_cls = WanT2V_1_3B_SamplingParams payload = { + **task_fields, "prompt": original, "seed": 123, "negative_prompt": "blur", @@ -307,6 +313,8 @@ def test_http_routes_enhance_once_before_sampling_and_preserve_options( assert len(server.batches) == 1 batch = server.batches[0] assert batch.prompt == expected + assert batch.task_type.name.lower() == task + assert server.args.pipeline_config.task_type.name.lower() == task assert batch.seed == 123 assert batch.negative_prompt == "blur" assert batch.num_outputs_per_prompt == 2 diff --git a/python/sglang/multimodal_gen/test/unit/test_task_type_config_args.py b/python/sglang/multimodal_gen/test/unit/test_task_type_config_args.py new file mode 100644 index 0000000000..481c2d8bc6 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_task_type_config_args.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +from types import SimpleNamespace + +import pytest + +from sglang.multimodal_gen import registry +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task + + +class MultiConfig(PipelineConfig): + supported_task_types = (Task.T2V, Task.I2V) + + +@pytest.mark.parametrize("prefix", ["", "custom"]) +def test_request_task_and_capabilities_are_not_pipeline_overrides(prefix): + config = MultiConfig(task_type=Task.T2V) + stem = prefix + "." if prefix else "" + args = { + stem + "task_type": "i2v", + stem + "supported_task_types": (Task.T2I,), + stem + "dit_precision": "fp32", + } + config.update_config_from_dict(args, prefix) + assert config.task_type == Task.T2V + assert config.get_supported_task_types() == (Task.T2V, Task.I2V) + assert config.dit_precision == "fp32" + assert args[stem + "task_type"] == "i2v" + + +def test_config_construction_preserves_nested_default_and_request_task(monkeypatch): + monkeypatch.setattr( + registry, + "get_model_info", + lambda *a, **kw: SimpleNamespace(pipeline_config_cls=MultiConfig), + ) + kwargs = { + "model_path": "test-model", + "task_type": "i2v", + "pipeline_config": {"task_type": Task.T2V, "supported_task_types": (Task.I2V,)}, + } + config = PipelineConfig.from_kwargs(kwargs) + assert config.task_type == Task.T2V + assert config.get_supported_task_types() == (Task.T2V, Task.I2V) + assert kwargs["task_type"] == "i2v" + + +def test_request_cannot_expand_legacy_capabilities(): + config = PipelineConfig(task_type=Task.T2V) + config.update_config_from_dict({"task_type": "i2v"}) + assert config.get_supported_task_types() == (Task.T2V,) + with pytest.raises(ValueError, match="Unsupported task_type"): + config.resolve_task_type("i2v")