diff --git a/docs/cookbook/diffusion/Tencent/HunyuanImage3.mdx b/docs/cookbook/diffusion/Tencent/HunyuanImage3.mdx
new file mode 100644
index 0000000000..3310e79e05
--- /dev/null
+++ b/docs/cookbook/diffusion/Tencent/HunyuanImage3.mdx
@@ -0,0 +1,76 @@
+---
+title: HunyuanImage-3.0-Instruct
+description: Deploy HunyuanImage-3.0-Instruct for text-to-image generation and reference-image editing.
+tag: NEW
+---
+
+import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx';
+
+
+
+## 1. Quick start
+
+Follow the [SGLang Diffusion installation guide](/docs/sglang-diffusion/installation). The command builder below targets Linux/CUDA; use the [Ascend installation guide](/docs/hardware-platforms/ascend-npus/getting-started/installation) for NPU environments. Exact deployment recipes remain unverified unless explicitly marked otherwise.
+
+```bash
+uv pip install -e 'python[diffusion]'
+```
+
+import { Deployment } from '/src/snippets/_deployment.jsx';
+import { config } from '/src/snippets/configs/tencent/hunyuan-image3.jsx';
+
+
+
+## 2. Model capabilities
+
+[HunyuanImage-3.0-Instruct](https://huggingface.co/tencent/HunyuanImage-3.0-Instruct) combines a text-and-image transformer with diffusion image generation. It supports text-to-image requests and instruction-guided editing with reference images. The native pipeline reuses the transformer's text prefix during denoising and uses a SigLIP2 encoder plus a convolutional VAE for image conditioning.
+
+Choose it when one checkpoint must handle both generation and reference-based editing. Its 80B total parameters, despite approximately 13B active per token, make weight residency a major deployment constraint. This recipe covers the Instruct checkpoint, not the separate distilled release. Start with tensor parallelism and the default 50-step, guidance-2.5 sampling profile before testing acceleration options.
+
+## 3. Reference-image editing
+
+Use the same server with the images edit endpoint. Repeat `image[]` for multiple reference images. The file paths below are local to the client.
+
+```bash
+curl -sS http://localhost:30010/v1/images/edits \
+ -F model=tencent/HunyuanImage-3.0-Instruct \
+ -F 'prompt=Change the background to a quiet seaside town; preserve the subject.' \
+ -F 'image[]=@reference.png' \
+ -F num_inference_steps=50 \
+ -F guidance_scale=2.5 \
+ -F seed=42 \
+ -F response_format=b64_json
+```
+
+For offline generation, pass the same model and topology to `sglang generate`:
+
+```bash
+sglang generate --model-path tencent/HunyuanImage-3.0-Instruct \
+ --num-gpus 2 --tp-size 2 --ulysses-degree 1 \
+ --prompt "Change the background to a quiet seaside town; preserve the subject." \
+ --image-path reference.png --num-inference-steps 50 --guidance-scale 2.5 \
+ --seed 42 --save-output
+```
+
+## 4. Request semantics
+
+- **Resolution:** `width` and `height` select an aspect ratio by default, not an exact generation canvas. The processor selects a native bucket; decoding crops or pads it. Editing without an explicit size uses the first reference image's dimensions. Use `output_size_mode="exact_size"` when exact output dimensions matter; this resamples the decoded result rather than changing its native bucket.
+- **Prompting:** `bot_task="image"` and `system_prompt="en_unified"` are the defaults. Other `bot_task` values change the response prefix. They do not run a separate autoregressive reasoning or recaptioning loop. `cot_text` accepts an already-generated reasoning/recaption prefix.
+- **Multiple outputs:** `n` on the HTTP API or `--num-outputs-per-prompt` offline controls output count. More outputs require additional activation memory.
+
+## 5. Deployment boundaries
+
+For an Ascend environment with four visible NPU devices and sufficient aggregate memory, the equivalent TP4 launch is:
+
+```bash
+sglang serve --model-path tencent/HunyuanImage-3.0-Instruct \
+ --num-gpus 4 --tp-size 4 --ulysses-degree 1 --port 30010
+```
+
+Here `--num-gpus` counts visible accelerator devices, including NPUs. Follow the platform installation guide for compatible PyTorch/CANN packages; do not install the CUDA extra into an NPU environment.
+
+Use TP to shard transformer weights; keep Ulysses and Ring at 1. This pipeline does not implement sequence-parallel attention or CFG-parallel execution. The vision encoder and VAE remain replicated, so increasing TP does not divide every component's memory usage. Approximately 160 GB of BF16 transformer weights is only a capacity estimate, not a peak-VRAM measurement: leave room for those components, activations, and loading buffers.
+
+Compatible requests can be grouped with `--batching-max-size` and `--batching-delay-ms`. Begin at batch size 1, then validate peak memory at the largest reference-image count and output size you intend to serve. Different prompt/seed values can share a batch; incompatible conditioning and sampling settings cannot.
+
+The pipeline has a model-specific Cache-DiT adapter. Caching reuses intermediate results and is not numerically lossless; measure quality and latency on your workload before enabling it. See [Cache-DiT](/docs/sglang-diffusion/cache_dit). SP, CFG parallel, quantization, and full-checkpoint CUDA performance are not claimed as verified by this recipe.
diff --git a/docs/cookbook/diffusion/intro.mdx b/docs/cookbook/diffusion/intro.mdx
index 48067a89fd..037efc3a29 100644
--- a/docs/cookbook/diffusion/intro.mdx
+++ b/docs/cookbook/diffusion/intro.mdx
@@ -13,6 +13,12 @@ Choose a recipe by output modality. The sidebar stays organized by model family,
Image models generate one image request as a bounded denoising job, usually with bidirectional attention over the whole latent sequence.
+
({ tp_size: Number(s.gpus_per_node), ulysses_degree: 1, ring_degree: 1 }),
+ validateTopology: (s, topology) => {
+ const errors = [];
+ if (Number(s.nodes) !== 1 || topology.tp_size !== Number(s.gpus_per_node)) {
+ errors.push("Use a single node with TP equal to the number of GPUs.");
+ }
+ if (![2, 4, 8].includes(topology.tp_size)) errors.push("This recipe supports TP 2, 4, or 8.");
+ if (topology.ulysses_degree !== 1 || topology.ring_degree !== 1) {
+ errors.push("HunyuanImage-3 does not implement sequence-parallel attention.");
+ }
+ return errors;
+ },
+ },
+ resolveDeployment: (s) => {
+ const resource = config.commandBuilder.resource;
+ const topology = s.topology_mode === "manual"
+ ? { tp_size: Number(s.tp_size), ulysses_degree: Number(s.ulysses_degree), ring_degree: Number(s.ring_degree) }
+ : resource.autoTopology(s);
+ return {
+ match: { hw: s.hw }, nnodes: 1, verified: false,
+ flags: ["--model-path {{MODEL_NAME}}", `--num-gpus ${Number(s.gpus_per_node)}`,
+ `--tp-size ${topology.tp_size}`, "--ulysses-degree 1", "--ring-degree 1",
+ "--host {{HOST_IP}}", "--port {{PORT}}"],
+ builder: {
+ topology,
+ topologySummary: `TP ${topology.tp_size}; replicated vision encoder and VAE`,
+ errors: resource.validateTopology(s, topology),
+ warnings: ["Full-checkpoint end-to-end verification of this CUDA recipe is pending."],
+ verification: { serve: "unverified", request: "unverified" },
+ },
+ };
+ },
+ },
+ modelNames: { default: "tencent/HunyuanImage-3.0-Instruct" },
+ placeholders: {
+ HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
+ PORT: { target: "command", label: "Bind port", default: "30010" },
+ CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
+ CURL_PORT: { target: "curl", label: "Server port", default: "30010" },
+ },
+ curl: (s) => `curl -sS http://{{CURL_HOST}}:{{CURL_PORT}}/v1/images/generations \\
+ -H 'Content-Type: application/json' \\
+ -d '${JSON.stringify({ model: "{{MODEL_NAME}}", prompt: "A blue ceramic teapot on a wooden table, soft daylight.", n: Number(s.outputs), num_inference_steps: 50, guidance_scale: 2.5, seed: 42, width: 1024, height: 1024, response_format: "b64_json" }, null, 2)}'`,
+ cells: [],
+};
diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx
index 3cc3cb8f78..fd1d5a2c3c 100644
--- a/docs/src/snippets/diffusion/model-catalog.jsx
+++ b/docs/src/snippets/diffusion/model-catalog.jsx
@@ -3,6 +3,11 @@
export const DiffusionModelCatalog = ({ category }) => {
const MODEL_CATALOG = {
image: [
+ {
+ name: "HunyuanImage-3.0",
+ modelIds: ["tencent/HunyuanImage-3.0", "tencent/HunyuanImage-3.0-Instruct"],
+ cookbook: "/cookbook/diffusion/Tencent/HunyuanImage3",
+ },
{
name: "Anima",
modelIds: ["circlestone-labs/Anima-Base-v1.0-Diffusers"],
diff --git a/python/sglang/multimodal_gen/configs/models/dits/hunyuan_image3.py b/python/sglang/multimodal_gen/configs/models/dits/hunyuan_image3.py
new file mode 100644
index 0000000000..ec2f9af9bb
--- /dev/null
+++ b/python/sglang/multimodal_gen/configs/models/dits/hunyuan_image3.py
@@ -0,0 +1,58 @@
+from dataclasses import dataclass, field
+
+from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
+
+
+@dataclass
+class HunyuanImage3ArchConfig(DiTArchConfig):
+ """Architecture config for HunyuanImage-3 AR transformer backbone."""
+
+ # AR transformer params
+ hidden_size: int = 4096
+ num_hidden_layers: int = 32
+ num_attention_heads: int = 32
+ num_key_value_heads: int = 8
+ attention_head_dim: int = 128
+ vocab_size: int = 133120
+ intermediate_size: int = 3072
+
+ # MoE params
+ num_experts: int = 64
+ moe_topk: int = 8
+ num_shared_expert: int = 1
+ use_mixed_mlp_moe: bool = True
+ norm_topk_prob: bool = True
+
+ # CLA (Cross-Layer Attention)
+ use_cla: bool = False
+ cla_share_factor: int = 2
+
+ # QK Norm
+ use_qk_norm: bool = True
+
+ # RoPE
+ rope_theta: float = 10000.0
+ rope_scaling_type: str = "custom"
+ max_position_embeddings: int = 22800
+
+ # Image generation
+ vae_downsample_factor: tuple[int, int] = (16, 16)
+ latent_channels: int = 32
+
+ stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list)
+
+ param_names_mapping: dict = field(default_factory=dict)
+
+ def __post_init__(self):
+ super().__post_init__()
+ self.num_layers = self.num_hidden_layers
+ self.in_channels = self.latent_channels
+ self.out_channels = self.latent_channels
+ self.num_channels_latents = self.latent_channels
+
+
+@dataclass
+class HunyuanImage3DitConfig(DiTConfig):
+ arch_config: DiTArchConfig = field(default_factory=HunyuanImage3ArchConfig)
+
+ prefix: str = "hunyuan_image3"
diff --git a/python/sglang/multimodal_gen/configs/models/vaes/hunyuan_image3.py b/python/sglang/multimodal_gen/configs/models/vaes/hunyuan_image3.py
new file mode 100644
index 0000000000..360afa460c
--- /dev/null
+++ b/python/sglang/multimodal_gen/configs/models/vaes/hunyuan_image3.py
@@ -0,0 +1,57 @@
+from dataclasses import dataclass, field
+
+from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
+
+
+@dataclass
+class HunyuanImage3VAEArchConfig(VAEArchConfig):
+ spatial_compression_ratio: int = 16
+
+ # VAE architecture params
+ block_out_channels: tuple[int, ...] = (128, 256, 512, 1024, 1024)
+ layers_per_block: int = 2
+ latent_channels: int = 32
+ in_channels: int = 3
+ out_channels: int = 3
+
+ # Scaling factors
+ scaling_factor: float = 0.562679178327931
+ ffactor_spatial: int = 16
+ ffactor_temporal: int = 4
+
+ # Latent statistics
+ latents_mean: tuple[float, ...] | None = None
+ latents_std: tuple[float, ...] | None = None
+ shift_factor: float | None = None
+
+ # Temporal downsampling
+ temperal_downsample: tuple[bool, ...] = (False, True, True)
+
+ is_residual: bool = False
+ patch_size: int | None = None
+ clip_output: bool = True
+
+
+@dataclass
+class HunyuanImage3VAEConfig(VAEConfig):
+ arch_config: HunyuanImage3VAEArchConfig = field(
+ default_factory=HunyuanImage3VAEArchConfig
+ )
+
+ use_feature_cache: bool = False
+ use_tiling: bool = False
+ use_temporal_tiling: bool = False
+ use_parallel_tiling: bool = False
+
+ def __post_init__(self):
+ # Blend on both sides of each temporal tile (diffusers 3D-VAE convention).
+ self.blend_num_frames = (
+ self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames
+ ) * 2
+
+ def get_vae_scale_factor(self):
+ return self.arch_config.ffactor_spatial
+
+ def post_init(self):
+ self.arch_config.vae_scale_factor = self.arch_config.ffactor_spatial
+ self.arch_config.spatial_compression_ratio = self.arch_config.ffactor_spatial
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py b/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py
index b84cfedde8..b111249a88 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py
@@ -28,6 +28,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import (
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import (
Hunyuan3D2PipelineConfig,
)
+from sglang.multimodal_gen.configs.pipeline_configs.hunyuan_image3 import (
+ HunyuanImage3PipelineConfig,
+)
from sglang.multimodal_gen.configs.pipeline_configs.ideogram import (
Ideogram4DistilledPipelineConfig,
Ideogram4PipelineConfig,
@@ -77,6 +80,7 @@ __all__ = [
"HunyuanConfig",
"FastHunyuanConfig",
"Hunyuan3D2PipelineConfig",
+ "HunyuanImage3PipelineConfig",
"Ideogram4PipelineConfig",
"Ideogram4DistilledPipelineConfig",
"FluxPipelineConfig",
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py
index 7ad38d1ad9..5fcb33e8d6 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py
@@ -479,6 +479,10 @@ class PipelineConfig:
"""Return whether dynamic batches should run as grouped Req lists."""
return False
+ def supports_batching_image_conditioning(self):
+ """Return whether image-conditioned requests may join dynamic batches."""
+ return False
+
def supports_sequential_dit_inference(self):
"""Return whether batched AR is followed by per-request DiT inference."""
return False
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan_image3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan_image3.py
new file mode 100644
index 0000000000..8e38a16278
--- /dev/null
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan_image3.py
@@ -0,0 +1,90 @@
+from dataclasses import dataclass, field
+
+from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
+from sglang.multimodal_gen.configs.models.dits.hunyuan_image3 import (
+ HunyuanImage3DitConfig,
+)
+from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig
+from sglang.multimodal_gen.configs.models.encoders.t5 import T5ArchConfig, T5Config
+from sglang.multimodal_gen.configs.models.vaes.hunyuan_image3 import (
+ HunyuanImage3VAEConfig,
+)
+from sglang.multimodal_gen.configs.pipeline_configs.base import (
+ ModelTaskType,
+ SpatialImagePipelineConfig,
+)
+from sglang.multimodal_gen.runtime.platforms import current_platform
+
+
+@dataclass
+class HunyuanImage3PipelineConfig(SpatialImagePipelineConfig):
+ vae_precision: str = "fp32"
+
+ should_use_guidance: bool = True
+ task_type: ModelTaskType = ModelTaskType.TI2I
+
+ vae_tiling: bool = False
+ vae_sp: bool = False
+
+ dit_config: DiTConfig = field(default_factory=HunyuanImage3DitConfig)
+
+ vae_config: VAEConfig = field(default_factory=HunyuanImage3VAEConfig)
+
+ text_encoder_configs: tuple[EncoderConfig, ...] = field(
+ default_factory=lambda: (T5Config(T5ArchConfig(num_heads=6)),)
+ )
+
+ enable_autocast: bool = False
+
+ def __post_init__(self):
+ self.vae_scale_factor = self.vae_config.get_vae_scale_factor()
+
+ def supports_dynamic_batching(self):
+ # The AR stage batches compatible requests in one diffusion loop.
+ return True
+
+ def supports_native_grouped_requests(self):
+ return True
+
+ def calculate_condition_image_size(self, image, width, height):
+ """Let the native processor choose the conditional-image bucket.
+
+ The generic resize snaps to a ``2 * vae_scale`` grid, but this model's
+ processor derives VAE and vision inputs from the original image and
+ accepts a 16-px grid; resizing first can alter the aspect ratio.
+ """
+ del image, width, height
+ return None
+
+ def prepare_calculated_size(self, image):
+ """Keep HunyuanImage-3 output resolution under native AR-stage control."""
+ del image
+ return None
+
+ def supports_batching_image_conditioning(self):
+ # TI2I requests carry per-request conditioning (masks/scatter/RoPE);
+ # requests are bucketed by resolution and condition-image count.
+ return True
+
+ def supports_sequential_dit_inference(self):
+ return True
+
+ def supports_sequential_multi_output_inference(self):
+ return current_platform.is_npu()
+
+
+def register():
+ from sglang.multimodal_gen.configs.sample.hunyuan_image3 import (
+ HunyuanImage3SamplingParams,
+ )
+ from sglang.multimodal_gen.registry import register_configs
+
+ register_configs(
+ sampling_param_cls=HunyuanImage3SamplingParams,
+ pipeline_config_cls=HunyuanImage3PipelineConfig,
+ hf_model_paths=[
+ "tencent/HunyuanImage-3.0",
+ "tencent/HunyuanImage-3.0-Instruct",
+ ],
+ model_detectors=[lambda hf_id: "hunyuanimage-3" in hf_id.lower()],
+ )
diff --git a/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py b/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py
new file mode 100644
index 0000000000..f11ad64dc6
--- /dev/null
+++ b/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py
@@ -0,0 +1,120 @@
+import math
+from dataclasses import dataclass, field
+
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT = 16
+
+VALID_BOT_TASKS = {
+ "auto",
+ "image",
+ "think",
+ "recaption",
+ "think_recaption",
+ "img_ratio",
+ "none",
+}
+
+SYSTEM_PROMPT_PRESETS = {
+ "none",
+ "en_unified",
+ "en_vanilla",
+ "en_recaption",
+ "en_think_recaption",
+ "dynamic",
+ "auto",
+}
+
+
+@dataclass
+class HunyuanImage3SamplingParams(SamplingParams):
+ negative_prompt: str = ""
+ num_frames: int = 1
+ guidance_scale: float = 2.5
+ num_inference_steps: int = 50
+
+ # Tokenizer bot_task: controls the bot response prefix; "image" adds
+ # no bot prefix in gen_image mode.
+ bot_task: str = "image"
+
+ # Preset name (see SYSTEM_PROMPT_PRESETS) or raw custom text.
+ system_prompt: str | None = "en_unified"
+
+ # Pre-generated CoT text from AR stage (think/recaption output)
+ cot_text: str | None = None
+
+ # Output geometry is separate from the generation canvas: the AR
+ # processor picks its own bucket, then the decoder applies this
+ # request-scoped policy to the decoded pixels.
+ output_size_mode: str = "aspect_ratio"
+ output_strategy: str = "native_crop"
+ output_ratio_policy: str = "exact"
+ output_crop_anchor: tuple[float, float] = (0.5, 0.5)
+ output_max_ratio_error: float = 0.0005
+ output_pad_value: float = 0.0
+
+ # Supported resolutions (height, width) - must be divisible by 16
+ supported_resolutions: list[tuple[int, int]] | None = field(
+ default_factory=lambda: [
+ (1024, 1024), # 1:1
+ (768, 1024), # 3:4 portrait
+ (1024, 768), # 4:3 landscape
+ (720, 1280), # 9:16 portrait
+ (1280, 720), # 16:9 landscape
+ ]
+ )
+
+ def _adjust(self, server_args):
+ # Pre-aligning each dimension to a 16-px grid here would distort the
+ # processor's aspect-ratio bucket selection.
+ if self.bot_task not in VALID_BOT_TASKS:
+ logger.warning(
+ f"Invalid bot_task '{self.bot_task}'. Must be one of {VALID_BOT_TASKS}. "
+ f"Defaulting to 'image'."
+ )
+ self.bot_task = "image"
+ self._validate_output_geometry()
+ super()._adjust(server_args)
+
+ def _validate_output_geometry(self) -> None:
+ if self.output_size_mode not in {"aspect_ratio", "exact_size"}:
+ raise ValueError("output_size_mode must be 'aspect_ratio' or 'exact_size'")
+ if self.output_strategy not in {"native_crop", "native_pad"}:
+ raise ValueError("output_strategy must be 'native_crop' or 'native_pad'")
+ if self.output_ratio_policy not in {"exact", "approximate"}:
+ raise ValueError("output_ratio_policy must be 'exact' or 'approximate'")
+ if len(self.output_crop_anchor) != 2 or any(
+ not math.isfinite(float(value)) or not 0.0 <= float(value) <= 1.0
+ for value in self.output_crop_anchor
+ ):
+ raise ValueError(
+ "output_crop_anchor must contain two finite values in [0, 1]"
+ )
+ if (
+ not math.isfinite(self.output_max_ratio_error)
+ or not 0.0 <= (self.output_max_ratio_error) < 1.0
+ ):
+ raise ValueError("output_max_ratio_error must be a finite value in [0, 1)")
+ if (
+ not math.isfinite(self.output_pad_value)
+ or not 0.0 <= (self.output_pad_value) <= 1.0
+ ):
+ raise ValueError("output_pad_value must be a finite value in [0, 1]")
+
+
+def align_hunyuan_image3_dimension(value: int) -> int:
+ """Round a HunyuanImage-3 dimension up to a supported multiple."""
+ return max(
+ HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT,
+ (value + HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT - 1)
+ // HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT
+ * HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT,
+ )
+
+
+def align_hunyuan_image3_resolution(width: int, height: int) -> tuple[int, int]:
+ """Align both width and height to supported multiples."""
+ return align_hunyuan_image3_dimension(width), align_hunyuan_image3_dimension(height)
diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py
index f292b216ab..0536dcdf06 100644
--- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py
+++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py
@@ -243,7 +243,12 @@ class SamplingParams:
# All fields below are copied from ForwardBatch
# Image inputs
- image_path: str | list[str] | None = None
+ # Per-request conditioning input: excluded from the dynamic-batch
+ # signature so image-conditioned requests can still group (the grouping
+ # gate is supports_batching_image_conditioning()).
+ image_path: str | list[str] | None = field(
+ default=None, metadata={"batch_sig_exclude": True}
+ )
# Video inputs (video-to-video conditioning)
video_path: str | list[str] | None = None
@@ -1640,6 +1645,29 @@ class SamplingParams:
type=int,
help="Upscaling factor (default: 4).",
)
+ # HunyuanImage-3 and similar model-specific tokenizer/prompt arguments
+ add_argument(
+ "--bot-task",
+ dest="bot_task",
+ type=str,
+ help=(
+ "Tokenizer bot task (model-specific). For HunyuanImage-3: "
+ "auto, image, think, recaption, think_recaption, img_ratio, none. "
+ "Controls the bot response prefix in the tokenizer."
+ ),
+ )
+ add_argument(
+ "--system-prompt",
+ dest="system_prompt",
+ type=str,
+ help=(
+ "System prompt: preset name or raw custom text. "
+ "Presets: none, en_unified, en_vanilla, en_recaption, "
+ "en_think_recaption, dynamic, auto. "
+ "Any other string is used directly as the system prompt. "
+ "Default: en_unified."
+ ),
+ )
return parser
@classmethod
diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py
index 8cfa2a1cd8..5d2089017f 100644
--- a/python/sglang/multimodal_gen/envs.py
+++ b/python/sglang/multimodal_gen/envs.py
@@ -105,6 +105,14 @@ if TYPE_CHECKING:
SGLANG_CACHE_DIT_SECONDARY_MC: int = 3
SGLANG_CACHE_DIT_SECONDARY_TAYLORSEER: bool = False
SGLANG_CACHE_DIT_SECONDARY_TS_ORDER: int = 1
+ # HunyuanImage-3: batch all condition images of a request group into one
+ # VAE/ViT encode call (halve-on-OOM backoff). Opt-in, off by default.
+ SGLANG_HI3_COND_ENCODE_BATCHING: bool = False
+ # HunyuanImage-3: apply the post-decode output geometry (native crop/pad
+ # and exact-size resample) to hit the requested aspect ratio. On by
+ # default; set 0 to skip the plan and return the full decoded native
+ # bucket untouched.
+ SGLANG_HI3_OUTPUT_CROP: bool = True
SGLANG_CACHE_DIT_SECONDARY_DMD: bool = False
SGLANG_CACHE_DIT_SECONDARY_DMD_HISTORY: int = 6
SGLANG_CACHE_DIT_SECONDARY_DMD_RANK: int = 0
@@ -461,6 +469,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
"SGLANG_CACHE_DIT_SCM_CACHE_BINS": _lazy_str("SGLANG_CACHE_DIT_SCM_CACHE_BINS"),
# SCM policy: dynamic or static
"SGLANG_CACHE_DIT_SCM_POLICY": _lazy_str("SGLANG_CACHE_DIT_SCM_POLICY", "dynamic"),
+ # HunyuanImage-3: batch all condition images of a request group into one
+ # VAE/ViT encode call (halve-on-OOM backoff on OOM). Opt-in, off by
+ # default; the AR backbone resident on the same device makes full-batch
+ # encoding tight on memory, so it must be explicitly enabled.
+ "SGLANG_HI3_COND_ENCODE_BATCHING": _lazy_bool("SGLANG_HI3_COND_ENCODE_BATCHING"),
+ # HunyuanImage-3: post-decode output geometry (native crop/pad plus
+ # exact-size resample). On by default; 0 returns the raw native bucket.
+ "SGLANG_HI3_OUTPUT_CROP": _lazy_bool("SGLANG_HI3_OUTPUT_CROP", "true"),
# model loading
"SGLANG_USE_RUNAI_MODEL_STREAMER": _lazy_bool(
"SGLANG_USE_RUNAI_MODEL_STREAMER", "true"
diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py
index 6ef80039ad..acf67e4885 100644
--- a/python/sglang/multimodal_gen/registry.py
+++ b/python/sglang/multimodal_gen/registry.py
@@ -186,6 +186,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
"pi0.5": "Pi05Pipeline",
"flux-3-action": "Flux3ActionPipeline",
"hunyuan3d": "Hunyuan3D2Pipeline",
+ "hunyuanimage-3": "HunyuanImage3Pipeline",
"flux.2-dev-nvfp4": "Flux2NvfpPipeline",
"fal/ideogram-v4-fast": "Ideogram4FastPipeline",
"fal/ideogram-v4-instant": "Ideogram4InstantPipeline",
diff --git a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py
index 99fcb918f4..4f34019c86 100644
--- a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py
+++ b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py
@@ -66,6 +66,7 @@ _REPLICA: GroupCoordinator | None = None
# Corresponding TP lanes across the replicated encoder copies in one pipeline
# replica. None means the encoder has only one TP copy, so batch DP cannot run.
_ENCODER_DP: GroupCoordinator | None = None
+_SRT_MOE_EP = None
_VAE_DECODE: GroupCoordinator | None = None
_DIT: ProcessGroup | None = None
_VAE: ProcessGroup | None = None
@@ -112,6 +113,28 @@ def _clear_srt_world_group() -> None:
srt_parallel_state._WORLD = None
+def _init_srt_moe_ep_group():
+ """SRT's MOE EP group for FusedMoE layers.
+
+ multimodal_gen has no expert parallelism: every rank keeps all experts
+ and TP shards the weights inside each expert (``_MOE_TP = _TP``). The EP
+ group must therefore be single-rank (``moe_ep_size=1``); aliasing
+ ``_MOE_EP`` to ``_TP`` would partition experts across ranks and break
+ FusedMoE dispatch.
+ """
+ import sglang.srt.distributed.parallel_state as srt_parallel_state
+
+ world_size = torch.distributed.get_world_size()
+ return srt_parallel_state.init_model_parallel_group(
+ group_ranks=[[r] for r in range(world_size)],
+ local_rank=get_world_group().local_rank,
+ backend=torch.distributed.get_backend(get_world_group().device_group),
+ use_pynccl=False,
+ use_custom_allreduce=False,
+ group_name="moe_ep",
+ )
+
+
def _sync_srt_tp_group() -> None:
"""Expose this package's TP group, widths, and ranks to shared SRT layers.
@@ -119,6 +142,7 @@ def _sync_srt_tp_group() -> None:
the TP size in the dummy SRT configuration. Other parallel dimensions are
one. Overrides also work before SRT configuration is published.
"""
+ global _SRT_MOE_EP
import sglang.srt.distributed.parallel_state as srt_parallel_state
from sglang.srt.runtime_context import derive_parallel_widths, get_parallel
@@ -126,6 +150,16 @@ def _sync_srt_tp_group() -> None:
srt_parallel_state._TP = _TP
if srt_parallel_state._ATTN_TP is None:
srt_parallel_state._ATTN_TP = _TP
+ if srt_parallel_state._TP is _TP and srt_parallel_state._MOE_TP is None:
+ srt_parallel_state._MOE_TP = _TP
+ if (
+ _TP is not None
+ and srt_parallel_state._TP is _TP
+ and srt_parallel_state._MOE_TP is _TP
+ and srt_parallel_state._MOE_EP is None
+ ):
+ _SRT_MOE_EP = _init_srt_moe_ep_group()
+ srt_parallel_state._MOE_EP = _SRT_MOE_EP
if srt_parallel_state._ATTN_TP is _TP:
get_parallel().override_permanently(
tp_group=_TP,
@@ -150,6 +184,7 @@ def _sync_srt_tp_group() -> None:
def _clear_srt_tp_group() -> None:
+ global _SRT_MOE_EP
import sglang.srt.distributed.parallel_state as srt_parallel_state
from sglang.srt.runtime_context import get_parallel
@@ -161,6 +196,13 @@ def _clear_srt_tp_group() -> None:
get_parallel().override_permanently(world_group=srt_parallel_state._WORLD)
if srt_parallel_state._TP is _TP:
srt_parallel_state._TP = None
+ if srt_parallel_state._MOE_TP is _TP:
+ srt_parallel_state._MOE_TP = None
+ if _SRT_MOE_EP is not None:
+ _SRT_MOE_EP.destroy()
+ if srt_parallel_state._MOE_EP is _SRT_MOE_EP:
+ srt_parallel_state._MOE_EP = None
+ _SRT_MOE_EP = None
def init_parallel_group_coordinator(
diff --git a/python/sglang/multimodal_gen/runtime/managers/scheduler.py b/python/sglang/multimodal_gen/runtime/managers/scheduler.py
index 2a3c82de0e..7301887664 100644
--- a/python/sglang/multimodal_gen/runtime/managers/scheduler.py
+++ b/python/sglang/multimodal_gen/runtime/managers/scheduler.py
@@ -626,10 +626,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
candidate_req.prompt, str
):
return "prompt_type"
- if (
- getattr(base_req, "image_path", None) is not None
- or getattr(candidate_req, "image_path", None) is not None
- ):
+ if self._has_unbatchable_image_conditioning(base_req, candidate_req):
return "image_conditioning"
if base_req.return_file_paths_only != candidate_req.return_file_paths_only:
return "return_file_paths_only"
@@ -648,6 +645,15 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
def _has_realtime_session(req: Req) -> bool:
return bool(req.realtime_session_id) or req.session is not None
+ def _has_unbatchable_image_conditioning(
+ self, base_req: Req, candidate_req: Req
+ ) -> bool:
+ """True when either request carries image conditioning that this
+ pipeline cannot keep per-request inside a merged dynamic batch."""
+ if self.server_args.pipeline_config.supports_batching_image_conditioning():
+ return False
+ return base_req.image_path is not None or candidate_req.image_path is not None
+
def _requires_sequential_multi_output(self, *reqs: Req) -> bool:
pipeline_config = self.server_args.pipeline_config
return (
@@ -687,10 +693,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
):
return False
- if (
- getattr(base_req, "image_path", None) is not None
- or getattr(candidate_req, "image_path", None) is not None
- ):
+ if self._has_unbatchable_image_conditioning(base_req, candidate_req):
return False
if base_req.return_file_paths_only != candidate_req.return_file_paths_only:
return False
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py b/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py
new file mode 100644
index 0000000000..11e792e320
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py
@@ -0,0 +1,1388 @@
+"""HunyuanImage-3 AR backbone + diffusion I/O for multimodal_gen.
+
+Ported from the official HunyuanImage-3 repository (``modeling_hunyuan_image_3.py``).
+"""
+
+import re
+import types
+from typing import Iterable, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+from einops import rearrange
+from torch import nn
+from transformers import PretrainedConfig
+
+from sglang.multimodal_gen.configs.models.dits.hunyuan_image3 import (
+ HunyuanImage3DitConfig,
+)
+from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
+from sglang.multimodal_gen.runtime.layers.attention import LocalAttention
+from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm
+from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig
+from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
+ VocabParallelEmbedding,
+)
+from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader
+from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.layers.linear import (
+ ColumnParallelLinear,
+ QKVParallelLinear,
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
+from sglang.srt.layers.moe.topk import TopK
+from sglang.srt.layers.rotary_embedding import get_rope
+from sglang.srt.models.hunyuan import (
+ HunYuanMLP,
+ _get_cla_factor,
+ _is_moe,
+)
+
+from .hunyuan_image3_utils import (
+ CachedRoPE,
+ HunYuanRotary2DEmbedder,
+ ImageKVCacheManager,
+ create_hunyuan_image_attention_meta,
+ timestep_embedding,
+)
+
+logger = init_logger(__name__)
+
+# Non-AR checkpoint weights (VAE, ViT), skipped during backbone loading.
+UNEXPECTED_KEYWORDS = [
+ "vae",
+ "vision_aligner",
+ "vision_model",
+]
+
+# Diffusion I/O helpers, ported from the official HunyuanImage-3 repository.
+
+
+def _conv_nd(dims, *args, **kwargs):
+ if dims == 1:
+ return nn.Conv1d(*args, **kwargs)
+ elif dims == 2:
+ return nn.Conv2d(*args, **kwargs)
+ elif dims == 3:
+ return nn.Conv3d(*args, **kwargs)
+ raise ValueError(f"unsupported dimensions: {dims}")
+
+
+def _zero_module(module):
+ for p in module.parameters():
+ p.detach().zero_()
+ return module
+
+
+def _normalization(channels, **kwargs):
+ return nn.GroupNorm(32, channels, **kwargs)
+
+
+class _Upsample(nn.Module):
+ """Upsample layer with optional convolution (dims=3 for spatial 2D)."""
+
+ def __init__(
+ self, channels, use_conv, dims=2, out_channels=None, device=None, dtype=None
+ ):
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super().__init__()
+ self.channels = channels
+ self.out_channels = out_channels or channels
+ self.use_conv = use_conv
+ self.dims = dims
+ if use_conv:
+ self.conv = _conv_nd(
+ dims, self.channels, self.out_channels, 3, padding=1, **factory_kwargs
+ )
+
+ def forward(self, x):
+ assert x.shape[1] == self.channels
+ if self.dims == 3:
+ x = F.interpolate(
+ x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), mode="nearest"
+ )
+ else:
+ x = F.interpolate(x, scale_factor=2, mode="nearest")
+ if self.use_conv:
+ x = self.conv(x)
+ return x
+
+
+class _Downsample(nn.Module):
+ """Downsample layer with optional convolution (dims=3 for spatial 2D)."""
+
+ def __init__(
+ self, channels, use_conv, dims=2, out_channels=None, device=None, dtype=None
+ ):
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super().__init__()
+ self.channels = channels
+ self.out_channels = out_channels or channels
+ self.use_conv = use_conv
+ self.dims = dims
+ stride = 2 if dims != 3 else (1, 2, 2)
+ if use_conv:
+ self.op = _conv_nd(
+ dims,
+ self.channels,
+ self.out_channels,
+ 3,
+ stride=stride,
+ padding=1,
+ **factory_kwargs,
+ )
+ else:
+ assert self.channels == self.out_channels
+ self.op = nn.AvgPool2d(kernel_size=stride, stride=stride)
+
+ def forward(self, x):
+ assert x.shape[1] == self.channels
+ return self.op(x)
+
+
+class _ResBlock(nn.Module):
+ """Residual block with timestep embedding conditioning."""
+
+ def __init__(
+ self,
+ in_channels,
+ emb_channels,
+ out_channels=None,
+ dropout=0.0,
+ use_conv=False,
+ dims=2,
+ up=False,
+ down=False,
+ device=None,
+ dtype=None,
+ ):
+ factory_kwargs = {"dtype": dtype, "device": device}
+ super().__init__()
+ self.in_channels = in_channels
+ self.dropout = dropout
+ self.out_channels = out_channels or self.in_channels
+
+ self.in_layers = nn.Sequential(
+ _normalization(self.in_channels, **factory_kwargs),
+ nn.SiLU(),
+ _conv_nd(
+ dims,
+ self.in_channels,
+ self.out_channels,
+ 3,
+ padding=1,
+ **factory_kwargs,
+ ),
+ )
+
+ self.updown = up or down
+ if up:
+ self.h_upd = _Upsample(self.in_channels, False, dims, **factory_kwargs)
+ self.x_upd = _Upsample(self.in_channels, False, dims, **factory_kwargs)
+ elif down:
+ self.h_upd = _Downsample(self.in_channels, False, dims, **factory_kwargs)
+ self.x_upd = _Downsample(self.in_channels, False, dims, **factory_kwargs)
+ else:
+ self.h_upd = self.x_upd = nn.Identity()
+
+ self.emb_layers = nn.Sequential(
+ nn.SiLU(),
+ nn.Linear(emb_channels, 2 * self.out_channels, **factory_kwargs),
+ )
+
+ self.out_layers = nn.Sequential(
+ _normalization(self.out_channels, **factory_kwargs),
+ nn.SiLU(),
+ nn.Dropout(p=dropout),
+ _zero_module(
+ _conv_nd(
+ dims,
+ self.out_channels,
+ self.out_channels,
+ 3,
+ padding=1,
+ **factory_kwargs,
+ )
+ ),
+ )
+
+ if self.out_channels == self.in_channels:
+ self.skip_connection = nn.Identity()
+ elif use_conv:
+ self.skip_connection = _conv_nd(
+ dims,
+ self.in_channels,
+ self.out_channels,
+ 3,
+ padding=1,
+ **factory_kwargs,
+ )
+ else:
+ self.skip_connection = _conv_nd(
+ dims, self.in_channels, self.out_channels, 1, **factory_kwargs
+ )
+
+ def forward(self, x, emb):
+ if self.updown:
+ in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1]
+ h = in_rest(x)
+ h = self.h_upd(h)
+ x = self.x_upd(x)
+ h = in_conv(h)
+ else:
+ h = self.in_layers(x)
+
+ emb_out = self.emb_layers(emb)
+ while len(emb_out.shape) < len(h.shape):
+ emb_out = emb_out[..., None]
+
+ out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
+ scale, shift = torch.chunk(emb_out, 2, dim=1)
+ h = out_norm(h) * (1.0 + scale) + shift
+ h = out_rest(h)
+
+ return self.skip_connection(x) + h
+
+
+class TimestepEmbedder(nn.Module):
+ def __init__(
+ self,
+ hidden_size,
+ act_layer=nn.GELU,
+ frequency_embedding_size=256,
+ max_period=10000,
+ out_size=None,
+ dtype=None,
+ device=None,
+ ):
+ factory_kwargs = {"dtype": dtype, "device": device}
+ super().__init__()
+ self.frequency_embedding_size = frequency_embedding_size
+ self.max_period = max_period
+ if out_size is None:
+ out_size = hidden_size
+ self.mlp = nn.Sequential(
+ nn.Linear(
+ frequency_embedding_size, hidden_size, bias=True, **factory_kwargs
+ ),
+ act_layer(),
+ nn.Linear(hidden_size, out_size, bias=True, **factory_kwargs),
+ )
+
+ def forward(self, t):
+ t_freq = timestep_embedding(t, self.frequency_embedding_size, self.max_period)
+ t_freq = t_freq.type(self.mlp[0].weight.dtype)
+ return self.mlp(t_freq)
+
+
+class UNetDown(nn.Module):
+ """Patch embed: converts noise latents (B, C, H, W) into sequence embeddings."""
+
+ def __init__(
+ self,
+ patch_size,
+ in_channels,
+ emb_channels,
+ hidden_channels,
+ out_channels,
+ dropout=0.0,
+ device=None,
+ dtype=None,
+ ):
+ factory_kwargs = {"dtype": dtype, "device": device}
+ super().__init__()
+ self.patch_size = patch_size
+
+ self.model = nn.ModuleList(
+ [
+ _conv_nd(
+ 2,
+ in_channels=in_channels,
+ out_channels=hidden_channels,
+ kernel_size=3,
+ padding=1,
+ **factory_kwargs,
+ )
+ ]
+ )
+ if self.patch_size == 1:
+ self.model.append(
+ _ResBlock(
+ in_channels=hidden_channels,
+ emb_channels=emb_channels,
+ out_channels=out_channels,
+ dropout=dropout,
+ **factory_kwargs,
+ )
+ )
+ else:
+ for i in range(self.patch_size // 2):
+ self.model.append(
+ _ResBlock(
+ in_channels=hidden_channels,
+ emb_channels=emb_channels,
+ out_channels=(
+ hidden_channels
+ if (i + 1) * 2 != self.patch_size
+ else out_channels
+ ),
+ dropout=dropout,
+ down=True,
+ **factory_kwargs,
+ )
+ )
+
+ def forward(self, x, t):
+ assert x.shape[2] % self.patch_size == 0 and x.shape[3] % self.patch_size == 0
+ for module in self.model:
+ if isinstance(module, _ResBlock):
+ x = module(x, t)
+ else:
+ x = module(x)
+ _, _, token_h, token_w = x.shape
+ x = rearrange(x, "b c h w -> b (h w) c")
+ return x, token_h, token_w
+
+
+class UNetUp(nn.Module):
+ """Final layer: converts backbone output sequence into noise predictions."""
+
+ def __init__(
+ self,
+ patch_size,
+ in_channels,
+ emb_channels,
+ hidden_channels,
+ out_channels,
+ dropout=0.0,
+ device=None,
+ dtype=None,
+ out_norm=False,
+ ):
+ factory_kwargs = {"dtype": dtype, "device": device}
+ super().__init__()
+ self.patch_size = patch_size
+ self.model = nn.ModuleList()
+
+ if self.patch_size == 1:
+ self.model.append(
+ _ResBlock(
+ in_channels=in_channels,
+ emb_channels=emb_channels,
+ out_channels=hidden_channels,
+ dropout=dropout,
+ **factory_kwargs,
+ )
+ )
+ else:
+ for i in range(self.patch_size // 2):
+ self.model.append(
+ _ResBlock(
+ in_channels=in_channels if i == 0 else hidden_channels,
+ emb_channels=emb_channels,
+ out_channels=hidden_channels,
+ dropout=dropout,
+ up=True,
+ **factory_kwargs,
+ )
+ )
+
+ if out_norm:
+ self.model.append(
+ nn.Sequential(
+ _normalization(hidden_channels, **factory_kwargs),
+ nn.SiLU(),
+ _conv_nd(
+ 2,
+ hidden_channels,
+ out_channels,
+ kernel_size=3,
+ padding=1,
+ **factory_kwargs,
+ ),
+ )
+ )
+ else:
+ self.model.append(
+ _conv_nd(
+ 2,
+ hidden_channels,
+ out_channels,
+ kernel_size=3,
+ padding=1,
+ **factory_kwargs,
+ )
+ )
+
+ def forward(self, x, t, token_h, token_w):
+ x = rearrange(x, "b (h w) c -> b c h w", h=token_h, w=token_w)
+ for module in self.model:
+ if isinstance(module, _ResBlock):
+ x = module(x, t)
+ else:
+ x = module(x)
+ return x
+
+
+def _make_rope(
+ config: PretrainedConfig, head_dim: int, rope_theta, rope_scaling, max_position
+):
+ if rope_scaling is not None:
+ rope_scaling = dict(rope_scaling)
+ rope_scaling["rope_type"] = "default"
+ return get_rope(
+ head_dim,
+ rotary_dim=head_dim,
+ max_position=max_position,
+ base=rope_theta,
+ rope_scaling=rope_scaling,
+ is_neox_style=True,
+ )
+
+
+def _get_layer_value(config: PretrainedConfig, field: str, layer_id: int, default=None):
+ value = getattr(config, field, default)
+ if isinstance(value, list):
+ assert layer_id >= 0 and len(value) > layer_id, f"{field}[{layer_id}] missing"
+ return value[layer_id]
+ return value
+
+
+class HunYuanSparseMoeBlock(nn.Module):
+ """Sparse MoE block: TopK does softmax + top-k routing, FusedMoE the experts.
+
+ A separate shared MLP (when present) is always applied to all tokens.
+ """
+
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ layer_id: int,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ assert layer_id >= 0
+ self.tp_size = get_tp_world_size()
+ self.n_routed_experts = config.num_experts
+ self.layer_id = layer_id
+
+ top_k = _get_layer_value(config, "moe_topk", layer_id)
+ intermediate_size = _get_layer_value(config, "intermediate_size", layer_id, 0)
+ if getattr(config, "moe_intermediate_size", None) is not None:
+ intermediate_size = _get_layer_value(
+ config, "moe_intermediate_size", layer_id
+ )
+
+ self.gate = ReplicatedLinear(
+ config.hidden_size,
+ config.num_experts,
+ bias=False,
+ quant_config=quant_config,
+ prefix=f"{prefix}.gate",
+ )
+
+ self.topk = TopK(
+ top_k=top_k,
+ layer_id=layer_id,
+ )
+
+ if getattr(config, "use_mixed_mlp_moe", 0) > 0:
+ num_shared_expert = _get_layer_value(config, "num_shared_expert", layer_id)
+ self.shared_mlp = HunYuanMLP(
+ hidden_size=config.hidden_size,
+ intermediate_size=intermediate_size * num_shared_expert,
+ hidden_act=config.hidden_act,
+ quant_config=quant_config,
+ prefix=f"{prefix}.shared_mlp",
+ reduce_results=True,
+ )
+ else:
+ self.shared_mlp = None
+
+ self.experts = FusedMoE(
+ num_experts=config.num_experts,
+ top_k=top_k,
+ hidden_size=config.hidden_size,
+ intermediate_size=intermediate_size,
+ reduce_results=True,
+ quant_config=quant_config,
+ layer_id=layer_id,
+ prefix=f"{prefix}.experts",
+ )
+
+ def forward(self, hidden_states):
+ orig_shape = hidden_states.shape
+ hidden_dim = hidden_states.shape[-1]
+ hidden_states = hidden_states.view(-1, hidden_dim)
+
+ router_logits, _ = self.gate(hidden_states)
+ topk_output = self.topk(hidden_states, router_logits)
+ final_hidden_states = self.experts(hidden_states, topk_output)
+
+ if self.shared_mlp is not None:
+ _shared_out = self.shared_mlp(hidden_states)
+ final_hidden_states = final_hidden_states + _shared_out
+
+ # NPU: AscendTPDispatcher all-gathers the FusedMoE output internally;
+ # the shared MLP's down_proj all-reduces its own output.
+
+ return final_hidden_states.view(orig_shape)
+
+
+class HunYuanAttention(nn.Module):
+ """Self-attention; CLA followers attend to the master's K/V via ``kv_states``."""
+
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ hidden_size: int,
+ num_heads: int,
+ num_kv_heads: int,
+ layer_id: int = 0,
+ rope_theta: float = 10000,
+ rope_scaling: Optional[dict] = None,
+ max_position_embeddings: int = 8192,
+ quant_config: Optional[QuantizationConfig] = None,
+ bias: bool = False,
+ prefix: str = "",
+ is_cross_attention: bool = False,
+ ) -> None:
+ super().__init__()
+ tp_size = get_tp_world_size()
+ self.hidden_size = hidden_size
+ self.is_cross_attention = is_cross_attention
+ self.total_num_heads = num_heads
+ assert self.total_num_heads % tp_size == 0
+ self.num_heads = self.total_num_heads // tp_size
+ self.total_num_kv_heads = num_kv_heads
+ if self.total_num_kv_heads >= tp_size:
+ assert self.total_num_kv_heads % tp_size == 0
+ else:
+ assert tp_size % self.total_num_kv_heads == 0
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
+
+ self.head_dim = hidden_size // self.total_num_heads
+ self.q_size = self.num_heads * self.head_dim
+ self.kv_size = self.num_kv_heads * self.head_dim
+ self.scaling = self.head_dim**-0.5
+ self.use_qk_norm = getattr(config, "use_qk_norm", False)
+ self.layer_id = layer_id
+
+ if is_cross_attention:
+ # CLA follower: project only Q; K/V are reused from the master layer
+ # via ``kv_states``. The weight loader routes follower ``.q_proj``
+ # weights here instead of the fused-qkv mapping.
+ self.q_proj = ColumnParallelLinear(
+ hidden_size,
+ hidden_size,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.q_proj",
+ )
+ else:
+ self.qkv_proj = QKVParallelLinear(
+ hidden_size,
+ self.head_dim,
+ self.total_num_heads,
+ self.total_num_kv_heads,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.qkv_proj",
+ )
+
+ self.o_proj = RowParallelLinear(
+ self.total_num_heads * self.head_dim,
+ hidden_size,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.o_proj",
+ )
+
+ self.rotary_emb = _make_rope(
+ config, self.head_dim, rope_theta, rope_scaling, max_position_embeddings
+ )
+ self.attn = LocalAttention(
+ num_heads=self.num_heads,
+ head_size=self.head_dim,
+ num_kv_heads=self.num_kv_heads,
+ softmax_scale=self.scaling,
+ causal=True,
+ )
+
+ self.image_attn = ImageKVCacheManager()
+ self.image_rope2d_emb = HunYuanRotary2DEmbedder(
+ num_heads=self.num_heads,
+ num_kv_heads=self.num_kv_heads,
+ head_dim=self.head_dim,
+ )
+
+ if self.use_qk_norm:
+ self.rms_norm_eps = getattr(config, "rms_norm_eps", 1e-5)
+ self.query_layernorm = RMSNorm(self.head_dim, eps=self.rms_norm_eps)
+ self.key_layernorm = RMSNorm(self.head_dim, eps=self.rms_norm_eps)
+
+ def _apply_rope(
+ self,
+ positions,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ hidden_states: torch.Tensor,
+ custom_pos_emb,
+ ):
+ """Apply the 2D image RoPE (diffusion path) or the 1D text RoPE."""
+ if custom_pos_emb is not None:
+ return self.image_rope2d_emb(q, k, hidden_states, custom_pos_emb)
+ return self.rotary_emb(positions, q, k)
+
+ def forward(
+ self,
+ positions,
+ hidden_states,
+ forward_batch,
+ kv_states=None,
+ attn_meta=None,
+ attention_mask=None,
+ custom_pos_emb=None,
+ ):
+ q_len, hidden_size = hidden_states.size()
+ hidden_states = hidden_states.reshape(-1, hidden_size)
+
+ if self.is_cross_attention:
+ ori_k, v = kv_states
+ k = ori_k
+ q, _ = self.q_proj(hidden_states)
+ q, _ = self._apply_rope(
+ positions,
+ q,
+ torch.empty_like(k),
+ hidden_states,
+ custom_pos_emb,
+ )
+ else:
+ qkv, _ = self.qkv_proj(hidden_states)
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q, k = self._apply_rope(positions, q, k, hidden_states, custom_pos_emb)
+ ori_k = k
+
+ if self.use_qk_norm:
+ q = self.query_layernorm(
+ q.view(-1, self.num_heads, self.head_dim).contiguous()
+ )
+ k = self.key_layernorm(
+ k.view(-1, self.num_kv_heads, self.head_dim).contiguous()
+ )
+
+ if attn_meta is not None:
+ attn_output = self.image_attn(
+ q, k, v, attn_meta, attention_mask=attention_mask
+ )
+ else:
+ q = q.view(-1, self.num_heads, self.head_dim)
+ k = k.view(-1, self.num_kv_heads, self.head_dim)
+ v = v.view(-1, self.num_kv_heads, self.head_dim)
+ attn_output = self.attn(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0))
+
+ attn_output = attn_output.view(q.shape[0], -1)
+ output, _ = self.o_proj(attn_output)
+ output = output.reshape(q_len, -1)
+ return output, (ori_k, v)
+
+
+class HunyuanImage3DecoderLayer(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ layer_id: int,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ self.layer_id = layer_id
+ self.hidden_size = config.hidden_size
+ # intermediate_size may be a scalar or a per-layer list in the config
+ intermediate_size = getattr(config, "intermediate_size", 0)
+ if isinstance(intermediate_size, list):
+ intermediate_size = intermediate_size[layer_id]
+ self.intermediate_size = intermediate_size
+ rope_theta = getattr(config, "rope_theta", 10000)
+ rope_scaling = getattr(config, "rope_scaling", None)
+ if rope_scaling is not None and getattr(
+ config, "original_max_position_embeddings", None
+ ):
+ rope_scaling = dict(rope_scaling)
+ rope_scaling["original_max_position_embeddings"] = (
+ config.original_max_position_embeddings
+ )
+ max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
+ config, "bias", False
+ )
+
+ cla_factor = _get_cla_factor(config)
+ attention_type = "cross" if layer_id % cla_factor != 0 else "self"
+ attn_kwargs = dict(
+ config=config,
+ hidden_size=self.hidden_size,
+ num_heads=config.num_attention_heads,
+ num_kv_heads=getattr(
+ config, "num_key_value_heads", config.num_attention_heads
+ ),
+ layer_id=layer_id,
+ rope_theta=rope_theta,
+ rope_scaling=rope_scaling,
+ max_position_embeddings=max_position_embeddings,
+ quant_config=quant_config,
+ bias=attention_bias,
+ prefix=f"{prefix}.self_attn",
+ )
+ self.self_attn = HunYuanAttention(
+ **attn_kwargs, is_cross_attention=attention_type == "cross"
+ )
+
+ if _is_moe(config):
+ self.mlp = HunYuanSparseMoeBlock(
+ config=config,
+ layer_id=layer_id,
+ quant_config=quant_config,
+ prefix=f"{prefix}.mlp",
+ )
+ else:
+ self.mlp = HunYuanMLP(
+ hidden_size=self.hidden_size,
+ intermediate_size=self.intermediate_size,
+ hidden_act=config.hidden_act,
+ quant_config=quant_config,
+ bias=getattr(config, "mlp_bias", False),
+ prefix=f"{prefix}.mlp",
+ )
+
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+
+ def forward(
+ self,
+ positions,
+ hidden_states,
+ forward_batch,
+ residual,
+ kv_states=None,
+ attn_meta=None,
+ attention_mask=None,
+ custom_pos_emb=None,
+ ):
+ if attention_mask is not None:
+ residual = hidden_states
+ hidden_states = self.input_layernorm(hidden_states)
+
+ hidden_states, ori_kv_states = self.self_attn(
+ positions=positions,
+ hidden_states=hidden_states,
+ forward_batch=forward_batch,
+ kv_states=kv_states,
+ attn_meta=attn_meta,
+ attention_mask=attention_mask,
+ custom_pos_emb=custom_pos_emb,
+ )
+ hidden_states = residual + hidden_states
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+ else:
+ if residual is None:
+ residual = hidden_states
+ hidden_states = self.input_layernorm(hidden_states)
+ else:
+ hidden_states, residual = self.input_layernorm(hidden_states, residual)
+ hidden_states, ori_kv_states = self.self_attn(
+ positions=positions,
+ hidden_states=hidden_states,
+ forward_batch=forward_batch,
+ kv_states=kv_states,
+ )
+ hidden_states, residual = self.post_attention_layernorm(
+ hidden_states, residual
+ )
+ hidden_states = self.mlp(hidden_states)
+ return hidden_states, residual, ori_kv_states
+
+
+class HunyuanImage3Model(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.config = config
+ self.quant_config = quant_config
+ self.vocab_size = config.vocab_size
+
+ self.embed_tokens = VocabParallelEmbedding(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=quant_config,
+ prefix=f"{prefix}.embed_tokens",
+ )
+ self.layers = nn.ModuleList(
+ [
+ HunyuanImage3DecoderLayer(
+ config=config,
+ layer_id=i,
+ quant_config=quant_config,
+ prefix=f"{prefix}.layers.{i}",
+ )
+ for i in range(config.num_hidden_layers)
+ ]
+ )
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ def get_input_embeddings(self, input_ids):
+ return self.embed_tokens(input_ids)
+
+ @torch.no_grad()
+ def forward(self, input_ids, positions, forward_batch, input_embeds=None):
+ if input_embeds is not None:
+ hidden_states = input_embeds
+ else:
+ hidden_states = self.get_input_embeddings(input_ids)
+ residual = None
+
+ cla_factor = _get_cla_factor(self.config)
+ prev_kv_states = None
+ for i, layer in enumerate(self.layers):
+ hidden_states, residual, kv_states = layer(
+ positions,
+ hidden_states,
+ forward_batch,
+ residual,
+ prev_kv_states,
+ )
+ if getattr(self.config, "use_cla", False) and i % cla_factor == 0:
+ prev_kv_states = kv_states
+ else:
+ prev_kv_states = None
+
+ hidden_states, _ = self.norm(hidden_states, residual)
+ return hidden_states
+
+ def forward_block(
+ self,
+ hidden_states,
+ attention_mask,
+ custom_pos_emb,
+ attn_meta=None,
+ num_image_tokens=None,
+ first_step=False,
+ ):
+ if attn_meta is None:
+ attn_meta = create_hunyuan_image_attention_meta(
+ attention_mask, num_image_tokens, first_step
+ )
+
+ residual = None
+ cla_factor = _get_cla_factor(self.config)
+ prev_kv_states = None
+ for i, layer in enumerate(self.layers):
+ hidden_states, residual, kv_states = layer(
+ None,
+ hidden_states,
+ None,
+ residual,
+ prev_kv_states,
+ attn_meta,
+ attention_mask,
+ custom_pos_emb,
+ )
+ if getattr(self.config, "use_cla", False) and i % cla_factor == 0:
+ prev_kv_states = kv_states
+ else:
+ prev_kv_states = None
+
+ return hidden_states.contiguous()
+
+ def _split_qkv_weight(self, qkv):
+ num_attention_heads = self.config.num_attention_heads
+ num_kv_heads = getattr(
+ self.config, "num_key_value_heads", self.config.num_attention_heads
+ )
+ num_key_value_groups = num_attention_heads // num_kv_heads
+ hidden_size = self.config.hidden_size
+ attention_head_dim = hidden_size // num_attention_heads
+
+ qkv = qkv.reshape(
+ num_kv_heads, num_key_value_groups + 2, attention_head_dim, hidden_size
+ )
+ q, k, v = torch.split(qkv, (num_key_value_groups, 1, 1), dim=1)
+ q = q.reshape(-1, hidden_size)
+ k = k.reshape(-1, hidden_size)
+ v = v.reshape(-1, hidden_size)
+ return torch.concat((q, k, v))
+
+
+class HunyuanImage3ForCausalMM(CachableDiT):
+ def __init__(
+ self,
+ config: HunyuanImage3DitConfig,
+ prefix: str = "",
+ **kwargs,
+ ):
+ super().__init__(config=config, **kwargs)
+
+ arch_config = self.config
+
+ self.model = HunyuanImage3Model(
+ arch_config,
+ prefix=f"{prefix}.model",
+ )
+
+ self.unpadded_vocab_size = arch_config.vocab_size
+ # multimodal_gen has no LM-head layer; the vocab-parallel embedding
+ # shares its layout and only ``.weight`` is consumed downstream.
+ self.lm_head = VocabParallelEmbedding(
+ self.unpadded_vocab_size,
+ arch_config.hidden_size,
+ org_num_embeddings=self.unpadded_vocab_size,
+ prefix=f"{prefix}.lm_head",
+ )
+ if getattr(arch_config, "tie_word_embeddings", False):
+ self.lm_head.weight = self.model.embed_tokens.weight
+
+ patch_size = getattr(arch_config, "patch_size", 1)
+ patch_embed_hidden_dim = getattr(arch_config, "patch_embed_hidden_dim", 1024)
+ img_proj_type = getattr(arch_config, "img_proj_type", "unet")
+ # latent_channels may be top-level or nested under arch_config.vae
+ if isinstance(getattr(arch_config, "vae", None), dict):
+ latent_channels = arch_config.vae["latent_channels"]
+ else:
+ latent_channels = arch_config.latent_channels
+
+ if img_proj_type == "unet":
+ self.timestep_emb = TimestepEmbedder(hidden_size=arch_config.hidden_size)
+ self.patch_embed = UNetDown(
+ patch_size=patch_size,
+ emb_channels=arch_config.hidden_size,
+ in_channels=latent_channels,
+ hidden_channels=patch_embed_hidden_dim,
+ out_channels=arch_config.hidden_size,
+ )
+ self.time_embed = TimestepEmbedder(hidden_size=arch_config.hidden_size)
+ self.final_layer = UNetUp(
+ patch_size=patch_size,
+ emb_channels=arch_config.hidden_size,
+ in_channels=arch_config.hidden_size,
+ hidden_channels=patch_embed_hidden_dim,
+ out_channels=latent_channels,
+ out_norm=True,
+ )
+ self.time_embed_2 = TimestepEmbedder(hidden_size=arch_config.hidden_size)
+ else:
+ raise ValueError(f"Unknown img_proj_type: {img_proj_type}")
+
+ head_dim = arch_config.hidden_size // arch_config.num_attention_heads
+ self.cached_rope = CachedRoPE(
+ rope_theta=arch_config.rope_theta,
+ head_dim=head_dim,
+ rope_type=getattr(arch_config, "rope_type", "2d"),
+ )
+
+ def forward(
+ self, hidden_states, timestep=None, encoder_hidden_states=None, **kwargs
+ ):
+ """DiT-style forward for denoising stage."""
+ return hidden_states
+
+ def forward_block(
+ self,
+ hidden_states,
+ attention_mask,
+ custom_pos_emb,
+ num_image_tokens=None,
+ first_step=False,
+ timestep=None,
+ ):
+ # TeaCache gate: skip similar steps and reuse the cached residual;
+ # one decision covers the packed CFG batch.
+ if timestep is not None and self.should_skip_forward_for_cached_states(
+ timestep=timestep
+ ):
+ return self.retrieve_cached_states(hidden_states).contiguous()
+
+ output = self.model.forward_block(
+ hidden_states,
+ attention_mask,
+ custom_pos_emb,
+ num_image_tokens=num_image_tokens,
+ first_step=first_step,
+ )
+
+ if timestep is not None:
+ self.maybe_cache_states(output, hidden_states)
+ return output
+
+ def get_embed_and_head(self):
+ return self.model.embed_tokens.weight, self.lm_head.weight
+
+ def set_embed_and_head(self, embed, head):
+ del self.model.embed_tokens.weight
+ del self.lm_head.weight
+ self.model.embed_tokens.weight = embed
+ self.lm_head.weight = head
+ torch.cuda.empty_cache()
+ torch.cuda.synchronize()
+
+ # TeaCache: see runtime/cache/teacache.py
+
+ def should_skip_forward_for_cached_states(self, **kwargs) -> bool:
+ ctx = self._get_teacache_context()
+ # Gate consumed by forward_block / maybe_cache_states.
+ self.enable_teacache = ctx is not None
+ if ctx is None:
+ return False
+ # Cond/uncond rows share one packed forward (same timestep), so skip
+ # boundaries are step-unit based; no CFG doubling.
+ start_skipping, end_skipping = ctx.teacache_params.get_skip_boundaries(
+ ctx.num_inference_steps, do_cfg=False
+ )
+ is_boundary_step = (
+ ctx.current_timestep < start_skipping
+ or ctx.current_timestep >= end_skipping
+ )
+ # Timestep-conditioned input for the L1 similarity decision.
+ modulated_inp = self.time_embed(kwargs["timestep"])
+ should_calc = self._compute_teacache_decision(
+ modulated_inp=modulated_inp,
+ is_boundary_step=is_boundary_step,
+ coefficients=ctx.coefficients,
+ teacache_thresh=ctx.teacache_thresh,
+ )
+ return not should_calc
+
+ def maybe_cache_states(
+ self, hidden_states: torch.Tensor, original_hidden_states: torch.Tensor
+ ) -> None:
+ """Cache the backbone residual for the packed [tokens, hidden] tensor."""
+ if not self.enable_teacache:
+ return
+ self.previous_residual = hidden_states - original_hidden_states
+
+ def retrieve_cached_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ return hidden_states + self.previous_residual
+
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+ stacked_params_mapping = [
+ (".qkv_proj", ".q_proj", "q"),
+ (".qkv_proj", ".k_proj", "k"),
+ (".qkv_proj", ".v_proj", "v"),
+ (".gate_up_proj", ".gate_proj", 0),
+ (".gate_up_proj", ".up_proj", 1),
+ ]
+
+ num_attention_heads = self.config.num_attention_heads
+ num_kv_heads = getattr(
+ self.config, "num_key_value_heads", self.config.num_attention_heads
+ )
+ split_params_mapping = [
+ (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None),
+ (
+ ".qkv_proj",
+ ".qkv_proj",
+ num_attention_heads + num_kv_heads * 2,
+ [("q", num_attention_heads), ("k", num_kv_heads), ("v", num_kv_heads)],
+ self.model._split_qkv_weight,
+ ),
+ ]
+
+ cla_factor = _get_cla_factor(self.config)
+
+ # FusedMoE expert mapping, remapped to the fused gate_and_up_proj
+ # checkpoint keys.
+ expert_weights_remapping = {
+ "gate_proj": ("gate_and_up_proj", 1, 2),
+ "up_proj": ("gate_and_up_proj", 0, 2),
+ }
+ expert_params_mapping = []
+ if _is_moe(self.config):
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
+ ckpt_gate_proj_name="gate_proj",
+ ckpt_down_proj_name="down_proj",
+ ckpt_up_proj_name="up_proj",
+ num_experts=self.config.num_experts,
+ )
+
+ params_dict = dict(self.named_parameters())
+ loaded_params: set = set()
+ _ckpt_dtype_logged = False
+
+ for name, loaded_weight in weights:
+ if not _ckpt_dtype_logged:
+ logger.info(
+ " checkpoint weight dtype: %s (param dtype: %s)",
+ loaded_weight.dtype,
+ next(iter(params_dict.values())).dtype if params_dict else "?",
+ )
+ _ckpt_dtype_logged = True
+ if any(keyword in name for keyword in UNEXPECTED_KEYWORDS):
+ continue
+ if "rotary_emb.inv_freq" in name:
+ continue
+ if "gate_proj_bias" in name:
+ name = name.replace("gate_proj_bias", "gate_proj.bias")
+ if "up_proj_bias" in name:
+ name = name.replace("up_proj_bias", "up_proj.bias")
+ if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
+ continue
+ if (
+ getattr(self.config, "tie_word_embeddings", False)
+ and "lm_head.weight" in name
+ ):
+ continue
+
+ if name.endswith("wte.weight"):
+ name = name.replace("wte.weight", "embed_tokens.weight")
+ if name.endswith("ln_f.weight"):
+ name = name.replace("ln_f.weight", "norm.weight")
+ if "mlp.gate.wg." in name:
+ name = name.replace("wg.", "")
+
+ is_found = False
+ for param_name, weight_name, shard_id in stacked_params_mapping:
+ if weight_name not in name:
+ continue
+ if "mlp.experts" in name:
+ continue
+ if weight_name == ".q_proj" and cla_factor > 1:
+ match = re.search(r"layers\.(\d+)", name)
+ if match and int(match.group(1)) % cla_factor != 0:
+ continue
+ name = name.replace(weight_name, param_name)
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+ if name not in params_dict:
+ continue
+ param = params_dict[name]
+ weight_loader = param.weight_loader
+ weight_loader(param, loaded_weight, shard_id)
+ loaded_params.add(name)
+ is_found = True
+ break
+ if is_found:
+ continue
+
+ for param_name, weight_name, den, split_param, func in split_params_mapping:
+ if weight_name not in name:
+ continue
+ if "mlp.experts" in name:
+ continue
+ name = name.replace(weight_name, param_name)
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+ if name not in params_dict:
+ continue
+ assert loaded_weight.shape[0] % den == 0
+ units = loaded_weight.shape[0] // den
+ param = params_dict[name]
+ weight_loader = param.weight_loader
+ chunk = func(loaded_weight) if func is not None else loaded_weight
+ offset = 0
+ for shard_id, num in split_param:
+ new_offset = offset + num * units
+ weight_loader(param, chunk[offset:new_offset], shard_id)
+ offset = new_offset
+ loaded_params.add(name)
+ is_found = True
+ break
+ if is_found:
+ continue
+
+ # Expert weights: expert_params_mapping + expert_weights_remapping
+ # handle the fused gate_and_up_proj format (the format the released
+ # checkpoints ship; separate gate_proj/up_proj keys are rejected
+ # below instead of being silently dropped).
+ is_expert_weight = False
+ is_found = False
+ found_num = 0
+ if _is_moe(self.config) and "mlp.experts" in name:
+ if not getattr(self, "_expert_ckpt_logged", False):
+ logger.info(" expert ckpt key sample: %s", name)
+ self._expert_ckpt_logged = True
+ for mapping in expert_params_mapping:
+ param_name, weight_name, expert_id, shard_id = mapping
+ offset = 0
+ den = 1
+ for (
+ mapped_weight_substr,
+ origin_weight_info,
+ ) in expert_weights_remapping.items():
+ if mapped_weight_substr in weight_name:
+ origin_weight_name, offset, den = origin_weight_info
+ weight_name = weight_name.replace(
+ mapped_weight_substr, origin_weight_name
+ )
+ break
+ if weight_name not in name:
+ continue
+ is_expert_weight = True
+ name_mapped = name.replace(weight_name, param_name)
+ found_num += 1
+ if name_mapped not in params_dict:
+ continue
+ param = params_dict[name_mapped]
+ weight_loader = param.weight_loader
+
+ if den > 1:
+ assert loaded_weight.shape[0] % den == 0
+ units = loaded_weight.shape[0] // den
+ loaded_weight_shard = loaded_weight[
+ offset * units : offset * units + units
+ ]
+ else:
+ loaded_weight_shard = loaded_weight
+
+ weight_loader(
+ param,
+ loaded_weight_shard,
+ name_mapped,
+ shard_id=shard_id,
+ expert_id=expert_id,
+ )
+ loaded_params.add(name_mapped)
+ is_found = True
+ if found_num == den:
+ break
+ if is_found:
+ continue
+ if is_expert_weight:
+ # Recognised as expert weight but not mapped locally
+ continue
+ if _is_moe(self.config) and "mlp.experts" in name:
+ raise KeyError(
+ f"Unsupported expert checkpoint key '{name}': expected fused "
+ "'gate_and_up_proj'/'down_proj' expert keys (separate "
+ "gate_proj/up_proj checkpoints are not supported; re-export "
+ "the checkpoint with fused gate_and_up_proj)."
+ )
+
+ if name.endswith(".bias") and name not in params_dict:
+ continue
+ if name not in params_dict:
+ continue
+ param = params_dict[name]
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
+ weight_loader(param, loaded_weight)
+ loaded_params.add(name)
+
+ all_param_names = set(params_dict.keys())
+ missing = all_param_names - loaded_params
+ if missing:
+ significant_missing = [
+ n for n in missing if not any(k in n for k in ["rotary_emb", "lm_head"])
+ ]
+ if significant_missing:
+ logger.warning(
+ "Weight loading: %d/%d params loaded, %d MISSING:",
+ len(loaded_params),
+ len(all_param_names),
+ len(significant_missing),
+ )
+ for n in sorted(significant_missing)[:30]:
+ logger.warning(" MISSING: %s", n)
+ if len(significant_missing) > 30:
+ logger.warning(" ... and %d more", len(significant_missing) - 30)
+ else:
+ logger.info(
+ "Weight loading: %d/%d params loaded (all accounted for)",
+ len(loaded_params),
+ len(all_param_names),
+ )
+ else:
+ logger.info(
+ "Weight loading: %d/%d params loaded (complete)",
+ len(loaded_params),
+ len(all_param_names),
+ )
+
+ return loaded_params
+
+
+class LightProjector(nn.Module):
+ """ViT embedding -> transformer dim projection."""
+
+ def __init__(self, config):
+ config = types.SimpleNamespace(**config)
+ super().__init__()
+
+ if config.projector_type == "linear":
+ self.layers = nn.Linear(config.input_dim, config.n_embed)
+ elif config.projector_type == "mlp_gelu":
+ modules = [nn.Linear(config.input_dim, config.n_embed)]
+ for _ in range(1, config.depth):
+ modules.append(nn.GELU())
+ modules.append(nn.Linear(config.n_embed, config.n_embed))
+ self.layers = nn.Sequential(*modules)
+ else:
+ raise ValueError(f"Unknown projector type: {config.projector_type}")
+
+ def forward(self, x):
+ return self.layers(x)
+
+
+class _Hi3CacheBlock(nn.Module):
+ """Pattern_3 view of one decoder layer for cache-dit.
+
+ cache-dit drives blocks as ``(hidden_states, *conds) -> hidden_states``
+ (ForwardPattern.Pattern_3); in the masked diffusion path the real layer
+ resets ``residual = hidden`` internally and CLA is off, so it is a pure
+ ``hidden -> hidden`` map. Held by plain reference (object.__setattr__)
+ so parameters stay registered only under ``HunyuanImage3Model.layers``.
+ """
+
+ def __init__(self, layer: nn.Module):
+ super().__init__()
+ object.__setattr__(self, "_layer", layer)
+
+ def forward(self, hidden_states, attention_mask, custom_pos_emb):
+ # Model.forward_block no longer threads attn_meta; ImageKVCacheManager
+ # only reads query_lens from it, so build the metadata here.
+ attn_meta = create_hunyuan_image_attention_meta(attention_mask, None, False)
+ hidden_states, _, _ = self._layer(
+ None,
+ hidden_states,
+ None,
+ None,
+ None,
+ attn_meta,
+ attention_mask,
+ custom_pos_emb,
+ )
+ return hidden_states
+
+
+class Hi3CacheBlockAdapter(nn.Module):
+ """cache-dit-wrappable view of the diffusion block loop (Pattern_3).
+
+ Threads ``attention_mask`` / ``custom_pos_emb`` through every block
+ unchanged. The name must NOT start with ``HunyuanImage``: cache-dit's
+ ``BlockAdapterRegister.is_supported`` prefix-matches class names and would
+ mis-resolve this AR backbone to its built-in diffusers HunyuanImage
+ adapter instead of the custom Pattern_3 spec.
+ """
+
+ def __init__(self, model: "HunyuanImage3Model"):
+ super().__init__()
+ self.blocks = nn.ModuleList([_Hi3CacheBlock(layer) for layer in model.layers])
+
+ def forward(self, hidden_states, attention_mask, custom_pos_emb):
+ for block in self.blocks:
+ hidden_states = block(hidden_states, attention_mask, custom_pos_emb)
+ return hidden_states.contiguous()
+
+
+EntryClass = [HunyuanImage3ForCausalMM]
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3_utils.py b/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3_utils.py
new file mode 100644
index 0000000000..088aa4b3da
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3_utils.py
@@ -0,0 +1,290 @@
+"""Attention metadata and 2D RoPE helpers for HunyuanImage-3.
+
+Ported from the official repository (``modeling_hunyuan_image_3.py``).
+"""
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+
+
+def timestep_embedding(t, dim, max_period=10000):
+ half = dim // 2
+ freqs = torch.exp(
+ -math.log(max_period)
+ * torch.arange(start=0, end=half, dtype=torch.float32)
+ / half
+ ).to(device=t.device)
+ args = t[:, None].float() * freqs[None]
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+ if dim % 2:
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+ return embedding
+
+
+@dataclass
+class HunYuanImageAttentionMeta:
+ query_lens: List[int]
+ seq_lens: List[int]
+ num_image_tokens: int
+ first_step: bool
+
+
+def create_hunyuan_image_attention_meta(
+ attention_mask: torch.Tensor, num_image_tokens: int, first_step: bool
+) -> HunYuanImageAttentionMeta:
+ b, _, q_len1, seq_len = attention_mask.shape
+ return HunYuanImageAttentionMeta(
+ query_lens=[q_len1] * b,
+ seq_lens=[seq_len] * b,
+ num_image_tokens=num_image_tokens,
+ first_step=first_step,
+ )
+
+
+def rotate_half(x):
+ x1, x2 = x.chunk(2, dim=-1)
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin):
+ """Apply 2D RoPE with the official tiled angle layout.
+
+ ``cos``/``sin`` from ``build_2d_rope`` are already expanded to full
+ head_dim ([seq, head_dim] or [batch, seq, head_dim]); batched tables
+ broadcast per row so requests with different layouts keep their own
+ positions. ``q``/``k`` are [batch, seq, heads, head_dim].
+ """
+ if cos.dim() == 2:
+ cos = cos.unsqueeze(0)
+ sin = sin.unsqueeze(0)
+ cos = cos.unsqueeze(2) # broadcast over heads: [b, s, 1, head_dim]
+ sin = sin.unsqueeze(2)
+ return q * cos + rotate_half(q) * sin, k * cos + rotate_half(k) * sin
+
+
+class HunYuanRotary2DEmbedder:
+ def __init__(self, num_heads: int, num_kv_heads: int, head_dim: int):
+ self.num_heads = num_heads
+ self.num_kv_heads = num_kv_heads
+ self.head_dim = head_dim
+
+ def __call__(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ hidden_states: torch.Tensor,
+ custom_pos_emb: tuple[torch.Tensor, torch.Tensor],
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ hidden_states_shape = hidden_states.shape
+ hidden_states = hidden_states.reshape(-1, hidden_states_shape[-1])
+
+ cos, sin = custom_pos_emb
+ bs, q_len = cos.shape[0], cos.shape[1]
+
+ assert hidden_states.shape[0] == bs * q_len, (
+ f"{hidden_states.shape[0]} != {bs * q_len}"
+ )
+
+ # [B*L, H*D] -> [B, L, H, D] for apply_rotary_pos_emb
+ q = q.reshape(bs, q_len, self.num_heads, self.head_dim)
+ k = k.reshape(bs, q_len, self.num_kv_heads, self.head_dim)
+
+ q, k = apply_rotary_pos_emb(q.to(torch.float32), k.to(torch.float32), cos, sin)
+
+ q = q.reshape(hidden_states.shape[0], self.num_heads * self.head_dim).to(
+ torch.bfloat16
+ )
+ k = k.reshape(hidden_states.shape[0], self.num_kv_heads * self.head_dim).to(
+ torch.bfloat16
+ )
+ hidden_states = hidden_states.reshape(hidden_states_shape)
+ return q, k
+
+
+def build_2d_rope(
+ seq_len: int,
+ n_elem: int,
+ image_infos: Optional[List[Tuple[slice, Tuple[int, int]]]] = None,
+ device: Optional[torch.device] = None,
+ base: int = 10000,
+):
+ """Build 2D RoPE cos/sin tables ([seq_len, n_elem]); text uses 1D,
+ image positions use 2D grid indices.
+ """
+ assert n_elem % 4 == 0, f"n_elem must be divisible by 4, but got {n_elem}."
+
+ theta = 1.0 / (base ** (torch.arange(0, n_elem, 2, device=device).float() / n_elem))
+ theta = theta.reshape(1, n_elem // 4, 2) # [1, half_d, 2]
+
+ if image_infos is None:
+ image_infos = []
+
+ image_infos_list = [image_infos]
+ sample_seq_lens = [seq_len]
+
+ x_sections = []
+ y_sections = []
+ for sample_id, sample_image_infos in enumerate(image_infos_list):
+ last_pos = 0
+ for sec_slice, (h, w) in sample_image_infos:
+ L = sec_slice.start
+ if last_pos < L:
+ y_sections.append(torch.arange(last_pos, L, device=device))
+ x_sections.append(torch.arange(last_pos, L, device=device))
+ elif h is None:
+ y_sections.append(
+ torch.arange(sec_slice.start, sec_slice.stop, device=device)
+ )
+ x_sections.append(
+ torch.arange(sec_slice.start, sec_slice.stop, device=device)
+ )
+ continue
+ else:
+ pass
+ beta_y = L + (w * h - h) / 2
+ beta_x = L + (w * h - w) / 2
+ # linspace(a, b, n+1)[:n] == arange(a, a+n); kept for bit-parity
+ # with the official implementation
+ y_axis = torch.linspace(
+ beta_y, beta_y + h, h + 1, dtype=torch.float32, device=device
+ )[:h]
+ x_axis = torch.linspace(
+ beta_x, beta_x + w, w + 1, dtype=torch.float32, device=device
+ )[:w]
+ grid = torch.stack(
+ torch.meshgrid(y_axis, x_axis, indexing="ij"), dim=0
+ ).reshape(2, -1)
+ y_sections.append(grid[0])
+ x_sections.append(grid[1])
+ last_pos = L + w * h
+ y_sections.append(
+ torch.arange(last_pos, sample_seq_lens[sample_id], device=device)
+ )
+ x_sections.append(
+ torch.arange(last_pos, sample_seq_lens[sample_id], device=device)
+ )
+
+ x_pos = torch.cat(x_sections).long()
+ y_pos = torch.cat(y_sections).long()
+ x_pos = x_pos[:seq_len]
+ y_pos = y_pos[:seq_len]
+ all_pos = (
+ torch.stack((y_pos, x_pos), dim=1).unsqueeze(1).to(device)
+ ) # [seq_len, 1, 2]
+
+ # Tile the half-dim angles up to head_dim (official layout): rotate_half
+ # pairs dim i with dim i + n_elem/2, both carrying the SAME angle;
+ # interleaved expansion pairs mismatched frequencies and garbles positions.
+ idx_theta = (all_pos * theta).reshape(all_pos.shape[0], n_elem // 2).repeat(1, 2)
+ cos = torch.cos(idx_theta)
+ sin = torch.sin(idx_theta)
+ return cos, sin
+
+
+def build_batch_2d_rope(
+ seq_len: int,
+ n_elem: int,
+ image_infos: Optional[List[List[Tuple[slice, Tuple[int, int]]]]] = None,
+ device: Optional[torch.device] = None,
+ base: int = 10000,
+):
+ cos_list, sin_list = [], []
+ if image_infos is None:
+ image_infos = [None]
+ for image_info in image_infos:
+ cos, sin = build_2d_rope(
+ seq_len,
+ n_elem,
+ image_infos=image_info,
+ device=device,
+ base=base,
+ )
+ cos_list.append(cos)
+ sin_list.append(sin)
+
+ return torch.stack(cos_list, dim=0), torch.stack(sin_list, dim=0)
+
+
+class CachedRoPE:
+ def __init__(self, rope_theta: float, head_dim: int, rope_type: str = "2d"):
+ # "2d" and "default" are the official spellings of the same scheme;
+ # anything else in a checkpoint config would silently get wrong RoPE.
+ if rope_type not in ("2d", "default"):
+ raise NotImplementedError(f"rope_type `{rope_type}` not supported")
+ self.rope_theta = rope_theta
+ self.head_dim = head_dim
+ self.cos_cache = None
+ self.sin_cache = None
+ self.seq_len = None
+ self.rope_image_info = None
+
+ def __call__(self, seq_len, device, rope_image_info=None):
+ if (self.seq_len != seq_len) or (
+ rope_image_info is not None and self.rope_image_info != rope_image_info
+ ):
+ self.cos_cache, self.sin_cache = build_batch_2d_rope(
+ image_infos=rope_image_info,
+ seq_len=seq_len,
+ n_elem=self.head_dim,
+ device=device,
+ base=self.rope_theta,
+ )
+ self.seq_len = seq_len
+ self.rope_image_info = rope_image_info
+
+ return self.cos_cache, self.sin_cache
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+ if n_rep == 1:
+ return hidden_states
+ hidden_states = hidden_states[:, :, None, :, :].expand(
+ batch, num_key_value_heads, n_rep, slen, head_dim
+ )
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class ImageKVCacheManager:
+ """Full attention over the text+image sequence (no KV caching); packed
+ ``[tokens, heads, dim]`` inputs, 4-D bool mask.
+ """
+
+ def __call__(self, query, key, value, attn_metadata, attention_mask=None):
+ total_tokens = query.shape[0]
+ bs = len(attn_metadata.query_lens)
+ q_len = total_tokens // bs
+
+ head_num_per_rank = query.shape[1]
+ kv_head_num_per_rank = key.shape[1]
+ repeat_num = head_num_per_rank // kv_head_num_per_rank
+ head_dim = query.shape[2]
+
+ query = query.reshape(bs, q_len, head_num_per_rank, head_dim)
+ key = key.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
+ value = value.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
+
+ query = query.transpose(1, 2).contiguous()
+ key = key.transpose(1, 2).contiguous()
+ value = value.transpose(1, 2).contiguous()
+
+ key = repeat_kv(key, repeat_num)
+ value = repeat_kv(value, repeat_num)
+
+ attn_output = F.scaled_dot_product_attention(
+ query,
+ key,
+ value,
+ attn_mask=attention_mask.contiguous(),
+ dropout_p=0.0,
+ is_causal=False,
+ scale=1.0 / (query.shape[-1] ** 0.5),
+ )
+
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.reshape(total_tokens, head_num_per_rank, head_dim)
+ return attn_output
diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan_image3.py b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan_image3.py
new file mode 100644
index 0000000000..38edb77a23
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan_image3.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import torch
+import torch.nn.functional as F
+from transformers import Siglip2VisionConfig
+
+from sglang.srt.models.siglip2 import Siglip2Model
+
+
+class _PerImageSDPA(torch.nn.Module):
+ """Keep independent images separate without a quadratic packed-sequence mask."""
+
+ def forward(self, q, k, v, cu_seqlens, **kwargs):
+ boundaries = cu_seqlens.tolist()
+ output = torch.empty_like(q)
+ for start, end in zip(boundaries[:-1], boundaries[1:]):
+ output[start:end] = (
+ F.scaled_dot_product_attention(
+ q[start:end].transpose(0, 1).unsqueeze(0),
+ k[start:end].transpose(0, 1).unsqueeze(0),
+ v[start:end].transpose(0, 1).unsqueeze(0),
+ is_causal=False,
+ )
+ .squeeze(0)
+ .transpose(0, 1)
+ )
+ return output
+
+
+class HunyuanImage3VisionModel(Siglip2Model):
+ """Adapt padded HunyuanImage-3 conditioning inputs to SRT's packed SigLIP2."""
+
+ def __init__(self, config: dict):
+ config = dict(config, hidden_act="gelu")
+ # The checkpoint stores a legacy read-only Transformers property.
+ config.pop("use_return_dict", None)
+ super().__init__(
+ Siglip2VisionConfig(**config),
+ qkv_backend="sdpa",
+ use_data_parallel=True,
+ )
+ for layer in self.vision_model.encoder.layers:
+ layer.self_attn.attn.qkv_backend = _PerImageSDPA()
+
+ def forward(self, pixel_values, attention_mask, spatial_shapes):
+ batch_size, max_patches, _ = pixel_values.shape
+ mask = attention_mask.reshape(batch_size, max_patches).bool()
+ # shape metadata stays on CPU and is shared by every encoder layer
+ spatial_shapes = spatial_shapes.reshape(-1, 2).cpu()
+ lengths = spatial_shapes.prod(dim=1).to(torch.int32)
+ cu_seqlens = torch.nn.functional.pad(
+ lengths.cumsum(0, dtype=torch.int32), (1, 0)
+ )
+ hidden_states = (
+ super()
+ .forward(
+ pixel_values_packed=pixel_values[mask],
+ spatial_shapes=spatial_shapes,
+ cu_seqlens=cu_seqlens,
+ max_seqlen=int(lengths.max()),
+ )
+ .squeeze(0)
+ )
+ output = hidden_states.new_zeros(
+ batch_size, max_patches, hidden_states.shape[-1]
+ )
+ output[mask] = hidden_states
+ return output
+
+ def load_weights(self, weights):
+ return super().load_weights(
+ (f"vision_model.{name}", weight) for name, weight in weights
+ )
diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_conv3d_hunyuan_image3.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_conv3d_hunyuan_image3.py
new file mode 100644
index 0000000000..d047bf9a6b
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_conv3d_hunyuan_image3.py
@@ -0,0 +1,764 @@
+# Copied from the official HunyuanImage-3 repository (autoencoder_kl_3d.py);
+# distributed helpers omitted.
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
+from diffusers.models.modeling_outputs import AutoencoderKLOutput
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.utils import BaseOutput
+from einops import rearrange
+from torch import Tensor, nn
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class DecoderOutput(BaseOutput):
+ sample: torch.FloatTensor
+ posterior: Optional[DiagonalGaussianDistribution] = None
+
+
+def swish(x: Tensor) -> Tensor:
+ return x * torch.sigmoid(x)
+
+
+def forward_with_checkpointing(module, *inputs, use_checkpointing=False):
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs)
+
+ return custom_forward
+
+ if use_checkpointing:
+ return torch.utils.checkpoint.checkpoint(
+ create_custom_forward(module), *inputs, use_reentrant=False
+ )
+ else:
+ return module(*inputs)
+
+
+class Conv3d(nn.Conv3d):
+ """Perform Conv3d on patches with numerical differences from nn.Conv3d
+ within 1e-5. Only symmetric padding is supported."""
+
+ def forward(self, input):
+ B, C, T, H, W = input.shape
+ memory_count = (C * T * H * W) * 2 / 1024**3
+ if memory_count > 2:
+ n_split = math.ceil(memory_count / 2)
+ chunks = torch.chunk(input, chunks=n_split, dim=-3)
+ padded_chunks = []
+ for i in range(len(chunks)):
+ if self.padding[0] > 0:
+ padded_chunk = F.pad(
+ chunks[i],
+ (0, 0, 0, 0, self.padding[0], self.padding[0]),
+ mode="constant"
+ if self.padding_mode == "zeros"
+ else self.padding_mode,
+ value=0,
+ )
+ if i > 0:
+ padded_chunk[:, :, : self.padding[0]] = chunks[i - 1][
+ :, :, -self.padding[0] :
+ ]
+ if i < len(chunks) - 1:
+ padded_chunk[:, :, -self.padding[0] :] = chunks[i + 1][
+ :, :, : self.padding[0]
+ ]
+ else:
+ padded_chunk = chunks[i]
+ padded_chunks.append(padded_chunk)
+ padding_bak = self.padding
+ self.padding = (0, self.padding[1], self.padding[2])
+ outputs = []
+ for i in range(len(padded_chunks)):
+ outputs.append(super().forward(padded_chunks[i]))
+ self.padding = padding_bak
+ return torch.cat(outputs, dim=-3)
+ else:
+ return super().forward(input)
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels: int):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm = nn.GroupNorm(
+ num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
+ )
+ self.q = Conv3d(in_channels, in_channels, kernel_size=1)
+ self.k = Conv3d(in_channels, in_channels, kernel_size=1)
+ self.v = Conv3d(in_channels, in_channels, kernel_size=1)
+ self.proj_out = Conv3d(in_channels, in_channels, kernel_size=1)
+
+ def attention(self, h_: Tensor) -> Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+ b, c, f, h, w = q.shape
+ q = rearrange(q, "b c f h w -> b 1 (f h w) c").contiguous()
+ k = rearrange(k, "b c f h w -> b 1 (f h w) c").contiguous()
+ v = rearrange(v, "b c f h w -> b 1 (f h w) c").contiguous()
+ h_ = nn.functional.scaled_dot_product_attention(q, k, v)
+ return rearrange(h_, "b 1 (f h w) c -> b c f h w", f=f, h=h, w=w, c=c, b=b)
+
+ def forward(self, x: Tensor) -> Tensor:
+ return x + self.proj_out(self.attention(x))
+
+
+class ResnetBlock(nn.Module):
+ def __init__(self, in_channels: int, out_channels: int):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.norm1 = nn.GroupNorm(
+ num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
+ )
+ self.conv1 = Conv3d(
+ in_channels, out_channels, kernel_size=3, stride=1, padding=1
+ )
+ self.norm2 = nn.GroupNorm(
+ num_groups=32, num_channels=out_channels, eps=1e-6, affine=True
+ )
+ self.conv2 = Conv3d(
+ out_channels, out_channels, kernel_size=3, stride=1, padding=1
+ )
+ if self.in_channels != self.out_channels:
+ self.nin_shortcut = Conv3d(
+ in_channels, out_channels, kernel_size=1, stride=1, padding=0
+ )
+
+ def forward(self, x):
+ h = x
+ h = self.norm1(h)
+ h = swish(h)
+ h = self.conv1(h)
+ h = self.norm2(h)
+ h = swish(h)
+ h = self.conv2(h)
+ if self.in_channels != self.out_channels:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels: int, add_temporal_downsample: bool = True):
+ super().__init__()
+ self.add_temporal_downsample = add_temporal_downsample
+ stride = (2, 2, 2) if add_temporal_downsample else (1, 2, 2)
+ self.conv = Conv3d(
+ in_channels, in_channels, kernel_size=3, stride=stride, padding=0
+ )
+
+ def forward(self, x: Tensor):
+ spatial_pad = (0, 1, 0, 1, 0, 0)
+ x = nn.functional.pad(x, spatial_pad, mode="constant", value=0)
+ temporal_pad = (
+ (0, 0, 0, 0, 0, 1) if self.add_temporal_downsample else (0, 0, 0, 0, 1, 1)
+ )
+ x = nn.functional.pad(x, temporal_pad, mode="replicate")
+ x = self.conv(x)
+ return x
+
+
+class DownsampleDCAE(nn.Module):
+ def __init__(
+ self, in_channels: int, out_channels: int, add_temporal_downsample: bool = True
+ ):
+ super().__init__()
+ factor = 2 * 2 * 2 if add_temporal_downsample else 1 * 2 * 2
+ self.conv = Conv3d(
+ in_channels, out_channels // factor, kernel_size=3, stride=1, padding=1
+ )
+ self.add_temporal_downsample = add_temporal_downsample
+ self.group_size = factor * in_channels // out_channels
+
+ def forward(self, x: Tensor):
+ r1 = 2 if self.add_temporal_downsample else 1
+ h = self.conv(x)
+ h = rearrange(
+ h, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2
+ )
+ shortcut = rearrange(
+ x, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2
+ )
+ B, C, T, H, W = shortcut.shape
+ shortcut = shortcut.view(B, h.shape[1], self.group_size, T, H, W).mean(dim=2)
+ return h + shortcut
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels: int, add_temporal_upsample: bool = True):
+ super().__init__()
+ self.add_temporal_upsample = add_temporal_upsample
+ self.scale_factor = (2, 2, 2) if add_temporal_upsample else (1, 2, 2)
+ self.conv = Conv3d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x: Tensor):
+ x = nn.functional.interpolate(x, scale_factor=self.scale_factor, mode="nearest")
+ x = self.conv(x)
+ return x
+
+
+class UpsampleDCAE(nn.Module):
+ def __init__(
+ self, in_channels: int, out_channels: int, add_temporal_upsample: bool = True
+ ):
+ super().__init__()
+ factor = 2 * 2 * 2 if add_temporal_upsample else 1 * 2 * 2
+ self.conv = Conv3d(
+ in_channels, out_channels * factor, kernel_size=3, stride=1, padding=1
+ )
+ self.add_temporal_upsample = add_temporal_upsample
+ self.repeats = factor * out_channels // in_channels
+
+ def forward(self, x: Tensor):
+ r1 = 2 if self.add_temporal_upsample else 1
+ h = self.conv(x)
+ h = rearrange(
+ h, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2
+ )
+ shortcut = x.repeat_interleave(repeats=self.repeats, dim=1)
+ shortcut = rearrange(
+ shortcut,
+ "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)",
+ r1=r1,
+ r2=2,
+ r3=2,
+ )
+ return h + shortcut
+
+
+class Encoder(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ z_channels: int,
+ block_out_channels: Tuple[int, ...],
+ num_res_blocks: int,
+ ffactor_spatial: int,
+ ffactor_temporal: int,
+ downsample_match_channel: bool = True,
+ ):
+ super().__init__()
+
+ self.z_channels = z_channels
+ self.block_out_channels = block_out_channels
+ self.num_res_blocks = num_res_blocks
+
+ self.conv_in = Conv3d(
+ in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1
+ )
+
+ self.down = nn.ModuleList()
+ block_in = block_out_channels[0]
+ for i_level, ch in enumerate(block_out_channels):
+ block = nn.ModuleList()
+ block_out = ch
+ for _ in range(self.num_res_blocks):
+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
+ block_in = block_out
+ down = nn.Module()
+ down.block = block
+
+ add_spatial_downsample = bool(i_level < np.log2(ffactor_spatial))
+ add_temporal_downsample = add_spatial_downsample and bool(
+ i_level >= np.log2(ffactor_spatial // ffactor_temporal)
+ )
+ if add_spatial_downsample or add_temporal_downsample:
+ block_out = (
+ block_out_channels[i_level + 1]
+ if downsample_match_channel
+ else block_in
+ )
+ down.downsample = DownsampleDCAE(
+ block_in, block_out, add_temporal_downsample
+ )
+ block_in = block_out
+ self.down.append(down)
+
+ self.mid = nn.Module()
+ self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+ self.mid.attn_1 = AttnBlock(block_in)
+ self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+
+ self.norm_out = nn.GroupNorm(
+ num_groups=32, num_channels=block_in, eps=1e-6, affine=True
+ )
+ self.conv_out = Conv3d(
+ block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1
+ )
+
+ self.gradient_checkpointing = False
+
+ def forward(self, x: Tensor) -> Tensor:
+ with torch.no_grad():
+ use_checkpointing = bool(self.training and self.gradient_checkpointing)
+
+ h = self.conv_in(x)
+ for i_level in range(len(self.block_out_channels)):
+ for i_block in range(self.num_res_blocks):
+ h = forward_with_checkpointing(
+ self.down[i_level].block[i_block],
+ h,
+ use_checkpointing=use_checkpointing,
+ )
+ if hasattr(self.down[i_level], "downsample"):
+ h = forward_with_checkpointing(
+ self.down[i_level].downsample,
+ h,
+ use_checkpointing=use_checkpointing,
+ )
+
+ h = forward_with_checkpointing(
+ self.mid.block_1, h, use_checkpointing=use_checkpointing
+ )
+ h = forward_with_checkpointing(
+ self.mid.attn_1, h, use_checkpointing=use_checkpointing
+ )
+ h = forward_with_checkpointing(
+ self.mid.block_2, h, use_checkpointing=use_checkpointing
+ )
+
+ group_size = self.block_out_channels[-1] // (2 * self.z_channels)
+ shortcut = rearrange(h, "b (c r) f h w -> b c r f h w", r=group_size).mean(
+ dim=2
+ )
+ h = self.norm_out(h)
+ h = swish(h)
+ h = self.conv_out(h)
+ h += shortcut
+ return h
+
+
+class Decoder(nn.Module):
+ def __init__(
+ self,
+ z_channels: int,
+ out_channels: int,
+ block_out_channels: Tuple[int, ...],
+ num_res_blocks: int,
+ ffactor_spatial: int,
+ ffactor_temporal: int,
+ upsample_match_channel: bool = True,
+ ):
+ super().__init__()
+
+ self.z_channels = z_channels
+ self.block_out_channels = block_out_channels
+ self.num_res_blocks = num_res_blocks
+
+ block_in = block_out_channels[0]
+ self.conv_in = Conv3d(z_channels, block_in, kernel_size=3, stride=1, padding=1)
+
+ self.mid = nn.Module()
+ self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+ self.mid.attn_1 = AttnBlock(block_in)
+ self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+
+ self.up = nn.ModuleList()
+ for i_level, ch in enumerate(block_out_channels):
+ block = nn.ModuleList()
+ block_out = ch
+ for _ in range(self.num_res_blocks + 1):
+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
+ block_in = block_out
+ up = nn.Module()
+ up.block = block
+
+ add_spatial_upsample = bool(i_level < np.log2(ffactor_spatial))
+ add_temporal_upsample = bool(i_level < np.log2(ffactor_temporal))
+ if add_spatial_upsample or add_temporal_upsample:
+ block_out = (
+ block_out_channels[i_level + 1]
+ if upsample_match_channel
+ else block_in
+ )
+ up.upsample = UpsampleDCAE(block_in, block_out, add_temporal_upsample)
+ block_in = block_out
+ self.up.append(up)
+
+ self.norm_out = nn.GroupNorm(
+ num_groups=32, num_channels=block_in, eps=1e-6, affine=True
+ )
+ self.conv_out = Conv3d(
+ block_in, out_channels, kernel_size=3, stride=1, padding=1
+ )
+
+ self.gradient_checkpointing = False
+
+ def forward(self, z: Tensor) -> Tensor:
+ with torch.no_grad():
+ use_checkpointing = bool(self.training and self.gradient_checkpointing)
+ repeats = self.block_out_channels[0] // self.z_channels
+ h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1)
+
+ h = forward_with_checkpointing(
+ self.mid.block_1, h, use_checkpointing=use_checkpointing
+ )
+ h = forward_with_checkpointing(
+ self.mid.attn_1, h, use_checkpointing=use_checkpointing
+ )
+ h = forward_with_checkpointing(
+ self.mid.block_2, h, use_checkpointing=use_checkpointing
+ )
+
+ for i_level in range(len(self.block_out_channels)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = forward_with_checkpointing(
+ self.up[i_level].block[i_block],
+ h,
+ use_checkpointing=use_checkpointing,
+ )
+ if hasattr(self.up[i_level], "upsample"):
+ h = forward_with_checkpointing(
+ self.up[i_level].upsample,
+ h,
+ use_checkpointing=use_checkpointing,
+ )
+
+ h = self.norm_out(h)
+ h = swish(h)
+ h = self.conv_out(h)
+ return h
+
+
+class AutoencoderKLConv3D(ModelMixin, ConfigMixin):
+ """3D VAE copied from the official HunyuanImage-3 repository."""
+
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ latent_channels: int,
+ block_out_channels: Tuple[int, ...],
+ layers_per_block: int,
+ ffactor_spatial: int,
+ ffactor_temporal: int,
+ sample_size: int,
+ sample_tsize: int,
+ scaling_factor: float = None,
+ shift_factor: Optional[float] = None,
+ downsample_match_channel: bool = True,
+ upsample_match_channel: bool = True,
+ only_encoder: bool = False,
+ only_decoder: bool = False,
+ ):
+ super().__init__()
+ self.ffactor_spatial = ffactor_spatial
+ self.ffactor_temporal = ffactor_temporal
+ self.scaling_factor = scaling_factor
+ self.shift_factor = shift_factor
+
+ if not only_decoder:
+ self.encoder = Encoder(
+ in_channels=in_channels,
+ z_channels=latent_channels,
+ block_out_channels=block_out_channels,
+ num_res_blocks=layers_per_block,
+ ffactor_spatial=ffactor_spatial,
+ ffactor_temporal=ffactor_temporal,
+ downsample_match_channel=downsample_match_channel,
+ )
+ if not only_encoder:
+ self.decoder = Decoder(
+ z_channels=latent_channels,
+ out_channels=out_channels,
+ block_out_channels=list(reversed(block_out_channels)),
+ num_res_blocks=layers_per_block,
+ ffactor_spatial=ffactor_spatial,
+ ffactor_temporal=ffactor_temporal,
+ upsample_match_channel=upsample_match_channel,
+ )
+
+ self.use_slicing = False
+ self.slicing_bsz = 1
+ self.use_spatial_tiling = False
+ self.use_temporal_tiling = False
+ self.use_tiling_during_training = False
+
+ self.tile_sample_min_size = sample_size
+ self.tile_latent_min_size = sample_size // ffactor_spatial
+ self.tile_sample_min_tsize = sample_tsize
+ self.tile_latent_min_tsize = sample_tsize // ffactor_temporal
+ self.tile_overlap_factor = 0.125
+
+ self.use_compile = False
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, (Encoder, Decoder)):
+ module.gradient_checkpointing = value
+
+ def enable_tiling_during_training(self, use_tiling: bool = True):
+ self.use_tiling_during_training = use_tiling
+
+ def disable_tiling_during_training(self):
+ self.enable_tiling_during_training(False)
+
+ def enable_temporal_tiling(self, use_tiling: bool = True):
+ self.use_temporal_tiling = use_tiling
+
+ def disable_temporal_tiling(self):
+ self.enable_temporal_tiling(False)
+
+ def enable_spatial_tiling(self, use_tiling: bool = True):
+ self.use_spatial_tiling = use_tiling
+
+ def disable_spatial_tiling(self):
+ self.enable_spatial_tiling(False)
+
+ def enable_tiling(self, use_tiling: bool = True):
+ self.enable_spatial_tiling(use_tiling)
+
+ def disable_tiling(self):
+ self.disable_spatial_tiling()
+
+ def enable_slicing(self):
+ self.use_slicing = True
+
+ def disable_slicing(self):
+ self.use_slicing = False
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
+ blend_extent = min(a.shape[-1], b.shape[-1], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (
+ 1 - x / blend_extent
+ ) + b[:, :, :, :, x] * (x / blend_extent)
+ return b
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
+ blend_extent = min(a.shape[-2], b.shape[-2], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (
+ 1 - y / blend_extent
+ ) + b[:, :, :, y, :] * (y / blend_extent)
+ return b
+
+ def blend_t(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
+ blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (
+ 1 - x / blend_extent
+ ) + b[:, :, x, :, :] * (x / blend_extent)
+ return b
+
+ def spatial_tiled_encode(self, x: torch.Tensor):
+ B, C, T, H, W = x.shape
+ overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor))
+ blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor)
+ row_limit = self.tile_latent_min_size - blend_extent
+
+ rows = []
+ for i in range(0, H, overlap_size):
+ row = []
+ for j in range(0, W, overlap_size):
+ tile = x[
+ :,
+ :,
+ :,
+ i : i + self.tile_sample_min_size,
+ j : j + self.tile_sample_min_size,
+ ]
+ tile = self.encoder(tile)
+ row.append(tile)
+ rows.append(row)
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent)
+ result_row.append(tile[:, :, :, :row_limit, :row_limit])
+ result_rows.append(torch.cat(result_row, dim=-1))
+ moments = torch.cat(result_rows, dim=-2)
+ return moments
+
+ def temporal_tiled_encode(self, x: torch.Tensor):
+ B, C, T, H, W = x.shape
+ overlap_size = int(self.tile_sample_min_tsize * (1 - self.tile_overlap_factor))
+ blend_extent = int(self.tile_latent_min_tsize * self.tile_overlap_factor)
+ t_limit = self.tile_latent_min_tsize - blend_extent
+
+ row = []
+ for i in range(0, T, overlap_size):
+ tile = x[:, :, i : i + self.tile_sample_min_tsize, :, :]
+ if self.use_spatial_tiling and (
+ tile.shape[-1] > self.tile_sample_min_size
+ or tile.shape[-2] > self.tile_sample_min_size
+ ):
+ tile = self.spatial_tiled_encode(tile)
+ else:
+ tile = self.encoder(tile)
+ row.append(tile)
+ result_row = []
+ for i, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_t(row[i - 1], tile, blend_extent)
+ result_row.append(tile[:, :, :t_limit, :, :])
+ moments = torch.cat(result_row, dim=-3)
+ return moments
+
+ def spatial_tiled_decode(self, z: torch.Tensor):
+ B, C, T, H, W = z.shape
+ overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))
+ blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor)
+ row_limit = self.tile_sample_min_size - blend_extent
+
+ rows = []
+ for i in range(0, H, overlap_size):
+ row = []
+ for j in range(0, W, overlap_size):
+ tile = z[
+ :,
+ :,
+ :,
+ i : i + self.tile_latent_min_size,
+ j : j + self.tile_latent_min_size,
+ ]
+ decoded = self.decoder(tile)
+ row.append(decoded)
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent)
+ result_row.append(tile[:, :, :, :row_limit, :row_limit])
+ result_rows.append(torch.cat(result_row, dim=-1))
+ dec = torch.cat(result_rows, dim=-2)
+ return dec
+
+ def temporal_tiled_decode(self, z: torch.Tensor):
+ B, C, T, H, W = z.shape
+ overlap_size = int(self.tile_latent_min_tsize * (1 - self.tile_overlap_factor))
+ blend_extent = int(self.tile_sample_min_tsize * self.tile_overlap_factor)
+ t_limit = self.tile_sample_min_tsize - blend_extent
+ assert 0 < overlap_size < self.tile_latent_min_tsize
+
+ row = []
+ for i in range(0, T, overlap_size):
+ tile = z[:, :, i : i + self.tile_latent_min_tsize, :, :]
+ if self.use_spatial_tiling and (
+ tile.shape[-1] > self.tile_latent_min_size
+ or tile.shape[-2] > self.tile_latent_min_size
+ ):
+ decoded = self.spatial_tiled_decode(tile)
+ else:
+ decoded = self.decoder(tile)
+ row.append(decoded)
+
+ result_row = []
+ for i, tile in enumerate(row):
+ if i > 0:
+ tile = self.blend_t(row[i - 1], tile, blend_extent)
+ result_row.append(tile[:, :, :t_limit, :, :])
+ dec = torch.cat(result_row, dim=-3)
+ return dec
+
+ def encode(self, x: Tensor, return_dict: bool = True):
+ def _encode(x):
+ if self.use_temporal_tiling and x.shape[-3] > self.tile_sample_min_tsize:
+ return self.temporal_tiled_encode(x)
+ if self.use_spatial_tiling and (
+ x.shape[-1] > self.tile_sample_min_size
+ or x.shape[-2] > self.tile_sample_min_size
+ ):
+ return self.spatial_tiled_encode(x)
+ if self.use_compile:
+
+ @torch.compile
+ def encoder(x):
+ return self.encoder(x)
+
+ return encoder(x)
+ return self.encoder(x)
+
+ if len(x.shape) != 5: # (B, C, T, H, W)
+ x = x[:, :, None]
+ if x.shape[2] == 1:
+ x = x.expand(-1, -1, self.ffactor_temporal, -1, -1)
+ else:
+ assert (
+ x.shape[2] != self.ffactor_temporal
+ and x.shape[2] % self.ffactor_temporal == 0
+ )
+
+ if self.use_slicing and x.shape[0] > 1:
+ if self.slicing_bsz == 1:
+ encoded_slices = [_encode(x_slice) for x_slice in x.split(1)]
+ else:
+ sections = [self.slicing_bsz] * (x.shape[0] // self.slicing_bsz)
+ if x.shape[0] % self.slicing_bsz != 0:
+ sections.append(x.shape[0] % self.slicing_bsz)
+ encoded_slices = [_encode(x_slice) for x_slice in x.split(sections)]
+ h = torch.cat(encoded_slices)
+ else:
+ h = _encode(x)
+ posterior = DiagonalGaussianDistribution(h)
+
+ if not return_dict:
+ return (posterior,)
+ return AutoencoderKLOutput(latent_dist=posterior)
+
+ def decode(self, z: Tensor, return_dict: bool = True, generator=None):
+ def _decode(z):
+ if self.use_temporal_tiling and z.shape[-3] > self.tile_latent_min_tsize:
+ return self.temporal_tiled_decode(z)
+ if self.use_spatial_tiling and (
+ z.shape[-1] > self.tile_latent_min_size
+ or z.shape[-2] > self.tile_latent_min_size
+ ):
+ return self.spatial_tiled_decode(z)
+ return self.decoder(z)
+
+ if self.use_slicing and z.shape[0] > 1:
+ decoded_slices = [_decode(z_slice) for z_slice in z.split(1)]
+ decoded = torch.cat(decoded_slices)
+ else:
+ decoded = _decode(z)
+
+ if z.shape[-3] == 1:
+ decoded = decoded[:, :, -1:]
+ if not return_dict:
+ return (decoded,)
+ return DecoderOutput(sample=decoded)
+
+ def forward(
+ self,
+ sample: torch.Tensor,
+ sample_posterior: bool = False,
+ return_posterior: bool = True,
+ return_dict: bool = True,
+ ):
+ posterior = self.encode(sample).latent_dist
+ z = posterior.sample() if sample_posterior else posterior.mode()
+ dec = self.decode(z).sample
+ return (
+ DecoderOutput(sample=dec, posterior=posterior)
+ if return_dict
+ else (dec, posterior)
+ )
diff --git a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_image3.py b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_image3.py
new file mode 100644
index 0000000000..2be4fc3c55
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_image3.py
@@ -0,0 +1,372 @@
+import json
+import os
+from typing import Any
+
+import torch
+from torch.distributed import init_device_mesh
+from torch.distributed.fsdp import MixedPrecisionPolicy
+from transformers import AutoTokenizer
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
+from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
+from sglang.multimodal_gen.runtime.layers.attention.selector import (
+ component_attn_backend_context_manager,
+)
+from sglang.multimodal_gen.runtime.loader.fsdp_load import shard_model
+from sglang.multimodal_gen.runtime.loader.transformer_load_utils import (
+ resolve_transformer_checkpoint_files,
+)
+from sglang.multimodal_gen.runtime.loader.utils import (
+ load_model_state_dict,
+ set_default_torch_dtype,
+)
+from sglang.multimodal_gen.runtime.loader.weight_utils import (
+ safetensors_weights_iterator,
+)
+from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
+ get_global_component_residency_manager,
+)
+from sglang.multimodal_gen.runtime.models.dits.hunyuan_image3 import (
+ HunyuanImage3ForCausalMM,
+ LightProjector,
+)
+from sglang.multimodal_gen.runtime.models.encoders.hunyuan_image3 import (
+ HunyuanImage3VisionModel,
+)
+from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
+ FlowMatchEulerDiscreteScheduler,
+)
+from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_conv3d_hunyuan_image3 import (
+ AutoencoderKLConv3D,
+)
+from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline
+from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
+ ComposedPipelineBase,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3 import (
+ HunyuanImage3AR,
+ HunyuanImage3DecodingStage,
+)
+from sglang.multimodal_gen.runtime.platforms import current_platform
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
+ get_hf_config,
+ load_dict,
+ maybe_download_model,
+)
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy
+from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE
+
+logger = init_logger(__name__)
+
+
+def _module_memory_gb(module: torch.nn.Module) -> float:
+ total_bytes = sum(p.numel() * p.element_size() for p in module.parameters())
+ total_bytes += sum(b.numel() * b.element_size() for b in module.buffers())
+ return total_bytes / (1024**3)
+
+
+class HunyuanImage3Pipeline(LoRAPipeline, ComposedPipelineBase):
+ pipeline_name = "HunyuanImage3Pipeline"
+
+ _required_config_modules = [
+ "text_encoder",
+ "tokenizer",
+ "vae",
+ "processor",
+ "transformer",
+ "scheduler",
+ "vision_model",
+ "vision_aligner",
+ ]
+
+ def load_modules(
+ self,
+ server_args: ServerArgs,
+ loaded_modules: dict[str, torch.nn.Module] | None = None,
+ ) -> dict[str, Any]:
+ pipeline_config = server_args.pipeline_config
+
+ model_path = maybe_download_model(self.model_path)
+ self.model_path = model_path
+ logger.info("Loading HunyuanImage-3 components from %s", model_path)
+
+ hf_config = get_hf_config(
+ model_path,
+ trust_remote_code=True,
+ revision=server_args.revision,
+ )
+ config_dict = hf_config.to_dict()
+
+ pipeline_config.dit_config.update_model_arch(config_dict)
+ vae_config_dict = dict(config_dict["vae"])
+ pipeline_config.vae_config.update_model_arch(vae_config_dict)
+ pipeline_config.vae_config.post_init()
+
+ flow_shift = self._read_flow_shift(model_path)
+ pipeline_config.flow_shift = flow_shift
+
+ ar_model = self._load_ar_model(
+ server_args, pipeline_config, model_path, config_dict
+ )
+
+ return {
+ "transformer": ar_model,
+ "text_encoder": ar_model,
+ "vae": self._load_vae(
+ server_args, pipeline_config, model_path, vae_config_dict
+ ),
+ "tokenizer": self._load_tokenizer(server_args, model_path),
+ "scheduler": FlowMatchEulerDiscreteScheduler(shift=flow_shift),
+ "processor": self._load_processor(server_args, model_path, hf_config),
+ "vision_model": self._load_component(
+ HunyuanImage3VisionModel(config_dict["vit"]),
+ "vision_model",
+ model_path,
+ server_args,
+ ),
+ "vision_aligner": self._load_component(
+ LightProjector(config_dict["vit_aligner"]),
+ "vision_aligner",
+ model_path,
+ server_args,
+ ),
+ }
+
+ def _load_ar_model(
+ self,
+ server_args: ServerArgs,
+ pipeline_config: Any,
+ model_path: str,
+ config_dict: dict[str, Any],
+ ) -> torch.nn.Module:
+ # resolve_transformer_checkpoint_files handles the --transformer-
+ # weights-path override (incl. GGUF/mixed-export selection) and the
+ # plain component directory, returning the shard list plus an
+ # optional override config path (unused here; the AR config comes
+ # from the unified checkpoint's config.json).
+ safetensors_list = list(
+ resolve_transformer_checkpoint_files(server_args, model_path).safetensors
+ )
+
+ local_torch_device = get_local_torch_device()
+ cpu_offload = bool(server_args.dit_cpu_offload)
+ checkpoint_load_device = (
+ torch.device("cpu") if cpu_offload else local_torch_device
+ )
+ fsdp_inference = bool(server_args.use_fsdp_inference)
+ if fsdp_inference and current_platform.is_mps():
+ logger.warning("Disabling FSDP for MPS platform as it's not compatible")
+ fsdp_inference = False
+
+ param_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
+ logger.info(
+ "Loading AR model: %d shard(s), %s",
+ len(safetensors_list),
+ param_dtype,
+ )
+
+ attn_backend, matched_backend_key = (
+ server_args.resolve_component_attention_backend("transformer")
+ )
+ with component_attn_backend_context_manager(
+ attn_backend, component_name=matched_backend_key or "transformer"
+ ):
+ with (
+ set_default_torch_dtype(param_dtype),
+ torch.device(checkpoint_load_device),
+ ):
+ model = HunyuanImage3ForCausalMM(
+ config=pipeline_config.dit_config,
+ hf_config=config_dict,
+ )
+
+ weights_to_load = {name for name, _ in model.named_parameters()}
+ loaded_weights = model.load_weights(
+ safetensors_weights_iterator(safetensors_list)
+ )
+ weights_not_loaded = weights_to_load - loaded_weights
+ if weights_not_loaded:
+ raise ValueError(
+ f"AR weights not initialized from checkpoint: "
+ f"{sorted(weights_not_loaded)}"
+ )
+
+ model.post_load_weights()
+ for param in model.parameters():
+ param.requires_grad = False
+
+ if fsdp_inference:
+ self._shard_ar_model(
+ model,
+ server_args=server_args,
+ cpu_offload=cpu_offload,
+ param_dtype=param_dtype,
+ )
+ model.eval()
+ self.memory_usages["transformer"] = _module_memory_gb(model)
+ return model
+
+ def _shard_ar_model(
+ self,
+ model: torch.nn.Module,
+ server_args: ServerArgs,
+ cpu_offload: bool,
+ param_dtype: torch.dtype,
+ ) -> None:
+ mp_policy = MixedPrecisionPolicy(
+ param_dtype=param_dtype,
+ reduce_dtype=torch.float32,
+ cast_forward_inputs=False,
+ )
+ set_mixed_precision_policy(
+ param_dtype=param_dtype,
+ reduce_dtype=torch.float32,
+ mp_policy=mp_policy,
+ )
+ device_mesh = init_device_mesh(
+ current_platform.device_type,
+ mesh_shape=(server_args.hsdp_replicate_dim, server_args.hsdp_shard_dim),
+ mesh_dim_names=("replicate", "shard"),
+ )
+ shard_model(
+ model,
+ cpu_offload=cpu_offload,
+ reshard_after_forward=True,
+ mp_policy=mp_policy,
+ mesh=device_mesh,
+ fsdp_shard_conditions=model._fsdp_shard_conditions,
+ pin_cpu_memory=server_args.pin_cpu_memory,
+ )
+
+ def _load_vae(
+ self,
+ server_args: ServerArgs,
+ pipeline_config: Any,
+ model_path: str,
+ vae_config_dict: dict[str, Any],
+ ) -> torch.nn.Module:
+ # config.json omits a few constructor args
+ vae_params = dict(vae_config_dict)
+ vae_params.setdefault("in_channels", 3)
+ vae_params.setdefault("out_channels", 3)
+ vae_params.setdefault("ffactor_temporal", 4)
+ dtype = PRECISION_TO_TYPE[pipeline_config.vae_precision]
+ vae = self._load_component(
+ AutoencoderKLConv3D(**vae_params).to(dtype=dtype),
+ "vae",
+ model_path,
+ server_args,
+ )
+ if server_args.pipeline_config.vae_tiling:
+ vae.enable_tiling()
+ if server_args.pipeline_config.vae_slicing:
+ vae.enable_slicing()
+ return vae
+
+ @staticmethod
+ def _load_tokenizer(server_args: ServerArgs, model_path: str):
+ return AutoTokenizer.from_pretrained(
+ model_path,
+ revision=server_args.revision,
+ trust_remote_code=True,
+ )
+
+ def _load_processor(self, server_args: ServerArgs, model_path: str, hf_config):
+ processor_cls = get_class_from_dynamic_module(
+ "image_processor.HunyuanImage3ImageProcessor",
+ model_path,
+ revision=server_args.revision,
+ )
+ return processor_cls(hf_config)
+
+ def _load_component(
+ self,
+ model: torch.nn.Module,
+ component_name: str,
+ model_path: str,
+ server_args: ServerArgs,
+ ) -> torch.nn.Module:
+ state_dict = self._collect_prefixed_weights(model_path, f"{component_name}.")
+ if isinstance(model, HunyuanImage3VisionModel):
+ loaded = model.load_weights(state_dict.items())
+ missing = set(dict(model.named_parameters())) - loaded
+ if missing:
+ raise ValueError(f"Missing {component_name} weights: {sorted(missing)}")
+ else:
+ load_model_state_dict(model, state_dict)
+ device = (
+ torch.device("cpu")
+ if server_args.should_cpu_offload_component(component_name)
+ else get_local_torch_device()
+ )
+ model.to(device=device).requires_grad_(False).eval()
+ self.memory_usages[component_name] = _module_memory_gb(model)
+ return model
+
+ @staticmethod
+ def _collect_prefixed_weights(
+ model_path: str, prefix: str
+ ) -> dict[str, torch.Tensor]:
+ index_path = os.path.join(model_path, "model.safetensors.index.json")
+ with open(index_path) as f:
+ weight_map = json.load(f)["weight_map"]
+ shard_names = sorted(
+ {shard for key, shard in weight_map.items() if key.startswith(prefix)}
+ )
+ shard_paths = [os.path.join(model_path, name) for name in shard_names]
+
+ state_dict: dict[str, torch.Tensor] = {}
+ for name, tensor in safetensors_weights_iterator(shard_paths):
+ if name.startswith(prefix):
+ state_dict[name[len(prefix) :]] = tensor
+ return state_dict
+
+ @staticmethod
+ def _read_flow_shift(model_path: str) -> float:
+ return float(
+ load_dict(os.path.join(model_path, "generation_config.json"))["flow_shift"]
+ )
+
+ def create_pipeline_stages(self, server_args: ServerArgs):
+ self.add_stage(
+ HunyuanImage3AR(
+ ar_model=self.get_module("transformer"),
+ vae=self.get_module("vae"),
+ tokenizer=self.get_module("tokenizer"),
+ processor=self.get_module("processor"),
+ scheduler=self.get_module("scheduler"),
+ model_path=self.model_path,
+ vision_model=self.get_module("vision_model"),
+ vision_aligner=self.get_module("vision_aligner"),
+ # Reuse the scheduler's group ceiling as the initial cond-encode
+ # chunk cap; 1 (the no-grouping default) means unlimited.
+ max_cond_encode_chunk=(
+ server_args.batching_max_size
+ if server_args.batching_max_size > 1
+ else None
+ ),
+ ),
+ "hunyuan_image3_ar",
+ )
+
+ self.add_stage_factory(
+ RoleType.DECODER,
+ lambda: HunyuanImage3DecodingStage(
+ vae=self.get_module("vae"), pipeline=self
+ ),
+ "decoding_stage",
+ )
+
+ def forward_batch(self, batches, server_args: ServerArgs):
+ if len(batches) > 1 and self.executor.component_residency_manager is None:
+ self.component_residency_manager = get_global_component_residency_manager(
+ self, server_args
+ )
+ self.executor.component_residency_manager = self.component_residency_manager
+ return super().forward_batch(batches, server_args)
+
+
+EntryClass = [HunyuanImage3Pipeline]
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py
index b70920d3ac..3567a6d2d1 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py
@@ -468,6 +468,13 @@ class Req:
save_output: {self.save_output}
output_file_path: {self.output_file_path()}
""" # type: ignore[attr-defined]
+ # Add model-specific fields if present
+ bot_task = getattr(self, "bot_task", None)
+ system_prompt = getattr(self, "system_prompt", None)
+ if bot_task is not None:
+ debug_str += f" bot_task: {bot_task}\n"
+ if system_prompt is not None:
+ debug_str += f" system_prompt: {system_prompt}\n"
logger.info(debug_str)
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/__init__.py
new file mode 100644
index 0000000000..385819d978
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/__init__.py
@@ -0,0 +1,6 @@
+"""HunyuanImage-3 native pipeline stages."""
+
+from .ar_stage import HunyuanImage3AR
+from .decoding import HunyuanImage3DecodingStage
+
+__all__ = ["HunyuanImage3AR", "HunyuanImage3DecodingStage"]
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/ar_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/ar_stage.py
new file mode 100644
index 0000000000..b8f287db9b
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/ar_stage.py
@@ -0,0 +1,1411 @@
+"""Native AR stage for HunyuanImage-3."""
+
+import copy
+import os
+from typing import Any
+
+import torch
+from PIL import Image as PILImage
+
+from sglang.multimodal_gen import envs
+from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
+ CacheDitConfig,
+ enable_cache_on_transformer,
+ refresh_context_on_transformer,
+)
+from sglang.multimodal_gen.runtime.distributed import (
+ get_local_torch_device,
+ get_tp_group,
+ model_parallel_is_initialized,
+)
+from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
+from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
+ ComponentUse,
+)
+from sglang.multimodal_gen.runtime.models.dits.hunyuan_image3 import (
+ Hi3CacheBlockAdapter,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
+ PipelineStage,
+ StageParallelismType,
+)
+from sglang.multimodal_gen.runtime.platforms import current_platform
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import load_dict
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.multimodal_gen.runtime.utils.vision import load_image
+
+from .decoding import (
+ OUTPUT_GEOMETRY_EXTRA_KEY,
+ build_hunyuan_image3_output_geometry,
+ resolve_hunyuan_image3_output_resolution,
+)
+from .prompts import resolve_system_prompt
+from .tokenizer import (
+ HunyuanImage3TokenizerWrapper,
+ ImageInfo,
+ JointImageInfo,
+ TokenizerEncodeOutput,
+)
+
+logger = init_logger(__name__)
+
+
+def _is_oom_error(exc: BaseException) -> bool:
+ if isinstance(exc, torch.OutOfMemoryError):
+ return True
+ return isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower()
+
+
+def _empty_device_cache() -> None:
+ empty_cache = getattr(torch.get_device_module(), "empty_cache", None)
+ if empty_cache is not None:
+ empty_cache()
+
+
+def _seed_for_output(seed, output_idx: int):
+ if isinstance(seed, list):
+ if output_idx < len(seed):
+ return int(seed[output_idx])
+ return int(seed[0]) + output_idx
+ return int(seed) + output_idx
+
+
+def _cond_image_to_pil(raw_img):
+ if isinstance(raw_img, torch.Tensor):
+ return PILImage.fromarray(raw_img.cpu().permute(1, 2, 0).numpy())
+ try:
+ return load_image(raw_img)
+ except ValueError as e:
+ logger.warning("Failed to load condition image %r: %s", raw_img, e)
+ return None
+
+
+def _vae_downsample_factors(config: Any) -> tuple[int, int]:
+ vae_factor = getattr(config, "vae_downsample_factor", [16, 16])
+ if isinstance(vae_factor, (list, tuple)):
+ vae_h = vae_factor[0]
+ vae_w = vae_factor[1] if len(vae_factor) > 1 else vae_factor[0]
+ else:
+ vae_h = vae_w = int(vae_factor)
+ return vae_h, vae_w
+
+
+def _build_causal_attention_mask(
+ batch_size: int,
+ seq_len: int,
+ image_slices: list[list[slice]],
+ device: torch.device,
+) -> torch.Tensor:
+ # Build the causal base in one allocation via an index compare instead of
+ # ones().tril(), which transiently holds two full [L, L] bool tensors.
+ idx = torch.arange(seq_len, device=device)
+ mask = (idx.unsqueeze(0) <= idx.unsqueeze(1)).repeat(batch_size, 1, 1)
+
+ for i in range(batch_size):
+ for image_slice in image_slices[i]:
+ mask[i, image_slice, image_slice] = True
+
+ return mask.unsqueeze(1)
+
+
+def _section_shapes(
+ sections_row: list | None, fallback_h: int, fallback_w: int
+) -> list[tuple[int, int]]:
+ shapes: list[tuple[int, int]] = []
+ for section in sections_row or []:
+ stype = section.get("type", "")
+ if "image" not in stype:
+ continue
+ t_h = section.get("token_height", fallback_h)
+ t_w = section.get("token_width", fallback_w)
+ if isinstance(t_h, list):
+ # joint_image: list of [vae, vit] dims
+ for h_i, w_i in zip(t_h, t_w):
+ shapes.append((int(h_i), int(w_i)))
+ else:
+ shapes.append((int(t_h), int(t_w)))
+ return shapes
+
+
+def _build_rope_image_info(
+ tokenizer_output: TokenizerEncodeOutput,
+ batch_size: int,
+ token_h: int,
+ token_w: int,
+ image_info: ImageInfo,
+ sections: list | None = None,
+) -> list[list[tuple[slice, tuple[int, int]]]]:
+ th = image_info.token_height
+ tw = image_info.token_width
+
+ per_row_sections = bool(sections) and isinstance(sections[0], list)
+ shared_shapes = [] if per_row_sections else _section_shapes(sections, th, tw)
+
+ rope_image_info: list[list[tuple[slice, tuple[int, int]]]] = []
+ for b in range(batch_size):
+ section_shapes = (
+ _section_shapes(sections[b], th, tw) if per_row_sections else shared_shapes
+ )
+ batch_info: list[tuple[slice, tuple[int, int]]] = []
+ shape_idx = 0
+
+ def _take(s: slice, fallback_shape: tuple[int, int]):
+ nonlocal shape_idx
+ if shape_idx < len(section_shapes):
+ batch_info.append((s, section_shapes[shape_idx]))
+ shape_idx += 1
+ else:
+ batch_info.append((s, fallback_shape))
+
+ # The section shape stream follows sequence order: each joint image
+ # contributes [vae, vit] dims and the tokenizer emits VAE tokens then
+ # ViT tokens inside every joint block. Cond spans arrive grouped by
+ # type (all VAE slices, then all ViT slices), so sort them by sequence
+ # position to restore per-image pairing -- consuming the groups as-is
+ # hands image 1's VAE span image 0's ViT dims whenever two cond images
+ # differ in size.
+ cond_spans: list[tuple[int, slice]] = [
+ (s.start, s) for s in tokenizer_output.cond_vae_image_slices[b]
+ ] + [(s.start, s) for s in tokenizer_output.cond_vit_image_slices[b]]
+ for _, s in sorted(cond_spans):
+ _take(s, (token_h, token_w))
+
+ # Cond (joint) images precede gen images in the sequence
+ for s in tokenizer_output.gen_image_slices[b]:
+ _take(s, (th, tw))
+
+ rope_image_info.append(batch_info)
+ return rope_image_info
+
+
+def _register_hi3_cache_dit_spec() -> None:
+ from cache_dit import ForwardPattern
+
+ from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
+ _CUSTOM_BLOCK_ADAPTER_SPECS,
+ CustomBlockAdapterSpec,
+ )
+
+ _CUSTOM_BLOCK_ADAPTER_SPECS.setdefault(
+ Hi3CacheBlockAdapter.__name__,
+ CustomBlockAdapterSpec(
+ blocks_attr="blocks", forward_pattern=ForwardPattern.Pattern_3
+ ),
+ )
+
+
+class HunyuanImage3AR(PipelineStage):
+ def __init__(
+ self,
+ ar_model,
+ vae,
+ tokenizer,
+ processor,
+ scheduler,
+ model_path: str,
+ vision_model,
+ vision_aligner,
+ max_cond_encode_chunk: int | None = None,
+ ):
+ super().__init__()
+ self.ar_model = ar_model
+ self._vae = vae
+ self._processor = processor
+ self._scheduler = scheduler
+ self._model_path = model_path
+ self._vision_model = vision_model
+ self._vision_aligner = vision_aligner
+ # Initial cond-encode chunk cap from --batching-max-size.
+ self._max_cond_encode_chunk = max_cond_encode_chunk
+ self._custom_tokenizer = HunyuanImage3TokenizerWrapper(tokenizer)
+ self._gen_config_cache: dict | None = None
+ self._cache_dit_adapter = None
+ self._cache_dit_num_steps: int | None = None
+
+ def _generation_config(self) -> dict:
+ if self._gen_config_cache is None:
+ self._gen_config_cache = load_dict(
+ os.path.join(self._model_path, "generation_config.json")
+ )
+ return self._gen_config_cache
+
+ def component_uses(
+ self, server_args: ServerArgs, stage_name: str | None = None
+ ) -> list[ComponentUse]:
+ return [
+ ComponentUse(
+ self._component_stage_name(stage_name),
+ "transformer",
+ memory_intensive=True,
+ )
+ ]
+
+ @property
+ def parallelism_type(self) -> StageParallelismType:
+ return StageParallelismType.REPLICATED
+
+ def _rebuild_image_info(self, image_info):
+ # Re-create as this module's ImageInfo so isinstance checks pass.
+ if isinstance(image_info, ImageInfo):
+ return image_info
+ new_info = ImageInfo.__new__(ImageInfo)
+ new_info.__dict__.update(image_info.__dict__)
+ return new_info
+
+ def _build_cache_dit_config(self, num_inference_steps: int) -> CacheDitConfig:
+ return CacheDitConfig(
+ enabled=True,
+ Fn_compute_blocks=envs.SGLANG_CACHE_DIT_FN,
+ Bn_compute_blocks=envs.SGLANG_CACHE_DIT_BN,
+ max_warmup_steps=envs.SGLANG_CACHE_DIT_WARMUP,
+ residual_diff_threshold=envs.SGLANG_CACHE_DIT_RDT,
+ max_continuous_cached_steps=envs.SGLANG_CACHE_DIT_MC,
+ enable_taylorseer=envs.SGLANG_CACHE_DIT_TAYLORSEER,
+ taylorseer_order=envs.SGLANG_CACHE_DIT_TS_ORDER,
+ num_inference_steps=num_inference_steps,
+ )
+
+ def _maybe_enable_cache_dit(self, num_inference_steps: int) -> None:
+ """Mount cache-dit on the diffusion block loop (env-gated, idempotent).
+
+ The AR backbone is not a diffusers DiT, so it is exposed via
+ Hi3CacheBlockAdapter (see the adapter's docstring for the naming
+ constraint); SGLANG_CACHE_DIT_* then apply as for GLM-Image/Wan.
+ """
+ if not envs.SGLANG_CACHE_DIT_ENABLED:
+ return
+ if self._cache_dit_adapter is None:
+ _register_hi3_cache_dit_spec()
+ self._cache_dit_adapter = Hi3CacheBlockAdapter(self.ar_model.model)
+ tp_group = None
+ if model_parallel_is_initialized():
+ group = get_tp_group()
+ tp_group = group.device_group if group.world_size > 1 else None
+ enable_cache_on_transformer(
+ self._cache_dit_adapter,
+ self._build_cache_dit_config(num_inference_steps),
+ model_name="hunyuan_image3",
+ tp_group=tp_group,
+ has_separate_cfg=True,
+ )
+ self.log_info(
+ "cache-dit enabled on HunyuanImage-3 (Fn=%d Bn=%d W=%d R=%.2f "
+ "MC=%d TaylorSeer=%s)",
+ envs.SGLANG_CACHE_DIT_FN,
+ envs.SGLANG_CACHE_DIT_BN,
+ envs.SGLANG_CACHE_DIT_WARMUP,
+ envs.SGLANG_CACHE_DIT_RDT,
+ envs.SGLANG_CACHE_DIT_MC,
+ envs.SGLANG_CACHE_DIT_TAYLORSEER,
+ )
+ # Refresh every batch: a new generation resets cache-dit's step counter.
+ refresh_context_on_transformer(self._cache_dit_adapter, num_inference_steps)
+ self._cache_dit_num_steps = num_inference_steps
+
+ def _backbone_forward(
+ self,
+ num_image_tokens: int,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ custom_pos_emb: tuple[torch.Tensor, torch.Tensor],
+ first_step: bool,
+ timestep: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ hidden_size = hidden_states.shape[-1]
+ hidden_states = hidden_states.reshape(-1, hidden_size).contiguous()
+ attention_mask = attention_mask.contiguous()
+ cos, sin = custom_pos_emb
+ cos = cos.contiguous()
+ sin = sin.contiguous()
+
+ # hidden_states changes every step, so its TP sync stays here;
+ # request-static inputs are broadcast once before the loop.
+ if model_parallel_is_initialized():
+ tp_group = get_tp_group()
+ if tp_group.world_size > 1:
+ hidden_states = tp_group.broadcast(hidden_states, src=0)
+
+ if self._cache_dit_adapter is not None:
+ output = self._cache_dit_adapter(hidden_states, attention_mask, (cos, sin))
+ else:
+ output = self.ar_model.forward_block(
+ hidden_states,
+ attention_mask,
+ (cos, sin),
+ num_image_tokens=num_image_tokens,
+ first_step=first_step,
+ timestep=timestep,
+ )
+ # batch_size may differ after TP broadcast
+ actual_batch = attention_mask.shape[0]
+ actual_seq_len = output.shape[0] // actual_batch
+ return output.view(actual_batch, actual_seq_len, hidden_size)
+
+ def _broadcast_static_inputs(
+ self,
+ attention_mask: torch.Tensor,
+ custom_pos_emb: tuple[torch.Tensor, torch.Tensor],
+ ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+ attention_mask = attention_mask.contiguous()
+ cos, sin = custom_pos_emb
+ cos = cos.contiguous()
+ sin = sin.contiguous()
+ if model_parallel_is_initialized():
+ tp_group = get_tp_group()
+ if tp_group.world_size > 1:
+ attention_mask = tp_group.broadcast(attention_mask, src=0)
+ cos = tp_group.broadcast(cos, src=0)
+ sin = tp_group.broadcast(sin, src=0)
+ return attention_mask, (cos, sin)
+
+ def _instantiate_vae_tokens_first_step(
+ self,
+ hidden_states: torch.Tensor,
+ images: torch.Tensor,
+ timesteps: torch.Tensor,
+ image_mask: torch.Tensor,
+ ) -> torch.Tensor:
+ bsz, seqlen, n_embd = hidden_states.shape
+ t_emb = self.ar_model.time_embed(timesteps)
+ image_seq, token_h, token_w = self.ar_model.patch_embed(images, t_emb)
+ image_scatter_index = (
+ torch.arange(seqlen, device=hidden_states.device)
+ .unsqueeze(0)
+ .expand(bsz, -1)
+ )
+ image_scatter_index = image_scatter_index.masked_select(
+ image_mask.bool()
+ ).reshape(bsz, -1)
+ hidden_states.scatter_(
+ dim=1,
+ index=image_scatter_index.unsqueeze(-1).expand(-1, -1, n_embd),
+ src=image_seq,
+ )
+ return hidden_states
+
+ def _instantiate_timestep_tokens(
+ self,
+ hidden_states: torch.Tensor,
+ timesteps: torch.Tensor,
+ timestep_index: torch.Tensor,
+ ) -> torch.Tensor:
+ bsz, seqlen, n_embd = hidden_states.shape
+ timestep_emb = self.ar_model.timestep_emb(timesteps).reshape(bsz, -1, n_embd)
+
+ if timestep_index.dtype == torch.bool:
+ index = (
+ torch.arange(seqlen, device=hidden_states.device)
+ .unsqueeze(0)
+ .expand(bsz, -1)
+ )
+ ts_scatter_index = index.masked_select(timestep_index).reshape(bsz, -1)
+ else:
+ ts_scatter_index = timestep_index.long()
+
+ num_positions = ts_scatter_index.shape[1]
+ if ts_scatter_index.shape[0] != bsz or timestep_emb.shape[1] != num_positions:
+ # Covers both the gen-timestep and cond-timestep callers; without
+ # this the expand below fails with a cryptic broadcast error.
+ logger.error(
+ "timestep scatter mismatch: bsz=%d seqlen=%d "
+ "timestep_index=%s dtype=%s timesteps=%s "
+ "timestep_emb=%s",
+ bsz,
+ seqlen,
+ tuple(timestep_index.shape),
+ timestep_index.dtype,
+ tuple(timesteps.shape),
+ tuple(timestep_emb.shape),
+ )
+ raise RuntimeError(
+ "timestep scatter mismatch: "
+ f"timestep_emb rows {timestep_emb.shape[1]} vs "
+ f"index positions {num_positions} "
+ f"(bsz {ts_scatter_index.shape[0]} vs {bsz})"
+ )
+ timestep_emb = timestep_emb.expand(-1, num_positions, -1)
+ hidden_states.scatter_(
+ dim=1,
+ index=ts_scatter_index.unsqueeze(-1).expand(-1, -1, n_embd),
+ src=timestep_emb,
+ )
+ return hidden_states
+
+ def _extract_diffusion_pred(
+ self,
+ hidden_states: torch.Tensor,
+ timesteps: torch.Tensor,
+ image_mask: torch.Tensor,
+ token_h: int,
+ token_w: int,
+ ) -> torch.Tensor:
+ n_embd = hidden_states.size(-1)
+ t_emb = self.ar_model.time_embed_2(timesteps)
+
+ image_output = hidden_states.masked_select(
+ image_mask.unsqueeze(-1).bool()
+ ).reshape(-1, token_h * token_w, n_embd)
+
+ return self.ar_model.final_layer(image_output, t_emb, token_h, token_w)
+
+ @staticmethod
+ def _resize_and_crop_center(image, target_width: int, target_height: int):
+ tw, th = target_width, target_height
+ w, h = image.size
+ tr = th / tw
+ r = h / w
+ if r < tr:
+ resize_height = th
+ resize_width = int(round(th / h * w))
+ else:
+ resize_width = tw
+ resize_height = int(round(tw / w * h))
+ resized = image.resize(
+ (resize_width, resize_height), PILImage.Resampling.LANCZOS
+ )
+ crop_left = int(round((resize_width - tw) / 2.0))
+ crop_top = int(round((resize_height - th) / 2.0))
+ return resized.crop((crop_left, crop_top, crop_left + tw, crop_top + th))
+
+ def _preprocess_cond_image(self, pil_image):
+ processor = self._processor
+ pil_image = pil_image.convert("RGB")
+ orig_width, orig_height = pil_image.size
+
+ vae_h_factor, vae_w_factor = _vae_downsample_factors(self.ar_model.config)
+
+ base_size, ratio_idx = processor.vae_reso_group.get_base_size_and_ratio_index(
+ orig_width, orig_height
+ )
+ base_size = int(base_size)
+ ratio_idx = int(ratio_idx)
+ reso = processor.vae_reso_group[ratio_idx]
+ target_width = int(reso.width)
+ target_height = int(reso.height)
+
+ vae_input = self._resize_and_crop_center(pil_image, target_width, target_height)
+ vae_tensor = processor.pil_image_to_tensor(vae_input)
+
+ vae_info = ImageInfo(
+ image_type="vae",
+ image_width=target_width,
+ image_height=target_height,
+ token_width=target_width // vae_w_factor,
+ token_height=target_height // vae_h_factor,
+ base_size=base_size,
+ ratio_index=ratio_idx,
+ )
+
+ vit_processor = processor.vit_processor
+ vit_inputs = vit_processor(pil_image, return_tensors="pt")
+ vit_tensor = vit_inputs["pixel_values"].squeeze(0)
+ spatial_shapes = vit_inputs["spatial_shapes"].squeeze(0)
+ pixel_attention_mask = vit_inputs["pixel_attention_mask"].squeeze(0)
+ vit_token_h = int(spatial_shapes[0].item())
+ vit_token_w = int(spatial_shapes[1].item())
+ vit_patch_size = getattr(vit_processor, "patch_size", 1)
+ if isinstance(vit_patch_size, (tuple, list)):
+ vit_patch_size = int(vit_patch_size[0])
+
+ vit_info = ImageInfo(
+ image_type="siglip2",
+ image_width=vit_token_w * vit_patch_size,
+ image_height=vit_token_h * vit_patch_size,
+ token_width=vit_token_w,
+ token_height=vit_token_h,
+ image_token_length=int(vit_tensor.shape[0]),
+ )
+
+ joint_info = JointImageInfo(
+ vae_image_info=vae_info,
+ vision_image_info=vit_info,
+ vision_encoder_kwargs={
+ "spatial_shapes": spatial_shapes,
+ "pixel_attention_mask": pixel_attention_mask,
+ },
+ )
+ vae_info.image_tensor = vae_tensor
+ vit_info.image_tensor = vit_tensor
+ return joint_info
+
+ def _vae_encode_cond_image(self, vae_tensor, device):
+ vae = self._vae
+ vae_tensor = self._cond_vae_tensor_5d(vae_tensor).to(device)
+ with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=True):
+ latents = vae.encode(vae_tensor).latent_dist.sample()
+ if vae.shift_factor:
+ latents.sub_(vae.shift_factor)
+ if vae.scaling_factor:
+ latents.mul_(vae.scaling_factor)
+
+ latents = latents.squeeze(2)
+ t = torch.zeros((latents.shape[0],))
+ return t, latents.squeeze(0)
+
+ def _preprocess_cond_images(self, per_request_raw_conds: list[list]) -> list[list]:
+ per_request_joint_infos: list[list] = []
+ for raw_conds in per_request_raw_conds:
+ joints = []
+ for raw_img in raw_conds or []:
+ pil_img = _cond_image_to_pil(raw_img)
+ if pil_img is None:
+ logger.warning(
+ "Skipping unsupported condition image (%s)",
+ type(raw_img).__name__,
+ )
+ continue
+ joints.append(self._preprocess_cond_image(pil_img))
+ per_request_joint_infos.append(joints)
+ return per_request_joint_infos
+
+ def _encode_conditions(self, per_request_joint_infos, device):
+ flat_infos = [info for joints in per_request_joint_infos for info in joints]
+ request_bounds = []
+ offset = 0
+ for joints in per_request_joint_infos:
+ request_bounds.append((offset, offset + len(joints)))
+ offset += len(joints)
+
+ if not flat_infos:
+ empty_embeds: list[list] = [[] for _ in per_request_joint_infos]
+ return (
+ empty_embeds,
+ [None] * len(per_request_joint_infos),
+ [[] for _ in per_request_joint_infos],
+ )
+
+ # Batched cond encoding is opt-in: the AR backbone is resident on the
+ # same device, so full-batch encoding is memory-tight by default.
+ batched = None
+ if envs.SGLANG_HI3_COND_ENCODE_BATCHING:
+ try:
+ batched = self._encode_conditions_batched(flat_infos, device)
+ except (torch.OutOfMemoryError, RuntimeError) as e:
+ if (
+ isinstance(e, RuntimeError)
+ and "out of memory" not in str(e).lower()
+ ):
+ raise
+ logger.warning("Batched cond encoding OOM; falling back to sequential")
+ _empty_device_cache()
+ if batched is None:
+ batched = self._encode_conditions_sequential(flat_infos, device)
+ vae_embeds, t_values, vit_embeds = batched
+
+ per_request_vae_embeds: list[list] = []
+ per_request_t: list = []
+ per_request_vit_embeds: list[list] = []
+ for start, end in request_bounds:
+ per_request_vae_embeds.append(vae_embeds[start:end])
+ t_slice = t_values[start:end]
+ per_request_t.append(torch.cat(t_slice, dim=0) if t_slice else None)
+ per_request_vit_embeds.append(vit_embeds[start:end])
+ return per_request_vae_embeds, per_request_t, per_request_vit_embeds
+
+ @staticmethod
+ def _cond_vae_tensor_5d(image_tensor) -> torch.Tensor:
+ tensor = image_tensor
+ if tensor.ndim == 3:
+ tensor = tensor.unsqueeze(0)
+ if tensor.ndim == 4:
+ tensor = tensor.unsqueeze(2)
+ return tensor
+
+ def _encode_conditions_sequential(self, flat_infos, device):
+ vae_embeds: list = []
+ t_values: list = []
+ vit_embeds: list = []
+ # Autocast matches the diffusion loop for bit-identical embeddings.
+ with torch.autocast(
+ device_type=current_platform.device_type,
+ dtype=torch.bfloat16,
+ enabled=True,
+ ):
+ for info in flat_infos:
+ t_i, latents_i = self._vae_encode_cond_image(
+ info.vae_image_info.image_tensor, device
+ )
+ # Keep t 1D: time_embed output feeds the ResBlock scale/shift split.
+ t_i_emb = self.ar_model.time_embed(t_i.to(device))
+ if latents_i.dim() == 3:
+ latents_i = latents_i.unsqueeze(0)
+ image_i_seq, _, _ = self.ar_model.patch_embed(
+ latents_i.to(device), t_i_emb
+ )
+ vae_embeds.append(image_i_seq)
+ t_values.append(t_i)
+
+ vit_kwargs = self._cond_vit_kwargs(info)
+ # Reference Siglip2VisionTransformer returns padded embeddings.
+ pixels = info.vision_image_info.image_tensor
+ image_embed = self._vision_model(
+ pixels.unsqueeze(0).to(device),
+ attention_mask=vit_kwargs["attention_mask"].to(device),
+ spatial_shapes=vit_kwargs["spatial_shapes"],
+ )
+ image_embed = self._vision_aligner(image_embed)
+ vit_embeds.append(image_embed[0])
+ return vae_embeds, t_values, vit_embeds
+
+ @staticmethod
+ def _cond_vit_kwargs(info) -> dict:
+ vit_kwargs = {
+ "spatial_shapes": info.vision_encoder_kwargs["spatial_shapes"],
+ "attention_mask": info.vision_encoder_kwargs["pixel_attention_mask"],
+ }
+ if vit_kwargs["spatial_shapes"].ndim == 1:
+ vit_kwargs["spatial_shapes"] = vit_kwargs["spatial_shapes"].unsqueeze(0)
+ if vit_kwargs["attention_mask"].ndim == 1:
+ vit_kwargs["attention_mask"] = vit_kwargs["attention_mask"].unsqueeze(0)
+ return vit_kwargs
+
+ def _encode_adaptive(self, count, run_batch, max_chunk=None):
+ """Encode items in the largest batch that fits; on OOM halve and retry.
+
+ Chunking is bit-safe: VAE conv is per-row and the ViT isolates each
+ image via its mask + spatial_shapes. A single-item failure propagates
+ to the sequential fallback in ``_encode_conditions``.
+ """
+ if count <= 0:
+ return []
+ results: list = []
+ start = 0
+ chunk = count if max_chunk is None else min(count, max_chunk)
+ while start < count:
+ size = min(chunk, count - start)
+ try:
+ results.extend(run_batch(list(range(start, start + size))))
+ start += size
+ except (torch.OutOfMemoryError, RuntimeError) as e:
+ if not _is_oom_error(e):
+ raise
+ _empty_device_cache()
+ if size <= 1:
+ raise
+ chunk = size // 2
+ return results
+
+ def _encode_conditions_batched(self, flat_infos, device):
+ # Returns None when the images cannot be stacked (shape mismatch).
+ vae_tensors = [
+ self._cond_vae_tensor_5d(info.vae_image_info.image_tensor)
+ for info in flat_infos
+ ]
+ if len({tuple(t.shape) for t in vae_tensors}) > 1:
+ return None
+
+ vit_tensors = [info.vision_image_info.image_tensor for info in flat_infos]
+ feat_dim = vit_tensors[0].shape[-1]
+ if any(t.ndim != 2 or t.shape[-1] != feat_dim for t in vit_tensors):
+ return None
+
+ # Stackable only when every image pads to the same max_patches.
+ if len({t.shape[0] for t in vit_tensors}) > 1:
+ return None
+ vit_mask_rows: list = []
+ spatial_shape_rows: list = []
+ for info in flat_infos:
+ vit_kwargs = self._cond_vit_kwargs(info)
+ vit_mask_rows.append(vit_kwargs["attention_mask"][0])
+ spatial_shape_rows.append(vit_kwargs["spatial_shapes"][0])
+
+ # Autocast matches the diffusion loop for bit-identical embeddings.
+ with torch.autocast(
+ device_type=current_platform.device_type,
+ dtype=torch.bfloat16,
+ enabled=True,
+ ):
+
+ def _run_vae(idxs):
+ batched_vae = torch.cat(
+ [vae_tensors[i].to(dtype=vae_tensors[0].dtype) for i in idxs],
+ dim=0,
+ )
+ t_chunk, latents_chunk = self._vae_encode_cond_image(
+ batched_vae, device
+ )
+ # Keep t 1D: time_embed output feeds the ResBlock scale/shift split.
+ t_emb = self.ar_model.time_embed(t_chunk.to(device))
+ if latents_chunk.dim() == 3:
+ latents_chunk = latents_chunk.unsqueeze(0)
+ image_seq, _, _ = self.ar_model.patch_embed(
+ latents_chunk.to(device), t_emb
+ )
+ return [(image_seq[k], t_chunk[k : k + 1]) for k in range(len(idxs))]
+
+ max_chunk = self._max_cond_encode_chunk
+ vae_results = self._encode_adaptive(
+ len(flat_infos), _run_vae, max_chunk=max_chunk
+ )
+ vae_embeds = [r[0] for r in vae_results]
+ t_values = [r[1] for r in vae_results]
+
+ # ViT: the reference ViT unpacks per image internally.
+ def _run_vit(idxs):
+ pixel_values = torch.stack(
+ [
+ vit_tensors[i].to(device=device, dtype=vit_tensors[0].dtype)
+ for i in idxs
+ ],
+ dim=0,
+ )
+ attention_mask = torch.stack(
+ [vit_mask_rows[i].to(device) for i in idxs], dim=0
+ )
+ spatial_shapes = torch.stack(
+ [spatial_shape_rows[i] for i in idxs], dim=0
+ )
+ image_embed = self._vision_model(
+ pixel_values,
+ attention_mask=attention_mask,
+ spatial_shapes=spatial_shapes,
+ )
+ image_embed = self._vision_aligner(image_embed)
+ return [image_embed[b] for b in range(len(idxs))]
+
+ vit_embeds = self._encode_adaptive(
+ len(flat_infos), _run_vit, max_chunk=max_chunk
+ )
+ return vae_embeds, t_values, vit_embeds
+
+ def _scatter_cond_vae_tokens_batched(
+ self,
+ hidden_states,
+ per_request_vae_embeds,
+ cond_vae_slices_rows,
+ n_req,
+ do_cfg,
+ ):
+ n_embd = hidden_states.shape[-1]
+ for r, embeds in enumerate(per_request_vae_embeds):
+ if not embeds:
+ continue
+ target_rows = [r] + ([n_req + r] if do_cfg else [])
+ for row in target_rows:
+ slices_row = cond_vae_slices_rows[row]
+ for i, s in enumerate(slices_row):
+ positions = torch.arange(
+ s.start, s.stop, device=hidden_states.device
+ )
+ hidden_states[row, positions] = embeds[i].reshape(-1, n_embd)
+ return hidden_states
+
+ def _scatter_cond_vit_tokens_batched(
+ self,
+ hidden_states,
+ per_request_vit_embeds,
+ cond_vit_slices_rows,
+ n_req,
+ do_cfg,
+ ):
+ # The CFG-packed uncond half keeps the cond (joint) image sections,
+ # so scatter ViT embeddings into those rows too.
+ for r, embeds in enumerate(per_request_vit_embeds):
+ if not embeds:
+ continue
+ target_rows = [r] + ([n_req + r] if do_cfg else [])
+ for row in target_rows:
+ for i, s in enumerate(cond_vit_slices_rows[row]):
+ positions = torch.arange(
+ s.start, s.stop, device=hidden_states.device
+ )
+ hidden_states[row, positions] = embeds[i][: s.stop - s.start].to(
+ hidden_states.dtype
+ )
+ return hidden_states
+
+ @staticmethod
+ def _normalize_bot_task(bot_task: str) -> str:
+ # Mirrors vllm-omni's _normalize_single_stage_bot_task: "none" falls
+ # back to "auto" (which maps to the vanilla system prompt under the
+ # "dynamic" preset), "vanilla" to "image", and the composite
+ # think_recaption to its single-stage "think" form.
+ if bot_task == "none":
+ return "auto"
+ if bot_task == "vanilla":
+ return "image"
+ if bot_task == "think_recaption":
+ return "think"
+ return bot_task
+
+ def _resolve_generation_params(self, reqs: list[Req], raw_conds_rows: list):
+ head = reqs[0]
+ target_sizes = [
+ self._effective_resolution(req, raw_cond_images)
+ for req, raw_cond_images in zip(reqs, raw_conds_rows)
+ ]
+ for req, target_size in zip(reqs, target_sizes):
+ req.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = build_hunyuan_image3_output_geometry(
+ *target_size,
+ size_mode=getattr(req, "output_size_mode", "aspect_ratio"),
+ strategy=getattr(req, "output_strategy", "native_crop"),
+ ratio_policy=getattr(req, "output_ratio_policy", "exact"),
+ crop_anchor=getattr(req, "output_crop_anchor", (0.5, 0.5)),
+ max_ratio_error=getattr(req, "output_max_ratio_error", 0.0005),
+ pad_value=getattr(req, "output_pad_value", 0.0),
+ )
+ width, height = target_sizes[0]
+ image_info = self._processor.build_gen_image_info(f"{height}x{width}")
+ height = image_info.image_height
+ width = image_info.image_width
+ token_h = image_info.token_height
+ token_w = image_info.token_width
+ image_info = self._rebuild_image_info(image_info)
+ for req in reqs:
+ req.extra[OUTPUT_GEOMETRY_EXTRA_KEY]["native_bucket_size"] = [
+ width,
+ height,
+ ]
+ req.width, req.height = width, height
+
+ guidance_scale = head.guidance_scale
+ num_inference_steps = head.num_inference_steps
+ return (
+ width,
+ height,
+ token_h,
+ token_w,
+ image_info,
+ guidance_scale,
+ num_inference_steps,
+ )
+
+ def _build_tokenizer_kwargs(
+ self, reqs: list[Req], image_info, tokenizer_bot_task: str, cfg_factor: int
+ ) -> dict[str, Any]:
+ gen_config = self._generation_config()
+ tokenizer_kwargs: dict[str, Any] = dict(
+ batch_prompt=[req.prompt for req in reqs],
+ # mode="gen_image" is required: "gen_text" omits the gen_image
+ # assistant message and leaves gen_image_mask / gen slices unset.
+ mode="gen_image",
+ bot_task=tokenizer_bot_task,
+ sequence_template=gen_config.get("sequence_template", "pretrain"),
+ drop_think=gen_config.get("drop_think", False),
+ cfg_factor=cfg_factor,
+ )
+ resolved_prompt = resolve_system_prompt(
+ reqs[0].system_prompt, bot_task=tokenizer_bot_task
+ )
+ if resolved_prompt is not None:
+ tokenizer_kwargs["batch_system_prompt"] = [resolved_prompt.strip()] * len(
+ reqs
+ )
+ cot_texts = [req.cot_text for req in reqs]
+ if any(cot is not None for cot in cot_texts):
+ tokenizer_kwargs["batch_cot_text"] = cot_texts
+ tokenizer_kwargs["batch_gen_image_info"] = [image_info] * len(reqs)
+ return tokenizer_kwargs
+
+ @staticmethod
+ def _tok_field(
+ tokenizer_output: TokenizerEncodeOutput, name: str, device: torch.device
+ ):
+ value = getattr(tokenizer_output, name)
+ return value.to(device) if torch.is_tensor(value) else value
+
+ def _parse_tokenizer_output(
+ self, tokenizer_output: TokenizerEncodeOutput, device: torch.device
+ ):
+ input_ids = tokenizer_output.tokens.to(device)
+ return dict(
+ input_ids=input_ids,
+ actual_batch_size=input_ids.shape[0],
+ seq_len=input_ids.shape[1],
+ image_mask=tokenizer_output.gen_image_mask.to(device),
+ timestep_index=self._tok_field(
+ tokenizer_output, "gen_timestep_scatter_index", device
+ ),
+ cond_timestep_scatter_index=self._tok_field(
+ tokenizer_output, "cond_timestep_scatter_index", device
+ ),
+ cond_vae_slices_rows=tokenizer_output.cond_vae_image_slices,
+ cond_vit_slices_rows=tokenizer_output.cond_vit_image_slices,
+ )
+
+ def _build_attention_and_rope(
+ self,
+ tokenizer_output,
+ tokenizer_sections,
+ actual_batch_size: int,
+ seq_len: int,
+ token_h: int,
+ token_w: int,
+ image_info,
+ device,
+ do_cfg: bool = False,
+ ):
+ gen_slices = tokenizer_output.gen_image_slices
+ joint_slices = tokenizer_output.joint_image_slices
+ image_slices = [
+ joint_slices[i] + gen_slices[i] for i in range(actual_batch_size)
+ ]
+
+ # Only one half's mask is live at a time; when both halves share an
+ # identical image-token layout, build one half-size mask and reuse it
+ # (byte-identical; any layout mismatch falls back to the full mask).
+ half_bs = actual_batch_size // 2
+ mask_shared = (
+ do_cfg
+ and actual_batch_size % 2 == 0
+ and image_slices[:half_bs] == image_slices[half_bs:]
+ )
+ mask_bs = half_bs if mask_shared else actual_batch_size
+ mask_slices = image_slices[:half_bs] if mask_shared else image_slices
+ attention_mask = _build_causal_attention_mask(
+ mask_bs, seq_len, mask_slices, device
+ )
+
+ rope_image_info = _build_rope_image_info(
+ tokenizer_output,
+ actual_batch_size,
+ token_h,
+ token_w,
+ image_info,
+ sections=tokenizer_sections,
+ )
+ cos, sin = self.ar_model.cached_rope(
+ seq_len, device, rope_image_info=rope_image_info
+ )
+ return attention_mask, cos, sin, mask_shared
+
+ def _latent_dims(self, height: int, width: int) -> tuple[int, int, int]:
+ arch_config = self.ar_model.config
+ if isinstance(getattr(arch_config, "vae", None), dict):
+ latent_channels = arch_config.vae["latent_channels"]
+ else:
+ latent_channels = arch_config.latent_channels
+ vae_h, vae_w = _vae_downsample_factors(arch_config)
+ return latent_channels, height // vae_h, width // vae_w
+
+ def _prepare_noise(
+ self,
+ reqs: list[Req],
+ latent_channels: int,
+ latent_h: int,
+ latent_w: int,
+ device: torch.device,
+ ) -> torch.Tensor:
+ # One generator per request keeps output bit-identical to its
+ # single-request run.
+ noise_rows = []
+ for req in reqs:
+ generator = torch.Generator(device=device)
+ # Seeds arrive normalized to per-output scalars by
+ # _expand_multi_output.
+ if req.seed is not None:
+ generator.manual_seed(req.seed)
+ noise_rows.append(
+ torch.randn(
+ 1,
+ latent_channels,
+ latent_h,
+ latent_w,
+ generator=generator,
+ device=device,
+ dtype=torch.bfloat16,
+ )
+ )
+ return torch.cat(noise_rows, dim=0)
+
+ @staticmethod
+ def _collect_raw_cond_images(req: Req):
+ raw_cond_images = req.condition_image
+ if raw_cond_images is None and req.image_path is not None:
+ image_path = req.image_path
+ raw_cond_images = (
+ image_path if isinstance(image_path, list) else [image_path]
+ )
+ if raw_cond_images is not None and not isinstance(
+ raw_cond_images, (list, tuple)
+ ):
+ raw_cond_images = [raw_cond_images]
+ return raw_cond_images
+
+ @staticmethod
+ def _effective_resolution(req: Req, raw_cond_images) -> tuple[int, int]:
+ # Use the input-validation snapshot, not the preprocessed image.
+ user_explicit_fields = getattr(req.sampling_params, "_explicit_fields", set())
+ reference_size = req.original_condition_image_size
+ if reference_size is None and raw_cond_images:
+ first_cond_pil = _cond_image_to_pil(raw_cond_images[0])
+ reference_size = first_cond_pil.size if first_cond_pil is not None else None
+ return resolve_hunyuan_image3_output_resolution(
+ req.width,
+ req.height,
+ user_explicit_fields,
+ reference_size,
+ )
+
+ @torch.no_grad()
+ def forward(self, batch: Req, server_args: ServerArgs) -> Req:
+ clones = self._expand_multi_output(batch)
+ outputs = self._forward_batched(clones)
+ if len(outputs) == 1:
+ return outputs[0]
+ batch.latents = torch.cat([out.latents for out in outputs], dim=0)
+ batch.width, batch.height = outputs[0].width, outputs[0].height
+ output_extra = getattr(outputs[0], "extra", {})
+ if OUTPUT_GEOMETRY_EXTRA_KEY in output_extra:
+ batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = output_extra[
+ OUTPUT_GEOMETRY_EXTRA_KEY
+ ]
+ return batch
+
+ def run_grouped_requests(
+ self,
+ batches: list[Req],
+ server_args: ServerArgs,
+ ) -> list[Req]:
+ flat: list[Req] = []
+ for batch in batches:
+ flat.extend(self._expand_multi_output(batch))
+
+ if len(flat) <= 1:
+ return [self(req, server_args) for req in flat]
+
+ results: list[Req | None] = [None] * len(flat)
+ batchable: list[tuple[int, Req]] = []
+ for index, req in enumerate(flat):
+ if self._is_batchable(req):
+ batchable.append((index, req))
+ else:
+ results[index] = self(req, server_args)
+
+ buckets: dict[Any, list[tuple[int, Req]]] = {}
+ for index, req in batchable:
+ buckets.setdefault(self._batch_bucket_key(req), []).append((index, req))
+
+ for group in buckets.values():
+ if len(group) == 1:
+ index, req = group[0]
+ results[index] = self(req, server_args)
+ continue
+ outputs = self._forward_batched([req for _, req in group])
+ for (index, _), output in zip(group, outputs):
+ results[index] = output
+
+ return [result for result in results if result is not None]
+
+ @staticmethod
+ def _expand_multi_output(req: Req) -> list[Req]:
+ num_outputs = int(req.num_outputs_per_prompt)
+ if num_outputs == 1:
+ # No clone loop for n=1, so normalize the seed here: downstream
+ # noise preparation assumes per-output scalar seeds.
+ if req.seed is not None:
+ req.seed = _seed_for_output(req.seed, 0)
+ return [req]
+ clones: list[Req] = []
+ for output_idx in range(num_outputs):
+ clone = copy.copy(req)
+ clone.sampling_params = copy.copy(req.sampling_params)
+ clone.extra = dict(req.extra)
+ clone.metrics = copy.deepcopy(req.metrics)
+ clone.num_outputs_per_prompt = 1
+ clone.seed = _seed_for_output(req.seed, output_idx)
+ clone.seeds = None
+ clone.generator = None
+ if req.request_id is not None:
+ clone.request_id = f"{req.request_id}:{output_idx}"
+ if clone.metrics is not None:
+ clone.metrics.request_id = clone.request_id
+ clones.append(clone)
+ return clones
+
+ @staticmethod
+ def _is_batchable(req: Req) -> bool:
+ return not req.is_warmup
+
+ def _batch_bucket_key(self, req: Req):
+ raw_cond_images = self._collect_raw_cond_images(req)
+ n_cond = len(raw_cond_images) if raw_cond_images else 0
+ width, height = self._effective_resolution(req, raw_cond_images)
+ return (
+ width,
+ height,
+ req.num_inference_steps,
+ req.guidance_scale,
+ self._normalize_bot_task(req.bot_task),
+ req.system_prompt,
+ n_cond,
+ req.output_size_mode,
+ req.output_strategy,
+ req.output_ratio_policy,
+ tuple(req.output_crop_anchor),
+ req.output_max_ratio_error,
+ req.output_pad_value,
+ )
+
+ @torch.no_grad()
+ def _forward_batched(self, reqs: list[Req]) -> list[Req]:
+ tokenizer = self._custom_tokenizer
+ n_req = len(reqs)
+ head = reqs[0]
+
+ per_request_raw_conds = [self._collect_raw_cond_images(req) for req in reqs]
+ has_cond = any(bool(conds) for conds in per_request_raw_conds)
+ (
+ width,
+ height,
+ token_h,
+ token_w,
+ image_info,
+ guidance_scale,
+ num_inference_steps,
+ ) = self._resolve_generation_params(reqs, per_request_raw_conds)
+ do_cfg = guidance_scale > 1.0
+ cfg_factor = 2 if do_cfg else 1
+
+ # cpu_offload may leave weights on CPU
+ device = get_local_torch_device()
+ model_device = self.ar_model.model.embed_tokens.weight.device
+ if model_device.type == "cpu":
+ self.log_info("Moving AR model from CPU to %s", device)
+ self.ar_model.to(device)
+ else:
+ device = model_device
+
+ self._maybe_enable_cache_dit(num_inference_steps)
+
+ tokenizer_bot_task = self._normalize_bot_task(head.bot_task)
+ tokenizer_kwargs = self._build_tokenizer_kwargs(
+ reqs, image_info, tokenizer_bot_task, cfg_factor
+ )
+
+ per_request_joint_infos: list[list] = []
+ if has_cond:
+ per_request_joint_infos = self._preprocess_cond_images(
+ per_request_raw_conds
+ )
+ tokenizer_kwargs["batch_cond_image_info"] = per_request_joint_infos
+
+ tokenizer_output_dict = tokenizer.apply_chat_template(**tokenizer_kwargs)
+ tokenizer_output = tokenizer_output_dict["output"]
+ tokenizer_sections = tokenizer_output_dict["sections"]
+
+ tok = self._parse_tokenizer_output(tokenizer_output, device)
+ input_ids = tok["input_ids"]
+ actual_batch_size = tok["actual_batch_size"]
+ image_mask = tok["image_mask"]
+ timestep_index = tok["timestep_index"]
+
+ attention_mask, cos, sin, mask_shared = self._build_attention_and_rope(
+ tokenizer_output,
+ tokenizer_sections,
+ actual_batch_size,
+ tok["seq_len"],
+ token_h,
+ token_w,
+ image_info,
+ device,
+ do_cfg,
+ )
+ attention_mask, (cos, sin) = self._broadcast_static_inputs(
+ attention_mask, (cos, sin)
+ )
+
+ scheduler = self._scheduler
+ scheduler.set_timesteps(num_inference_steps)
+ timesteps = scheduler.timesteps
+
+ latent_channels, latent_h, latent_w = self._latent_dims(height, width)
+ latents = self._prepare_noise(reqs, latent_channels, latent_h, latent_w, device)
+
+ per_request_vae_embeds: list[list] = []
+ per_request_t: list = []
+ per_request_vit_embeds: list[list] = []
+ has_cond_encoded = False
+ if has_cond and any(per_request_joint_infos):
+ self._vision_model.to(device)
+ self._vision_model.eval()
+ self._vision_aligner.to(device)
+ self._vision_aligner.eval()
+ (
+ per_request_vae_embeds,
+ per_request_t,
+ per_request_vit_embeds,
+ ) = self._encode_conditions(per_request_joint_infos, device)
+ has_cond_encoded = True
+
+ cond_vae_slices_rows = tok["cond_vae_slices_rows"]
+ cond_vit_slices_rows = tok["cond_vit_slices_rows"]
+ cond_timestep_scatter_index = tok["cond_timestep_scatter_index"]
+
+ # Concatenate per-request cond timesteps once, outside the denoising
+ # loop. Requests without cond images contribute None entries; they are
+ # skipped so batch composition cannot surface as an opaque TypeError
+ # mid-loop. An all-None group with a stale scatter index skips
+ # instantiation (warned); a partial group fails the expected-count
+ # check inside the loop with a clear diagnostic.
+ all_cond_t = None
+ if has_cond_encoded and cond_timestep_scatter_index is not None:
+ cond_t_rows = [t for t in per_request_t if t is not None]
+ if cond_t_rows:
+ all_cond_t = torch.cat(cond_t_rows, dim=0).repeat(cfg_factor)
+ else:
+ logger.warning(
+ "cond_timestep_scatter_index present but no request in the "
+ "group produced cond timesteps; skipping cond timestep "
+ "instantiation"
+ )
+
+ num_image_tokens = token_h * token_w
+
+ for step_idx, t in enumerate(self.progress_bar(timesteps, batch=head)):
+ latent_model_input = scheduler.scale_model_input(latents, t)
+ if do_cfg:
+ latent_model_input = torch.cat([latent_model_input] * cfg_factor, dim=0)
+ latent_bs = latent_model_input.shape[0]
+ t_expand = t.repeat(latent_bs).to(device)
+
+ with (
+ torch.autocast(
+ device_type=current_platform.device_type,
+ dtype=torch.bfloat16,
+ enabled=True,
+ ),
+ set_forward_context(
+ current_timestep=step_idx,
+ attn_metadata=None,
+ forward_batch=head,
+ ),
+ ):
+ # Re-embed the full input_ids every step; shortening produces garbage.
+ hidden_states = self.ar_model.model.get_input_embeddings(input_ids)
+ hidden_states = self._instantiate_vae_tokens_first_step(
+ hidden_states,
+ latent_model_input,
+ t_expand,
+ image_mask,
+ )
+ if timestep_index is not None:
+ hidden_states = self._instantiate_timestep_tokens(
+ hidden_states,
+ t_expand,
+ timestep_index,
+ )
+
+ if has_cond_encoded:
+ hidden_states = self._scatter_cond_vae_tokens_batched(
+ hidden_states,
+ per_request_vae_embeds,
+ cond_vae_slices_rows,
+ n_req,
+ do_cfg,
+ )
+ hidden_states = self._scatter_cond_vit_tokens_batched(
+ hidden_states,
+ per_request_vit_embeds,
+ cond_vit_slices_rows,
+ n_req,
+ do_cfg,
+ )
+ if (
+ cond_timestep_scatter_index is not None
+ and all_cond_t is not None
+ ):
+ cond_ts_index = cond_timestep_scatter_index
+ # Legacy tokenizer shapes: 1-D [P] or [1, P] must be
+ # expanded to one index row per sequence row.
+ if cond_ts_index.ndim == 1:
+ cond_ts_index = cond_ts_index.unsqueeze(0).expand(
+ actual_batch_size, -1
+ )
+ elif cond_ts_index.shape[0] == 1 and actual_batch_size > 1:
+ cond_ts_index = cond_ts_index.expand(actual_batch_size, -1)
+ expected_t = cond_ts_index.shape[0] * cond_ts_index.shape[1]
+ if all_cond_t.shape[0] != expected_t:
+ logger.error(
+ "cond timestep scatter mismatch: "
+ "all_cond_t=%d n_req=%d cfg_factor=%d "
+ "conds_per_req=%s scatter_index=%s "
+ "cond_vae_slices=%s",
+ all_cond_t.shape[0],
+ n_req,
+ cfg_factor,
+ [len(j) for j in per_request_joint_infos],
+ tuple(cond_timestep_scatter_index.shape),
+ [len(s) for s in cond_vae_slices_rows],
+ )
+ raise RuntimeError(
+ "cond timestep scatter mismatch: "
+ f"{all_cond_t.shape[0]} t values for "
+ f"{cond_ts_index.shape} scatter index"
+ )
+ hidden_states = self._instantiate_timestep_tokens(
+ hidden_states,
+ all_cond_t.to(hidden_states.device),
+ cond_ts_index,
+ )
+
+ # CFG: run cond/uncond halves separately (halves peak attention memory).
+ if do_cfg:
+ half = hidden_states.shape[0] // 2
+ # With a shared half-mask, reuse it for both calls instead of
+ # slicing a full-batch mask whose two halves are never both live.
+ cond_mask = attention_mask if mask_shared else attention_mask[:half]
+ uncond_mask = (
+ attention_mask if mask_shared else attention_mask[half:]
+ )
+ out_cond = self._backbone_forward(
+ num_image_tokens,
+ hidden_states[:half],
+ cond_mask,
+ (cos[:half], sin[:half]),
+ True,
+ timestep=t_expand[:half],
+ )
+ out_uncond = self._backbone_forward(
+ num_image_tokens,
+ hidden_states[half:],
+ uncond_mask,
+ (cos[half:], sin[half:]),
+ True,
+ timestep=t_expand[half:],
+ )
+ backbone_out = torch.cat([out_cond, out_uncond], dim=0)
+ else:
+ backbone_out = self._backbone_forward(
+ num_image_tokens,
+ hidden_states,
+ attention_mask,
+ (cos, sin),
+ True,
+ timestep=t_expand,
+ )
+
+ pred = self._extract_diffusion_pred(
+ backbone_out,
+ t_expand,
+ image_mask,
+ token_h,
+ token_w,
+ )
+
+ pred = pred.float()
+
+ if do_cfg:
+ pred_cond, pred_uncond = pred.chunk(2)
+ pred = pred_uncond + guidance_scale * (pred_cond - pred_uncond)
+
+ latent_dtype = latents.dtype
+ latents = scheduler.step(pred, t, latents, return_dict=False)[0].to(
+ dtype=latent_dtype
+ )
+
+ # [B, C, H, W] -> [B, C, 1, H, W]
+ latents = latents.to(torch.bfloat16)
+ for i, req in enumerate(reqs):
+ req.latents = latents[i : i + 1].unsqueeze(2)
+
+ self.log_info(
+ "AR stage: %d req(s) -> latents %s (%dx%d)",
+ n_req,
+ tuple(latents.shape),
+ height,
+ width,
+ )
+ return reqs
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/decoding.py
new file mode 100644
index 0000000000..4118192c56
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/decoding.py
@@ -0,0 +1,353 @@
+"""Post-decode output geometry for HunyuanImage-3."""
+
+import math
+from dataclasses import dataclass
+
+import torch
+import torch.nn.functional as F
+
+from sglang.multimodal_gen import envs
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+ OutputBatch,
+ Req,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+OUTPUT_GEOMETRY_EXTRA_KEY = "hunyuan_image3_output_geometry"
+
+
+def build_hunyuan_image3_output_geometry(
+ width: int,
+ height: int,
+ *,
+ size_mode: str = "aspect_ratio",
+ strategy: str = "native_crop",
+ ratio_policy: str = "exact",
+ crop_anchor: tuple[float, float] = (0.5, 0.5),
+ max_ratio_error: float = 0.0005,
+ pad_value: float = 0.0,
+) -> dict[str, object]:
+ """Create a request-local output contract before native bucketing.
+
+ ``width``/``height`` are the user-visible target geometry, not the
+ processor-selected bucket. Kept on ``Req.extra`` so it is safe with
+ cloned requests and decoder disaggregation.
+ """
+ width, height = int(width), int(height)
+ if width <= 0 or height <= 0:
+ raise ValueError("HunyuanImage-3 target dimensions must be positive")
+ if size_mode not in {"aspect_ratio", "exact_size"}:
+ raise ValueError("size_mode must be 'aspect_ratio' or 'exact_size'")
+ if strategy not in {"native_crop", "native_pad"}:
+ raise ValueError("strategy must be 'native_crop' or 'native_pad'")
+ if ratio_policy not in {"exact", "approximate"}:
+ raise ValueError("ratio_policy must be 'exact' or 'approximate'")
+ if len(crop_anchor) != 2 or any(
+ not math.isfinite(float(value)) or not 0.0 <= float(value) <= 1.0
+ for value in crop_anchor
+ ):
+ raise ValueError("crop_anchor must contain two finite values in [0, 1]")
+ if not math.isfinite(max_ratio_error) or not 0.0 <= max_ratio_error < 1.0:
+ raise ValueError("max_ratio_error must be a finite value in [0, 1)")
+ if not math.isfinite(pad_value) or not 0.0 <= pad_value <= 1.0:
+ raise ValueError("pad_value must be a finite value in [0, 1]")
+ if size_mode == "exact_size" and ratio_policy == "approximate":
+ raise ValueError(
+ "exact_size requires ratio_policy='exact'; approximate cropping "
+ "would require non-uniform resampling to reach the requested size"
+ )
+
+ divisor = math.gcd(width, height)
+ return {
+ "requested_size": [width, height],
+ "requested_aspect_ratio": [width // divisor, height // divisor],
+ "size_mode": size_mode,
+ "strategy": strategy,
+ "ratio_policy": ratio_policy,
+ "crop_anchor": [float(crop_anchor[0]), float(crop_anchor[1])],
+ "max_ratio_error": float(max_ratio_error),
+ "pad_value": float(pad_value),
+ }
+
+
+def resolve_hunyuan_image3_output_resolution(
+ width: int,
+ height: int,
+ explicit_fields: set[str],
+ reference_size: tuple[int, int] | None = None,
+) -> tuple[int, int]:
+ """Return the raw target size used for native aspect-ratio bucketing.
+
+ Explicit dimensions take precedence; editing without an explicit size
+ inherits the reference size instead of the 1280x720 pipeline default.
+ """
+ if reference_size is not None and not {"width", "height"} & explicit_fields:
+ width, height = reference_size
+ return int(width), int(height)
+
+
+@dataclass(frozen=True)
+class _SpatialPlan:
+ source_size: tuple[int, int]
+ crop_box: tuple[int, int, int, int] | None = None
+ padding: tuple[int, int, int, int] | None = None
+ pad_value: float = 0.0
+ resize_size: tuple[int, int] | None = None
+
+
+def _round_half_up(value: float) -> int:
+ return math.floor(value + 0.5)
+
+
+def _crop_box(
+ width: int,
+ height: int,
+ crop_width: int,
+ crop_height: int,
+ anchor: tuple[float, float],
+) -> tuple[int, int, int, int]:
+ excess_x, excess_y = width - crop_width, height - crop_height
+ left = min(max(_round_half_up(excess_x * anchor[0]), 0), excess_x)
+ top = min(max(_round_half_up(excess_y * anchor[1]), 0), excess_y)
+ return left, top, left + crop_width, top + crop_height
+
+
+def _maximum_exact_crop_size(
+ width: int, height: int, p: int, q: int
+) -> tuple[int, int]:
+ """Largest integer crop whose ratio matches p:q within pixel rounding.
+
+ A strict p:q-multiple crop degenerates for coprime reference sizes
+ (e.g. 1361x907): the multiplier collapses to 0 or discards most of the
+ image. Anchoring on the limiting dimension and rounding keeps max area
+ with at most half a pixel of ratio deviation per axis.
+ """
+ ratio = p / q
+ if width / height >= ratio:
+ crop_height = height
+ crop_width = min(width, _round_half_up(crop_height * ratio))
+ else:
+ crop_width = width
+ crop_height = min(height, _round_half_up(crop_width / ratio))
+ if crop_width <= 0 or crop_height <= 0:
+ raise ValueError(
+ f"Cannot crop decoded {width}x{height} pixels to ratio {p}:{q}"
+ )
+ return crop_width, crop_height
+
+
+def _maximum_approximate_crop_size(
+ width: int,
+ height: int,
+ p: int,
+ q: int,
+ max_ratio_error: float,
+) -> tuple[int, int]:
+ """Return max-area integer crop within the declared relative ratio error."""
+ best: tuple[int, int] | None = None
+ best_area, best_error = -1, math.inf
+ for crop_height in range(1, height + 1):
+ nominal_width = crop_height * p / q
+ lower = max(1, math.ceil(nominal_width * (1.0 - max_ratio_error)))
+ upper = min(width, math.floor(nominal_width * (1.0 + max_ratio_error)))
+ if lower > upper:
+ continue
+ crop_width = upper
+ ratio_error = abs((crop_width * q) / (crop_height * p) - 1.0)
+ area = crop_width * crop_height
+ if area > best_area or (area == best_area and ratio_error < best_error):
+ best = crop_width, crop_height
+ best_area, best_error = area, ratio_error
+ if best is None:
+ raise ValueError(
+ f"Cannot crop decoded {width}x{height} pixels within ratio-error "
+ f"limit {max_ratio_error} for {p}:{q}"
+ )
+ return best
+
+
+def _minimum_exact_canvas_size(
+ width: int, height: int, p: int, q: int
+) -> tuple[int, int]:
+ multiplier = max(math.ceil(width / p), math.ceil(height / q))
+ return multiplier * p, multiplier * q
+
+
+def _build_spatial_plan(
+ decoded_size: tuple[int, int], geometry: dict[str, object]
+) -> tuple[_SpatialPlan, dict[str, object]]:
+ width, height = decoded_size
+ p, q = (int(value) for value in geometry["requested_aspect_ratio"])
+ anchor = tuple(float(value) for value in geometry["crop_anchor"])
+ strategy = str(geometry["strategy"])
+ ratio_policy = str(geometry["ratio_policy"])
+ size_mode = str(geometry["size_mode"])
+
+ metadata = dict(geometry)
+ metadata.setdefault("native_bucket_size", [width, height])
+ metadata["decoded_size"] = [width, height]
+ metadata["resampled"] = False
+
+ if strategy == "native_crop":
+ if ratio_policy == "exact":
+ crop_width, crop_height = _maximum_exact_crop_size(width, height, p, q)
+ else:
+ crop_width, crop_height = _maximum_approximate_crop_size(
+ width,
+ height,
+ p,
+ q,
+ float(geometry["max_ratio_error"]),
+ )
+ crop_box = _crop_box(width, height, crop_width, crop_height, anchor)
+ plan = _SpatialPlan(source_size=decoded_size, crop_box=crop_box)
+ retained_fraction = crop_width * crop_height / (width * height)
+ metadata["crop_box"] = list(crop_box)
+ metadata["retained_pixel_fraction"] = retained_fraction
+ theoretical_area = (
+ height * height * p / q
+ if width * q >= height * p
+ else width * width * q / p
+ )
+ metadata["integer_constraint_extra_loss"] = max(
+ 0.0, (theoretical_area - crop_width * crop_height) / (width * height)
+ )
+ cropped_size = crop_width, crop_height
+ elif strategy == "native_pad":
+ canvas_width, canvas_height = _minimum_exact_canvas_size(width, height, p, q)
+ left, top, right, bottom = _crop_box(
+ canvas_width, canvas_height, width, height, anchor
+ )
+ padding = left, canvas_width - right, top, canvas_height - bottom
+ plan = _SpatialPlan(
+ source_size=decoded_size,
+ padding=padding,
+ pad_value=float(geometry["pad_value"]),
+ )
+ metadata["pad"] = list(padding)
+ metadata["retained_pixel_fraction"] = 1.0
+ metadata["integer_constraint_extra_loss"] = 0.0
+ cropped_size = canvas_width, canvas_height
+ else:
+ raise ValueError(f"Unsupported HunyuanImage-3 output strategy: {strategy}")
+
+ ratio_error = abs((cropped_size[0] * q) / (cropped_size[1] * p) - 1.0)
+ metadata["relative_ratio_error"] = ratio_error
+ if size_mode == "exact_size":
+ requested_size = tuple(int(value) for value in geometry["requested_size"])
+ plan = _SpatialPlan(
+ source_size=plan.source_size,
+ crop_box=plan.crop_box,
+ padding=plan.padding,
+ pad_value=plan.pad_value,
+ resize_size=requested_size,
+ )
+ metadata["resampled"] = cropped_size != requested_size
+ if metadata["resampled"]:
+ metadata["resample_mode"] = "bicubic_antialias"
+ output_size = requested_size
+ elif size_mode == "aspect_ratio":
+ output_size = cropped_size
+ else:
+ raise ValueError(f"Unsupported HunyuanImage-3 output size mode: {size_mode}")
+ metadata["output_size"] = list(output_size)
+ return plan, metadata
+
+
+def _resize_frames(frames: torch.Tensor, output_size: tuple[int, int]) -> torch.Tensor:
+ output_width, output_height = output_size
+ input_height, input_width = frames.shape[-2:]
+ if (input_width, input_height) == output_size:
+ return frames
+
+ original_dtype = frames.dtype
+ if frames.ndim == 5:
+ batch_size, channels, num_frames, _, _ = frames.shape
+ flat_frames = frames.permute(0, 2, 1, 3, 4).reshape(
+ batch_size * num_frames, channels, input_height, input_width
+ )
+ elif frames.ndim == 4:
+ batch_size, channels, _, _ = frames.shape
+ num_frames = None
+ flat_frames = frames
+ else:
+ raise ValueError(
+ "HunyuanImage-3 decoded frames must be BCHW or BCFHW, "
+ f"but got shape {tuple(frames.shape)}"
+ )
+
+ compute_dtype = torch.float64 if original_dtype == torch.float64 else torch.float32
+ resized = F.interpolate(
+ flat_frames.to(compute_dtype),
+ size=(output_height, output_width),
+ mode="bicubic",
+ align_corners=False,
+ antialias=True,
+ ).clamp_(0, 1)
+ if original_dtype.is_floating_point:
+ resized = resized.to(original_dtype)
+ if num_frames is None:
+ return resized
+ return (
+ resized.reshape(batch_size, num_frames, channels, output_height, output_width)
+ .permute(0, 2, 1, 3, 4)
+ .contiguous()
+ )
+
+
+def apply_hunyuan_image3_spatial_plan(
+ frames: torch.Tensor, plan: _SpatialPlan
+) -> torch.Tensor:
+ """Apply the exact same post-decode spatial transform to images or paths."""
+ if frames.ndim not in (4, 5):
+ raise ValueError(
+ "HunyuanImage-3 decoded frames must be BCHW or BCFHW, "
+ f"but got shape {tuple(frames.shape)}"
+ )
+ source_width, source_height = plan.source_size
+ if tuple(frames.shape[-2:]) != (source_height, source_width):
+ raise ValueError(
+ "HunyuanImage-3 output and trajectory decoded to different spatial "
+ f"sizes: expected {source_width}x{source_height}, got "
+ f"{frames.shape[-1]}x{frames.shape[-2]}"
+ )
+ if plan.crop_box is not None:
+ left, top, right, bottom = plan.crop_box
+ frames = frames[..., top:bottom, left:right]
+ if plan.padding is not None:
+ frames = F.pad(frames, plan.padding, value=plan.pad_value)
+ if plan.resize_size is not None:
+ frames = _resize_frames(frames, plan.resize_size)
+ return frames
+
+
+class HunyuanImage3DecodingStage(DecodingStage):
+ """Decode a complete native bucket, then apply request-scoped geometry."""
+
+ @torch.no_grad()
+ def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
+ output_batch = super().forward(batch, server_args)
+ geometry = batch.extra.get(OUTPUT_GEOMETRY_EXTRA_KEY)
+ if geometry is None or not isinstance(output_batch.output, torch.Tensor):
+ return output_batch
+ if not envs.SGLANG_HI3_OUTPUT_CROP:
+ # Env-gated escape hatch: return the full decoded bucket
+ # uncropped; the geometry contract stays in extra.
+ batch.width = output_batch.output.shape[-1]
+ batch.height = output_batch.output.shape[-2]
+ return output_batch
+
+ source_size = output_batch.output.shape[-1], output_batch.output.shape[-2]
+ plan, metadata = _build_spatial_plan(source_size, geometry)
+ output_batch.output = apply_hunyuan_image3_spatial_plan(
+ output_batch.output, plan
+ )
+ if output_batch.trajectory_decoded is not None:
+ output_batch.trajectory_decoded = [
+ apply_hunyuan_image3_spatial_plan(frames, plan)
+ for frames in output_batch.trajectory_decoded
+ ]
+ batch.width, batch.height = metadata["output_size"]
+ batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = metadata
+ return output_batch
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/prompts.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/prompts.py
new file mode 100644
index 0000000000..1b6a320897
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/prompts.py
@@ -0,0 +1,180 @@
+"""System prompt presets for HunyuanImage-3 (from vllm_omni system_prompt.py)."""
+
+_SYSTEM_PROMPTS = {
+ "en_vanilla": """\
+You are an advanced AI text-to-image generation system. Given a detailed text prompt, your task is to create a high-quality, visually compelling image that accurately represents the described scene, characters, or objects. Pay careful attention to style, color, lighting, perspective, and any specific instructions provided.
+""",
+ "en_recaption": """\
+You are a world-class image generation prompt expert. Your task is to rewrite a user's simple description into a **structured, objective, and detail-rich** professional-level prompt.
+
+The final output must be wrapped in `` tags.
+
+### **Universal Core Principles**
+
+When rewriting the prompt (inside the `` tags), you must adhere to the following principles:
+
+1. **Absolute Objectivity**: Describe only what is visually present. Avoid subjective words like "beautiful" or "sad". Convey aesthetic qualities through specific descriptions of color, light, shadow, and composition.
+2. **Physical and Logical Consistency**: All scene elements (e.g., gravity, light, shadows, reflections, spatial relationships, object proportions) must strictly adhere to real-world physics and common sense. For example, tennis players must be on opposite sides of the net; objects cannot float without a cause.
+3. **Structured Description**: Strictly follow a logical order: from general to specific, background to foreground, and primary to secondary elements. Use directional terms like "foreground," "mid-ground," "background," and "left side of the frame" to clearly define the spatial layout.
+4. **Use Present Tense**: Describe the scene from an observer's perspective using the present tense, such as "A man stands..." or "Light shines on..."
+5. **Use Rich and Specific Descriptive Language**: Use precise adjectives to describe the quantity, size, shape, color, and other attributes of objects, subjects, and text. Vague expressions are strictly prohibited.
+
+If the user specifies a style (e.g., oil painting, anime, UI design, text rendering), strictly adhere to that style. Otherwise, first infer a suitable style from the user's input. If there is no clear stylistic preference, default to an **ultra-realistic photographic style**. Then, generate the detailed rewritten prompt according to the **Style-Specific Creation Guide** below:
+
+### **Style-Specific Creation Guide**
+
+Based on the determined artistic style, apply the corresponding professional knowledge.
+
+**1. Photography and Realism Style**
+* Utilize professional photography terms (e.g., lighting, lens, composition) and meticulously detail material textures, physical attributes of subjects, and environmental details.
+
+**2. Illustration and Painting Style**
+* Clearly specify the artistic school (e.g., Japanese Cel Shading, Impasto Oil Painting) and focus on describing its unique medium characteristics, such as line quality, brushstroke texture, or paint properties.
+
+**3. Graphic/UI/APP Design Style**
+* Objectively describe the final product, clearly defining the layout, elements, and color palette. All text on the interface must be enclosed in double quotes `""` to specify its exact content (e.g., "Login"). Vague descriptions are strictly forbidden.
+
+**4. Typographic Art**
+* The text must be described as a complete physical object. The description must begin with the text itself. Use a straightforward front-on or top-down perspective to ensure the entire text is visible without cropping.
+
+### **Final Output Requirements**
+
+1. **Output the Final Prompt Only**: Do not show any thought process, Markdown formatting, or line breaks.
+2. **Adhere to the Input**: You must retain the core concepts, attributes, and any specified text from the user's input.
+3. **Style Reinforcement**: Mention the core style 3-5 times within the prompt and conclude with a style declaration sentence.
+4. **Avoid Self-Reference**: Describe the image content directly. Remove redundant phrases like "This image shows..." or "The scene depicts..."
+5. **The final output must be wrapped in `xxxx` tags.**
+
+The user will now provide an input prompt. You will provide the expanded prompt.
+""",
+ "en_think_recaption": """\
+You will act as a top-tier Text-to-Image AI. Your core task is to deeply analyze the user's text input and transform it into a detailed, artistic, and fully user-intent-compliant image.
+
+Your workflow is divided into two phases:
+
+1. Thinking Phase (): In the tag, you need to conduct a structured thinking process, progressively breaking down and enriching the constituent elements of the image. This process must include, but is not limited to, the following dimensions:
+
+Subject: Clearly define the core character(s) or object(s) in the scene, including their appearance, posture, expression, and emotion.
+Composition: Set the camera angle and layout, such as close-up, long shot, bird's-eye view, golden ratio composition, etc.
+Environment/Background: Describe the scene where the subject is located, including the location, time of day, weather, and other elements in the background.
+Lighting: Define the type, direction, and quality of the light source, such as soft afternoon sunlight, cool tones of neon lights, dramatic Rembrandt lighting, etc., to create a specific atmosphere.
+Color Palette: Set the main color tone and color scheme of the image, such as vibrant and saturated, low-saturation Morandi colors, black and white, etc.
+Quality/Style: Determine the artistic style and technical details of the image. This includes user-specified styles (e.g., anime, oil painting) or the default realistic style, as well as camera parameters (e.g., focal length, aperture, depth of field).
+Details: Add minute elements that enhance the realism and narrative quality of the image, such as a character's accessories, the texture of a surface, dust particles in the air, etc.
+
+
+2. Recaption Phase (): In the tag, merge all the key details from the thinking process into a coherent, precise, and visually evocative final description. This description is the direct instruction for generating the image, so it must be clear, unambiguous, and organized in a way that is most suitable for an image generation engine to understand.
+
+Absolutely Objective: Describe only what is visually present. Avoid subjective words like "beautiful" or "sad." Convey aesthetic sense through concrete descriptions of colors, light, shadow, and composition.
+
+Physical and Logical Consistency: All scene elements (e.g., gravity, light and shadow, reflections, spatial relationships, object proportions) must strictly adhere to the physical laws of the real world and common sense. For example, in a tennis match, players must be on opposite sides of the net; objects cannot float without reason.
+
+Structured Description: Strictly follow a logical order: from whole to part, background to foreground, and primary to secondary. Use directional words like "foreground," "mid-ground," "background," "left side of the frame" to clearly define the spatial layout.
+
+Use Present Tense: Describe from an observer's perspective using the present tense, such as "a man stands," "light shines on..."
+Use Rich and Specific Descriptive Language: Use precise adjectives to describe the quantity, size, shape, color, and other attributes of objects/characters/text. Absolutely avoid any vague expressions.
+
+
+Output Format:
+Thinking processRefined image descriptionGenerate Image
+
+
+You must strictly adhere to the following rules:
+
+1. Faithful to Intent, Reasonable Expansion: You can creatively add details to the user's description to enhance the image's realism and artistic quality. However, all additions must be highly consistent with the user's core intent and never introduce irrelevant or conflicting elements.
+2. Style Handling: When the user does not specify a style, you must default to an "Ultra-realistic, Photorealistic" style. If the user explicitly specifies a style (e.g., anime, watercolor, oil painting, cyberpunk, etc.), both your thinking process and final description must strictly follow and reflect that specified style.
+3. Text Rendering: If specific text needs to appear in the image (such as words on a sign, a book title), you must enclose this text in English double quotes (""). Descriptive text must not use double quotes.
+4. Design-related Images: You need to specify all text and graphical elements that appear in the image and clearly describe their design details, including font, color, size, position, arrangement, visual effects, etc.
+""",
+ "en_unified": """You are an advanced multimodal model whose core mission is to analyze user intent and generate high-quality text and images.
+
+#### Four Core Capabilities
+1. **Text-to-Text (T2T):** Generate coherent text responses from text prompts.
+2. **Text-to-Image (T2I):** Generate high-quality images from text prompts.
+3. **Text & Image to Text (TI2T):** Generate accurate text responses based on a combination of images and text.
+4. **Text & Image to Image (TI2I):** Generate modified images based on a reference image and editing instructions.
+
+---
+### Image Generation Protocol (for T2I & TI2I)
+You will operate in one of two modes, determined by the user's starting tag:
+#### ** Mode (Prompt Rewriting)**:
+* **Trigger:** Input begins with ``.
+* **Task:** Immediately rewrite the user's text into a structured, objective, and detail-rich professional-grade prompt.
+* **Output:** Output only the rewritten prompt within `` tags: `Rewritten professional-grade prompt`
+
+#### ** Mode (Think + Rewrite)**:
+* **Trigger:** Input begins with ``.
+* **Task:** First, conduct a structured analysis of the request within `` tags. Then, output the professional prompt, rewritten based on the analysis, within `` tags.
+* **Output:** Strictly adhere to the format: `Analysis processRewritten prompt`
+
+---
+### Execution Standards and Guidelines
+#### **`` Phase: Analysis Guidelines**
+**For T2I (New Image Generation):**
+Deconstruct the user's request into the following core visual components:
+* **Subject:** Key features of the main character/object, including appearance, pose, expression, and emotion.
+* **Composition:** Camera angle, lens type, and layout.
+* **Environment/Background:** The setting, time of day, weather, and background elements.
+* **Lighting:** Technical details such as light source type, direction, and quality.
+* **Color Palette:** The dominant hues and overall color scheme.
+* **Style/Quality:** The artistic style, clarity, depth of field, and other technical details.
+* **Text:** Identify any text to be rendered in the image, including its content, style, and position.
+* **Details:** Small elements that add narrative depth and realism.
+
+**For TI2I (Image Editing):**
+Adopt a task-diagnostic approach:
+1. **Diagnose Task:** Identify the edit type and analyze key requirements.
+2. **Prioritize Analysis:**
+ * **Adding:** Analyze the new element's position and appearance, ensuring seamless integration with the original image's lighting, shadows, and style.
+ * **Removing:** Identify the target for removal and determine how to logically fill the resulting space using surrounding textures and lighting.
+ * **Modifying:** Analyze what to change and what it should become, while emphasizing which elements must remain unchanged.
+ * **Style Transfer:** Deconstruct the target style into specific features (e.g., brushstrokes, color palette) and apply them to the original image.
+ * **Text Editing:** Ensure correct content and format. Consider the text's visual style (e.g., font, color, material) and how it adapts to the surface's perspective, curvature, and lighting.
+ * **Reference Editing:** Extract specific visual elements (e.g., appearance, posture, composition, lines, depth) from the reference image to generate an image that aligns with the text description while also incorporating the referenced content.
+ * **Inferential Editing:** Identify vague requests (e.g., "make it more professional") and translate them into concrete visual descriptions.
+
+#### `` Phase: Professional-Grade Prompt Generation Rules
+**General Rewriting Principles (for T2I & TI2I):**
+1. **Structure & Logic:** Start with a global description. Use positional words (e.g., "foreground", "background") to define the layout.
+2. **Absolute Objectivity:** Avoid subjective terms. Convey aesthetics through precise descriptions of color, light, shadow, and materials.
+3. **Physical & Logical Consistency:** Ensure all descriptions adhere to the laws of physics and common sense.
+4. **Fidelity to User Intent:** Preserve the user's core concepts, subjects, and attributes. Text to be rendered in the image **must be enclosed in double quotes ("")**.
+5. **Camera & Resolution:** Translate camera parameters into descriptions of visual effects. Convert resolution information into natural language.
+
+**T2I-Specific Guidelines:**
+* **Style Adherence & Inference:** Strictly follow the specified style. If none is given, infer the most appropriate style and detail it using professional terminology.
+* **Style Detailing:**
+ * **Photography/Realism:** Use professional photography terms to describe lighting, lens effects, and material textures.
+ * **Painting/Illustration:** Specify the art movement or medium's characteristics.
+ * **UI/Design:** Objectively describe the final product. Define layout, elements, and typography. Text content must be specific and unambiguous.
+
+**TI2I-Specific Guidelines:**
+* **Preserve Unchanged Elements:** Emphasize elements that **remain unchanged**. Unless explicitly instructed, never alter a character's identity/appearance, the core background, camera angle, or overall style.
+* **Clear Editing Instructions:**
+ * **Replacement:** Use the logic "**replace B with A**," and provide a detailed description of A.
+ * **Addition:** Clearly state what to add, where, and what it looks like.
+* **Unambiguous Referencing:** Avoid vague references (e.g., "that person"). Use specific descriptions of appearance.
+""",
+}
+
+
+def resolve_system_prompt(
+ system_prompt: str | None,
+ bot_task: str = "image",
+) -> str | None:
+ """Resolve system prompt: preset name -> prompt text, or raw text as-is."""
+ if system_prompt is None or system_prompt == "none":
+ return None
+ if system_prompt in _SYSTEM_PROMPTS:
+ return _SYSTEM_PROMPTS[system_prompt]
+ if system_prompt == "dynamic":
+ if bot_task == "think":
+ return _SYSTEM_PROMPTS["en_think_recaption"]
+ elif bot_task == "recaption":
+ return _SYSTEM_PROMPTS["en_recaption"]
+ elif bot_task in ("image", "auto"):
+ return _SYSTEM_PROMPTS["en_vanilla"]
+ return None
+ if system_prompt == "auto":
+ return _SYSTEM_PROMPTS["en_unified"]
+ return system_prompt
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/tokenizer.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/tokenizer.py
new file mode 100644
index 0000000000..d5ba090277
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan_image3/tokenizer.py
@@ -0,0 +1,1079 @@
+"""Tokenizer wrapper for HunyuanImage-3 within sglang.
+
+``HunyuanImage3TokenizerWrapper`` wraps a base ``PreTrainedTokenizerFast``
+(loaded via ``AutoTokenizer`` without ``trust_remote_code``) and implements
+the multimodal ``apply_chat_template`` entry point needed for AR tokenization.
+Mirrors vllm-omni's ``TokenizerWrapper`` but is entirely self-contained.
+"""
+
+import random
+import warnings
+from collections import defaultdict
+from copy import deepcopy
+from dataclasses import dataclass
+
+import torch
+import torch.nn.functional as F
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+class ImageInfo:
+ """Image metadata for the tokenizer (mirrors vllm-omni's class)."""
+
+ def __init__(
+ self,
+ image_type: str = None,
+ image_width: int = None,
+ image_height: int = None,
+ token_width: int = None,
+ token_height: int = None,
+ image_token_length: int = None,
+ base_size: int = None,
+ ratio_index: int = None,
+ **kwargs,
+ ):
+ self.image_type = image_type
+ self.image_width = image_width
+ self.w = image_width
+ self.image_height = image_height
+ self.h = image_height
+ self.token_width = token_width
+ self.tk_w = token_width
+ self.token_height = token_height
+ self.tk_h = token_height
+ self.image_token_length = (
+ image_token_length
+ if image_token_length is not None
+ else (
+ token_width * token_height
+ if token_width is not None and token_height is not None
+ else None
+ )
+ )
+ self.base_size = base_size
+ self.ratio_index = ratio_index
+
+ self.add_timestep_token = kwargs.get("add_timestep_token", True)
+ self.add_guidance_token = kwargs.get("add_guidance_token", False)
+ self.use_front_boi_token = kwargs.get("use_front_boi_token", True)
+ self.add_image_shape_token = kwargs.get("add_image_shape_token", True)
+
+ def __getitem__(self, key: str):
+ if hasattr(self, key):
+ return getattr(self, key)
+ raise KeyError(f"Key '{key}' not found in ImageInfo")
+
+ def __contains__(self, key: str) -> bool:
+ return hasattr(self, key)
+
+ @property
+ def meta_info(self):
+ if self.image_type in ["vae", "gen_image", "joint_image"]:
+ return dict(
+ token_length=self.image_token_length,
+ add_timestep_token=self.add_timestep_token,
+ add_guidance_token=self.add_guidance_token,
+ use_front_boi_token=self.use_front_boi_token,
+ add_image_shape_token=self.add_image_shape_token,
+ base_size=self.base_size,
+ ratio_idx=self.ratio_index,
+ token_height=self.token_height,
+ token_width=self.token_width,
+ image_height=self.image_height,
+ image_width=self.image_width,
+ )
+ raise ValueError(f"Unknown image type '{self.image_type}'")
+
+
+class JointImageInfo:
+ """Dual VAE + ViT metadata for a conditional (joint) image.
+
+ ``meta_info`` returns ``token_length`` as ``[vae_len, vit_len]`` so
+ ``encode_sequence`` can lay out both token regions within one
+ ``joint_image`` section.
+ """
+
+ def __init__(
+ self,
+ vae_image_info: ImageInfo,
+ vision_image_info: ImageInfo,
+ vision_encoder_kwargs: dict = None,
+ ):
+ self.vae_image_info = vae_image_info
+ self.vision_image_info = vision_image_info
+ self.vision_encoder_kwargs = vision_encoder_kwargs or {}
+
+ # Uniformity with ImageInfo
+ self.image_type = "joint_image"
+ self.image_token_length = (
+ vae_image_info.image_token_length + vision_image_info.image_token_length
+ )
+ self.add_timestep_token = vae_image_info.add_timestep_token
+ self.use_front_boi_token = vae_image_info.use_front_boi_token
+ self.add_image_shape_token = vae_image_info.add_image_shape_token
+
+ @property
+ def meta_info(self):
+ return dict(
+ token_length=[
+ self.vae_image_info.image_token_length,
+ self.vision_image_info.image_token_length,
+ ],
+ add_timestep_token=self.add_timestep_token,
+ use_front_boi_token=self.use_front_boi_token,
+ add_image_shape_token=self.add_image_shape_token,
+ base_size=self.vae_image_info.base_size,
+ ratio_idx=self.vae_image_info.ratio_index,
+ token_height=[
+ self.vae_image_info.token_height,
+ self.vision_image_info.token_height,
+ ],
+ token_width=[
+ self.vae_image_info.token_width,
+ self.vision_image_info.token_width,
+ ],
+ image_height=[
+ self.vae_image_info.image_height,
+ self.vision_image_info.image_height,
+ ],
+ image_width=[
+ self.vae_image_info.image_width,
+ self.vision_image_info.image_width,
+ ],
+ )
+
+
+@dataclass
+class TokenizerEncodeOutput:
+ tokens: torch.Tensor = None
+ timestep_scatter_index: torch.Tensor = None
+ guidance_scatter_index: torch.Tensor = None
+ text_slices: list = None
+ gen_image_slices: list = None
+ text_mask: torch.Tensor = None
+ gen_image_mask: torch.Tensor = None
+ real_pos: torch.Tensor = None
+ all_image_slices: list = None
+ cond_timestep_scatter_index: torch.Tensor = None
+ gen_timestep_scatter_index: torch.Tensor = None
+ think_recaption_end_pos: list = None
+ uncond_cfg_start_pos: list = None
+ # Conditional (joint) image tracking for TI2I / I2I
+ cond_vae_image_mask: torch.Tensor = None
+ cond_vit_image_mask: torch.Tensor = None
+ cond_vae_image_slices: list = None
+ cond_vit_image_slices: list = None
+ joint_image_slices: list = None
+
+
+class _Conversation:
+ roles: list = ["User", "Assistant"]
+ sep: str = "\n\n"
+
+
+def _default(value, default_value):
+ return value if value is not None else default_value
+
+
+class HunyuanImage3TokenizerWrapper:
+ """HF tokenizer with multimodal ``apply_chat_template`` (vllm-omni parity)."""
+
+ def __init__(self, tokenizer):
+ from transformers import AutoTokenizer
+
+ if isinstance(tokenizer, str):
+ self.tokenizer = AutoTokenizer.from_pretrained(tokenizer)
+ else:
+ self.tokenizer = tokenizer
+
+ self.bos_token_id = self.tokenizer.bos_token_id
+ self.eos_token_id = self.tokenizer.eos_token_id
+ self.pad_token_id = self.tokenizer.pad_token_id
+ self.boi_token_id = self.tokenizer.convert_tokens_to_ids("")
+ self.eoi_token_id = self.tokenizer.convert_tokens_to_ids("")
+ self.img_token_id = self.tokenizer.convert_tokens_to_ids("
")
+ self.cfg_token_id = self.tokenizer.convert_tokens_to_ids("")
+ self.end_answer_token_id = self.tokenizer.convert_tokens_to_ids("")
+ self.end_recaption_token_id = self.tokenizer.convert_tokens_to_ids(
+ ""
+ )
+ self.end_think_token_id = self.tokenizer.convert_tokens_to_ids("")
+ self.ratio_token_offset = self.tokenizer.convert_tokens_to_ids("")
+ self.joint_img_sep_token_id = self.tokenizer.convert_tokens_to_ids(
+ ""
+ )
+ self.special_token_map = self.tokenizer.added_tokens_encoder
+
+ @staticmethod
+ def _pad(tensors, dim=0, pad_val=0):
+ max_len = max(t.shape[dim] for t in tensors)
+ out = []
+ for t in tensors:
+ if t.shape[dim] < max_len:
+ t = F.pad(t, (0, max_len - t.shape[dim]), value=pad_val)
+ out.append(t)
+ return out
+
+ def _get_cot_sections(self, cot_text, uncond_kwargs, drop_think=False):
+ """Parse / or / blocks."""
+ if not cot_text:
+ return []
+ if "" in cot_text and "" in cot_text:
+ before = cot_text.split("")[0]
+ think = cot_text.split("")[1].split("")[0]
+ after = cot_text.split("")[1]
+ return (
+ self._get_cot_sections(before, uncond_kwargs, drop_think)
+ + (
+ [
+ dict(type="text", text=""),
+ dict(type="text", text=think, **uncond_kwargs),
+ dict(type="text", text=""),
+ ]
+ if not drop_think
+ else []
+ )
+ + self._get_cot_sections(after, uncond_kwargs, drop_think)
+ )
+ if "" in cot_text and "" in cot_text:
+ before = cot_text.split("")[0]
+ recaption = cot_text.split("")[1].split("")[0]
+ after = cot_text.split("")[1]
+ return (
+ self._get_cot_sections(before, uncond_kwargs, drop_think)
+ + [
+ dict(type="text", text=""),
+ dict(type="text", text=recaption, **uncond_kwargs),
+ dict(type="text", text=""),
+ ]
+ + self._get_cot_sections(after, uncond_kwargs, drop_think)
+ )
+ return [dict(type="text", text=cot_text, **uncond_kwargs)]
+
+ def encode_text(
+ self,
+ *texts,
+ uncond_enabled=None,
+ uncond_p=None,
+ max_length=None,
+ pad=None,
+ return_lengths=False,
+ ):
+ """Encode text(s), optionally replacing with ```` tokens."""
+ if pad is not None:
+ assert max_length is not None
+
+ if uncond_enabled is None:
+ uncond_enabled = [True] * len(texts)
+ elif isinstance(uncond_enabled, bool):
+ uncond_enabled = [uncond_enabled] * len(texts)
+ assert len(uncond_enabled) == len(texts)
+
+ do_uncond_drop = (uncond_p is not None) and (random.random() < uncond_p)
+ text_tokens, lengths = [], []
+ cum_length = 0
+ for text, uncond_flag in zip(texts, uncond_enabled):
+ if max_length is not None and cum_length >= max_length:
+ warnings.warn(f"Text exceeds max_length({max_length}), truncating.")
+ break
+ if isinstance(text, str):
+ text_token = self.tokenizer.encode(text, add_special_tokens=False)
+ else:
+ text_token = text
+ if uncond_flag and do_uncond_drop:
+ text_token = [self.cfg_token_id] * len(text_token)
+ if max_length is not None and (cum_length + len(text_token)) > max_length:
+ text_token = text_token[: max_length - cum_length]
+ text_tokens.extend(text_token)
+ lengths.append(len(text_token))
+ cum_length += len(text_token)
+
+ if pad is not None and (pad_length := max_length - len(text_tokens)) > 0:
+ if pad == "left":
+ text_tokens = [self.pad_token_id] * pad_length + text_tokens
+ elif pad == "right":
+ text_tokens = text_tokens + [self.pad_token_id] * pad_length
+ else:
+ raise ValueError(f"Unsupported pad: {pad}")
+
+ if return_lengths:
+ return text_tokens, lengths
+ return text_tokens
+
+ def encode_sequence(
+ self,
+ template,
+ token_source,
+ total_length=None,
+ add_timestep_token=False,
+ add_guidance_token=False,
+ add_eos=True,
+ use_front_boi_token=True,
+ add_pad=True,
+ add_bos=True,
+ drop_last="auto",
+ add_image_shape_token=False,
+ ):
+ """Assemble token sequence from *template* (e.g. ``text-text-gen_image-text``)."""
+ keys = template.split("-")
+ modal_length = len(keys)
+ index_indicator = {k: 0 for k in token_source}
+ for v in token_source.values():
+ assert isinstance(v, (list, tuple))
+
+ assert set(keys) == set(token_source.keys())
+ _key_counts = {k: 0 for k in keys}
+ for k in keys:
+ _key_counts[k] += 1
+ for k, c in _key_counts.items():
+ assert len(token_source[k]) == c
+
+ token_seq = []
+ token_count = 0
+ extra = defaultdict(list)
+
+ if add_bos:
+ token_seq.append(self.bos_token_id)
+ token_count += 1
+
+ drop_last_break = False
+ for i, key in enumerate(keys):
+ source = token_source[key][index_indicator[key]]
+
+ if key == "text":
+ token_seq.extend(source)
+ extra["_start"].append(token_count)
+ if (
+ "_start" not in extra
+ and len(source) > 0
+ and source[0] == self.cfg_token_id
+ ):
+ extra["_start"].append(token_count)
+ token_count += len(source)
+ extra["_end"].append(token_count - 1)
+ if len(source) > 0 and source[-1] == self.end_think_token_id:
+ extra["_end"].append(token_count - 1)
+ if len(source) > 0 and source[-1] == self.end_recaption_token_id:
+ extra["_end"].append(token_count - 1)
+
+ elif key == "gen_image":
+ if isinstance(source, int):
+ source = {"length": source}
+ extra_count = (
+ 2
+ + (1 if source.get("timestep", add_timestep_token) else 0)
+ + (1 if source.get("guidance", add_guidance_token) else 0)
+ + (2 if source.get("image_shape", add_image_shape_token) else 0)
+ )
+ if (
+ drop_last is True
+ and total_length is not None
+ and token_count + extra_count + source["length"] > total_length
+ ):
+ drop_last_break = True
+ break
+ if source.get("front_boi", use_front_boi_token):
+ token_seq.append(self.boi_token_id)
+ extra["boi"].append(token_count)
+ token_count += 1
+ token_count = self._add_image_meta_info_token(
+ token_seq,
+ token_count,
+ extra,
+ add_timestep_token=source.get("timestep", add_timestep_token),
+ add_guidance_token=source.get("guidance", add_guidance_token),
+ add_image_shape_token=source.get(
+ "image_shape", add_image_shape_token
+ ),
+ base_size=source.get("base_size"),
+ ratio_idx=source.get("ratio_idx"),
+ image_type=key,
+ )
+ if not source.get("front_boi", use_front_boi_token):
+ token_seq.append(self.boi_token_id)
+ extra["boi"].append(token_count)
+ token_count += 1
+ token_seq.extend(
+ [self.img_token_id] * source["length"] + [self.eoi_token_id]
+ )
+ extra["
_start"].append(token_count)
+ extra["_start"].append(token_count)
+ token_count += source["length"]
+ extra["
_end"].append(token_count - 1)
+ extra["_end"].append(token_count - 1)
+ extra["eoi"].append(token_count)
+ token_count += 1
+
+ elif key == "joint_image":
+ # joint_image has dual layout: VAE tokens + + ViT tokens
+ assert (
+ isinstance(source["length"], list) and len(source["length"]) == 2
+ ), (
+ "joint_image length should be a list of two integers [vae_len, vit_len]"
+ )
+ vae_len, vit_len = source["length"]
+ extra_count = (
+ 2 # boi + eoi
+ + 1 # joint_img_sep
+ + (1 if source.get("timestep", add_timestep_token) else 0)
+ + (2 if source.get("image_shape", add_image_shape_token) else 0)
+ )
+ if (
+ drop_last is True
+ and total_length is not None
+ and token_count + extra_count + vae_len + vit_len > total_length
+ ):
+ drop_last_break = True
+ break
+ if source.get("front_boi", use_front_boi_token):
+ token_seq.append(self.boi_token_id)
+ extra["boi"].append(token_count)
+ token_count += 1
+ # image meta (timestep, image_shape; no guidance for joint_image)
+ token_count = self._add_image_meta_info_token(
+ token_seq,
+ token_count,
+ extra,
+ add_timestep_token=source.get("timestep", add_timestep_token),
+ add_guidance_token=False,
+ add_image_shape_token=source.get(
+ "image_shape", add_image_shape_token
+ ),
+ base_size=source.get("base_size"),
+ ratio_idx=source.get("ratio_idx"),
+ image_type=key,
+ )
+ if not source.get("front_boi", use_front_boi_token):
+ token_seq.append(self.boi_token_id)
+ extra["boi"].append(token_count)
+ token_count += 1
+ token_seq.extend([self.img_token_id] * vae_len)
+ extra["_start"].append(token_count)
+ extra["_start"].append(token_count)
+ extra["_start"].append(token_count)
+ token_count += vae_len
+ extra["_end"].append(token_count - 1)
+ extra["_end"].append(token_count - 1)
+ token_seq.append(self.joint_img_sep_token_id)
+ extra["joint_img_sep"].append(token_count)
+ token_count += 1
+ token_seq.extend([self.img_token_id] * vit_len)
+ extra["_start"].append(token_count)
+ extra["_start"].append(token_count)
+ token_count += vit_len
+ extra["_end"].append(token_count - 1)
+ extra["_end"].append(token_count - 1)
+ extra["_end"].append(token_count - 1)
+ token_seq.append(self.eoi_token_id)
+ extra["eoi"].append(token_count)
+ token_count += 1
+
+ else:
+ raise ValueError(f"Unsupported key: {key}")
+ index_indicator[key] += 1
+
+ if add_eos is True and not drop_last_break:
+ token_seq.append(self.eos_token_id)
+ extra["eos"].append(token_count)
+ token_count += 1
+ elif add_eos == "auto" and not drop_last_break:
+ if token_seq[-1] != self.eos_token_id and (
+ total_length is None or token_count < total_length
+ ):
+ token_seq.append(self.eos_token_id)
+ extra["eos"].append(token_count)
+ token_count += 1
+
+ if total_length:
+ if token_count > total_length and drop_last:
+ for sk, ek in [
+ ("
_start", "
_end"),
+ ("_start", "_end"),
+ ("_start", "_end"),
+ ("_start", "_end"),
+ ]:
+ if sk in extra and ek in extra:
+ assert all(
+ s > total_length or e + 1 < total_length
+ for s, e in zip(extra[sk], extra[ek])
+ ), "Clip position in the middle of image tokens!"
+ token_seq = token_seq[:total_length]
+ pad_num = max(0, total_length - len(token_seq))
+ if add_pad and pad_num:
+ token_seq.extend([self.pad_token_id] * pad_num)
+ extra["first_pad"].append(token_count)
+
+ return token_seq, extra
+
+ def _add_image_meta_info_token(
+ self,
+ token_seq,
+ token_count,
+ extra_token_pos,
+ add_timestep_token=False,
+ add_image_shape_token=False,
+ add_guidance_token=False,
+ base_size=None,
+ ratio_idx=None,
+ image_type=None,
+ ):
+ if add_image_shape_token:
+ token_seq.extend(
+ [
+ self.special_token_map[f""],
+ self.special_token_map[f""],
+ ]
+ )
+ token_count += 2
+ if add_timestep_token:
+ token_seq.extend([self.special_token_map[""]])
+ extra_token_pos["timestep"].append(token_count)
+ if image_type == "gen_image":
+ extra_token_pos["gen_timestep"].append(token_count)
+ elif image_type == "joint_image":
+ extra_token_pos["cond_timestep"].append(token_count)
+ token_count += 1
+ if add_guidance_token:
+ token_seq.extend([self.special_token_map[""]])
+ extra_token_pos["guidance"].append(token_count)
+ token_count += 1
+ return token_count
+
+ def encode_general(
+ self,
+ sections,
+ max_token_length=None,
+ add_eos="auto",
+ use_text_mask=True,
+ add_pad="auto",
+ add_bos=True,
+ drop_last="auto",
+ ):
+ """Encode a list of section dicts into a ``TokenizerEncodeOutput``."""
+ sections = deepcopy(sections)
+ template = "-".join(s["type"] for s in sections)
+
+ token_source = defaultdict(list)
+ text_mask_specs = []
+ for section in sections:
+ if section["type"] == "text":
+ text = self.encode_text(
+ section.get("text", section.get("tokens")),
+ uncond_enabled=section.get("uncond_enabled"),
+ uncond_p=section.get("uncond_p"),
+ max_length=section.get("max_length"),
+ )
+ token_source["text"].append(text)
+ text_mask_specs.append(
+ dict(
+ ignore=section.get("ignore", False),
+ start_offset=section.get("start_offset", 0),
+ end_offset=section.get("end_offset", 0),
+ )
+ )
+ elif section["type"] == "gen_image":
+ token_source["gen_image"].append(
+ dict(
+ length=section["token_length"],
+ timestep=section.get("add_timestep_token", False),
+ guidance=section.get("add_guidance_token", False),
+ front_boi=section.get("use_front_boi_token", False),
+ image_shape=section.get("add_image_shape_token", False),
+ base_size=section.get("base_size"),
+ ratio_idx=section.get("ratio_idx"),
+ )
+ )
+ elif section["type"] == "joint_image":
+ token_source["joint_image"].append(
+ dict(
+ length=section["token_length"],
+ timestep=section.get("add_timestep_token", False),
+ guidance=section.get("add_guidance_token", False),
+ front_boi=section.get("use_front_boi_token", False),
+ image_shape=section.get("add_image_shape_token", False),
+ base_size=section.get("base_size"),
+ ratio_idx=section.get("ratio_idx"),
+ )
+ )
+ else:
+ raise ValueError(f"Invalid section type: {section['type']}")
+
+ full_token_seq, extra = self.encode_sequence(
+ template=template,
+ token_source=dict(token_source),
+ total_length=max_token_length,
+ add_eos=add_eos,
+ add_pad=add_pad,
+ add_bos=add_bos,
+ drop_last=drop_last,
+ )
+ full_tensor = torch.tensor(full_token_seq, dtype=torch.long)
+
+ timestep_idx = (
+ torch.tensor(extra["timestep"], dtype=torch.long)
+ if "timestep" in extra
+ else None
+ )
+ gen_ts_idx = (
+ torch.tensor(extra["gen_timestep"], dtype=torch.long)
+ if "gen_timestep" in extra
+ else None
+ )
+ cond_ts_idx = (
+ torch.tensor(extra["cond_timestep"], dtype=torch.long)
+ if "cond_timestep" in extra
+ else None
+ )
+
+ gen_image_slices = []
+ gen_image_mask = None
+ if "
_start" in extra and "
_end" in extra:
+ gen_image_slices = [
+ slice(s, e + 1)
+ for s, e in zip(extra["
_start"], extra["
_end"])
+ ]
+ gen_image_mask = torch.zeros_like(full_tensor, dtype=torch.bool)
+ for sl in gen_image_slices:
+ gen_image_mask[sl] = True
+
+ joint_image_slices = []
+ cond_vae_image_mask = None
+ cond_vit_image_mask = None
+ cond_vae_image_slices = []
+ cond_vit_image_slices = []
+ if "_start" in extra and "_end" in extra:
+ cond_vae_image_slices = [
+ slice(s, e + 1)
+ for s, e in zip(extra["_start"], extra["_end"])
+ ]
+ cond_vae_image_mask = torch.zeros_like(full_tensor, dtype=torch.bool)
+ for sl in cond_vae_image_slices:
+ cond_vae_image_mask[sl] = True
+ if "_start" in extra and "_end" in extra:
+ cond_vit_image_slices = [
+ slice(s, e + 1)
+ for s, e in zip(extra["_start"], extra["_end"])
+ ]
+ cond_vit_image_mask = torch.zeros_like(full_tensor, dtype=torch.bool)
+ for sl in cond_vit_image_slices:
+ cond_vit_image_mask[sl] = True
+ if "_start" in extra and "_end" in extra:
+ joint_image_slices = [
+ slice(s, e + 1)
+ for s, e in zip(extra["_start"], extra["_end"])
+ ]
+
+ all_image_slices = []
+ if "_start" in extra and "_end" in extra:
+ all_image_slices = [
+ slice(s, e + 1)
+ for s, e in zip(extra["_start"], extra["_end"])
+ ]
+
+ text_slices = []
+ if "_start" in extra and "_end" in extra:
+ text_slices = [
+ slice(s, e + 1)
+ for s, e in zip(extra["_start"], extra["_end"])
+ ]
+
+ text_mask = None
+ if use_text_mask:
+ text_mask = torch.zeros_like(full_tensor, dtype=torch.float32)
+ for sl, spec in zip(text_slices, text_mask_specs):
+ if not spec["ignore"]:
+ real = slice(
+ sl.start + spec["start_offset"], sl.stop + spec["end_offset"]
+ )
+ text_mask[real] = 1.0
+
+ real_pos = torch.tensor(
+ extra.get("first_pad", [full_tensor.shape[0]]), dtype=torch.long
+ )
+ think_end = extra.get("_end", [None])[0]
+ recaption_end = extra.get("_end", [None])[0]
+
+ return TokenizerEncodeOutput(
+ tokens=full_tensor,
+ timestep_scatter_index=timestep_idx,
+ text_slices=text_slices,
+ gen_image_slices=gen_image_slices,
+ text_mask=text_mask,
+ gen_image_mask=gen_image_mask,
+ real_pos=real_pos,
+ all_image_slices=all_image_slices,
+ gen_timestep_scatter_index=gen_ts_idx,
+ cond_timestep_scatter_index=cond_ts_idx,
+ cond_vae_image_mask=cond_vae_image_mask,
+ cond_vit_image_mask=cond_vit_image_mask,
+ cond_vae_image_slices=cond_vae_image_slices,
+ cond_vit_image_slices=cond_vit_image_slices,
+ joint_image_slices=joint_image_slices,
+ think_recaption_end_pos=[recaption_end or think_end],
+ uncond_cfg_start_pos=[extra.get("_start", [None])[0]],
+ )
+
+ def apply_chat_template(
+ self,
+ batch_prompt=None,
+ batch_message_list=None,
+ mode="gen_text",
+ batch_gen_image_info=None,
+ batch_cond_image_info=None,
+ batch_system_prompt=None,
+ batch_cot_text=None,
+ max_length=None,
+ bot_task="auto",
+ image_base_size=1024,
+ sequence_template="pretrain",
+ cfg_factor=1,
+ add_assistant_prefix=None,
+ drop_think=False,
+ ):
+ """Main entry point; mirrors vllm-omni ``apply_chat_template``."""
+ assert bot_task in ["image", "auto", "think", "recaption", "img_ratio"]
+
+ if batch_message_list is None:
+ batch_size = len(batch_prompt)
+ if not isinstance(batch_system_prompt, list):
+ batch_system_prompt = [batch_system_prompt] * batch_size
+ if not isinstance(batch_gen_image_info, list):
+ batch_gen_image_info = [batch_gen_image_info] * batch_size
+ batch_cot_text = batch_cot_text or [None] * batch_size
+ batch_cond_image_info = batch_cond_image_info or [
+ [] for _ in range(batch_size)
+ ]
+
+ batch_message_list = []
+ for prompt, sys_p, cot, img_info, cond_imgs in zip(
+ batch_prompt,
+ batch_system_prompt,
+ batch_cot_text,
+ batch_gen_image_info,
+ batch_cond_image_info,
+ ):
+ ml = []
+ if sys_p:
+ ml.append(
+ dict(
+ role="system",
+ type="text",
+ content=sys_p,
+ context_type="str",
+ )
+ )
+ if len(cond_imgs) > 0:
+ ml.extend(
+ [
+ dict(
+ role="user",
+ type="joint_image",
+ content=c,
+ context_type="image_info",
+ )
+ for c in cond_imgs
+ ]
+ )
+ ml.append(
+ dict(role="user", type="text", content=prompt, context_type="str")
+ )
+ if cot is not None:
+ ml.append(
+ dict(
+ role="assistant",
+ type="text",
+ content=cot,
+ context_type="str",
+ )
+ )
+ if mode == "gen_image":
+ ml.append(
+ dict(
+ role="assistant",
+ type="gen_image",
+ content=img_info,
+ context_type="image_info",
+ )
+ )
+ batch_message_list.append(ml)
+
+ output, sections = self._apply_general_template(
+ message_list=batch_message_list,
+ max_length=max_length,
+ add_assistant_prefix=_default(add_assistant_prefix, mode != "gen_image"),
+ bot_task=bot_task,
+ sequence_template=sequence_template,
+ cfg_factor=cfg_factor,
+ batchify=True,
+ image_base_size=image_base_size,
+ drop_think=drop_think,
+ )
+ return dict(output=output, sections=sections)
+
+ def _apply_general_template(
+ self,
+ message_list,
+ max_length=None,
+ add_assistant_prefix=False,
+ answer="auto",
+ bot_task="auto",
+ sequence_template="instruct",
+ uncond_p=0.0,
+ cfg_factor=1,
+ batchify=False,
+ image_base_size=1024,
+ drop_think=False,
+ ):
+ if batchify:
+ # One prompt slot per request: _batch_gen_infer zips prompt_list
+ # against the kwargs list; a shorter prompt_list would silently
+ # truncate the batch to its first request.
+ return self._batch_gen_infer(
+ infer_fn=self._apply_general_template,
+ prompt_list=[[]] * len(message_list),
+ infer_fn_kwargs_list=[
+ dict(
+ message_list=ml_i,
+ max_length=max_length,
+ add_assistant_prefix=add_assistant_prefix,
+ answer=answer,
+ bot_task=bot_task,
+ sequence_template=sequence_template,
+ image_base_size=image_base_size,
+ drop_think=drop_think,
+ )
+ for ml_i in message_list
+ ],
+ do_classifier_free_guidance=cfg_factor > 1,
+ )
+
+ conv = _Conversation()
+ uncond_kwargs = dict(uncond_enabled=uncond_p == 1.0, uncond_p=uncond_p)
+
+ if (answer == "auto" and sequence_template == "instruct") or answer is True:
+ answer_prefix, answer_suffix = "", ""
+ else:
+ answer_prefix, answer_suffix = "", ""
+
+ if sequence_template == "pretrain":
+ system_suffix = user_prefix = user_suffix = bot_prefix = bot_suffix = ""
+ else:
+ system_suffix = conv.sep
+ user_prefix = f"{conv.roles[0]}: "
+ user_suffix = conv.sep
+ bot_prefix = f"{conv.roles[1]}: "
+ bot_suffix = conv.sep
+
+ sections: list[dict] = []
+ cur_idx = 0
+ final_role = None
+ while cur_idx < len(message_list):
+ for role, pfx, sfx, apfx, asfx in [
+ ("system", "", system_suffix, "", ""),
+ ("user", user_prefix, user_suffix, "", ""),
+ ("assistant", bot_prefix, bot_suffix, answer_prefix, answer_suffix),
+ ]:
+ sub, cur_idx = self._process_successive(
+ message_list,
+ cur_idx,
+ role,
+ pfx,
+ sfx,
+ apfx,
+ asfx,
+ uncond_kwargs=uncond_kwargs,
+ )
+ sections.extend(sub)
+ if sub:
+ final_role = role
+
+ if add_assistant_prefix:
+ if final_role == "assistant":
+ _bot_prefix = ""
+ if sections and sections[-1].get("text") == bot_suffix:
+ sections = sections[:-1]
+ else:
+ _bot_prefix = bot_prefix
+ bot_response_prefix = {
+ "auto": _bot_prefix,
+ "image": "",
+ "think": f"{_bot_prefix}",
+ "recaption": f"{_bot_prefix}",
+ "img_ratio": f"{_bot_prefix}{answer_prefix}",
+ }[bot_task]
+ sections.append(dict(type="text", text=bot_response_prefix))
+
+ output = self.encode_general(
+ sections=sections, use_text_mask=False, add_eos=False, add_pad=False
+ )
+
+ if max_length is not None and output.tokens.shape[-1] > max_length:
+ raise ValueError(
+ f"Encoded length {output.tokens.shape[-1]} exceeds max_length {max_length}."
+ )
+ return output, sections
+
+ def _process_successive(
+ self,
+ message_list,
+ cur_idx,
+ role,
+ prefix,
+ suffix,
+ answer_prefix="",
+ answer_suffix="",
+ uncond_kwargs=None,
+ ):
+ if uncond_kwargs is None:
+ uncond_kwargs = {}
+ sub_sections: list[dict] = []
+ while cur_idx < len(message_list) and message_list[cur_idx]["role"] == role:
+ msg = message_list[cur_idx]
+ if msg["type"] == "text":
+ text = msg["content"]
+ if role == "system":
+ sub_sections.append(dict(type="text", text=text))
+ elif role == "assistant":
+ if ("" in text and "" in text) or (
+ "" in text and "" in text
+ ):
+ sub_sections.extend(self._get_cot_sections(text, uncond_kwargs))
+ else:
+ sub_sections.append(
+ dict(type="text", text=text, **uncond_kwargs)
+ )
+ else:
+ # User text: no answer tags, but apply uncond_kwargs for CFG
+ sub_sections.append(
+ dict(
+ type="text",
+ text=f"{answer_prefix}{text}{answer_suffix}",
+ **uncond_kwargs,
+ )
+ )
+ elif msg["type"] in ("gen_image", "joint_image"):
+ info = msg["content"]
+ expected_cls = (
+ JointImageInfo if msg["type"] == "joint_image" else ImageInfo
+ )
+ assert isinstance(info, expected_cls), (
+ f"Expected {expected_cls.__name__}, got {type(info).__name__}"
+ )
+ if role == "assistant" and msg["type"] == "gen_image":
+ sub_sections.append(dict(type="text", text=answer_prefix))
+ sub_sections.append(dict(type=msg["type"], **info.meta_info))
+ if role == "assistant" and msg["type"] == "gen_image":
+ sub_sections.append(dict(type="text", text=answer_suffix))
+ else:
+ raise ValueError(f"Unknown message type: {msg['type']}")
+ cur_idx += 1
+
+ if sub_sections:
+ sub_sections.insert(0, dict(type="text", text=prefix))
+ sub_sections.append(dict(type="text", text=suffix))
+ return sub_sections, cur_idx
+
+ def _batch_gen_infer(
+ self,
+ infer_fn,
+ prompt_list,
+ infer_fn_kwargs_list,
+ do_classifier_free_guidance=False,
+ condition_repeat_times=1,
+ uncondition_repeat_times=1,
+ ):
+ if infer_fn_kwargs_list is None:
+ infer_fn_kwargs_list = [{} for _ in prompt_list]
+ if len(prompt_list) != len(infer_fn_kwargs_list):
+ raise ValueError(
+ f"prompt_list ({len(prompt_list)}) and infer_fn_kwargs_list "
+ f"({len(infer_fn_kwargs_list)}) must have the same length"
+ )
+
+ cond_results_list = None
+ uncond_results_list = None
+ output_type_list = []
+
+ for prompt, kw in zip(prompt_list, infer_fn_kwargs_list):
+ if not isinstance(prompt, (list, tuple)):
+ prompt = [prompt]
+ cond_kw = {**kw, "uncond_p": 0.0} if do_classifier_free_guidance else kw
+ results = infer_fn(*prompt, **cond_kw)
+ output_type_list.append(
+ (
+ type(results),
+ len(results) if isinstance(results, (list, tuple)) else 1,
+ )
+ )
+ if not isinstance(results, (list, tuple)):
+ results = (results,)
+ if cond_results_list is None:
+ cond_results_list = [[] for _ in results]
+ uncond_results_list = [[] for _ in results]
+ for i, r in enumerate(results):
+ cond_results_list[i].append(r)
+
+ if do_classifier_free_guidance:
+ uncond_kw = {**kw, "uncond_p": 1.0}
+ uncond_results = infer_fn(*prompt, **uncond_kw)
+ if isinstance(uncond_results, TokenizerEncodeOutput):
+ uncond_results_list.append(uncond_results)
+ else:
+ if not isinstance(uncond_results, (list, tuple)):
+ uncond_results = (uncond_results,)
+ for i, r in enumerate(uncond_results):
+ uncond_results_list[i].append(r)
+
+ assert all(output_type_list[0] == n for n in output_type_list)
+
+ def _make_batch(cond_items, uncond_items):
+ first = cond_items[0]
+ if isinstance(first, torch.Tensor):
+ return torch.stack(
+ self._pad(
+ cond_items * condition_repeat_times
+ + uncond_items * uncondition_repeat_times,
+ )
+ )
+ if first is None:
+ return None
+ if isinstance(first, list):
+ return (
+ cond_items * condition_repeat_times
+ + uncond_items * uncondition_repeat_times
+ )
+ if isinstance(first, TokenizerEncodeOutput):
+ merged = {}
+ for key in list(first.__dataclass_fields__.keys()):
+ vals = [
+ getattr(c, key) for c in cond_items
+ ] * condition_repeat_times + [
+ getattr(u, key) for u in uncond_items
+ ] * uncondition_repeat_times
+ if isinstance(vals[0], torch.Tensor):
+ if "mask" in key:
+ pv = 0.0
+ elif key == "tokens":
+ pv = self.special_token_map.get("", self.pad_token_id)
+ else:
+ pv = False
+ merged[key] = torch.stack(self._pad(vals, pad_val=pv), dim=0)
+ elif isinstance(vals[0], list):
+ merged[key] = vals
+ elif vals[0] is None:
+ merged[key] = None
+ else:
+ merged[key] = vals
+ return TokenizerEncodeOutput(**merged)
+ raise TypeError(f"Cannot batch {type(first)}")
+
+ stacked = [
+ _make_batch(c, u) for c, u in zip(cond_results_list, uncond_results_list)
+ ]
+ _, num = output_type_list[0]
+ if num == 1:
+ return stacked[0]
+ return tuple(stacked)
diff --git a/python/sglang/multimodal_gen/test/runner/pytest_runner.py b/python/sglang/multimodal_gen/test/runner/pytest_runner.py
index 5e5e39b01f..b144dfce23 100644
--- a/python/sglang/multimodal_gen/test/runner/pytest_runner.py
+++ b/python/sglang/multimodal_gen/test/runner/pytest_runner.py
@@ -145,12 +145,14 @@ def _extract_short_test_summary(full_output: str) -> list[str]:
for line in full_output.splitlines():
stripped = line.strip()
if "short test summary info" in stripped:
+ summary_lines = []
in_summary = True
continue
if not in_summary:
continue
if stripped.startswith("="):
- break
+ in_summary = False
+ continue
if not stripped or stripped.startswith("!"):
continue
summary_lines.append(stripped)
diff --git a/python/sglang/multimodal_gen/test/unit/test_component_accuracy_parallel_runtime.py b/python/sglang/multimodal_gen/test/unit/test_component_accuracy_parallel_runtime.py
index 81b7d35829..4a955a30fc 100644
--- a/python/sglang/multimodal_gen/test/unit/test_component_accuracy_parallel_runtime.py
+++ b/python/sglang/multimodal_gen/test/unit/test_component_accuracy_parallel_runtime.py
@@ -23,6 +23,20 @@ from sglang.srt.runtime_context import ParallelContext, get_parallel
_UTILS = "sglang.multimodal_gen.test.single_test_file.component_accuracy.utils"
+@pytest.fixture(autouse=True)
+def isolate_srt_moe_groups():
+ with (
+ patch.object(parallel_state, "_SRT_MOE_EP", None),
+ patch.object(srt_parallel_state, "_TP", None),
+ patch.object(srt_parallel_state, "_ATTN_TP", None),
+ patch.object(srt_parallel_state, "_MOE_TP", None),
+ patch.object(srt_parallel_state, "_MOE_EP", None),
+ patch.object(parallel_state, "_init_srt_moe_ep_group") as create_group,
+ patch("sglang.srt.runtime_context._PARALLEL", ParallelContext()),
+ ):
+ yield create_group
+
+
def _tp_group(world_size: int = 1, rank_in_group: int = 0) -> SimpleNamespace:
"""A TP group handle carrying the two members the scope declares to the
runtime context (`use_tensor_parallel_group` overrides `tp_size` /
@@ -115,6 +129,8 @@ def test_destroy_releases_sequence_parallel_subgroups_after_partial_init():
"_VAE_DECODE",
"_DIT",
"_VAE",
+ "_REPLICA",
+ "_ENCODER_DP",
):
stack.enter_context(patch.object(parallel_state, name, None))
stack.enter_context(patch.object(PROCESS_GROUP, "ULYSSES_PG", ulysses_group))
@@ -178,6 +194,35 @@ def test_srt_owned_groups_are_not_overwritten_or_cleared():
assert srt_parallel_state._ATTN_TP is srt_attention_tp_group
+@pytest.mark.parametrize("ownership", ["diffusion", "srt", "replaced"])
+def test_srt_moe_ep_group_ownership(isolate_srt_moe_groups, ownership):
+ create_group = isolate_srt_moe_groups
+ owned = create_group.return_value
+ external = SimpleNamespace(destroy=lambda: pytest.fail("destroyed SRT-owned group"))
+ with patch.object(parallel_state, "_TP", _tp_group()):
+ if ownership == "srt":
+ srt_parallel_state._MOE_EP = external
+ parallel_state._sync_srt_tp_group()
+ parallel_state._sync_srt_tp_group()
+ if ownership == "replaced":
+ srt_parallel_state._MOE_EP = external
+
+ parallel_state._clear_srt_tp_group()
+ parallel_state._clear_srt_tp_group()
+
+ if ownership == "srt":
+ create_group.assert_not_called()
+ owned.destroy.assert_not_called()
+ else:
+ create_group.assert_called_once_with()
+ owned.destroy.assert_called_once_with()
+ assert parallel_state._SRT_MOE_EP is None
+ assert srt_parallel_state._MOE_TP is None
+ assert srt_parallel_state._MOE_EP is (
+ None if ownership == "diffusion" else external
+ )
+
+
@pytest.mark.parametrize("rank", [0, 1])
def test_srt_tp_groups_follow_encoder_folding_context(rank):
original_tp_group = _tp_group()
diff --git a/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_resolution.py b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_resolution.py
new file mode 100644
index 0000000000..0dabc379fc
--- /dev/null
+++ b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_resolution.py
@@ -0,0 +1,299 @@
+"""Resolution-contract tests for HunyuanImage-3 image generation and editing."""
+
+from types import SimpleNamespace
+
+import torch
+from PIL import Image
+
+from sglang.multimodal_gen import envs
+from sglang.multimodal_gen.configs.pipeline_configs.hunyuan_image3 import (
+ HunyuanImage3PipelineConfig,
+)
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
+from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
+from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
+ InputValidationStage,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3.ar_stage import (
+ HunyuanImage3AR,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3.decoding import (
+ OUTPUT_GEOMETRY_EXTRA_KEY,
+ HunyuanImage3DecodingStage,
+ _build_spatial_plan,
+ apply_hunyuan_image3_spatial_plan,
+ build_hunyuan_image3_output_geometry,
+ resolve_hunyuan_image3_output_resolution,
+)
+
+
+def test_edit_without_explicit_size_follows_reference_aspect_ratio():
+ width, height = resolve_hunyuan_image3_output_resolution(
+ width=1280,
+ height=720,
+ explicit_fields=set(),
+ reference_size=(1000, 333),
+ )
+
+ assert (width, height) == (1000, 333)
+
+
+def test_explicit_size_is_not_pre_aligned_before_processor():
+ width, height = resolve_hunyuan_image3_output_resolution(
+ width=1025,
+ height=577,
+ explicit_fields={"width", "height"},
+ )
+
+ assert (width, height) == (1025, 577)
+
+
+def test_explicit_size_takes_precedence_over_reference_image():
+ width, height = resolve_hunyuan_image3_output_resolution(
+ width=1025,
+ height=577,
+ explicit_fields={"width", "height"},
+ reference_size=(1000, 333),
+ )
+
+ assert (width, height) == (1025, 577)
+
+
+def test_one_explicit_dimension_prevents_reference_size_override():
+ width, height = resolve_hunyuan_image3_output_resolution(
+ width=640,
+ height=720,
+ explicit_fields={"width"},
+ reference_size=(333, 1000),
+ )
+
+ assert (width, height) == (640, 720)
+
+
+def test_text_to_image_without_reference_keeps_raw_request_size():
+ width, height = resolve_hunyuan_image3_output_resolution(
+ width=1025,
+ height=577,
+ explicit_fields=set(),
+ )
+
+ assert (width, height) == (1025, 577)
+
+
+def test_generation_saves_requested_ratio_before_selecting_native_bucket(monkeypatch):
+ requested_size = (1000, 700)
+ native_size = (1216, 832)
+ image_info = SimpleNamespace(
+ image_width=native_size[0],
+ image_height=native_size[1],
+ token_width=native_size[0] // 16,
+ token_height=native_size[1] // 16,
+ )
+ processor = SimpleNamespace(
+ build_gen_image_info=lambda image_size: image_info,
+ )
+ request = SimpleNamespace(
+ width=requested_size[0],
+ height=requested_size[1],
+ sampling_params=SimpleNamespace(_explicit_fields={"width", "height"}),
+ original_condition_image_size=None,
+ guidance_scale=2.5,
+ num_inference_steps=50,
+ extra={},
+ )
+ stage = object.__new__(HunyuanImage3AR)
+ stage._processor = processor
+ monkeypatch.setattr(stage, "_rebuild_image_info", lambda info: info)
+
+ width, height, *_ = stage._resolve_generation_params([request], [None])
+
+ assert (width, height) == native_size
+ assert (request.width, request.height) == native_size
+ assert request.extra[OUTPUT_GEOMETRY_EXTRA_KEY] == {
+ "requested_size": [1000, 700],
+ "requested_aspect_ratio": [10, 7],
+ "size_mode": "aspect_ratio",
+ "strategy": "native_crop",
+ "ratio_policy": "exact",
+ "crop_anchor": [0.5, 0.5],
+ "max_ratio_error": 0.0005,
+ "pad_value": 0.0,
+ "native_bucket_size": [1216, 832],
+ }
+
+
+def test_native_crop_is_maximum_area_within_pixel_rounding_and_preserves_pixels():
+ geometry = build_hunyuan_image3_output_geometry(1000, 700)
+ frames = torch.arange(1216 * 832, dtype=torch.float32).reshape(1, 1, 832, 1216)
+
+ plan, metadata = _build_spatial_plan((1216, 832), geometry)
+ cropped = apply_hunyuan_image3_spatial_plan(frames, plan)
+
+ assert plan.crop_box == (14, 0, 1203, 832)
+ assert metadata["crop_box"] == [14, 0, 1203, 832]
+ assert metadata["output_size"] == [1189, 832]
+ assert metadata["relative_ratio_error"] < 0.0005
+ assert metadata["resampled"] is False
+ assert cropped.shape == (1, 1, 832, 1189)
+ assert torch.equal(cropped, frames[..., 0:832, 14:1203])
+
+
+def test_native_crop_handles_coprime_ratios_without_discarding_pixels():
+ # gcd(1000, 701) == 1: a strict 1000:701-multiple crop would discard
+ # ~30% of the decoded image (or raise); the crop must stay near-exact
+ # and near-maximum-area instead.
+ geometry = build_hunyuan_image3_output_geometry(1000, 701)
+ frames = torch.arange(1216 * 832, dtype=torch.float32).reshape(1, 1, 832, 1216)
+
+ plan, metadata = _build_spatial_plan((1216, 832), geometry)
+ cropped = apply_hunyuan_image3_spatial_plan(frames, plan)
+
+ assert plan.crop_box == (15, 0, 1202, 832)
+ assert metadata["output_size"] == [1187, 832]
+ assert metadata["relative_ratio_error"] < 0.0005
+ retained = metadata["retained_pixel_fraction"]
+ assert retained > 0.97
+ assert cropped.shape == (1, 1, 832, 1187)
+
+
+def test_native_crop_reuses_the_same_plan_for_trajectory_frames():
+ geometry = build_hunyuan_image3_output_geometry(1000, 700)
+ image = torch.zeros(2, 3, 1, 832, 1216)
+ trajectory = torch.ones(2, 3, 1, 832, 1216)
+
+ plan, _ = _build_spatial_plan((1216, 832), geometry)
+
+ assert apply_hunyuan_image3_spatial_plan(image, plan).shape == (2, 3, 1, 832, 1189)
+ assert torch.equal(
+ apply_hunyuan_image3_spatial_plan(trajectory, plan),
+ trajectory[..., 0:832, 14:1203],
+ )
+
+
+def test_decoding_stage_crops_images_and_trajectories_with_one_plan(monkeypatch):
+ frames = torch.zeros(2, 3, 1, 832, 1216)
+ trajectory = torch.ones(2, 3, 1, 832, 1216)
+
+ def fake_decode(_stage, _batch, _server_args):
+ return OutputBatch(output=frames, trajectory_decoded=[trajectory])
+
+ monkeypatch.setattr(DecodingStage, "forward", fake_decode)
+ stage = object.__new__(HunyuanImage3DecodingStage)
+ batch = Req(sampling_params=SamplingParams(prompt="test", width=1000, height=700))
+ batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = build_hunyuan_image3_output_geometry(
+ 1000, 700
+ )
+
+ output = stage.forward(batch, SimpleNamespace())
+
+ assert output.output.shape == (2, 3, 1, 832, 1189)
+ assert output.trajectory_decoded[0].shape == (2, 3, 1, 832, 1189)
+ assert (batch.width, batch.height) == (1189, 832)
+ assert batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY]["crop_box"] == [14, 0, 1203, 832]
+
+
+def test_output_crop_env_disable_returns_full_native_bucket(monkeypatch):
+ frames = torch.zeros(2, 3, 1, 832, 1216)
+
+ def fake_decode(_stage, _batch, _server_args):
+ return OutputBatch(output=frames, trajectory_decoded=None)
+
+ monkeypatch.setattr(DecodingStage, "forward", fake_decode)
+ assert envs.SGLANG_HI3_OUTPUT_CROP is True
+ monkeypatch.setenv("SGLANG_HI3_OUTPUT_CROP", "0")
+ assert envs.SGLANG_HI3_OUTPUT_CROP is False
+ stage = object.__new__(HunyuanImage3DecodingStage)
+ batch = Req(sampling_params=SamplingParams(prompt="test", width=1000, height=700))
+ batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = build_hunyuan_image3_output_geometry(
+ 1000, 700
+ )
+
+ output = stage.forward(batch, SimpleNamespace())
+
+ # Full bucket untouched; the geometry contract stays in extra.
+ assert output.output.shape == (2, 3, 1, 832, 1216)
+ assert (batch.width, batch.height) == (1216, 832)
+ assert "crop_box" not in batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY]
+
+
+def test_exact_size_performs_one_uniform_final_resample():
+ geometry = build_hunyuan_image3_output_geometry(1000, 700, size_mode="exact_size")
+ frames = torch.zeros(1, 3, 1, 832, 1216)
+
+ plan, metadata = _build_spatial_plan((1216, 832), geometry)
+ output = apply_hunyuan_image3_spatial_plan(frames, plan)
+
+ assert plan.crop_box == (14, 0, 1203, 832)
+ assert plan.resize_size == (1000, 700)
+ assert output.shape == (1, 3, 1, 700, 1000)
+ assert metadata["resampled"] is True
+
+
+def test_native_pad_retains_all_decoded_pixels():
+ geometry = build_hunyuan_image3_output_geometry(1000, 700, strategy="native_pad")
+ frames = torch.ones(1, 1, 832, 1216)
+
+ plan, metadata = _build_spatial_plan((1216, 832), geometry)
+ padded = apply_hunyuan_image3_spatial_plan(frames, plan)
+
+ assert plan.padding == (2, 2, 11, 11)
+ assert padded.shape == (1, 1, 854, 1220)
+ assert torch.equal(padded[..., 11:843, 2:1218], frames)
+ assert metadata["retained_pixel_fraction"] == 1.0
+
+
+def test_multi_output_request_keeps_native_bucket_size(monkeypatch):
+ outputs = [
+ SimpleNamespace(
+ latents=torch.zeros(1, 1, 1, 1, 1),
+ width=1216,
+ height=832,
+ )
+ for _ in range(2)
+ ]
+ stage = object.__new__(HunyuanImage3AR)
+ monkeypatch.setattr(stage, "_expand_multi_output", lambda _batch: [object()] * 2)
+ monkeypatch.setattr(stage, "_forward_batched", lambda _batches: outputs)
+ batch = Req(
+ sampling_params=SamplingParams(
+ prompt="test",
+ width=1000,
+ height=700,
+ num_outputs_per_prompt=2,
+ )
+ )
+
+ output = stage.forward(batch, SimpleNamespace())
+
+ assert output is batch
+ assert output.latents.shape[0] == 2
+ assert (output.width, output.height) == (1216, 832)
+
+
+def test_pipeline_config_delegates_condition_image_sizing_to_native_stage():
+ config = HunyuanImage3PipelineConfig()
+ reference = Image.new("RGB", (1000, 333))
+
+ assert config.calculate_condition_image_size(reference, 1280, 720) is None
+ assert config.prepare_calculated_size(reference) is None
+
+
+def test_input_validation_does_not_apply_generic_32_pixel_edit_resize():
+ reference = Image.new("RGB", (1000, 333))
+ batch = Req(
+ sampling_params=SamplingParams(prompt="edit", width=1008, height=336),
+ condition_image=reference,
+ )
+ batch.extra["explicit_fields"] = ["width", "height"]
+ server_args = SimpleNamespace(pipeline_config=HunyuanImage3PipelineConfig())
+
+ InputValidationStage().preprocess_condition_image(
+ batch,
+ server_args,
+ condition_image_width=reference.width,
+ condition_image_height=reference.height,
+ )
+
+ assert batch.condition_image[0].size == (1000, 333)
+ assert (batch.width, batch.height) == (1008, 336)
diff --git a/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_rope_pairing.py b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_rope_pairing.py
new file mode 100644
index 0000000000..f4996299df
--- /dev/null
+++ b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_rope_pairing.py
@@ -0,0 +1,127 @@
+"""Unit tests for HunyuanImage-3 RoPE image-info span/shape pairing.
+
+The section shape stream follows sequence order -- each joint image
+contributes [vae, vit] dims and the tokenizer emits VAE tokens then ViT
+tokens inside every joint block -- while the cond slices arrive grouped by
+type. ``_build_rope_image_info`` must pair every span with its own shape
+even when consecutive cond images have different sizes.
+"""
+
+import unittest
+from types import SimpleNamespace
+
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3 import (
+ ar_stage,
+)
+from sglang.test.test_utils import CustomTestCase
+
+
+def _tokenizer_output(vae_slices, vit_slices, gen_slices):
+ return SimpleNamespace(
+ cond_vae_image_slices=[vae_slices],
+ cond_vit_image_slices=[vit_slices],
+ gen_image_slices=[gen_slices],
+ )
+
+
+def _joint_section(vae_hw, vit_hw):
+ return {
+ "type": "joint_image",
+ "token_height": [vae_hw[0], vit_hw[0]],
+ "token_width": [vae_hw[1], vit_hw[1]],
+ }
+
+
+class TestHunyuanImage3RoPEPairing(CustomTestCase):
+ def test_multi_image_different_sizes_pair_per_image(self):
+ # Two cond images with different grids: image 0 = 24x32 vae / 14x14
+ # vit, image 1 = 16x48 vae / 21x21 vit.
+ vae0, vit0, vae1, vit1 = (24, 32), (14, 14), (16, 48), (21, 21)
+ s_vae0, s_vit0 = slice(10, 20), slice(21, 30)
+ s_vae1, s_vit1 = slice(40, 55), slice(56, 60)
+ tok = _tokenizer_output(
+ vae_slices=[s_vae0, s_vae1],
+ vit_slices=[s_vit0, s_vit1],
+ gen_slices=[],
+ )
+ sections = [_joint_section(vae0, vit0), _joint_section(vae1, vit1)]
+
+ info = ar_stage._build_rope_image_info(
+ tokenizer_output=tok,
+ batch_size=1,
+ token_h=99,
+ token_w=99,
+ image_info=SimpleNamespace(token_height=32, token_width=32),
+ sections=sections,
+ )[0]
+
+ got = {s.start: shape for s, shape in info}
+ self.assertEqual(
+ got,
+ {
+ s_vae0.start: vae0,
+ s_vit0.start: vit0,
+ s_vae1.start: vae1,
+ s_vit1.start: vit1,
+ },
+ )
+ # Spans must stay in sequence order.
+ spans = [s for s, _ in info]
+ self.assertEqual(spans, sorted(spans, key=lambda s: s.start))
+
+ def test_single_image_pairing_unchanged(self):
+ vae, vit = (24, 32), (14, 14)
+ s_vae, s_vit = slice(10, 20), slice(21, 30)
+ tok = _tokenizer_output(vae_slices=[s_vae], vit_slices=[s_vit], gen_slices=[])
+ sections = [_joint_section(vae, vit)]
+
+ info = ar_stage._build_rope_image_info(
+ tokenizer_output=tok,
+ batch_size=1,
+ token_h=99,
+ token_w=99,
+ image_info=SimpleNamespace(token_height=32, token_width=32),
+ sections=sections,
+ )[0]
+
+ self.assertEqual(info, [(s_vae, vae), (s_vit, vit)])
+
+ def test_missing_shapes_fall_back_to_gen_dims(self):
+ # A cond span whose shape is absent from the section stream falls
+ # back to the gen-image dims instead of consuming the wrong shape.
+ vae, vit = (24, 32), (14, 14)
+ s_vae, s_vit = slice(10, 20), slice(21, 30)
+ tok = _tokenizer_output(vae_slices=[s_vae], vit_slices=[s_vit], gen_slices=[])
+ sections = [] # no image sections -> empty shape stream
+
+ info = ar_stage._build_rope_image_info(
+ tokenizer_output=tok,
+ batch_size=1,
+ token_h=99,
+ token_w=99,
+ image_info=SimpleNamespace(token_height=32, token_width=32),
+ sections=sections,
+ )[0]
+
+ self.assertEqual(info, [(s_vae, (99, 99)), (s_vit, (99, 99))])
+
+ def test_gen_images_take_their_own_shapes(self):
+ gen_hw = (40, 24)
+ s_gen = slice(70, 90)
+ tok = _tokenizer_output(vae_slices=[], vit_slices=[], gen_slices=[s_gen])
+ sections = [{"type": "gen_image", "token_height": 40, "token_width": 24}]
+
+ info = ar_stage._build_rope_image_info(
+ tokenizer_output=tok,
+ batch_size=1,
+ token_h=99,
+ token_w=99,
+ image_info=SimpleNamespace(token_height=32, token_width=32),
+ sections=sections,
+ )[0]
+
+ self.assertEqual(info, [(s_gen, gen_hw)])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py
new file mode 100644
index 0000000000..28665dde95
--- /dev/null
+++ b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py
@@ -0,0 +1,118 @@
+"""Unit tests for HunyuanImage-3 tensor-parallel input broadcasts."""
+
+import ast
+import inspect
+import textwrap
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3 import (
+ ar_stage,
+)
+from sglang.test.test_utils import CustomTestCase
+
+
+class _FakeTensor:
+ def __init__(self, shape):
+ self.shape = shape
+
+ def contiguous(self):
+ return self
+
+ def reshape(self, *_shape):
+ return self
+
+
+class _FakeTPGroup:
+ world_size = 2
+
+ def __init__(self):
+ self.broadcast_calls = []
+
+ def broadcast(self, tensor, src):
+ self.broadcast_calls.append((tensor, src))
+ return tensor
+
+
+class TestHunyuanImage3TPBroadcast(CustomTestCase):
+ def test_static_broadcast_happens_outside_denoising_loop(self):
+ source = textwrap.dedent(
+ inspect.getsource(ar_stage.HunyuanImage3AR._forward_batched)
+ )
+ function = ast.parse(source).body[0]
+ static_broadcast_calls = [
+ node
+ for node in ast.walk(function)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "_broadcast_static_inputs"
+ ]
+ self.assertEqual(len(static_broadcast_calls), 1)
+
+ denoising_loops = [
+ node
+ for node in ast.walk(function)
+ if isinstance(node, ast.For)
+ and any(
+ isinstance(child, ast.Name) and child.id == "timesteps"
+ for child in ast.walk(node.iter)
+ )
+ ]
+ self.assertEqual(len(denoising_loops), 1)
+ self.assertLess(
+ static_broadcast_calls[0].lineno,
+ denoising_loops[0].lineno,
+ "request-static inputs must be broadcast before the denoising loop",
+ )
+
+ def test_static_inputs_are_broadcast_once(self):
+ stage = ar_stage.HunyuanImage3AR.__new__(ar_stage.HunyuanImage3AR)
+ tp_group = _FakeTPGroup()
+ attention_mask = _FakeTensor((2, 1, 8, 8))
+ cos = _FakeTensor((2, 8, 4))
+ sin = _FakeTensor((2, 8, 4))
+
+ with (
+ patch.object(ar_stage, "model_parallel_is_initialized", return_value=True),
+ patch.object(ar_stage, "get_tp_group", return_value=tp_group),
+ ):
+ result = stage._broadcast_static_inputs(attention_mask, (cos, sin))
+
+ self.assertEqual(result, (attention_mask, (cos, sin)))
+ self.assertEqual(
+ tp_group.broadcast_calls,
+ [(attention_mask, 0), (cos, 0), (sin, 0)],
+ )
+
+ def test_backbone_forward_only_broadcasts_dynamic_hidden_states(self):
+ stage = ar_stage.HunyuanImage3AR.__new__(ar_stage.HunyuanImage3AR)
+ tp_group = _FakeTPGroup()
+ hidden_states = _FakeTensor((2, 3, 4))
+ attention_mask = _FakeTensor((2, 1, 3, 3))
+ cos = _FakeTensor((2, 3, 2))
+ sin = _FakeTensor((2, 3, 2))
+ output = MagicMock()
+ output.shape = (6, 4)
+ output.view.return_value = "reshaped-output"
+ stage._cache_dit_adapter = None
+ stage.ar_model = SimpleNamespace(forward_block=MagicMock(return_value=output))
+
+ with (
+ patch.object(ar_stage, "model_parallel_is_initialized", return_value=True),
+ patch.object(ar_stage, "get_tp_group", return_value=tp_group),
+ ):
+ result = stage._backbone_forward(
+ num_image_tokens=2,
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ custom_pos_emb=(cos, sin),
+ first_step=True,
+ )
+
+ self.assertEqual(result, "reshaped-output")
+ self.assertEqual(tp_group.broadcast_calls, [(hidden_states, 0)])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py b/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py
index ea2941a9ca..34cb583268 100644
--- a/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py
+++ b/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py
@@ -9,6 +9,7 @@ import pytest
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
from sglang.multimodal_gen.test.runner.pytest_runner import (
_estimate_failed_test_time,
+ _extract_short_test_summary,
_is_retryable_failure,
run_pytest,
)
@@ -60,6 +61,23 @@ def test_validation_failures_are_not_retryable(output):
assert not _is_retryable_failure(output)
+def test_failure_summary_uses_outer_pytest_session():
+ output = "\n".join(
+ [
+ "=== short test summary info ===",
+ "FAILED nested/test_retry.py::test_slow - AssertionError: [performance]",
+ "=== 1 failed ===",
+ "nested failure was expected; outer test PASSED",
+ "=== short test summary info ===",
+ "FAILED test_parallel.py::test_destroy - AssertionError",
+ "=== 1 failed, 10 passed ===",
+ ]
+ )
+ assert _extract_short_test_summary(output) == [
+ "FAILED test_parallel.py::test_destroy - AssertionError"
+ ]
+
+
@pytest.mark.parametrize(
"output",
[
diff --git a/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py b/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py
index ba16de6c8c..24dddff247 100644
--- a/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py
+++ b/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py
@@ -4,13 +4,19 @@ from unittest.mock import patch
import pytest
import torch
from torch import nn
+from transformers import Siglip2VisionConfig
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
from sglang.multimodal_gen.runtime.models.encoders import gemma_3
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
)
-from sglang.srt.models import siglip
+from sglang.multimodal_gen.runtime.models.encoders.hunyuan_image3 import (
+ HunyuanImage3VisionModel,
+ _PerImageSDPA,
+)
+from sglang.srt.models import siglip, siglip2
+from sglang.srt.runtime_context import get_parallel
def _vision_config():
@@ -79,6 +85,119 @@ def test_gemma3_maps_hf_siglip_projection_name():
assert mapped == "vision_tower.vision_model.encoder.layers.0.self_attn.proj.weight"
+@pytest.mark.parametrize("replicated", [False, True])
+def test_siglip2_propagates_backend_and_parallelism(replicated):
+ config = Siglip2VisionConfig(
+ hidden_size=16,
+ intermediate_size=32,
+ num_attention_heads=2,
+ num_hidden_layers=1,
+ num_patches=16,
+ )
+ with (
+ patch.object(siglip2, "VisionAttention", return_value=nn.Identity()) as attn,
+ patch.object(
+ siglip2, "ColumnParallelLinear", return_value=nn.Identity()
+ ) as fc1,
+ patch.object(siglip2, "RowParallelLinear", return_value=nn.Identity()) as fc2,
+ ):
+ kwargs = (
+ {"qkv_backend": "sdpa", "use_data_parallel": True} if replicated else {}
+ )
+ siglip2.Siglip2Model(config, **kwargs)
+
+ assert attn.call_args.kwargs["qkv_backend"] == ("sdpa" if replicated else None)
+ assert attn.call_args.kwargs["use_data_parallel"] is replicated
+ for linear in (fc1, fc2):
+ assert linear.call_args.kwargs["parallel_group"] == (
+ "replicated" if replicated else "tp"
+ )
+
+
+@pytest.mark.parametrize("mask_ndim", [1, 2, 3])
+def test_hunyuan_siglip2_preserves_padded_layout(mask_ndim):
+ # Exercise the processor boundary independently of device-specific kernels.
+ model = HunyuanImage3VisionModel.__new__(HunyuanImage3VisionModel)
+ nn.Module.__init__(model)
+ batch = 1 if mask_ndim == 1 else 2
+ pixels = torch.arange(batch * 8 * 3, dtype=torch.float32).reshape(batch, 8, 3)
+ shapes = torch.tensor([[2, 3], [1, 4]])[:batch]
+ mask = torch.arange(8)[None, :] < shapes.prod(dim=1)[:, None]
+ input_mask = mask.reshape(-1) if mask_ndim == 1 else mask
+ if mask_ndim == 3:
+ input_mask = input_mask[:, None, :]
+ packed = pixels[mask]
+
+ with patch.object(
+ siglip2.Siglip2Model, "forward", return_value=packed[None]
+ ) as forward:
+ output = model(pixels, input_mask, shapes)
+
+ torch.testing.assert_close(forward.call_args.kwargs["pixel_values_packed"], packed)
+ assert forward.call_args.kwargs["spatial_shapes"].device.type == "cpu"
+ assert forward.call_args.kwargs["cu_seqlens"].tolist() == (
+ [0, 6] if batch == 1 else [0, 6, 10]
+ )
+ assert forward.call_args.kwargs["max_seqlen"] == 6
+ torch.testing.assert_close(output[mask], packed)
+ assert torch.count_nonzero(output[~mask]) == 0
+
+
+def test_hunyuan_siglip2_attention_does_not_mix_images():
+ generator = torch.Generator().manual_seed(42)
+ q, k, v = [torch.randn(10, 2, 8, generator=generator) for _ in range(3)]
+ attention = _PerImageSDPA()
+ boundaries = torch.tensor([0, 6, 10], dtype=torch.int32)
+ actual = attention(q, k, v, boundaries)
+ expected = torch.cat(
+ [
+ torch.nn.functional.scaled_dot_product_attention(
+ q[start:end].transpose(0, 1),
+ k[start:end].transpose(0, 1),
+ v[start:end].transpose(0, 1),
+ ).transpose(0, 1)
+ for start, end in ((0, 6), (6, 10))
+ ]
+ )
+ torch.testing.assert_close(actual, expected)
+ v[6:] += 100
+ torch.testing.assert_close(attention(q, k, v, boundaries)[:6], actual[:6])
+
+
+def test_hunyuan_siglip2_loads_unfused_checkpoint_weights():
+ with get_parallel().override(tp_size=1):
+ model = HunyuanImage3VisionModel(
+ dict(
+ hidden_size=16,
+ intermediate_size=32,
+ num_attention_heads=2,
+ num_hidden_layers=1,
+ num_patches=16,
+ patch_size=2,
+ use_return_dict=True,
+ )
+ )
+ checkpoint = {}
+ expected = {}
+ for name, param in model.named_parameters():
+ value = torch.arange(param.numel(), dtype=param.dtype).reshape(param.shape)
+ expected[name] = value
+ checkpoint_name = name.removeprefix("vision_model.")
+ if "attn.qkv_proj" in name:
+ for projection, weight in zip(("q", "k", "v"), value.chunk(3)):
+ checkpoint[
+ checkpoint_name.replace("attn.qkv_proj", f"{projection}_proj")
+ ] = weight
+ else:
+ checkpoint[checkpoint_name.replace("attn.proj", "out_proj")] = value
+
+ loaded = model.load_weights(checkpoint.items())
+
+ assert loaded == set(expected)
+ for name, param in model.named_parameters():
+ torch.testing.assert_close(param, expected[name], rtol=0, atol=0)
+
+
@pytest.mark.parametrize(
"shard,normalized",
[("q", 0), ("k", 1), ("v", 2), (0, "q"), (1, "k"), (2, "v"), ("0", 0), ("01", 1)],
diff --git a/python/sglang/srt/models/siglip2.py b/python/sglang/srt/models/siglip2.py
index 6b218f1d7f..c436c75768 100644
--- a/python/sglang/srt/models/siglip2.py
+++ b/python/sglang/srt/models/siglip2.py
@@ -176,6 +176,8 @@ class Siglip2Attention(nn.Module):
config: Siglip2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
+ qkv_backend: Optional[str] = None,
+ use_data_parallel: bool = False,
):
super().__init__()
self.config = config
@@ -200,6 +202,8 @@ class Siglip2Attention(nn.Module):
flatten_batch=True, # For variable-length sequence support
quant_config=quant_config,
prefix=prefix,
+ qkv_backend=qkv_backend,
+ use_data_parallel=use_data_parallel,
)
def forward(
@@ -234,6 +238,7 @@ class Siglip2MLP(nn.Module):
config: Siglip2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
+ use_data_parallel: bool = False,
):
super().__init__()
self.config = config
@@ -244,12 +249,14 @@ class Siglip2MLP(nn.Module):
config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("fc1", prefix),
+ parallel_group="replicated" if use_data_parallel else "tp",
)
self.fc2 = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("fc2", prefix),
+ parallel_group="replicated" if use_data_parallel else "tp",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -267,6 +274,8 @@ class Siglip2EncoderLayer(nn.Module):
config: Siglip2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
+ qkv_backend: Optional[str] = None,
+ use_data_parallel: bool = False,
):
super().__init__()
self.embed_dim = config.hidden_size
@@ -275,12 +284,15 @@ class Siglip2EncoderLayer(nn.Module):
config,
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
+ qkv_backend=qkv_backend,
+ use_data_parallel=use_data_parallel,
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.mlp = Siglip2MLP(
config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
+ use_data_parallel=use_data_parallel,
)
def forward(
@@ -324,6 +336,8 @@ class Siglip2Encoder(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
num_hidden_layers_override: Optional[int] = None,
prefix: str = "",
+ qkv_backend: Optional[str] = None,
+ use_data_parallel: bool = False,
):
super().__init__()
self.config = config
@@ -339,6 +353,8 @@ class Siglip2Encoder(nn.Module):
config=config,
quant_config=quant_config,
prefix=add_prefix(f"layers.{idx}", prefix),
+ qkv_backend=qkv_backend,
+ use_data_parallel=use_data_parallel,
)
for idx in range(num_hidden_layers)
]
@@ -426,6 +442,8 @@ class Siglip2VisionTransformer(nn.Module):
num_hidden_layers_override: Optional[int] = None,
require_post_norm: Optional[bool] = None,
prefix: str = "",
+ qkv_backend: Optional[str] = None,
+ use_data_parallel: bool = False,
):
super().__init__()
embed_dim = config.hidden_size
@@ -436,6 +454,8 @@ class Siglip2VisionTransformer(nn.Module):
quant_config=quant_config,
num_hidden_layers_override=num_hidden_layers_override,
prefix=add_prefix("encoder", prefix),
+ qkv_backend=qkv_backend,
+ use_data_parallel=use_data_parallel,
)
num_hidden_layers = config.num_hidden_layers
if len(self.encoder.layers) > config.num_hidden_layers:
@@ -513,6 +533,8 @@ class Siglip2Model(nn.Module):
num_hidden_layers_override: Optional[int] = None,
require_post_norm: Optional[bool] = None,
prefix: str = "",
+ qkv_backend: Optional[str] = None,
+ use_data_parallel: bool = False,
):
super().__init__()
@@ -522,6 +544,8 @@ class Siglip2Model(nn.Module):
num_hidden_layers_override=num_hidden_layers_override,
require_post_norm=require_post_norm,
prefix=add_prefix("vision_model", prefix),
+ qkv_backend=qkv_backend,
+ use_data_parallel=use_data_parallel,
)
@property