[diffusion] model: support HunyuanImage-3.0-Instruct (#36311)
Co-authored-by: Elizaveta Martirosian <you@example.com> Co-authored-by: wes-lyu <294033513+wes-lyu@users.noreply.github.com> Co-authored-by: Wes <its.wesley.lyu@gmail.com> Co-authored-by: AuFlow <73925903+AuFlow@users.noreply.github.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
co-authored by
Elizaveta Martirosian
wes-lyu
Wes
AuFlow
Mick Qian
Mick
parent
506f140166
commit
436d73ddda
@@ -0,0 +1,58 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunyuanImage3ArchConfig(DiTArchConfig):
|
||||
"""Architecture config for HunyuanImage-3 AR transformer backbone."""
|
||||
|
||||
# AR transformer params
|
||||
hidden_size: int = 4096
|
||||
num_hidden_layers: int = 32
|
||||
num_attention_heads: int = 32
|
||||
num_key_value_heads: int = 8
|
||||
attention_head_dim: int = 128
|
||||
vocab_size: int = 133120
|
||||
intermediate_size: int = 3072
|
||||
|
||||
# MoE params
|
||||
num_experts: int = 64
|
||||
moe_topk: int = 8
|
||||
num_shared_expert: int = 1
|
||||
use_mixed_mlp_moe: bool = True
|
||||
norm_topk_prob: bool = True
|
||||
|
||||
# CLA (Cross-Layer Attention)
|
||||
use_cla: bool = False
|
||||
cla_share_factor: int = 2
|
||||
|
||||
# QK Norm
|
||||
use_qk_norm: bool = True
|
||||
|
||||
# RoPE
|
||||
rope_theta: float = 10000.0
|
||||
rope_scaling_type: str = "custom"
|
||||
max_position_embeddings: int = 22800
|
||||
|
||||
# Image generation
|
||||
vae_downsample_factor: tuple[int, int] = (16, 16)
|
||||
latent_channels: int = 32
|
||||
|
||||
stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
|
||||
param_names_mapping: dict = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
self.num_layers = self.num_hidden_layers
|
||||
self.in_channels = self.latent_channels
|
||||
self.out_channels = self.latent_channels
|
||||
self.num_channels_latents = self.latent_channels
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunyuanImage3DitConfig(DiTConfig):
|
||||
arch_config: DiTArchConfig = field(default_factory=HunyuanImage3ArchConfig)
|
||||
|
||||
prefix: str = "hunyuan_image3"
|
||||
@@ -0,0 +1,57 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunyuanImage3VAEArchConfig(VAEArchConfig):
|
||||
spatial_compression_ratio: int = 16
|
||||
|
||||
# VAE architecture params
|
||||
block_out_channels: tuple[int, ...] = (128, 256, 512, 1024, 1024)
|
||||
layers_per_block: int = 2
|
||||
latent_channels: int = 32
|
||||
in_channels: int = 3
|
||||
out_channels: int = 3
|
||||
|
||||
# Scaling factors
|
||||
scaling_factor: float = 0.562679178327931
|
||||
ffactor_spatial: int = 16
|
||||
ffactor_temporal: int = 4
|
||||
|
||||
# Latent statistics
|
||||
latents_mean: tuple[float, ...] | None = None
|
||||
latents_std: tuple[float, ...] | None = None
|
||||
shift_factor: float | None = None
|
||||
|
||||
# Temporal downsampling
|
||||
temperal_downsample: tuple[bool, ...] = (False, True, True)
|
||||
|
||||
is_residual: bool = False
|
||||
patch_size: int | None = None
|
||||
clip_output: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunyuanImage3VAEConfig(VAEConfig):
|
||||
arch_config: HunyuanImage3VAEArchConfig = field(
|
||||
default_factory=HunyuanImage3VAEArchConfig
|
||||
)
|
||||
|
||||
use_feature_cache: bool = False
|
||||
use_tiling: bool = False
|
||||
use_temporal_tiling: bool = False
|
||||
use_parallel_tiling: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
# Blend on both sides of each temporal tile (diffusers 3D-VAE convention).
|
||||
self.blend_num_frames = (
|
||||
self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames
|
||||
) * 2
|
||||
|
||||
def get_vae_scale_factor(self):
|
||||
return self.arch_config.ffactor_spatial
|
||||
|
||||
def post_init(self):
|
||||
self.arch_config.vae_scale_factor = self.arch_config.ffactor_spatial
|
||||
self.arch_config.spatial_compression_ratio = self.arch_config.ffactor_spatial
|
||||
@@ -28,6 +28,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import (
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import (
|
||||
Hunyuan3D2PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan_image3 import (
|
||||
HunyuanImage3PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ideogram import (
|
||||
Ideogram4DistilledPipelineConfig,
|
||||
Ideogram4PipelineConfig,
|
||||
@@ -77,6 +80,7 @@ __all__ = [
|
||||
"HunyuanConfig",
|
||||
"FastHunyuanConfig",
|
||||
"Hunyuan3D2PipelineConfig",
|
||||
"HunyuanImage3PipelineConfig",
|
||||
"Ideogram4PipelineConfig",
|
||||
"Ideogram4DistilledPipelineConfig",
|
||||
"FluxPipelineConfig",
|
||||
|
||||
@@ -479,6 +479,10 @@ class PipelineConfig:
|
||||
"""Return whether dynamic batches should run as grouped Req lists."""
|
||||
return False
|
||||
|
||||
def supports_batching_image_conditioning(self):
|
||||
"""Return whether image-conditioned requests may join dynamic batches."""
|
||||
return False
|
||||
|
||||
def supports_sequential_dit_inference(self):
|
||||
"""Return whether batched AR is followed by per-request DiT inference."""
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.hunyuan_image3 import (
|
||||
HunyuanImage3DitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig
|
||||
from sglang.multimodal_gen.configs.models.encoders.t5 import T5ArchConfig, T5Config
|
||||
from sglang.multimodal_gen.configs.models.vaes.hunyuan_image3 import (
|
||||
HunyuanImage3VAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
SpatialImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunyuanImage3PipelineConfig(SpatialImagePipelineConfig):
|
||||
vae_precision: str = "fp32"
|
||||
|
||||
should_use_guidance: bool = True
|
||||
task_type: ModelTaskType = ModelTaskType.TI2I
|
||||
|
||||
vae_tiling: bool = False
|
||||
vae_sp: bool = False
|
||||
|
||||
dit_config: DiTConfig = field(default_factory=HunyuanImage3DitConfig)
|
||||
|
||||
vae_config: VAEConfig = field(default_factory=HunyuanImage3VAEConfig)
|
||||
|
||||
text_encoder_configs: tuple[EncoderConfig, ...] = field(
|
||||
default_factory=lambda: (T5Config(T5ArchConfig(num_heads=6)),)
|
||||
)
|
||||
|
||||
enable_autocast: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
self.vae_scale_factor = self.vae_config.get_vae_scale_factor()
|
||||
|
||||
def supports_dynamic_batching(self):
|
||||
# The AR stage batches compatible requests in one diffusion loop.
|
||||
return True
|
||||
|
||||
def supports_native_grouped_requests(self):
|
||||
return True
|
||||
|
||||
def calculate_condition_image_size(self, image, width, height):
|
||||
"""Let the native processor choose the conditional-image bucket.
|
||||
|
||||
The generic resize snaps to a ``2 * vae_scale`` grid, but this model's
|
||||
processor derives VAE and vision inputs from the original image and
|
||||
accepts a 16-px grid; resizing first can alter the aspect ratio.
|
||||
"""
|
||||
del image, width, height
|
||||
return None
|
||||
|
||||
def prepare_calculated_size(self, image):
|
||||
"""Keep HunyuanImage-3 output resolution under native AR-stage control."""
|
||||
del image
|
||||
return None
|
||||
|
||||
def supports_batching_image_conditioning(self):
|
||||
# TI2I requests carry per-request conditioning (masks/scatter/RoPE);
|
||||
# requests are bucketed by resolution and condition-image count.
|
||||
return True
|
||||
|
||||
def supports_sequential_dit_inference(self):
|
||||
return True
|
||||
|
||||
def supports_sequential_multi_output_inference(self):
|
||||
return current_platform.is_npu()
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.hunyuan_image3 import (
|
||||
HunyuanImage3SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
register_configs(
|
||||
sampling_param_cls=HunyuanImage3SamplingParams,
|
||||
pipeline_config_cls=HunyuanImage3PipelineConfig,
|
||||
hf_model_paths=[
|
||||
"tencent/HunyuanImage-3.0",
|
||||
"tencent/HunyuanImage-3.0-Instruct",
|
||||
],
|
||||
model_detectors=[lambda hf_id: "hunyuanimage-3" in hf_id.lower()],
|
||||
)
|
||||
@@ -0,0 +1,120 @@
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT = 16
|
||||
|
||||
VALID_BOT_TASKS = {
|
||||
"auto",
|
||||
"image",
|
||||
"think",
|
||||
"recaption",
|
||||
"think_recaption",
|
||||
"img_ratio",
|
||||
"none",
|
||||
}
|
||||
|
||||
SYSTEM_PROMPT_PRESETS = {
|
||||
"none",
|
||||
"en_unified",
|
||||
"en_vanilla",
|
||||
"en_recaption",
|
||||
"en_think_recaption",
|
||||
"dynamic",
|
||||
"auto",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunyuanImage3SamplingParams(SamplingParams):
|
||||
negative_prompt: str = ""
|
||||
num_frames: int = 1
|
||||
guidance_scale: float = 2.5
|
||||
num_inference_steps: int = 50
|
||||
|
||||
# Tokenizer bot_task: controls the bot response prefix; "image" adds
|
||||
# no bot prefix in gen_image mode.
|
||||
bot_task: str = "image"
|
||||
|
||||
# Preset name (see SYSTEM_PROMPT_PRESETS) or raw custom text.
|
||||
system_prompt: str | None = "en_unified"
|
||||
|
||||
# Pre-generated CoT text from AR stage (think/recaption output)
|
||||
cot_text: str | None = None
|
||||
|
||||
# Output geometry is separate from the generation canvas: the AR
|
||||
# processor picks its own bucket, then the decoder applies this
|
||||
# request-scoped policy to the decoded pixels.
|
||||
output_size_mode: str = "aspect_ratio"
|
||||
output_strategy: str = "native_crop"
|
||||
output_ratio_policy: str = "exact"
|
||||
output_crop_anchor: tuple[float, float] = (0.5, 0.5)
|
||||
output_max_ratio_error: float = 0.0005
|
||||
output_pad_value: float = 0.0
|
||||
|
||||
# Supported resolutions (height, width) - must be divisible by 16
|
||||
supported_resolutions: list[tuple[int, int]] | None = field(
|
||||
default_factory=lambda: [
|
||||
(1024, 1024), # 1:1
|
||||
(768, 1024), # 3:4 portrait
|
||||
(1024, 768), # 4:3 landscape
|
||||
(720, 1280), # 9:16 portrait
|
||||
(1280, 720), # 16:9 landscape
|
||||
]
|
||||
)
|
||||
|
||||
def _adjust(self, server_args):
|
||||
# Pre-aligning each dimension to a 16-px grid here would distort the
|
||||
# processor's aspect-ratio bucket selection.
|
||||
if self.bot_task not in VALID_BOT_TASKS:
|
||||
logger.warning(
|
||||
f"Invalid bot_task '{self.bot_task}'. Must be one of {VALID_BOT_TASKS}. "
|
||||
f"Defaulting to 'image'."
|
||||
)
|
||||
self.bot_task = "image"
|
||||
self._validate_output_geometry()
|
||||
super()._adjust(server_args)
|
||||
|
||||
def _validate_output_geometry(self) -> None:
|
||||
if self.output_size_mode not in {"aspect_ratio", "exact_size"}:
|
||||
raise ValueError("output_size_mode must be 'aspect_ratio' or 'exact_size'")
|
||||
if self.output_strategy not in {"native_crop", "native_pad"}:
|
||||
raise ValueError("output_strategy must be 'native_crop' or 'native_pad'")
|
||||
if self.output_ratio_policy not in {"exact", "approximate"}:
|
||||
raise ValueError("output_ratio_policy must be 'exact' or 'approximate'")
|
||||
if len(self.output_crop_anchor) != 2 or any(
|
||||
not math.isfinite(float(value)) or not 0.0 <= float(value) <= 1.0
|
||||
for value in self.output_crop_anchor
|
||||
):
|
||||
raise ValueError(
|
||||
"output_crop_anchor must contain two finite values in [0, 1]"
|
||||
)
|
||||
if (
|
||||
not math.isfinite(self.output_max_ratio_error)
|
||||
or not 0.0 <= (self.output_max_ratio_error) < 1.0
|
||||
):
|
||||
raise ValueError("output_max_ratio_error must be a finite value in [0, 1)")
|
||||
if (
|
||||
not math.isfinite(self.output_pad_value)
|
||||
or not 0.0 <= (self.output_pad_value) <= 1.0
|
||||
):
|
||||
raise ValueError("output_pad_value must be a finite value in [0, 1]")
|
||||
|
||||
|
||||
def align_hunyuan_image3_dimension(value: int) -> int:
|
||||
"""Round a HunyuanImage-3 dimension up to a supported multiple."""
|
||||
return max(
|
||||
HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT,
|
||||
(value + HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT - 1)
|
||||
// HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT
|
||||
* HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT,
|
||||
)
|
||||
|
||||
|
||||
def align_hunyuan_image3_resolution(width: int, height: int) -> tuple[int, int]:
|
||||
"""Align both width and height to supported multiples."""
|
||||
return align_hunyuan_image3_dimension(width), align_hunyuan_image3_dimension(height)
|
||||
@@ -243,7 +243,12 @@ class SamplingParams:
|
||||
# All fields below are copied from ForwardBatch
|
||||
|
||||
# Image inputs
|
||||
image_path: str | list[str] | None = None
|
||||
# Per-request conditioning input: excluded from the dynamic-batch
|
||||
# signature so image-conditioned requests can still group (the grouping
|
||||
# gate is supports_batching_image_conditioning()).
|
||||
image_path: str | list[str] | None = field(
|
||||
default=None, metadata={"batch_sig_exclude": True}
|
||||
)
|
||||
|
||||
# Video inputs (video-to-video conditioning)
|
||||
video_path: str | list[str] | None = None
|
||||
@@ -1640,6 +1645,29 @@ class SamplingParams:
|
||||
type=int,
|
||||
help="Upscaling factor (default: 4).",
|
||||
)
|
||||
# HunyuanImage-3 and similar model-specific tokenizer/prompt arguments
|
||||
add_argument(
|
||||
"--bot-task",
|
||||
dest="bot_task",
|
||||
type=str,
|
||||
help=(
|
||||
"Tokenizer bot task (model-specific). For HunyuanImage-3: "
|
||||
"auto, image, think, recaption, think_recaption, img_ratio, none. "
|
||||
"Controls the bot response prefix in the tokenizer."
|
||||
),
|
||||
)
|
||||
add_argument(
|
||||
"--system-prompt",
|
||||
dest="system_prompt",
|
||||
type=str,
|
||||
help=(
|
||||
"System prompt: preset name or raw custom text. "
|
||||
"Presets: none, en_unified, en_vanilla, en_recaption, "
|
||||
"en_think_recaption, dynamic, auto. "
|
||||
"Any other string is used directly as the system prompt. "
|
||||
"Default: en_unified."
|
||||
),
|
||||
)
|
||||
return parser
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -105,6 +105,14 @@ if TYPE_CHECKING:
|
||||
SGLANG_CACHE_DIT_SECONDARY_MC: int = 3
|
||||
SGLANG_CACHE_DIT_SECONDARY_TAYLORSEER: bool = False
|
||||
SGLANG_CACHE_DIT_SECONDARY_TS_ORDER: int = 1
|
||||
# HunyuanImage-3: batch all condition images of a request group into one
|
||||
# VAE/ViT encode call (halve-on-OOM backoff). Opt-in, off by default.
|
||||
SGLANG_HI3_COND_ENCODE_BATCHING: bool = False
|
||||
# HunyuanImage-3: apply the post-decode output geometry (native crop/pad
|
||||
# and exact-size resample) to hit the requested aspect ratio. On by
|
||||
# default; set 0 to skip the plan and return the full decoded native
|
||||
# bucket untouched.
|
||||
SGLANG_HI3_OUTPUT_CROP: bool = True
|
||||
SGLANG_CACHE_DIT_SECONDARY_DMD: bool = False
|
||||
SGLANG_CACHE_DIT_SECONDARY_DMD_HISTORY: int = 6
|
||||
SGLANG_CACHE_DIT_SECONDARY_DMD_RANK: int = 0
|
||||
@@ -461,6 +469,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"SGLANG_CACHE_DIT_SCM_CACHE_BINS": _lazy_str("SGLANG_CACHE_DIT_SCM_CACHE_BINS"),
|
||||
# SCM policy: dynamic or static
|
||||
"SGLANG_CACHE_DIT_SCM_POLICY": _lazy_str("SGLANG_CACHE_DIT_SCM_POLICY", "dynamic"),
|
||||
# HunyuanImage-3: batch all condition images of a request group into one
|
||||
# VAE/ViT encode call (halve-on-OOM backoff on OOM). Opt-in, off by
|
||||
# default; the AR backbone resident on the same device makes full-batch
|
||||
# encoding tight on memory, so it must be explicitly enabled.
|
||||
"SGLANG_HI3_COND_ENCODE_BATCHING": _lazy_bool("SGLANG_HI3_COND_ENCODE_BATCHING"),
|
||||
# HunyuanImage-3: post-decode output geometry (native crop/pad plus
|
||||
# exact-size resample). On by default; 0 returns the raw native bucket.
|
||||
"SGLANG_HI3_OUTPUT_CROP": _lazy_bool("SGLANG_HI3_OUTPUT_CROP", "true"),
|
||||
# model loading
|
||||
"SGLANG_USE_RUNAI_MODEL_STREAMER": _lazy_bool(
|
||||
"SGLANG_USE_RUNAI_MODEL_STREAMER", "true"
|
||||
|
||||
@@ -186,6 +186,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
|
||||
"pi0.5": "Pi05Pipeline",
|
||||
"flux-3-action": "Flux3ActionPipeline",
|
||||
"hunyuan3d": "Hunyuan3D2Pipeline",
|
||||
"hunyuanimage-3": "HunyuanImage3Pipeline",
|
||||
"flux.2-dev-nvfp4": "Flux2NvfpPipeline",
|
||||
"fal/ideogram-v4-fast": "Ideogram4FastPipeline",
|
||||
"fal/ideogram-v4-instant": "Ideogram4InstantPipeline",
|
||||
|
||||
@@ -66,6 +66,7 @@ _REPLICA: GroupCoordinator | None = None
|
||||
# Corresponding TP lanes across the replicated encoder copies in one pipeline
|
||||
# replica. None means the encoder has only one TP copy, so batch DP cannot run.
|
||||
_ENCODER_DP: GroupCoordinator | None = None
|
||||
_SRT_MOE_EP = None
|
||||
_VAE_DECODE: GroupCoordinator | None = None
|
||||
_DIT: ProcessGroup | None = None
|
||||
_VAE: ProcessGroup | None = None
|
||||
@@ -112,6 +113,28 @@ def _clear_srt_world_group() -> None:
|
||||
srt_parallel_state._WORLD = None
|
||||
|
||||
|
||||
def _init_srt_moe_ep_group():
|
||||
"""SRT's MOE EP group for FusedMoE layers.
|
||||
|
||||
multimodal_gen has no expert parallelism: every rank keeps all experts
|
||||
and TP shards the weights inside each expert (``_MOE_TP = _TP``). The EP
|
||||
group must therefore be single-rank (``moe_ep_size=1``); aliasing
|
||||
``_MOE_EP`` to ``_TP`` would partition experts across ranks and break
|
||||
FusedMoE dispatch.
|
||||
"""
|
||||
import sglang.srt.distributed.parallel_state as srt_parallel_state
|
||||
|
||||
world_size = torch.distributed.get_world_size()
|
||||
return srt_parallel_state.init_model_parallel_group(
|
||||
group_ranks=[[r] for r in range(world_size)],
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=torch.distributed.get_backend(get_world_group().device_group),
|
||||
use_pynccl=False,
|
||||
use_custom_allreduce=False,
|
||||
group_name="moe_ep",
|
||||
)
|
||||
|
||||
|
||||
def _sync_srt_tp_group() -> None:
|
||||
"""Expose this package's TP group, widths, and ranks to shared SRT layers.
|
||||
|
||||
@@ -119,6 +142,7 @@ def _sync_srt_tp_group() -> None:
|
||||
the TP size in the dummy SRT configuration. Other parallel dimensions are
|
||||
one. Overrides also work before SRT configuration is published.
|
||||
"""
|
||||
global _SRT_MOE_EP
|
||||
import sglang.srt.distributed.parallel_state as srt_parallel_state
|
||||
from sglang.srt.runtime_context import derive_parallel_widths, get_parallel
|
||||
|
||||
@@ -126,6 +150,16 @@ def _sync_srt_tp_group() -> None:
|
||||
srt_parallel_state._TP = _TP
|
||||
if srt_parallel_state._ATTN_TP is None:
|
||||
srt_parallel_state._ATTN_TP = _TP
|
||||
if srt_parallel_state._TP is _TP and srt_parallel_state._MOE_TP is None:
|
||||
srt_parallel_state._MOE_TP = _TP
|
||||
if (
|
||||
_TP is not None
|
||||
and srt_parallel_state._TP is _TP
|
||||
and srt_parallel_state._MOE_TP is _TP
|
||||
and srt_parallel_state._MOE_EP is None
|
||||
):
|
||||
_SRT_MOE_EP = _init_srt_moe_ep_group()
|
||||
srt_parallel_state._MOE_EP = _SRT_MOE_EP
|
||||
if srt_parallel_state._ATTN_TP is _TP:
|
||||
get_parallel().override_permanently(
|
||||
tp_group=_TP,
|
||||
@@ -150,6 +184,7 @@ def _sync_srt_tp_group() -> None:
|
||||
|
||||
|
||||
def _clear_srt_tp_group() -> None:
|
||||
global _SRT_MOE_EP
|
||||
import sglang.srt.distributed.parallel_state as srt_parallel_state
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -161,6 +196,13 @@ def _clear_srt_tp_group() -> None:
|
||||
get_parallel().override_permanently(world_group=srt_parallel_state._WORLD)
|
||||
if srt_parallel_state._TP is _TP:
|
||||
srt_parallel_state._TP = None
|
||||
if srt_parallel_state._MOE_TP is _TP:
|
||||
srt_parallel_state._MOE_TP = None
|
||||
if _SRT_MOE_EP is not None:
|
||||
_SRT_MOE_EP.destroy()
|
||||
if srt_parallel_state._MOE_EP is _SRT_MOE_EP:
|
||||
srt_parallel_state._MOE_EP = None
|
||||
_SRT_MOE_EP = None
|
||||
|
||||
|
||||
def init_parallel_group_coordinator(
|
||||
|
||||
@@ -626,10 +626,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
|
||||
candidate_req.prompt, str
|
||||
):
|
||||
return "prompt_type"
|
||||
if (
|
||||
getattr(base_req, "image_path", None) is not None
|
||||
or getattr(candidate_req, "image_path", None) is not None
|
||||
):
|
||||
if self._has_unbatchable_image_conditioning(base_req, candidate_req):
|
||||
return "image_conditioning"
|
||||
if base_req.return_file_paths_only != candidate_req.return_file_paths_only:
|
||||
return "return_file_paths_only"
|
||||
@@ -648,6 +645,15 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
|
||||
def _has_realtime_session(req: Req) -> bool:
|
||||
return bool(req.realtime_session_id) or req.session is not None
|
||||
|
||||
def _has_unbatchable_image_conditioning(
|
||||
self, base_req: Req, candidate_req: Req
|
||||
) -> bool:
|
||||
"""True when either request carries image conditioning that this
|
||||
pipeline cannot keep per-request inside a merged dynamic batch."""
|
||||
if self.server_args.pipeline_config.supports_batching_image_conditioning():
|
||||
return False
|
||||
return base_req.image_path is not None or candidate_req.image_path is not None
|
||||
|
||||
def _requires_sequential_multi_output(self, *reqs: Req) -> bool:
|
||||
pipeline_config = self.server_args.pipeline_config
|
||||
return (
|
||||
@@ -687,10 +693,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
|
||||
):
|
||||
return False
|
||||
|
||||
if (
|
||||
getattr(base_req, "image_path", None) is not None
|
||||
or getattr(candidate_req, "image_path", None) is not None
|
||||
):
|
||||
if self._has_unbatchable_image_conditioning(base_req, candidate_req):
|
||||
return False
|
||||
if base_req.return_file_paths_only != candidate_req.return_file_paths_only:
|
||||
return False
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,290 @@
|
||||
"""Attention metadata and 2D RoPE helpers for HunyuanImage-3.
|
||||
|
||||
Ported from the official repository (``modeling_hunyuan_image_3.py``).
|
||||
"""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def timestep_embedding(t, dim, max_period=10000):
|
||||
half = dim // 2
|
||||
freqs = torch.exp(
|
||||
-math.log(max_period)
|
||||
* torch.arange(start=0, end=half, dtype=torch.float32)
|
||||
/ half
|
||||
).to(device=t.device)
|
||||
args = t[:, None].float() * freqs[None]
|
||||
embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
|
||||
if dim % 2:
|
||||
embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
|
||||
return embedding
|
||||
|
||||
|
||||
@dataclass
|
||||
class HunYuanImageAttentionMeta:
|
||||
query_lens: List[int]
|
||||
seq_lens: List[int]
|
||||
num_image_tokens: int
|
||||
first_step: bool
|
||||
|
||||
|
||||
def create_hunyuan_image_attention_meta(
|
||||
attention_mask: torch.Tensor, num_image_tokens: int, first_step: bool
|
||||
) -> HunYuanImageAttentionMeta:
|
||||
b, _, q_len1, seq_len = attention_mask.shape
|
||||
return HunYuanImageAttentionMeta(
|
||||
query_lens=[q_len1] * b,
|
||||
seq_lens=[seq_len] * b,
|
||||
num_image_tokens=num_image_tokens,
|
||||
first_step=first_step,
|
||||
)
|
||||
|
||||
|
||||
def rotate_half(x):
|
||||
x1, x2 = x.chunk(2, dim=-1)
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
def apply_rotary_pos_emb(q, k, cos, sin):
|
||||
"""Apply 2D RoPE with the official tiled angle layout.
|
||||
|
||||
``cos``/``sin`` from ``build_2d_rope`` are already expanded to full
|
||||
head_dim ([seq, head_dim] or [batch, seq, head_dim]); batched tables
|
||||
broadcast per row so requests with different layouts keep their own
|
||||
positions. ``q``/``k`` are [batch, seq, heads, head_dim].
|
||||
"""
|
||||
if cos.dim() == 2:
|
||||
cos = cos.unsqueeze(0)
|
||||
sin = sin.unsqueeze(0)
|
||||
cos = cos.unsqueeze(2) # broadcast over heads: [b, s, 1, head_dim]
|
||||
sin = sin.unsqueeze(2)
|
||||
return q * cos + rotate_half(q) * sin, k * cos + rotate_half(k) * sin
|
||||
|
||||
|
||||
class HunYuanRotary2DEmbedder:
|
||||
def __init__(self, num_heads: int, num_kv_heads: int, head_dim: int):
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_dim = head_dim
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
custom_pos_emb: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, hidden_states_shape[-1])
|
||||
|
||||
cos, sin = custom_pos_emb
|
||||
bs, q_len = cos.shape[0], cos.shape[1]
|
||||
|
||||
assert hidden_states.shape[0] == bs * q_len, (
|
||||
f"{hidden_states.shape[0]} != {bs * q_len}"
|
||||
)
|
||||
|
||||
# [B*L, H*D] -> [B, L, H, D] for apply_rotary_pos_emb
|
||||
q = q.reshape(bs, q_len, self.num_heads, self.head_dim)
|
||||
k = k.reshape(bs, q_len, self.num_kv_heads, self.head_dim)
|
||||
|
||||
q, k = apply_rotary_pos_emb(q.to(torch.float32), k.to(torch.float32), cos, sin)
|
||||
|
||||
q = q.reshape(hidden_states.shape[0], self.num_heads * self.head_dim).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
k = k.reshape(hidden_states.shape[0], self.num_kv_heads * self.head_dim).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
hidden_states = hidden_states.reshape(hidden_states_shape)
|
||||
return q, k
|
||||
|
||||
|
||||
def build_2d_rope(
|
||||
seq_len: int,
|
||||
n_elem: int,
|
||||
image_infos: Optional[List[Tuple[slice, Tuple[int, int]]]] = None,
|
||||
device: Optional[torch.device] = None,
|
||||
base: int = 10000,
|
||||
):
|
||||
"""Build 2D RoPE cos/sin tables ([seq_len, n_elem]); text uses 1D,
|
||||
image positions use 2D grid indices.
|
||||
"""
|
||||
assert n_elem % 4 == 0, f"n_elem must be divisible by 4, but got {n_elem}."
|
||||
|
||||
theta = 1.0 / (base ** (torch.arange(0, n_elem, 2, device=device).float() / n_elem))
|
||||
theta = theta.reshape(1, n_elem // 4, 2) # [1, half_d, 2]
|
||||
|
||||
if image_infos is None:
|
||||
image_infos = []
|
||||
|
||||
image_infos_list = [image_infos]
|
||||
sample_seq_lens = [seq_len]
|
||||
|
||||
x_sections = []
|
||||
y_sections = []
|
||||
for sample_id, sample_image_infos in enumerate(image_infos_list):
|
||||
last_pos = 0
|
||||
for sec_slice, (h, w) in sample_image_infos:
|
||||
L = sec_slice.start
|
||||
if last_pos < L:
|
||||
y_sections.append(torch.arange(last_pos, L, device=device))
|
||||
x_sections.append(torch.arange(last_pos, L, device=device))
|
||||
elif h is None:
|
||||
y_sections.append(
|
||||
torch.arange(sec_slice.start, sec_slice.stop, device=device)
|
||||
)
|
||||
x_sections.append(
|
||||
torch.arange(sec_slice.start, sec_slice.stop, device=device)
|
||||
)
|
||||
continue
|
||||
else:
|
||||
pass
|
||||
beta_y = L + (w * h - h) / 2
|
||||
beta_x = L + (w * h - w) / 2
|
||||
# linspace(a, b, n+1)[:n] == arange(a, a+n); kept for bit-parity
|
||||
# with the official implementation
|
||||
y_axis = torch.linspace(
|
||||
beta_y, beta_y + h, h + 1, dtype=torch.float32, device=device
|
||||
)[:h]
|
||||
x_axis = torch.linspace(
|
||||
beta_x, beta_x + w, w + 1, dtype=torch.float32, device=device
|
||||
)[:w]
|
||||
grid = torch.stack(
|
||||
torch.meshgrid(y_axis, x_axis, indexing="ij"), dim=0
|
||||
).reshape(2, -1)
|
||||
y_sections.append(grid[0])
|
||||
x_sections.append(grid[1])
|
||||
last_pos = L + w * h
|
||||
y_sections.append(
|
||||
torch.arange(last_pos, sample_seq_lens[sample_id], device=device)
|
||||
)
|
||||
x_sections.append(
|
||||
torch.arange(last_pos, sample_seq_lens[sample_id], device=device)
|
||||
)
|
||||
|
||||
x_pos = torch.cat(x_sections).long()
|
||||
y_pos = torch.cat(y_sections).long()
|
||||
x_pos = x_pos[:seq_len]
|
||||
y_pos = y_pos[:seq_len]
|
||||
all_pos = (
|
||||
torch.stack((y_pos, x_pos), dim=1).unsqueeze(1).to(device)
|
||||
) # [seq_len, 1, 2]
|
||||
|
||||
# Tile the half-dim angles up to head_dim (official layout): rotate_half
|
||||
# pairs dim i with dim i + n_elem/2, both carrying the SAME angle;
|
||||
# interleaved expansion pairs mismatched frequencies and garbles positions.
|
||||
idx_theta = (all_pos * theta).reshape(all_pos.shape[0], n_elem // 2).repeat(1, 2)
|
||||
cos = torch.cos(idx_theta)
|
||||
sin = torch.sin(idx_theta)
|
||||
return cos, sin
|
||||
|
||||
|
||||
def build_batch_2d_rope(
|
||||
seq_len: int,
|
||||
n_elem: int,
|
||||
image_infos: Optional[List[List[Tuple[slice, Tuple[int, int]]]]] = None,
|
||||
device: Optional[torch.device] = None,
|
||||
base: int = 10000,
|
||||
):
|
||||
cos_list, sin_list = [], []
|
||||
if image_infos is None:
|
||||
image_infos = [None]
|
||||
for image_info in image_infos:
|
||||
cos, sin = build_2d_rope(
|
||||
seq_len,
|
||||
n_elem,
|
||||
image_infos=image_info,
|
||||
device=device,
|
||||
base=base,
|
||||
)
|
||||
cos_list.append(cos)
|
||||
sin_list.append(sin)
|
||||
|
||||
return torch.stack(cos_list, dim=0), torch.stack(sin_list, dim=0)
|
||||
|
||||
|
||||
class CachedRoPE:
|
||||
def __init__(self, rope_theta: float, head_dim: int, rope_type: str = "2d"):
|
||||
# "2d" and "default" are the official spellings of the same scheme;
|
||||
# anything else in a checkpoint config would silently get wrong RoPE.
|
||||
if rope_type not in ("2d", "default"):
|
||||
raise NotImplementedError(f"rope_type `{rope_type}` not supported")
|
||||
self.rope_theta = rope_theta
|
||||
self.head_dim = head_dim
|
||||
self.cos_cache = None
|
||||
self.sin_cache = None
|
||||
self.seq_len = None
|
||||
self.rope_image_info = None
|
||||
|
||||
def __call__(self, seq_len, device, rope_image_info=None):
|
||||
if (self.seq_len != seq_len) or (
|
||||
rope_image_info is not None and self.rope_image_info != rope_image_info
|
||||
):
|
||||
self.cos_cache, self.sin_cache = build_batch_2d_rope(
|
||||
image_infos=rope_image_info,
|
||||
seq_len=seq_len,
|
||||
n_elem=self.head_dim,
|
||||
device=device,
|
||||
base=self.rope_theta,
|
||||
)
|
||||
self.seq_len = seq_len
|
||||
self.rope_image_info = rope_image_info
|
||||
|
||||
return self.cos_cache, self.sin_cache
|
||||
|
||||
|
||||
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
||||
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
|
||||
if n_rep == 1:
|
||||
return hidden_states
|
||||
hidden_states = hidden_states[:, :, None, :, :].expand(
|
||||
batch, num_key_value_heads, n_rep, slen, head_dim
|
||||
)
|
||||
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
|
||||
|
||||
|
||||
class ImageKVCacheManager:
|
||||
"""Full attention over the text+image sequence (no KV caching); packed
|
||||
``[tokens, heads, dim]`` inputs, 4-D bool mask.
|
||||
"""
|
||||
|
||||
def __call__(self, query, key, value, attn_metadata, attention_mask=None):
|
||||
total_tokens = query.shape[0]
|
||||
bs = len(attn_metadata.query_lens)
|
||||
q_len = total_tokens // bs
|
||||
|
||||
head_num_per_rank = query.shape[1]
|
||||
kv_head_num_per_rank = key.shape[1]
|
||||
repeat_num = head_num_per_rank // kv_head_num_per_rank
|
||||
head_dim = query.shape[2]
|
||||
|
||||
query = query.reshape(bs, q_len, head_num_per_rank, head_dim)
|
||||
key = key.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
|
||||
value = value.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
|
||||
|
||||
query = query.transpose(1, 2).contiguous()
|
||||
key = key.transpose(1, 2).contiguous()
|
||||
value = value.transpose(1, 2).contiguous()
|
||||
|
||||
key = repeat_kv(key, repeat_num)
|
||||
value = repeat_kv(value, repeat_num)
|
||||
|
||||
attn_output = F.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask.contiguous(),
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
scale=1.0 / (query.shape[-1] ** 0.5),
|
||||
)
|
||||
|
||||
attn_output = attn_output.transpose(1, 2).contiguous()
|
||||
attn_output = attn_output.reshape(total_tokens, head_num_per_rank, head_dim)
|
||||
return attn_output
|
||||
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import Siglip2VisionConfig
|
||||
|
||||
from sglang.srt.models.siglip2 import Siglip2Model
|
||||
|
||||
|
||||
class _PerImageSDPA(torch.nn.Module):
|
||||
"""Keep independent images separate without a quadratic packed-sequence mask."""
|
||||
|
||||
def forward(self, q, k, v, cu_seqlens, **kwargs):
|
||||
boundaries = cu_seqlens.tolist()
|
||||
output = torch.empty_like(q)
|
||||
for start, end in zip(boundaries[:-1], boundaries[1:]):
|
||||
output[start:end] = (
|
||||
F.scaled_dot_product_attention(
|
||||
q[start:end].transpose(0, 1).unsqueeze(0),
|
||||
k[start:end].transpose(0, 1).unsqueeze(0),
|
||||
v[start:end].transpose(0, 1).unsqueeze(0),
|
||||
is_causal=False,
|
||||
)
|
||||
.squeeze(0)
|
||||
.transpose(0, 1)
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class HunyuanImage3VisionModel(Siglip2Model):
|
||||
"""Adapt padded HunyuanImage-3 conditioning inputs to SRT's packed SigLIP2."""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
config = dict(config, hidden_act="gelu")
|
||||
# The checkpoint stores a legacy read-only Transformers property.
|
||||
config.pop("use_return_dict", None)
|
||||
super().__init__(
|
||||
Siglip2VisionConfig(**config),
|
||||
qkv_backend="sdpa",
|
||||
use_data_parallel=True,
|
||||
)
|
||||
for layer in self.vision_model.encoder.layers:
|
||||
layer.self_attn.attn.qkv_backend = _PerImageSDPA()
|
||||
|
||||
def forward(self, pixel_values, attention_mask, spatial_shapes):
|
||||
batch_size, max_patches, _ = pixel_values.shape
|
||||
mask = attention_mask.reshape(batch_size, max_patches).bool()
|
||||
# shape metadata stays on CPU and is shared by every encoder layer
|
||||
spatial_shapes = spatial_shapes.reshape(-1, 2).cpu()
|
||||
lengths = spatial_shapes.prod(dim=1).to(torch.int32)
|
||||
cu_seqlens = torch.nn.functional.pad(
|
||||
lengths.cumsum(0, dtype=torch.int32), (1, 0)
|
||||
)
|
||||
hidden_states = (
|
||||
super()
|
||||
.forward(
|
||||
pixel_values_packed=pixel_values[mask],
|
||||
spatial_shapes=spatial_shapes,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=int(lengths.max()),
|
||||
)
|
||||
.squeeze(0)
|
||||
)
|
||||
output = hidden_states.new_zeros(
|
||||
batch_size, max_patches, hidden_states.shape[-1]
|
||||
)
|
||||
output[mask] = hidden_states
|
||||
return output
|
||||
|
||||
def load_weights(self, weights):
|
||||
return super().load_weights(
|
||||
(f"vision_model.{name}", weight) for name, weight in weights
|
||||
)
|
||||
+764
@@ -0,0 +1,764 @@
|
||||
# Copied from the official HunyuanImage-3 repository (autoencoder_kl_3d.py);
|
||||
# distributed helpers omitted.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
|
||||
from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.utils import BaseOutput
|
||||
from einops import rearrange
|
||||
from torch import Tensor, nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DecoderOutput(BaseOutput):
|
||||
sample: torch.FloatTensor
|
||||
posterior: Optional[DiagonalGaussianDistribution] = None
|
||||
|
||||
|
||||
def swish(x: Tensor) -> Tensor:
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
def forward_with_checkpointing(module, *inputs, use_checkpointing=False):
|
||||
def create_custom_forward(module):
|
||||
def custom_forward(*inputs):
|
||||
return module(*inputs)
|
||||
|
||||
return custom_forward
|
||||
|
||||
if use_checkpointing:
|
||||
return torch.utils.checkpoint.checkpoint(
|
||||
create_custom_forward(module), *inputs, use_reentrant=False
|
||||
)
|
||||
else:
|
||||
return module(*inputs)
|
||||
|
||||
|
||||
class Conv3d(nn.Conv3d):
|
||||
"""Perform Conv3d on patches with numerical differences from nn.Conv3d
|
||||
within 1e-5. Only symmetric padding is supported."""
|
||||
|
||||
def forward(self, input):
|
||||
B, C, T, H, W = input.shape
|
||||
memory_count = (C * T * H * W) * 2 / 1024**3
|
||||
if memory_count > 2:
|
||||
n_split = math.ceil(memory_count / 2)
|
||||
chunks = torch.chunk(input, chunks=n_split, dim=-3)
|
||||
padded_chunks = []
|
||||
for i in range(len(chunks)):
|
||||
if self.padding[0] > 0:
|
||||
padded_chunk = F.pad(
|
||||
chunks[i],
|
||||
(0, 0, 0, 0, self.padding[0], self.padding[0]),
|
||||
mode="constant"
|
||||
if self.padding_mode == "zeros"
|
||||
else self.padding_mode,
|
||||
value=0,
|
||||
)
|
||||
if i > 0:
|
||||
padded_chunk[:, :, : self.padding[0]] = chunks[i - 1][
|
||||
:, :, -self.padding[0] :
|
||||
]
|
||||
if i < len(chunks) - 1:
|
||||
padded_chunk[:, :, -self.padding[0] :] = chunks[i + 1][
|
||||
:, :, : self.padding[0]
|
||||
]
|
||||
else:
|
||||
padded_chunk = chunks[i]
|
||||
padded_chunks.append(padded_chunk)
|
||||
padding_bak = self.padding
|
||||
self.padding = (0, self.padding[1], self.padding[2])
|
||||
outputs = []
|
||||
for i in range(len(padded_chunks)):
|
||||
outputs.append(super().forward(padded_chunks[i]))
|
||||
self.padding = padding_bak
|
||||
return torch.cat(outputs, dim=-3)
|
||||
else:
|
||||
return super().forward(input)
|
||||
|
||||
|
||||
class AttnBlock(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.norm = nn.GroupNorm(
|
||||
num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
|
||||
)
|
||||
self.q = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
self.k = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
self.v = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
self.proj_out = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
|
||||
def attention(self, h_: Tensor) -> Tensor:
|
||||
h_ = self.norm(h_)
|
||||
q = self.q(h_)
|
||||
k = self.k(h_)
|
||||
v = self.v(h_)
|
||||
b, c, f, h, w = q.shape
|
||||
q = rearrange(q, "b c f h w -> b 1 (f h w) c").contiguous()
|
||||
k = rearrange(k, "b c f h w -> b 1 (f h w) c").contiguous()
|
||||
v = rearrange(v, "b c f h w -> b 1 (f h w) c").contiguous()
|
||||
h_ = nn.functional.scaled_dot_product_attention(q, k, v)
|
||||
return rearrange(h_, "b 1 (f h w) c -> b c f h w", f=f, h=h, w=w, c=c, b=b)
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return x + self.proj_out(self.attention(x))
|
||||
|
||||
|
||||
class ResnetBlock(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
out_channels = in_channels if out_channels is None else out_channels
|
||||
self.out_channels = out_channels
|
||||
self.norm1 = nn.GroupNorm(
|
||||
num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv1 = Conv3d(
|
||||
in_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.norm2 = nn.GroupNorm(
|
||||
num_groups=32, num_channels=out_channels, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv2 = Conv3d(
|
||||
out_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
if self.in_channels != self.out_channels:
|
||||
self.nin_shortcut = Conv3d(
|
||||
in_channels, out_channels, kernel_size=1, stride=1, padding=0
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
h = x
|
||||
h = self.norm1(h)
|
||||
h = swish(h)
|
||||
h = self.conv1(h)
|
||||
h = self.norm2(h)
|
||||
h = swish(h)
|
||||
h = self.conv2(h)
|
||||
if self.in_channels != self.out_channels:
|
||||
x = self.nin_shortcut(x)
|
||||
return x + h
|
||||
|
||||
|
||||
class Downsample(nn.Module):
|
||||
def __init__(self, in_channels: int, add_temporal_downsample: bool = True):
|
||||
super().__init__()
|
||||
self.add_temporal_downsample = add_temporal_downsample
|
||||
stride = (2, 2, 2) if add_temporal_downsample else (1, 2, 2)
|
||||
self.conv = Conv3d(
|
||||
in_channels, in_channels, kernel_size=3, stride=stride, padding=0
|
||||
)
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
spatial_pad = (0, 1, 0, 1, 0, 0)
|
||||
x = nn.functional.pad(x, spatial_pad, mode="constant", value=0)
|
||||
temporal_pad = (
|
||||
(0, 0, 0, 0, 0, 1) if self.add_temporal_downsample else (0, 0, 0, 0, 1, 1)
|
||||
)
|
||||
x = nn.functional.pad(x, temporal_pad, mode="replicate")
|
||||
x = self.conv(x)
|
||||
return x
|
||||
|
||||
|
||||
class DownsampleDCAE(nn.Module):
|
||||
def __init__(
|
||||
self, in_channels: int, out_channels: int, add_temporal_downsample: bool = True
|
||||
):
|
||||
super().__init__()
|
||||
factor = 2 * 2 * 2 if add_temporal_downsample else 1 * 2 * 2
|
||||
self.conv = Conv3d(
|
||||
in_channels, out_channels // factor, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.add_temporal_downsample = add_temporal_downsample
|
||||
self.group_size = factor * in_channels // out_channels
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
r1 = 2 if self.add_temporal_downsample else 1
|
||||
h = self.conv(x)
|
||||
h = rearrange(
|
||||
h, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2
|
||||
)
|
||||
shortcut = rearrange(
|
||||
x, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2
|
||||
)
|
||||
B, C, T, H, W = shortcut.shape
|
||||
shortcut = shortcut.view(B, h.shape[1], self.group_size, T, H, W).mean(dim=2)
|
||||
return h + shortcut
|
||||
|
||||
|
||||
class Upsample(nn.Module):
|
||||
def __init__(self, in_channels: int, add_temporal_upsample: bool = True):
|
||||
super().__init__()
|
||||
self.add_temporal_upsample = add_temporal_upsample
|
||||
self.scale_factor = (2, 2, 2) if add_temporal_upsample else (1, 2, 2)
|
||||
self.conv = Conv3d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
x = nn.functional.interpolate(x, scale_factor=self.scale_factor, mode="nearest")
|
||||
x = self.conv(x)
|
||||
return x
|
||||
|
||||
|
||||
class UpsampleDCAE(nn.Module):
|
||||
def __init__(
|
||||
self, in_channels: int, out_channels: int, add_temporal_upsample: bool = True
|
||||
):
|
||||
super().__init__()
|
||||
factor = 2 * 2 * 2 if add_temporal_upsample else 1 * 2 * 2
|
||||
self.conv = Conv3d(
|
||||
in_channels, out_channels * factor, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.add_temporal_upsample = add_temporal_upsample
|
||||
self.repeats = factor * out_channels // in_channels
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
r1 = 2 if self.add_temporal_upsample else 1
|
||||
h = self.conv(x)
|
||||
h = rearrange(
|
||||
h, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2
|
||||
)
|
||||
shortcut = x.repeat_interleave(repeats=self.repeats, dim=1)
|
||||
shortcut = rearrange(
|
||||
shortcut,
|
||||
"b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)",
|
||||
r1=r1,
|
||||
r2=2,
|
||||
r3=2,
|
||||
)
|
||||
return h + shortcut
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int,
|
||||
z_channels: int,
|
||||
block_out_channels: Tuple[int, ...],
|
||||
num_res_blocks: int,
|
||||
ffactor_spatial: int,
|
||||
ffactor_temporal: int,
|
||||
downsample_match_channel: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.z_channels = z_channels
|
||||
self.block_out_channels = block_out_channels
|
||||
self.num_res_blocks = num_res_blocks
|
||||
|
||||
self.conv_in = Conv3d(
|
||||
in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
|
||||
self.down = nn.ModuleList()
|
||||
block_in = block_out_channels[0]
|
||||
for i_level, ch in enumerate(block_out_channels):
|
||||
block = nn.ModuleList()
|
||||
block_out = ch
|
||||
for _ in range(self.num_res_blocks):
|
||||
block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
|
||||
block_in = block_out
|
||||
down = nn.Module()
|
||||
down.block = block
|
||||
|
||||
add_spatial_downsample = bool(i_level < np.log2(ffactor_spatial))
|
||||
add_temporal_downsample = add_spatial_downsample and bool(
|
||||
i_level >= np.log2(ffactor_spatial // ffactor_temporal)
|
||||
)
|
||||
if add_spatial_downsample or add_temporal_downsample:
|
||||
block_out = (
|
||||
block_out_channels[i_level + 1]
|
||||
if downsample_match_channel
|
||||
else block_in
|
||||
)
|
||||
down.downsample = DownsampleDCAE(
|
||||
block_in, block_out, add_temporal_downsample
|
||||
)
|
||||
block_in = block_out
|
||||
self.down.append(down)
|
||||
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
self.mid.attn_1 = AttnBlock(block_in)
|
||||
self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
|
||||
self.norm_out = nn.GroupNorm(
|
||||
num_groups=32, num_channels=block_in, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv_out = Conv3d(
|
||||
block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
with torch.no_grad():
|
||||
use_checkpointing = bool(self.training and self.gradient_checkpointing)
|
||||
|
||||
h = self.conv_in(x)
|
||||
for i_level in range(len(self.block_out_channels)):
|
||||
for i_block in range(self.num_res_blocks):
|
||||
h = forward_with_checkpointing(
|
||||
self.down[i_level].block[i_block],
|
||||
h,
|
||||
use_checkpointing=use_checkpointing,
|
||||
)
|
||||
if hasattr(self.down[i_level], "downsample"):
|
||||
h = forward_with_checkpointing(
|
||||
self.down[i_level].downsample,
|
||||
h,
|
||||
use_checkpointing=use_checkpointing,
|
||||
)
|
||||
|
||||
h = forward_with_checkpointing(
|
||||
self.mid.block_1, h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
h = forward_with_checkpointing(
|
||||
self.mid.attn_1, h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
h = forward_with_checkpointing(
|
||||
self.mid.block_2, h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
|
||||
group_size = self.block_out_channels[-1] // (2 * self.z_channels)
|
||||
shortcut = rearrange(h, "b (c r) f h w -> b c r f h w", r=group_size).mean(
|
||||
dim=2
|
||||
)
|
||||
h = self.norm_out(h)
|
||||
h = swish(h)
|
||||
h = self.conv_out(h)
|
||||
h += shortcut
|
||||
return h
|
||||
|
||||
|
||||
class Decoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
z_channels: int,
|
||||
out_channels: int,
|
||||
block_out_channels: Tuple[int, ...],
|
||||
num_res_blocks: int,
|
||||
ffactor_spatial: int,
|
||||
ffactor_temporal: int,
|
||||
upsample_match_channel: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.z_channels = z_channels
|
||||
self.block_out_channels = block_out_channels
|
||||
self.num_res_blocks = num_res_blocks
|
||||
|
||||
block_in = block_out_channels[0]
|
||||
self.conv_in = Conv3d(z_channels, block_in, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
self.mid.attn_1 = AttnBlock(block_in)
|
||||
self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
|
||||
self.up = nn.ModuleList()
|
||||
for i_level, ch in enumerate(block_out_channels):
|
||||
block = nn.ModuleList()
|
||||
block_out = ch
|
||||
for _ in range(self.num_res_blocks + 1):
|
||||
block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
|
||||
block_in = block_out
|
||||
up = nn.Module()
|
||||
up.block = block
|
||||
|
||||
add_spatial_upsample = bool(i_level < np.log2(ffactor_spatial))
|
||||
add_temporal_upsample = bool(i_level < np.log2(ffactor_temporal))
|
||||
if add_spatial_upsample or add_temporal_upsample:
|
||||
block_out = (
|
||||
block_out_channels[i_level + 1]
|
||||
if upsample_match_channel
|
||||
else block_in
|
||||
)
|
||||
up.upsample = UpsampleDCAE(block_in, block_out, add_temporal_upsample)
|
||||
block_in = block_out
|
||||
self.up.append(up)
|
||||
|
||||
self.norm_out = nn.GroupNorm(
|
||||
num_groups=32, num_channels=block_in, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv_out = Conv3d(
|
||||
block_in, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(self, z: Tensor) -> Tensor:
|
||||
with torch.no_grad():
|
||||
use_checkpointing = bool(self.training and self.gradient_checkpointing)
|
||||
repeats = self.block_out_channels[0] // self.z_channels
|
||||
h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1)
|
||||
|
||||
h = forward_with_checkpointing(
|
||||
self.mid.block_1, h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
h = forward_with_checkpointing(
|
||||
self.mid.attn_1, h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
h = forward_with_checkpointing(
|
||||
self.mid.block_2, h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
|
||||
for i_level in range(len(self.block_out_channels)):
|
||||
for i_block in range(self.num_res_blocks + 1):
|
||||
h = forward_with_checkpointing(
|
||||
self.up[i_level].block[i_block],
|
||||
h,
|
||||
use_checkpointing=use_checkpointing,
|
||||
)
|
||||
if hasattr(self.up[i_level], "upsample"):
|
||||
h = forward_with_checkpointing(
|
||||
self.up[i_level].upsample,
|
||||
h,
|
||||
use_checkpointing=use_checkpointing,
|
||||
)
|
||||
|
||||
h = self.norm_out(h)
|
||||
h = swish(h)
|
||||
h = self.conv_out(h)
|
||||
return h
|
||||
|
||||
|
||||
class AutoencoderKLConv3D(ModelMixin, ConfigMixin):
|
||||
"""3D VAE copied from the official HunyuanImage-3 repository."""
|
||||
|
||||
_supports_gradient_checkpointing = True
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int,
|
||||
out_channels: int,
|
||||
latent_channels: int,
|
||||
block_out_channels: Tuple[int, ...],
|
||||
layers_per_block: int,
|
||||
ffactor_spatial: int,
|
||||
ffactor_temporal: int,
|
||||
sample_size: int,
|
||||
sample_tsize: int,
|
||||
scaling_factor: float = None,
|
||||
shift_factor: Optional[float] = None,
|
||||
downsample_match_channel: bool = True,
|
||||
upsample_match_channel: bool = True,
|
||||
only_encoder: bool = False,
|
||||
only_decoder: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.ffactor_spatial = ffactor_spatial
|
||||
self.ffactor_temporal = ffactor_temporal
|
||||
self.scaling_factor = scaling_factor
|
||||
self.shift_factor = shift_factor
|
||||
|
||||
if not only_decoder:
|
||||
self.encoder = Encoder(
|
||||
in_channels=in_channels,
|
||||
z_channels=latent_channels,
|
||||
block_out_channels=block_out_channels,
|
||||
num_res_blocks=layers_per_block,
|
||||
ffactor_spatial=ffactor_spatial,
|
||||
ffactor_temporal=ffactor_temporal,
|
||||
downsample_match_channel=downsample_match_channel,
|
||||
)
|
||||
if not only_encoder:
|
||||
self.decoder = Decoder(
|
||||
z_channels=latent_channels,
|
||||
out_channels=out_channels,
|
||||
block_out_channels=list(reversed(block_out_channels)),
|
||||
num_res_blocks=layers_per_block,
|
||||
ffactor_spatial=ffactor_spatial,
|
||||
ffactor_temporal=ffactor_temporal,
|
||||
upsample_match_channel=upsample_match_channel,
|
||||
)
|
||||
|
||||
self.use_slicing = False
|
||||
self.slicing_bsz = 1
|
||||
self.use_spatial_tiling = False
|
||||
self.use_temporal_tiling = False
|
||||
self.use_tiling_during_training = False
|
||||
|
||||
self.tile_sample_min_size = sample_size
|
||||
self.tile_latent_min_size = sample_size // ffactor_spatial
|
||||
self.tile_sample_min_tsize = sample_tsize
|
||||
self.tile_latent_min_tsize = sample_tsize // ffactor_temporal
|
||||
self.tile_overlap_factor = 0.125
|
||||
|
||||
self.use_compile = False
|
||||
|
||||
def _set_gradient_checkpointing(self, module, value=False):
|
||||
if isinstance(module, (Encoder, Decoder)):
|
||||
module.gradient_checkpointing = value
|
||||
|
||||
def enable_tiling_during_training(self, use_tiling: bool = True):
|
||||
self.use_tiling_during_training = use_tiling
|
||||
|
||||
def disable_tiling_during_training(self):
|
||||
self.enable_tiling_during_training(False)
|
||||
|
||||
def enable_temporal_tiling(self, use_tiling: bool = True):
|
||||
self.use_temporal_tiling = use_tiling
|
||||
|
||||
def disable_temporal_tiling(self):
|
||||
self.enable_temporal_tiling(False)
|
||||
|
||||
def enable_spatial_tiling(self, use_tiling: bool = True):
|
||||
self.use_spatial_tiling = use_tiling
|
||||
|
||||
def disable_spatial_tiling(self):
|
||||
self.enable_spatial_tiling(False)
|
||||
|
||||
def enable_tiling(self, use_tiling: bool = True):
|
||||
self.enable_spatial_tiling(use_tiling)
|
||||
|
||||
def disable_tiling(self):
|
||||
self.disable_spatial_tiling()
|
||||
|
||||
def enable_slicing(self):
|
||||
self.use_slicing = True
|
||||
|
||||
def disable_slicing(self):
|
||||
self.use_slicing = False
|
||||
|
||||
def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
|
||||
blend_extent = min(a.shape[-1], b.shape[-1], blend_extent)
|
||||
for x in range(blend_extent):
|
||||
b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (
|
||||
1 - x / blend_extent
|
||||
) + b[:, :, :, :, x] * (x / blend_extent)
|
||||
return b
|
||||
|
||||
def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
|
||||
blend_extent = min(a.shape[-2], b.shape[-2], blend_extent)
|
||||
for y in range(blend_extent):
|
||||
b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (
|
||||
1 - y / blend_extent
|
||||
) + b[:, :, :, y, :] * (y / blend_extent)
|
||||
return b
|
||||
|
||||
def blend_t(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
|
||||
blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)
|
||||
for x in range(blend_extent):
|
||||
b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (
|
||||
1 - x / blend_extent
|
||||
) + b[:, :, x, :, :] * (x / blend_extent)
|
||||
return b
|
||||
|
||||
def spatial_tiled_encode(self, x: torch.Tensor):
|
||||
B, C, T, H, W = x.shape
|
||||
overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor))
|
||||
blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor)
|
||||
row_limit = self.tile_latent_min_size - blend_extent
|
||||
|
||||
rows = []
|
||||
for i in range(0, H, overlap_size):
|
||||
row = []
|
||||
for j in range(0, W, overlap_size):
|
||||
tile = x[
|
||||
:,
|
||||
:,
|
||||
:,
|
||||
i : i + self.tile_sample_min_size,
|
||||
j : j + self.tile_sample_min_size,
|
||||
]
|
||||
tile = self.encoder(tile)
|
||||
row.append(tile)
|
||||
rows.append(row)
|
||||
result_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
result_row = []
|
||||
for j, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_v(rows[i - 1][j], tile, blend_extent)
|
||||
if j > 0:
|
||||
tile = self.blend_h(row[j - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :, :row_limit, :row_limit])
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
moments = torch.cat(result_rows, dim=-2)
|
||||
return moments
|
||||
|
||||
def temporal_tiled_encode(self, x: torch.Tensor):
|
||||
B, C, T, H, W = x.shape
|
||||
overlap_size = int(self.tile_sample_min_tsize * (1 - self.tile_overlap_factor))
|
||||
blend_extent = int(self.tile_latent_min_tsize * self.tile_overlap_factor)
|
||||
t_limit = self.tile_latent_min_tsize - blend_extent
|
||||
|
||||
row = []
|
||||
for i in range(0, T, overlap_size):
|
||||
tile = x[:, :, i : i + self.tile_sample_min_tsize, :, :]
|
||||
if self.use_spatial_tiling and (
|
||||
tile.shape[-1] > self.tile_sample_min_size
|
||||
or tile.shape[-2] > self.tile_sample_min_size
|
||||
):
|
||||
tile = self.spatial_tiled_encode(tile)
|
||||
else:
|
||||
tile = self.encoder(tile)
|
||||
row.append(tile)
|
||||
result_row = []
|
||||
for i, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_t(row[i - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :t_limit, :, :])
|
||||
moments = torch.cat(result_row, dim=-3)
|
||||
return moments
|
||||
|
||||
def spatial_tiled_decode(self, z: torch.Tensor):
|
||||
B, C, T, H, W = z.shape
|
||||
overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))
|
||||
blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor)
|
||||
row_limit = self.tile_sample_min_size - blend_extent
|
||||
|
||||
rows = []
|
||||
for i in range(0, H, overlap_size):
|
||||
row = []
|
||||
for j in range(0, W, overlap_size):
|
||||
tile = z[
|
||||
:,
|
||||
:,
|
||||
:,
|
||||
i : i + self.tile_latent_min_size,
|
||||
j : j + self.tile_latent_min_size,
|
||||
]
|
||||
decoded = self.decoder(tile)
|
||||
row.append(decoded)
|
||||
rows.append(row)
|
||||
|
||||
result_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
result_row = []
|
||||
for j, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_v(rows[i - 1][j], tile, blend_extent)
|
||||
if j > 0:
|
||||
tile = self.blend_h(row[j - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :, :row_limit, :row_limit])
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
dec = torch.cat(result_rows, dim=-2)
|
||||
return dec
|
||||
|
||||
def temporal_tiled_decode(self, z: torch.Tensor):
|
||||
B, C, T, H, W = z.shape
|
||||
overlap_size = int(self.tile_latent_min_tsize * (1 - self.tile_overlap_factor))
|
||||
blend_extent = int(self.tile_sample_min_tsize * self.tile_overlap_factor)
|
||||
t_limit = self.tile_sample_min_tsize - blend_extent
|
||||
assert 0 < overlap_size < self.tile_latent_min_tsize
|
||||
|
||||
row = []
|
||||
for i in range(0, T, overlap_size):
|
||||
tile = z[:, :, i : i + self.tile_latent_min_tsize, :, :]
|
||||
if self.use_spatial_tiling and (
|
||||
tile.shape[-1] > self.tile_latent_min_size
|
||||
or tile.shape[-2] > self.tile_latent_min_size
|
||||
):
|
||||
decoded = self.spatial_tiled_decode(tile)
|
||||
else:
|
||||
decoded = self.decoder(tile)
|
||||
row.append(decoded)
|
||||
|
||||
result_row = []
|
||||
for i, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_t(row[i - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :t_limit, :, :])
|
||||
dec = torch.cat(result_row, dim=-3)
|
||||
return dec
|
||||
|
||||
def encode(self, x: Tensor, return_dict: bool = True):
|
||||
def _encode(x):
|
||||
if self.use_temporal_tiling and x.shape[-3] > self.tile_sample_min_tsize:
|
||||
return self.temporal_tiled_encode(x)
|
||||
if self.use_spatial_tiling and (
|
||||
x.shape[-1] > self.tile_sample_min_size
|
||||
or x.shape[-2] > self.tile_sample_min_size
|
||||
):
|
||||
return self.spatial_tiled_encode(x)
|
||||
if self.use_compile:
|
||||
|
||||
@torch.compile
|
||||
def encoder(x):
|
||||
return self.encoder(x)
|
||||
|
||||
return encoder(x)
|
||||
return self.encoder(x)
|
||||
|
||||
if len(x.shape) != 5: # (B, C, T, H, W)
|
||||
x = x[:, :, None]
|
||||
if x.shape[2] == 1:
|
||||
x = x.expand(-1, -1, self.ffactor_temporal, -1, -1)
|
||||
else:
|
||||
assert (
|
||||
x.shape[2] != self.ffactor_temporal
|
||||
and x.shape[2] % self.ffactor_temporal == 0
|
||||
)
|
||||
|
||||
if self.use_slicing and x.shape[0] > 1:
|
||||
if self.slicing_bsz == 1:
|
||||
encoded_slices = [_encode(x_slice) for x_slice in x.split(1)]
|
||||
else:
|
||||
sections = [self.slicing_bsz] * (x.shape[0] // self.slicing_bsz)
|
||||
if x.shape[0] % self.slicing_bsz != 0:
|
||||
sections.append(x.shape[0] % self.slicing_bsz)
|
||||
encoded_slices = [_encode(x_slice) for x_slice in x.split(sections)]
|
||||
h = torch.cat(encoded_slices)
|
||||
else:
|
||||
h = _encode(x)
|
||||
posterior = DiagonalGaussianDistribution(h)
|
||||
|
||||
if not return_dict:
|
||||
return (posterior,)
|
||||
return AutoencoderKLOutput(latent_dist=posterior)
|
||||
|
||||
def decode(self, z: Tensor, return_dict: bool = True, generator=None):
|
||||
def _decode(z):
|
||||
if self.use_temporal_tiling and z.shape[-3] > self.tile_latent_min_tsize:
|
||||
return self.temporal_tiled_decode(z)
|
||||
if self.use_spatial_tiling and (
|
||||
z.shape[-1] > self.tile_latent_min_size
|
||||
or z.shape[-2] > self.tile_latent_min_size
|
||||
):
|
||||
return self.spatial_tiled_decode(z)
|
||||
return self.decoder(z)
|
||||
|
||||
if self.use_slicing and z.shape[0] > 1:
|
||||
decoded_slices = [_decode(z_slice) for z_slice in z.split(1)]
|
||||
decoded = torch.cat(decoded_slices)
|
||||
else:
|
||||
decoded = _decode(z)
|
||||
|
||||
if z.shape[-3] == 1:
|
||||
decoded = decoded[:, :, -1:]
|
||||
if not return_dict:
|
||||
return (decoded,)
|
||||
return DecoderOutput(sample=decoded)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
sample: torch.Tensor,
|
||||
sample_posterior: bool = False,
|
||||
return_posterior: bool = True,
|
||||
return_dict: bool = True,
|
||||
):
|
||||
posterior = self.encode(sample).latent_dist
|
||||
z = posterior.sample() if sample_posterior else posterior.mode()
|
||||
dec = self.decode(z).sample
|
||||
return (
|
||||
DecoderOutput(sample=dec, posterior=posterior)
|
||||
if return_dict
|
||||
else (dec, posterior)
|
||||
)
|
||||
@@ -0,0 +1,372 @@
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.distributed import init_device_mesh
|
||||
from torch.distributed.fsdp import MixedPrecisionPolicy
|
||||
from transformers import AutoTokenizer
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
component_attn_backend_context_manager,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.fsdp_load import shard_model
|
||||
from sglang.multimodal_gen.runtime.loader.transformer_load_utils import (
|
||||
resolve_transformer_checkpoint_files,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
load_model_state_dict,
|
||||
set_default_torch_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.weight_utils import (
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
get_global_component_residency_manager,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.hunyuan_image3 import (
|
||||
HunyuanImage3ForCausalMM,
|
||||
LightProjector,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.hunyuan_image3 import (
|
||||
HunyuanImage3VisionModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_conv3d_hunyuan_image3 import (
|
||||
AutoencoderKLConv3D,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3 import (
|
||||
HunyuanImage3AR,
|
||||
HunyuanImage3DecodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
get_hf_config,
|
||||
load_dict,
|
||||
maybe_download_model,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy
|
||||
from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _module_memory_gb(module: torch.nn.Module) -> float:
|
||||
total_bytes = sum(p.numel() * p.element_size() for p in module.parameters())
|
||||
total_bytes += sum(b.numel() * b.element_size() for b in module.buffers())
|
||||
return total_bytes / (1024**3)
|
||||
|
||||
|
||||
class HunyuanImage3Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
pipeline_name = "HunyuanImage3Pipeline"
|
||||
|
||||
_required_config_modules = [
|
||||
"text_encoder",
|
||||
"tokenizer",
|
||||
"vae",
|
||||
"processor",
|
||||
"transformer",
|
||||
"scheduler",
|
||||
"vision_model",
|
||||
"vision_aligner",
|
||||
]
|
||||
|
||||
def load_modules(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
loaded_modules: dict[str, torch.nn.Module] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
pipeline_config = server_args.pipeline_config
|
||||
|
||||
model_path = maybe_download_model(self.model_path)
|
||||
self.model_path = model_path
|
||||
logger.info("Loading HunyuanImage-3 components from %s", model_path)
|
||||
|
||||
hf_config = get_hf_config(
|
||||
model_path,
|
||||
trust_remote_code=True,
|
||||
revision=server_args.revision,
|
||||
)
|
||||
config_dict = hf_config.to_dict()
|
||||
|
||||
pipeline_config.dit_config.update_model_arch(config_dict)
|
||||
vae_config_dict = dict(config_dict["vae"])
|
||||
pipeline_config.vae_config.update_model_arch(vae_config_dict)
|
||||
pipeline_config.vae_config.post_init()
|
||||
|
||||
flow_shift = self._read_flow_shift(model_path)
|
||||
pipeline_config.flow_shift = flow_shift
|
||||
|
||||
ar_model = self._load_ar_model(
|
||||
server_args, pipeline_config, model_path, config_dict
|
||||
)
|
||||
|
||||
return {
|
||||
"transformer": ar_model,
|
||||
"text_encoder": ar_model,
|
||||
"vae": self._load_vae(
|
||||
server_args, pipeline_config, model_path, vae_config_dict
|
||||
),
|
||||
"tokenizer": self._load_tokenizer(server_args, model_path),
|
||||
"scheduler": FlowMatchEulerDiscreteScheduler(shift=flow_shift),
|
||||
"processor": self._load_processor(server_args, model_path, hf_config),
|
||||
"vision_model": self._load_component(
|
||||
HunyuanImage3VisionModel(config_dict["vit"]),
|
||||
"vision_model",
|
||||
model_path,
|
||||
server_args,
|
||||
),
|
||||
"vision_aligner": self._load_component(
|
||||
LightProjector(config_dict["vit_aligner"]),
|
||||
"vision_aligner",
|
||||
model_path,
|
||||
server_args,
|
||||
),
|
||||
}
|
||||
|
||||
def _load_ar_model(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
pipeline_config: Any,
|
||||
model_path: str,
|
||||
config_dict: dict[str, Any],
|
||||
) -> torch.nn.Module:
|
||||
# resolve_transformer_checkpoint_files handles the --transformer-
|
||||
# weights-path override (incl. GGUF/mixed-export selection) and the
|
||||
# plain component directory, returning the shard list plus an
|
||||
# optional override config path (unused here; the AR config comes
|
||||
# from the unified checkpoint's config.json).
|
||||
safetensors_list = list(
|
||||
resolve_transformer_checkpoint_files(server_args, model_path).safetensors
|
||||
)
|
||||
|
||||
local_torch_device = get_local_torch_device()
|
||||
cpu_offload = bool(server_args.dit_cpu_offload)
|
||||
checkpoint_load_device = (
|
||||
torch.device("cpu") if cpu_offload else local_torch_device
|
||||
)
|
||||
fsdp_inference = bool(server_args.use_fsdp_inference)
|
||||
if fsdp_inference and current_platform.is_mps():
|
||||
logger.warning("Disabling FSDP for MPS platform as it's not compatible")
|
||||
fsdp_inference = False
|
||||
|
||||
param_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
|
||||
logger.info(
|
||||
"Loading AR model: %d shard(s), %s",
|
||||
len(safetensors_list),
|
||||
param_dtype,
|
||||
)
|
||||
|
||||
attn_backend, matched_backend_key = (
|
||||
server_args.resolve_component_attention_backend("transformer")
|
||||
)
|
||||
with component_attn_backend_context_manager(
|
||||
attn_backend, component_name=matched_backend_key or "transformer"
|
||||
):
|
||||
with (
|
||||
set_default_torch_dtype(param_dtype),
|
||||
torch.device(checkpoint_load_device),
|
||||
):
|
||||
model = HunyuanImage3ForCausalMM(
|
||||
config=pipeline_config.dit_config,
|
||||
hf_config=config_dict,
|
||||
)
|
||||
|
||||
weights_to_load = {name for name, _ in model.named_parameters()}
|
||||
loaded_weights = model.load_weights(
|
||||
safetensors_weights_iterator(safetensors_list)
|
||||
)
|
||||
weights_not_loaded = weights_to_load - loaded_weights
|
||||
if weights_not_loaded:
|
||||
raise ValueError(
|
||||
f"AR weights not initialized from checkpoint: "
|
||||
f"{sorted(weights_not_loaded)}"
|
||||
)
|
||||
|
||||
model.post_load_weights()
|
||||
for param in model.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
if fsdp_inference:
|
||||
self._shard_ar_model(
|
||||
model,
|
||||
server_args=server_args,
|
||||
cpu_offload=cpu_offload,
|
||||
param_dtype=param_dtype,
|
||||
)
|
||||
model.eval()
|
||||
self.memory_usages["transformer"] = _module_memory_gb(model)
|
||||
return model
|
||||
|
||||
def _shard_ar_model(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
server_args: ServerArgs,
|
||||
cpu_offload: bool,
|
||||
param_dtype: torch.dtype,
|
||||
) -> None:
|
||||
mp_policy = MixedPrecisionPolicy(
|
||||
param_dtype=param_dtype,
|
||||
reduce_dtype=torch.float32,
|
||||
cast_forward_inputs=False,
|
||||
)
|
||||
set_mixed_precision_policy(
|
||||
param_dtype=param_dtype,
|
||||
reduce_dtype=torch.float32,
|
||||
mp_policy=mp_policy,
|
||||
)
|
||||
device_mesh = init_device_mesh(
|
||||
current_platform.device_type,
|
||||
mesh_shape=(server_args.hsdp_replicate_dim, server_args.hsdp_shard_dim),
|
||||
mesh_dim_names=("replicate", "shard"),
|
||||
)
|
||||
shard_model(
|
||||
model,
|
||||
cpu_offload=cpu_offload,
|
||||
reshard_after_forward=True,
|
||||
mp_policy=mp_policy,
|
||||
mesh=device_mesh,
|
||||
fsdp_shard_conditions=model._fsdp_shard_conditions,
|
||||
pin_cpu_memory=server_args.pin_cpu_memory,
|
||||
)
|
||||
|
||||
def _load_vae(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
pipeline_config: Any,
|
||||
model_path: str,
|
||||
vae_config_dict: dict[str, Any],
|
||||
) -> torch.nn.Module:
|
||||
# config.json omits a few constructor args
|
||||
vae_params = dict(vae_config_dict)
|
||||
vae_params.setdefault("in_channels", 3)
|
||||
vae_params.setdefault("out_channels", 3)
|
||||
vae_params.setdefault("ffactor_temporal", 4)
|
||||
dtype = PRECISION_TO_TYPE[pipeline_config.vae_precision]
|
||||
vae = self._load_component(
|
||||
AutoencoderKLConv3D(**vae_params).to(dtype=dtype),
|
||||
"vae",
|
||||
model_path,
|
||||
server_args,
|
||||
)
|
||||
if server_args.pipeline_config.vae_tiling:
|
||||
vae.enable_tiling()
|
||||
if server_args.pipeline_config.vae_slicing:
|
||||
vae.enable_slicing()
|
||||
return vae
|
||||
|
||||
@staticmethod
|
||||
def _load_tokenizer(server_args: ServerArgs, model_path: str):
|
||||
return AutoTokenizer.from_pretrained(
|
||||
model_path,
|
||||
revision=server_args.revision,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
|
||||
def _load_processor(self, server_args: ServerArgs, model_path: str, hf_config):
|
||||
processor_cls = get_class_from_dynamic_module(
|
||||
"image_processor.HunyuanImage3ImageProcessor",
|
||||
model_path,
|
||||
revision=server_args.revision,
|
||||
)
|
||||
return processor_cls(hf_config)
|
||||
|
||||
def _load_component(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
component_name: str,
|
||||
model_path: str,
|
||||
server_args: ServerArgs,
|
||||
) -> torch.nn.Module:
|
||||
state_dict = self._collect_prefixed_weights(model_path, f"{component_name}.")
|
||||
if isinstance(model, HunyuanImage3VisionModel):
|
||||
loaded = model.load_weights(state_dict.items())
|
||||
missing = set(dict(model.named_parameters())) - loaded
|
||||
if missing:
|
||||
raise ValueError(f"Missing {component_name} weights: {sorted(missing)}")
|
||||
else:
|
||||
load_model_state_dict(model, state_dict)
|
||||
device = (
|
||||
torch.device("cpu")
|
||||
if server_args.should_cpu_offload_component(component_name)
|
||||
else get_local_torch_device()
|
||||
)
|
||||
model.to(device=device).requires_grad_(False).eval()
|
||||
self.memory_usages[component_name] = _module_memory_gb(model)
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def _collect_prefixed_weights(
|
||||
model_path: str, prefix: str
|
||||
) -> dict[str, torch.Tensor]:
|
||||
index_path = os.path.join(model_path, "model.safetensors.index.json")
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
shard_names = sorted(
|
||||
{shard for key, shard in weight_map.items() if key.startswith(prefix)}
|
||||
)
|
||||
shard_paths = [os.path.join(model_path, name) for name in shard_names]
|
||||
|
||||
state_dict: dict[str, torch.Tensor] = {}
|
||||
for name, tensor in safetensors_weights_iterator(shard_paths):
|
||||
if name.startswith(prefix):
|
||||
state_dict[name[len(prefix) :]] = tensor
|
||||
return state_dict
|
||||
|
||||
@staticmethod
|
||||
def _read_flow_shift(model_path: str) -> float:
|
||||
return float(
|
||||
load_dict(os.path.join(model_path, "generation_config.json"))["flow_shift"]
|
||||
)
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs):
|
||||
self.add_stage(
|
||||
HunyuanImage3AR(
|
||||
ar_model=self.get_module("transformer"),
|
||||
vae=self.get_module("vae"),
|
||||
tokenizer=self.get_module("tokenizer"),
|
||||
processor=self.get_module("processor"),
|
||||
scheduler=self.get_module("scheduler"),
|
||||
model_path=self.model_path,
|
||||
vision_model=self.get_module("vision_model"),
|
||||
vision_aligner=self.get_module("vision_aligner"),
|
||||
# Reuse the scheduler's group ceiling as the initial cond-encode
|
||||
# chunk cap; 1 (the no-grouping default) means unlimited.
|
||||
max_cond_encode_chunk=(
|
||||
server_args.batching_max_size
|
||||
if server_args.batching_max_size > 1
|
||||
else None
|
||||
),
|
||||
),
|
||||
"hunyuan_image3_ar",
|
||||
)
|
||||
|
||||
self.add_stage_factory(
|
||||
RoleType.DECODER,
|
||||
lambda: HunyuanImage3DecodingStage(
|
||||
vae=self.get_module("vae"), pipeline=self
|
||||
),
|
||||
"decoding_stage",
|
||||
)
|
||||
|
||||
def forward_batch(self, batches, server_args: ServerArgs):
|
||||
if len(batches) > 1 and self.executor.component_residency_manager is None:
|
||||
self.component_residency_manager = get_global_component_residency_manager(
|
||||
self, server_args
|
||||
)
|
||||
self.executor.component_residency_manager = self.component_residency_manager
|
||||
return super().forward_batch(batches, server_args)
|
||||
|
||||
|
||||
EntryClass = [HunyuanImage3Pipeline]
|
||||
@@ -468,6 +468,13 @@ class Req:
|
||||
save_output: {self.save_output}
|
||||
output_file_path: {self.output_file_path()}
|
||||
""" # type: ignore[attr-defined]
|
||||
# Add model-specific fields if present
|
||||
bot_task = getattr(self, "bot_task", None)
|
||||
system_prompt = getattr(self, "system_prompt", None)
|
||||
if bot_task is not None:
|
||||
debug_str += f" bot_task: {bot_task}\n"
|
||||
if system_prompt is not None:
|
||||
debug_str += f" system_prompt: {system_prompt}\n"
|
||||
logger.info(debug_str)
|
||||
|
||||
|
||||
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
"""HunyuanImage-3 native pipeline stages."""
|
||||
|
||||
from .ar_stage import HunyuanImage3AR
|
||||
from .decoding import HunyuanImage3DecodingStage
|
||||
|
||||
__all__ = ["HunyuanImage3AR", "HunyuanImage3DecodingStage"]
|
||||
+1411
File diff suppressed because it is too large
Load Diff
+353
@@ -0,0 +1,353 @@
|
||||
"""Post-decode output geometry for HunyuanImage-3."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
|
||||
OutputBatch,
|
||||
Req,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
OUTPUT_GEOMETRY_EXTRA_KEY = "hunyuan_image3_output_geometry"
|
||||
|
||||
|
||||
def build_hunyuan_image3_output_geometry(
|
||||
width: int,
|
||||
height: int,
|
||||
*,
|
||||
size_mode: str = "aspect_ratio",
|
||||
strategy: str = "native_crop",
|
||||
ratio_policy: str = "exact",
|
||||
crop_anchor: tuple[float, float] = (0.5, 0.5),
|
||||
max_ratio_error: float = 0.0005,
|
||||
pad_value: float = 0.0,
|
||||
) -> dict[str, object]:
|
||||
"""Create a request-local output contract before native bucketing.
|
||||
|
||||
``width``/``height`` are the user-visible target geometry, not the
|
||||
processor-selected bucket. Kept on ``Req.extra`` so it is safe with
|
||||
cloned requests and decoder disaggregation.
|
||||
"""
|
||||
width, height = int(width), int(height)
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("HunyuanImage-3 target dimensions must be positive")
|
||||
if size_mode not in {"aspect_ratio", "exact_size"}:
|
||||
raise ValueError("size_mode must be 'aspect_ratio' or 'exact_size'")
|
||||
if strategy not in {"native_crop", "native_pad"}:
|
||||
raise ValueError("strategy must be 'native_crop' or 'native_pad'")
|
||||
if ratio_policy not in {"exact", "approximate"}:
|
||||
raise ValueError("ratio_policy must be 'exact' or 'approximate'")
|
||||
if len(crop_anchor) != 2 or any(
|
||||
not math.isfinite(float(value)) or not 0.0 <= float(value) <= 1.0
|
||||
for value in crop_anchor
|
||||
):
|
||||
raise ValueError("crop_anchor must contain two finite values in [0, 1]")
|
||||
if not math.isfinite(max_ratio_error) or not 0.0 <= max_ratio_error < 1.0:
|
||||
raise ValueError("max_ratio_error must be a finite value in [0, 1)")
|
||||
if not math.isfinite(pad_value) or not 0.0 <= pad_value <= 1.0:
|
||||
raise ValueError("pad_value must be a finite value in [0, 1]")
|
||||
if size_mode == "exact_size" and ratio_policy == "approximate":
|
||||
raise ValueError(
|
||||
"exact_size requires ratio_policy='exact'; approximate cropping "
|
||||
"would require non-uniform resampling to reach the requested size"
|
||||
)
|
||||
|
||||
divisor = math.gcd(width, height)
|
||||
return {
|
||||
"requested_size": [width, height],
|
||||
"requested_aspect_ratio": [width // divisor, height // divisor],
|
||||
"size_mode": size_mode,
|
||||
"strategy": strategy,
|
||||
"ratio_policy": ratio_policy,
|
||||
"crop_anchor": [float(crop_anchor[0]), float(crop_anchor[1])],
|
||||
"max_ratio_error": float(max_ratio_error),
|
||||
"pad_value": float(pad_value),
|
||||
}
|
||||
|
||||
|
||||
def resolve_hunyuan_image3_output_resolution(
|
||||
width: int,
|
||||
height: int,
|
||||
explicit_fields: set[str],
|
||||
reference_size: tuple[int, int] | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Return the raw target size used for native aspect-ratio bucketing.
|
||||
|
||||
Explicit dimensions take precedence; editing without an explicit size
|
||||
inherits the reference size instead of the 1280x720 pipeline default.
|
||||
"""
|
||||
if reference_size is not None and not {"width", "height"} & explicit_fields:
|
||||
width, height = reference_size
|
||||
return int(width), int(height)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SpatialPlan:
|
||||
source_size: tuple[int, int]
|
||||
crop_box: tuple[int, int, int, int] | None = None
|
||||
padding: tuple[int, int, int, int] | None = None
|
||||
pad_value: float = 0.0
|
||||
resize_size: tuple[int, int] | None = None
|
||||
|
||||
|
||||
def _round_half_up(value: float) -> int:
|
||||
return math.floor(value + 0.5)
|
||||
|
||||
|
||||
def _crop_box(
|
||||
width: int,
|
||||
height: int,
|
||||
crop_width: int,
|
||||
crop_height: int,
|
||||
anchor: tuple[float, float],
|
||||
) -> tuple[int, int, int, int]:
|
||||
excess_x, excess_y = width - crop_width, height - crop_height
|
||||
left = min(max(_round_half_up(excess_x * anchor[0]), 0), excess_x)
|
||||
top = min(max(_round_half_up(excess_y * anchor[1]), 0), excess_y)
|
||||
return left, top, left + crop_width, top + crop_height
|
||||
|
||||
|
||||
def _maximum_exact_crop_size(
|
||||
width: int, height: int, p: int, q: int
|
||||
) -> tuple[int, int]:
|
||||
"""Largest integer crop whose ratio matches p:q within pixel rounding.
|
||||
|
||||
A strict p:q-multiple crop degenerates for coprime reference sizes
|
||||
(e.g. 1361x907): the multiplier collapses to 0 or discards most of the
|
||||
image. Anchoring on the limiting dimension and rounding keeps max area
|
||||
with at most half a pixel of ratio deviation per axis.
|
||||
"""
|
||||
ratio = p / q
|
||||
if width / height >= ratio:
|
||||
crop_height = height
|
||||
crop_width = min(width, _round_half_up(crop_height * ratio))
|
||||
else:
|
||||
crop_width = width
|
||||
crop_height = min(height, _round_half_up(crop_width / ratio))
|
||||
if crop_width <= 0 or crop_height <= 0:
|
||||
raise ValueError(
|
||||
f"Cannot crop decoded {width}x{height} pixels to ratio {p}:{q}"
|
||||
)
|
||||
return crop_width, crop_height
|
||||
|
||||
|
||||
def _maximum_approximate_crop_size(
|
||||
width: int,
|
||||
height: int,
|
||||
p: int,
|
||||
q: int,
|
||||
max_ratio_error: float,
|
||||
) -> tuple[int, int]:
|
||||
"""Return max-area integer crop within the declared relative ratio error."""
|
||||
best: tuple[int, int] | None = None
|
||||
best_area, best_error = -1, math.inf
|
||||
for crop_height in range(1, height + 1):
|
||||
nominal_width = crop_height * p / q
|
||||
lower = max(1, math.ceil(nominal_width * (1.0 - max_ratio_error)))
|
||||
upper = min(width, math.floor(nominal_width * (1.0 + max_ratio_error)))
|
||||
if lower > upper:
|
||||
continue
|
||||
crop_width = upper
|
||||
ratio_error = abs((crop_width * q) / (crop_height * p) - 1.0)
|
||||
area = crop_width * crop_height
|
||||
if area > best_area or (area == best_area and ratio_error < best_error):
|
||||
best = crop_width, crop_height
|
||||
best_area, best_error = area, ratio_error
|
||||
if best is None:
|
||||
raise ValueError(
|
||||
f"Cannot crop decoded {width}x{height} pixels within ratio-error "
|
||||
f"limit {max_ratio_error} for {p}:{q}"
|
||||
)
|
||||
return best
|
||||
|
||||
|
||||
def _minimum_exact_canvas_size(
|
||||
width: int, height: int, p: int, q: int
|
||||
) -> tuple[int, int]:
|
||||
multiplier = max(math.ceil(width / p), math.ceil(height / q))
|
||||
return multiplier * p, multiplier * q
|
||||
|
||||
|
||||
def _build_spatial_plan(
|
||||
decoded_size: tuple[int, int], geometry: dict[str, object]
|
||||
) -> tuple[_SpatialPlan, dict[str, object]]:
|
||||
width, height = decoded_size
|
||||
p, q = (int(value) for value in geometry["requested_aspect_ratio"])
|
||||
anchor = tuple(float(value) for value in geometry["crop_anchor"])
|
||||
strategy = str(geometry["strategy"])
|
||||
ratio_policy = str(geometry["ratio_policy"])
|
||||
size_mode = str(geometry["size_mode"])
|
||||
|
||||
metadata = dict(geometry)
|
||||
metadata.setdefault("native_bucket_size", [width, height])
|
||||
metadata["decoded_size"] = [width, height]
|
||||
metadata["resampled"] = False
|
||||
|
||||
if strategy == "native_crop":
|
||||
if ratio_policy == "exact":
|
||||
crop_width, crop_height = _maximum_exact_crop_size(width, height, p, q)
|
||||
else:
|
||||
crop_width, crop_height = _maximum_approximate_crop_size(
|
||||
width,
|
||||
height,
|
||||
p,
|
||||
q,
|
||||
float(geometry["max_ratio_error"]),
|
||||
)
|
||||
crop_box = _crop_box(width, height, crop_width, crop_height, anchor)
|
||||
plan = _SpatialPlan(source_size=decoded_size, crop_box=crop_box)
|
||||
retained_fraction = crop_width * crop_height / (width * height)
|
||||
metadata["crop_box"] = list(crop_box)
|
||||
metadata["retained_pixel_fraction"] = retained_fraction
|
||||
theoretical_area = (
|
||||
height * height * p / q
|
||||
if width * q >= height * p
|
||||
else width * width * q / p
|
||||
)
|
||||
metadata["integer_constraint_extra_loss"] = max(
|
||||
0.0, (theoretical_area - crop_width * crop_height) / (width * height)
|
||||
)
|
||||
cropped_size = crop_width, crop_height
|
||||
elif strategy == "native_pad":
|
||||
canvas_width, canvas_height = _minimum_exact_canvas_size(width, height, p, q)
|
||||
left, top, right, bottom = _crop_box(
|
||||
canvas_width, canvas_height, width, height, anchor
|
||||
)
|
||||
padding = left, canvas_width - right, top, canvas_height - bottom
|
||||
plan = _SpatialPlan(
|
||||
source_size=decoded_size,
|
||||
padding=padding,
|
||||
pad_value=float(geometry["pad_value"]),
|
||||
)
|
||||
metadata["pad"] = list(padding)
|
||||
metadata["retained_pixel_fraction"] = 1.0
|
||||
metadata["integer_constraint_extra_loss"] = 0.0
|
||||
cropped_size = canvas_width, canvas_height
|
||||
else:
|
||||
raise ValueError(f"Unsupported HunyuanImage-3 output strategy: {strategy}")
|
||||
|
||||
ratio_error = abs((cropped_size[0] * q) / (cropped_size[1] * p) - 1.0)
|
||||
metadata["relative_ratio_error"] = ratio_error
|
||||
if size_mode == "exact_size":
|
||||
requested_size = tuple(int(value) for value in geometry["requested_size"])
|
||||
plan = _SpatialPlan(
|
||||
source_size=plan.source_size,
|
||||
crop_box=plan.crop_box,
|
||||
padding=plan.padding,
|
||||
pad_value=plan.pad_value,
|
||||
resize_size=requested_size,
|
||||
)
|
||||
metadata["resampled"] = cropped_size != requested_size
|
||||
if metadata["resampled"]:
|
||||
metadata["resample_mode"] = "bicubic_antialias"
|
||||
output_size = requested_size
|
||||
elif size_mode == "aspect_ratio":
|
||||
output_size = cropped_size
|
||||
else:
|
||||
raise ValueError(f"Unsupported HunyuanImage-3 output size mode: {size_mode}")
|
||||
metadata["output_size"] = list(output_size)
|
||||
return plan, metadata
|
||||
|
||||
|
||||
def _resize_frames(frames: torch.Tensor, output_size: tuple[int, int]) -> torch.Tensor:
|
||||
output_width, output_height = output_size
|
||||
input_height, input_width = frames.shape[-2:]
|
||||
if (input_width, input_height) == output_size:
|
||||
return frames
|
||||
|
||||
original_dtype = frames.dtype
|
||||
if frames.ndim == 5:
|
||||
batch_size, channels, num_frames, _, _ = frames.shape
|
||||
flat_frames = frames.permute(0, 2, 1, 3, 4).reshape(
|
||||
batch_size * num_frames, channels, input_height, input_width
|
||||
)
|
||||
elif frames.ndim == 4:
|
||||
batch_size, channels, _, _ = frames.shape
|
||||
num_frames = None
|
||||
flat_frames = frames
|
||||
else:
|
||||
raise ValueError(
|
||||
"HunyuanImage-3 decoded frames must be BCHW or BCFHW, "
|
||||
f"but got shape {tuple(frames.shape)}"
|
||||
)
|
||||
|
||||
compute_dtype = torch.float64 if original_dtype == torch.float64 else torch.float32
|
||||
resized = F.interpolate(
|
||||
flat_frames.to(compute_dtype),
|
||||
size=(output_height, output_width),
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
antialias=True,
|
||||
).clamp_(0, 1)
|
||||
if original_dtype.is_floating_point:
|
||||
resized = resized.to(original_dtype)
|
||||
if num_frames is None:
|
||||
return resized
|
||||
return (
|
||||
resized.reshape(batch_size, num_frames, channels, output_height, output_width)
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
def apply_hunyuan_image3_spatial_plan(
|
||||
frames: torch.Tensor, plan: _SpatialPlan
|
||||
) -> torch.Tensor:
|
||||
"""Apply the exact same post-decode spatial transform to images or paths."""
|
||||
if frames.ndim not in (4, 5):
|
||||
raise ValueError(
|
||||
"HunyuanImage-3 decoded frames must be BCHW or BCFHW, "
|
||||
f"but got shape {tuple(frames.shape)}"
|
||||
)
|
||||
source_width, source_height = plan.source_size
|
||||
if tuple(frames.shape[-2:]) != (source_height, source_width):
|
||||
raise ValueError(
|
||||
"HunyuanImage-3 output and trajectory decoded to different spatial "
|
||||
f"sizes: expected {source_width}x{source_height}, got "
|
||||
f"{frames.shape[-1]}x{frames.shape[-2]}"
|
||||
)
|
||||
if plan.crop_box is not None:
|
||||
left, top, right, bottom = plan.crop_box
|
||||
frames = frames[..., top:bottom, left:right]
|
||||
if plan.padding is not None:
|
||||
frames = F.pad(frames, plan.padding, value=plan.pad_value)
|
||||
if plan.resize_size is not None:
|
||||
frames = _resize_frames(frames, plan.resize_size)
|
||||
return frames
|
||||
|
||||
|
||||
class HunyuanImage3DecodingStage(DecodingStage):
|
||||
"""Decode a complete native bucket, then apply request-scoped geometry."""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
output_batch = super().forward(batch, server_args)
|
||||
geometry = batch.extra.get(OUTPUT_GEOMETRY_EXTRA_KEY)
|
||||
if geometry is None or not isinstance(output_batch.output, torch.Tensor):
|
||||
return output_batch
|
||||
if not envs.SGLANG_HI3_OUTPUT_CROP:
|
||||
# Env-gated escape hatch: return the full decoded bucket
|
||||
# uncropped; the geometry contract stays in extra.
|
||||
batch.width = output_batch.output.shape[-1]
|
||||
batch.height = output_batch.output.shape[-2]
|
||||
return output_batch
|
||||
|
||||
source_size = output_batch.output.shape[-1], output_batch.output.shape[-2]
|
||||
plan, metadata = _build_spatial_plan(source_size, geometry)
|
||||
output_batch.output = apply_hunyuan_image3_spatial_plan(
|
||||
output_batch.output, plan
|
||||
)
|
||||
if output_batch.trajectory_decoded is not None:
|
||||
output_batch.trajectory_decoded = [
|
||||
apply_hunyuan_image3_spatial_plan(frames, plan)
|
||||
for frames in output_batch.trajectory_decoded
|
||||
]
|
||||
batch.width, batch.height = metadata["output_size"]
|
||||
batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = metadata
|
||||
return output_batch
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
"""System prompt presets for HunyuanImage-3 (from vllm_omni system_prompt.py)."""
|
||||
|
||||
_SYSTEM_PROMPTS = {
|
||||
"en_vanilla": """\
|
||||
You are an advanced AI text-to-image generation system. Given a detailed text prompt, your task is to create a high-quality, visually compelling image that accurately represents the described scene, characters, or objects. Pay careful attention to style, color, lighting, perspective, and any specific instructions provided.
|
||||
""",
|
||||
"en_recaption": """\
|
||||
You are a world-class image generation prompt expert. Your task is to rewrite a user's simple description into a **structured, objective, and detail-rich** professional-level prompt.
|
||||
|
||||
The final output must be wrapped in `<recaption>` tags.
|
||||
|
||||
### **Universal Core Principles**
|
||||
|
||||
When rewriting the prompt (inside the `<recaption>` tags), you must adhere to the following principles:
|
||||
|
||||
1. **Absolute Objectivity**: Describe only what is visually present. Avoid subjective words like "beautiful" or "sad". Convey aesthetic qualities through specific descriptions of color, light, shadow, and composition.
|
||||
2. **Physical and Logical Consistency**: All scene elements (e.g., gravity, light, shadows, reflections, spatial relationships, object proportions) must strictly adhere to real-world physics and common sense. For example, tennis players must be on opposite sides of the net; objects cannot float without a cause.
|
||||
3. **Structured Description**: Strictly follow a logical order: from general to specific, background to foreground, and primary to secondary elements. Use directional terms like "foreground," "mid-ground," "background," and "left side of the frame" to clearly define the spatial layout.
|
||||
4. **Use Present Tense**: Describe the scene from an observer's perspective using the present tense, such as "A man stands..." or "Light shines on..."
|
||||
5. **Use Rich and Specific Descriptive Language**: Use precise adjectives to describe the quantity, size, shape, color, and other attributes of objects, subjects, and text. Vague expressions are strictly prohibited.
|
||||
|
||||
If the user specifies a style (e.g., oil painting, anime, UI design, text rendering), strictly adhere to that style. Otherwise, first infer a suitable style from the user's input. If there is no clear stylistic preference, default to an **ultra-realistic photographic style**. Then, generate the detailed rewritten prompt according to the **Style-Specific Creation Guide** below:
|
||||
|
||||
### **Style-Specific Creation Guide**
|
||||
|
||||
Based on the determined artistic style, apply the corresponding professional knowledge.
|
||||
|
||||
**1. Photography and Realism Style**
|
||||
* Utilize professional photography terms (e.g., lighting, lens, composition) and meticulously detail material textures, physical attributes of subjects, and environmental details.
|
||||
|
||||
**2. Illustration and Painting Style**
|
||||
* Clearly specify the artistic school (e.g., Japanese Cel Shading, Impasto Oil Painting) and focus on describing its unique medium characteristics, such as line quality, brushstroke texture, or paint properties.
|
||||
|
||||
**3. Graphic/UI/APP Design Style**
|
||||
* Objectively describe the final product, clearly defining the layout, elements, and color palette. All text on the interface must be enclosed in double quotes `""` to specify its exact content (e.g., "Login"). Vague descriptions are strictly forbidden.
|
||||
|
||||
**4. Typographic Art**
|
||||
* The text must be described as a complete physical object. The description must begin with the text itself. Use a straightforward front-on or top-down perspective to ensure the entire text is visible without cropping.
|
||||
|
||||
### **Final Output Requirements**
|
||||
|
||||
1. **Output the Final Prompt Only**: Do not show any thought process, Markdown formatting, or line breaks.
|
||||
2. **Adhere to the Input**: You must retain the core concepts, attributes, and any specified text from the user's input.
|
||||
3. **Style Reinforcement**: Mention the core style 3-5 times within the prompt and conclude with a style declaration sentence.
|
||||
4. **Avoid Self-Reference**: Describe the image content directly. Remove redundant phrases like "This image shows..." or "The scene depicts..."
|
||||
5. **The final output must be wrapped in `<recaption>xxxx</recaption>` tags.**
|
||||
|
||||
The user will now provide an input prompt. You will provide the expanded prompt.
|
||||
""",
|
||||
"en_think_recaption": """\
|
||||
You will act as a top-tier Text-to-Image AI. Your core task is to deeply analyze the user's text input and transform it into a detailed, artistic, and fully user-intent-compliant image.
|
||||
|
||||
Your workflow is divided into two phases:
|
||||
|
||||
1. Thinking Phase (<think>): In the <think> tag, you need to conduct a structured thinking process, progressively breaking down and enriching the constituent elements of the image. This process must include, but is not limited to, the following dimensions:
|
||||
|
||||
Subject: Clearly define the core character(s) or object(s) in the scene, including their appearance, posture, expression, and emotion.
|
||||
Composition: Set the camera angle and layout, such as close-up, long shot, bird's-eye view, golden ratio composition, etc.
|
||||
Environment/Background: Describe the scene where the subject is located, including the location, time of day, weather, and other elements in the background.
|
||||
Lighting: Define the type, direction, and quality of the light source, such as soft afternoon sunlight, cool tones of neon lights, dramatic Rembrandt lighting, etc., to create a specific atmosphere.
|
||||
Color Palette: Set the main color tone and color scheme of the image, such as vibrant and saturated, low-saturation Morandi colors, black and white, etc.
|
||||
Quality/Style: Determine the artistic style and technical details of the image. This includes user-specified styles (e.g., anime, oil painting) or the default realistic style, as well as camera parameters (e.g., focal length, aperture, depth of field).
|
||||
Details: Add minute elements that enhance the realism and narrative quality of the image, such as a character's accessories, the texture of a surface, dust particles in the air, etc.
|
||||
|
||||
|
||||
2. Recaption Phase (<recaption>): In the <recaption> tag, merge all the key details from the thinking process into a coherent, precise, and visually evocative final description. This description is the direct instruction for generating the image, so it must be clear, unambiguous, and organized in a way that is most suitable for an image generation engine to understand.
|
||||
|
||||
Absolutely Objective: Describe only what is visually present. Avoid subjective words like "beautiful" or "sad." Convey aesthetic sense through concrete descriptions of colors, light, shadow, and composition.
|
||||
|
||||
Physical and Logical Consistency: All scene elements (e.g., gravity, light and shadow, reflections, spatial relationships, object proportions) must strictly adhere to the physical laws of the real world and common sense. For example, in a tennis match, players must be on opposite sides of the net; objects cannot float without reason.
|
||||
|
||||
Structured Description: Strictly follow a logical order: from whole to part, background to foreground, and primary to secondary. Use directional words like "foreground," "mid-ground," "background," "left side of the frame" to clearly define the spatial layout.
|
||||
|
||||
Use Present Tense: Describe from an observer's perspective using the present tense, such as "a man stands," "light shines on..."
|
||||
Use Rich and Specific Descriptive Language: Use precise adjectives to describe the quantity, size, shape, color, and other attributes of objects/characters/text. Absolutely avoid any vague expressions.
|
||||
|
||||
|
||||
Output Format:
|
||||
<think>Thinking process</think><recaption>Refined image description</recaption>Generate Image
|
||||
|
||||
|
||||
You must strictly adhere to the following rules:
|
||||
|
||||
1. Faithful to Intent, Reasonable Expansion: You can creatively add details to the user's description to enhance the image's realism and artistic quality. However, all additions must be highly consistent with the user's core intent and never introduce irrelevant or conflicting elements.
|
||||
2. Style Handling: When the user does not specify a style, you must default to an "Ultra-realistic, Photorealistic" style. If the user explicitly specifies a style (e.g., anime, watercolor, oil painting, cyberpunk, etc.), both your thinking process and final description must strictly follow and reflect that specified style.
|
||||
3. Text Rendering: If specific text needs to appear in the image (such as words on a sign, a book title), you must enclose this text in English double quotes (""). Descriptive text must not use double quotes.
|
||||
4. Design-related Images: You need to specify all text and graphical elements that appear in the image and clearly describe their design details, including font, color, size, position, arrangement, visual effects, etc.
|
||||
""",
|
||||
"en_unified": """You are an advanced multimodal model whose core mission is to analyze user intent and generate high-quality text and images.
|
||||
|
||||
#### Four Core Capabilities
|
||||
1. **Text-to-Text (T2T):** Generate coherent text responses from text prompts.
|
||||
2. **Text-to-Image (T2I):** Generate high-quality images from text prompts.
|
||||
3. **Text & Image to Text (TI2T):** Generate accurate text responses based on a combination of images and text.
|
||||
4. **Text & Image to Image (TI2I):** Generate modified images based on a reference image and editing instructions.
|
||||
|
||||
---
|
||||
### Image Generation Protocol (for T2I & TI2I)
|
||||
You will operate in one of two modes, determined by the user's starting tag:
|
||||
#### **<recaption> Mode (Prompt Rewriting)**:
|
||||
* **Trigger:** Input begins with `<recaption>`.
|
||||
* **Task:** Immediately rewrite the user's text into a structured, objective, and detail-rich professional-grade prompt.
|
||||
* **Output:** Output only the rewritten prompt within `<recaption>` tags: `<recaption>Rewritten professional-grade prompt</recaption>`
|
||||
|
||||
#### **<think> Mode (Think + Rewrite)**:
|
||||
* **Trigger:** Input begins with `<think>`.
|
||||
* **Task:** First, conduct a structured analysis of the request within `<think>` tags. Then, output the professional prompt, rewritten based on the analysis, within `<recaption>` tags.
|
||||
* **Output:** Strictly adhere to the format: `<think>Analysis process</think><recaption>Rewritten prompt</recaption>`
|
||||
|
||||
---
|
||||
### Execution Standards and Guidelines
|
||||
#### **`<think>` Phase: Analysis Guidelines**
|
||||
**For T2I (New Image Generation):**
|
||||
Deconstruct the user's request into the following core visual components:
|
||||
* **Subject:** Key features of the main character/object, including appearance, pose, expression, and emotion.
|
||||
* **Composition:** Camera angle, lens type, and layout.
|
||||
* **Environment/Background:** The setting, time of day, weather, and background elements.
|
||||
* **Lighting:** Technical details such as light source type, direction, and quality.
|
||||
* **Color Palette:** The dominant hues and overall color scheme.
|
||||
* **Style/Quality:** The artistic style, clarity, depth of field, and other technical details.
|
||||
* **Text:** Identify any text to be rendered in the image, including its content, style, and position.
|
||||
* **Details:** Small elements that add narrative depth and realism.
|
||||
|
||||
**For TI2I (Image Editing):**
|
||||
Adopt a task-diagnostic approach:
|
||||
1. **Diagnose Task:** Identify the edit type and analyze key requirements.
|
||||
2. **Prioritize Analysis:**
|
||||
* **Adding:** Analyze the new element's position and appearance, ensuring seamless integration with the original image's lighting, shadows, and style.
|
||||
* **Removing:** Identify the target for removal and determine how to logically fill the resulting space using surrounding textures and lighting.
|
||||
* **Modifying:** Analyze what to change and what it should become, while emphasizing which elements must remain unchanged.
|
||||
* **Style Transfer:** Deconstruct the target style into specific features (e.g., brushstrokes, color palette) and apply them to the original image.
|
||||
* **Text Editing:** Ensure correct content and format. Consider the text's visual style (e.g., font, color, material) and how it adapts to the surface's perspective, curvature, and lighting.
|
||||
* **Reference Editing:** Extract specific visual elements (e.g., appearance, posture, composition, lines, depth) from the reference image to generate an image that aligns with the text description while also incorporating the referenced content.
|
||||
* **Inferential Editing:** Identify vague requests (e.g., "make it more professional") and translate them into concrete visual descriptions.
|
||||
|
||||
#### `<recaption>` Phase: Professional-Grade Prompt Generation Rules
|
||||
**General Rewriting Principles (for T2I & TI2I):**
|
||||
1. **Structure & Logic:** Start with a global description. Use positional words (e.g., "foreground", "background") to define the layout.
|
||||
2. **Absolute Objectivity:** Avoid subjective terms. Convey aesthetics through precise descriptions of color, light, shadow, and materials.
|
||||
3. **Physical & Logical Consistency:** Ensure all descriptions adhere to the laws of physics and common sense.
|
||||
4. **Fidelity to User Intent:** Preserve the user's core concepts, subjects, and attributes. Text to be rendered in the image **must be enclosed in double quotes ("")**.
|
||||
5. **Camera & Resolution:** Translate camera parameters into descriptions of visual effects. Convert resolution information into natural language.
|
||||
|
||||
**T2I-Specific Guidelines:**
|
||||
* **Style Adherence & Inference:** Strictly follow the specified style. If none is given, infer the most appropriate style and detail it using professional terminology.
|
||||
* **Style Detailing:**
|
||||
* **Photography/Realism:** Use professional photography terms to describe lighting, lens effects, and material textures.
|
||||
* **Painting/Illustration:** Specify the art movement or medium's characteristics.
|
||||
* **UI/Design:** Objectively describe the final product. Define layout, elements, and typography. Text content must be specific and unambiguous.
|
||||
|
||||
**TI2I-Specific Guidelines:**
|
||||
* **Preserve Unchanged Elements:** Emphasize elements that **remain unchanged**. Unless explicitly instructed, never alter a character's identity/appearance, the core background, camera angle, or overall style.
|
||||
* **Clear Editing Instructions:**
|
||||
* **Replacement:** Use the logic "**replace B with A**," and provide a detailed description of A.
|
||||
* **Addition:** Clearly state what to add, where, and what it looks like.
|
||||
* **Unambiguous Referencing:** Avoid vague references (e.g., "that person"). Use specific descriptions of appearance.
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
def resolve_system_prompt(
|
||||
system_prompt: str | None,
|
||||
bot_task: str = "image",
|
||||
) -> str | None:
|
||||
"""Resolve system prompt: preset name -> prompt text, or raw text as-is."""
|
||||
if system_prompt is None or system_prompt == "none":
|
||||
return None
|
||||
if system_prompt in _SYSTEM_PROMPTS:
|
||||
return _SYSTEM_PROMPTS[system_prompt]
|
||||
if system_prompt == "dynamic":
|
||||
if bot_task == "think":
|
||||
return _SYSTEM_PROMPTS["en_think_recaption"]
|
||||
elif bot_task == "recaption":
|
||||
return _SYSTEM_PROMPTS["en_recaption"]
|
||||
elif bot_task in ("image", "auto"):
|
||||
return _SYSTEM_PROMPTS["en_vanilla"]
|
||||
return None
|
||||
if system_prompt == "auto":
|
||||
return _SYSTEM_PROMPTS["en_unified"]
|
||||
return system_prompt
|
||||
+1079
File diff suppressed because it is too large
Load Diff
@@ -145,12 +145,14 @@ def _extract_short_test_summary(full_output: str) -> list[str]:
|
||||
for line in full_output.splitlines():
|
||||
stripped = line.strip()
|
||||
if "short test summary info" in stripped:
|
||||
summary_lines = []
|
||||
in_summary = True
|
||||
continue
|
||||
if not in_summary:
|
||||
continue
|
||||
if stripped.startswith("="):
|
||||
break
|
||||
in_summary = False
|
||||
continue
|
||||
if not stripped or stripped.startswith("!"):
|
||||
continue
|
||||
summary_lines.append(stripped)
|
||||
|
||||
@@ -23,6 +23,20 @@ from sglang.srt.runtime_context import ParallelContext, get_parallel
|
||||
_UTILS = "sglang.multimodal_gen.test.single_test_file.component_accuracy.utils"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_srt_moe_groups():
|
||||
with (
|
||||
patch.object(parallel_state, "_SRT_MOE_EP", None),
|
||||
patch.object(srt_parallel_state, "_TP", None),
|
||||
patch.object(srt_parallel_state, "_ATTN_TP", None),
|
||||
patch.object(srt_parallel_state, "_MOE_TP", None),
|
||||
patch.object(srt_parallel_state, "_MOE_EP", None),
|
||||
patch.object(parallel_state, "_init_srt_moe_ep_group") as create_group,
|
||||
patch("sglang.srt.runtime_context._PARALLEL", ParallelContext()),
|
||||
):
|
||||
yield create_group
|
||||
|
||||
|
||||
def _tp_group(world_size: int = 1, rank_in_group: int = 0) -> SimpleNamespace:
|
||||
"""A TP group handle carrying the two members the scope declares to the
|
||||
runtime context (`use_tensor_parallel_group` overrides `tp_size` /
|
||||
@@ -115,6 +129,8 @@ def test_destroy_releases_sequence_parallel_subgroups_after_partial_init():
|
||||
"_VAE_DECODE",
|
||||
"_DIT",
|
||||
"_VAE",
|
||||
"_REPLICA",
|
||||
"_ENCODER_DP",
|
||||
):
|
||||
stack.enter_context(patch.object(parallel_state, name, None))
|
||||
stack.enter_context(patch.object(PROCESS_GROUP, "ULYSSES_PG", ulysses_group))
|
||||
@@ -178,6 +194,35 @@ def test_srt_owned_groups_are_not_overwritten_or_cleared():
|
||||
assert srt_parallel_state._ATTN_TP is srt_attention_tp_group
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ownership", ["diffusion", "srt", "replaced"])
|
||||
def test_srt_moe_ep_group_ownership(isolate_srt_moe_groups, ownership):
|
||||
create_group = isolate_srt_moe_groups
|
||||
owned = create_group.return_value
|
||||
external = SimpleNamespace(destroy=lambda: pytest.fail("destroyed SRT-owned group"))
|
||||
with patch.object(parallel_state, "_TP", _tp_group()):
|
||||
if ownership == "srt":
|
||||
srt_parallel_state._MOE_EP = external
|
||||
parallel_state._sync_srt_tp_group()
|
||||
parallel_state._sync_srt_tp_group()
|
||||
if ownership == "replaced":
|
||||
srt_parallel_state._MOE_EP = external
|
||||
|
||||
parallel_state._clear_srt_tp_group()
|
||||
parallel_state._clear_srt_tp_group()
|
||||
|
||||
if ownership == "srt":
|
||||
create_group.assert_not_called()
|
||||
owned.destroy.assert_not_called()
|
||||
else:
|
||||
create_group.assert_called_once_with()
|
||||
owned.destroy.assert_called_once_with()
|
||||
assert parallel_state._SRT_MOE_EP is None
|
||||
assert srt_parallel_state._MOE_TP is None
|
||||
assert srt_parallel_state._MOE_EP is (
|
||||
None if ownership == "diffusion" else external
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
def test_srt_tp_groups_follow_encoder_folding_context(rank):
|
||||
original_tp_group = _tp_group()
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Resolution-contract tests for HunyuanImage-3 image generation and editing."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan_image3 import (
|
||||
HunyuanImage3PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
|
||||
InputValidationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3.ar_stage import (
|
||||
HunyuanImage3AR,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3.decoding import (
|
||||
OUTPUT_GEOMETRY_EXTRA_KEY,
|
||||
HunyuanImage3DecodingStage,
|
||||
_build_spatial_plan,
|
||||
apply_hunyuan_image3_spatial_plan,
|
||||
build_hunyuan_image3_output_geometry,
|
||||
resolve_hunyuan_image3_output_resolution,
|
||||
)
|
||||
|
||||
|
||||
def test_edit_without_explicit_size_follows_reference_aspect_ratio():
|
||||
width, height = resolve_hunyuan_image3_output_resolution(
|
||||
width=1280,
|
||||
height=720,
|
||||
explicit_fields=set(),
|
||||
reference_size=(1000, 333),
|
||||
)
|
||||
|
||||
assert (width, height) == (1000, 333)
|
||||
|
||||
|
||||
def test_explicit_size_is_not_pre_aligned_before_processor():
|
||||
width, height = resolve_hunyuan_image3_output_resolution(
|
||||
width=1025,
|
||||
height=577,
|
||||
explicit_fields={"width", "height"},
|
||||
)
|
||||
|
||||
assert (width, height) == (1025, 577)
|
||||
|
||||
|
||||
def test_explicit_size_takes_precedence_over_reference_image():
|
||||
width, height = resolve_hunyuan_image3_output_resolution(
|
||||
width=1025,
|
||||
height=577,
|
||||
explicit_fields={"width", "height"},
|
||||
reference_size=(1000, 333),
|
||||
)
|
||||
|
||||
assert (width, height) == (1025, 577)
|
||||
|
||||
|
||||
def test_one_explicit_dimension_prevents_reference_size_override():
|
||||
width, height = resolve_hunyuan_image3_output_resolution(
|
||||
width=640,
|
||||
height=720,
|
||||
explicit_fields={"width"},
|
||||
reference_size=(333, 1000),
|
||||
)
|
||||
|
||||
assert (width, height) == (640, 720)
|
||||
|
||||
|
||||
def test_text_to_image_without_reference_keeps_raw_request_size():
|
||||
width, height = resolve_hunyuan_image3_output_resolution(
|
||||
width=1025,
|
||||
height=577,
|
||||
explicit_fields=set(),
|
||||
)
|
||||
|
||||
assert (width, height) == (1025, 577)
|
||||
|
||||
|
||||
def test_generation_saves_requested_ratio_before_selecting_native_bucket(monkeypatch):
|
||||
requested_size = (1000, 700)
|
||||
native_size = (1216, 832)
|
||||
image_info = SimpleNamespace(
|
||||
image_width=native_size[0],
|
||||
image_height=native_size[1],
|
||||
token_width=native_size[0] // 16,
|
||||
token_height=native_size[1] // 16,
|
||||
)
|
||||
processor = SimpleNamespace(
|
||||
build_gen_image_info=lambda image_size: image_info,
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
width=requested_size[0],
|
||||
height=requested_size[1],
|
||||
sampling_params=SimpleNamespace(_explicit_fields={"width", "height"}),
|
||||
original_condition_image_size=None,
|
||||
guidance_scale=2.5,
|
||||
num_inference_steps=50,
|
||||
extra={},
|
||||
)
|
||||
stage = object.__new__(HunyuanImage3AR)
|
||||
stage._processor = processor
|
||||
monkeypatch.setattr(stage, "_rebuild_image_info", lambda info: info)
|
||||
|
||||
width, height, *_ = stage._resolve_generation_params([request], [None])
|
||||
|
||||
assert (width, height) == native_size
|
||||
assert (request.width, request.height) == native_size
|
||||
assert request.extra[OUTPUT_GEOMETRY_EXTRA_KEY] == {
|
||||
"requested_size": [1000, 700],
|
||||
"requested_aspect_ratio": [10, 7],
|
||||
"size_mode": "aspect_ratio",
|
||||
"strategy": "native_crop",
|
||||
"ratio_policy": "exact",
|
||||
"crop_anchor": [0.5, 0.5],
|
||||
"max_ratio_error": 0.0005,
|
||||
"pad_value": 0.0,
|
||||
"native_bucket_size": [1216, 832],
|
||||
}
|
||||
|
||||
|
||||
def test_native_crop_is_maximum_area_within_pixel_rounding_and_preserves_pixels():
|
||||
geometry = build_hunyuan_image3_output_geometry(1000, 700)
|
||||
frames = torch.arange(1216 * 832, dtype=torch.float32).reshape(1, 1, 832, 1216)
|
||||
|
||||
plan, metadata = _build_spatial_plan((1216, 832), geometry)
|
||||
cropped = apply_hunyuan_image3_spatial_plan(frames, plan)
|
||||
|
||||
assert plan.crop_box == (14, 0, 1203, 832)
|
||||
assert metadata["crop_box"] == [14, 0, 1203, 832]
|
||||
assert metadata["output_size"] == [1189, 832]
|
||||
assert metadata["relative_ratio_error"] < 0.0005
|
||||
assert metadata["resampled"] is False
|
||||
assert cropped.shape == (1, 1, 832, 1189)
|
||||
assert torch.equal(cropped, frames[..., 0:832, 14:1203])
|
||||
|
||||
|
||||
def test_native_crop_handles_coprime_ratios_without_discarding_pixels():
|
||||
# gcd(1000, 701) == 1: a strict 1000:701-multiple crop would discard
|
||||
# ~30% of the decoded image (or raise); the crop must stay near-exact
|
||||
# and near-maximum-area instead.
|
||||
geometry = build_hunyuan_image3_output_geometry(1000, 701)
|
||||
frames = torch.arange(1216 * 832, dtype=torch.float32).reshape(1, 1, 832, 1216)
|
||||
|
||||
plan, metadata = _build_spatial_plan((1216, 832), geometry)
|
||||
cropped = apply_hunyuan_image3_spatial_plan(frames, plan)
|
||||
|
||||
assert plan.crop_box == (15, 0, 1202, 832)
|
||||
assert metadata["output_size"] == [1187, 832]
|
||||
assert metadata["relative_ratio_error"] < 0.0005
|
||||
retained = metadata["retained_pixel_fraction"]
|
||||
assert retained > 0.97
|
||||
assert cropped.shape == (1, 1, 832, 1187)
|
||||
|
||||
|
||||
def test_native_crop_reuses_the_same_plan_for_trajectory_frames():
|
||||
geometry = build_hunyuan_image3_output_geometry(1000, 700)
|
||||
image = torch.zeros(2, 3, 1, 832, 1216)
|
||||
trajectory = torch.ones(2, 3, 1, 832, 1216)
|
||||
|
||||
plan, _ = _build_spatial_plan((1216, 832), geometry)
|
||||
|
||||
assert apply_hunyuan_image3_spatial_plan(image, plan).shape == (2, 3, 1, 832, 1189)
|
||||
assert torch.equal(
|
||||
apply_hunyuan_image3_spatial_plan(trajectory, plan),
|
||||
trajectory[..., 0:832, 14:1203],
|
||||
)
|
||||
|
||||
|
||||
def test_decoding_stage_crops_images_and_trajectories_with_one_plan(monkeypatch):
|
||||
frames = torch.zeros(2, 3, 1, 832, 1216)
|
||||
trajectory = torch.ones(2, 3, 1, 832, 1216)
|
||||
|
||||
def fake_decode(_stage, _batch, _server_args):
|
||||
return OutputBatch(output=frames, trajectory_decoded=[trajectory])
|
||||
|
||||
monkeypatch.setattr(DecodingStage, "forward", fake_decode)
|
||||
stage = object.__new__(HunyuanImage3DecodingStage)
|
||||
batch = Req(sampling_params=SamplingParams(prompt="test", width=1000, height=700))
|
||||
batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = build_hunyuan_image3_output_geometry(
|
||||
1000, 700
|
||||
)
|
||||
|
||||
output = stage.forward(batch, SimpleNamespace())
|
||||
|
||||
assert output.output.shape == (2, 3, 1, 832, 1189)
|
||||
assert output.trajectory_decoded[0].shape == (2, 3, 1, 832, 1189)
|
||||
assert (batch.width, batch.height) == (1189, 832)
|
||||
assert batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY]["crop_box"] == [14, 0, 1203, 832]
|
||||
|
||||
|
||||
def test_output_crop_env_disable_returns_full_native_bucket(monkeypatch):
|
||||
frames = torch.zeros(2, 3, 1, 832, 1216)
|
||||
|
||||
def fake_decode(_stage, _batch, _server_args):
|
||||
return OutputBatch(output=frames, trajectory_decoded=None)
|
||||
|
||||
monkeypatch.setattr(DecodingStage, "forward", fake_decode)
|
||||
assert envs.SGLANG_HI3_OUTPUT_CROP is True
|
||||
monkeypatch.setenv("SGLANG_HI3_OUTPUT_CROP", "0")
|
||||
assert envs.SGLANG_HI3_OUTPUT_CROP is False
|
||||
stage = object.__new__(HunyuanImage3DecodingStage)
|
||||
batch = Req(sampling_params=SamplingParams(prompt="test", width=1000, height=700))
|
||||
batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY] = build_hunyuan_image3_output_geometry(
|
||||
1000, 700
|
||||
)
|
||||
|
||||
output = stage.forward(batch, SimpleNamespace())
|
||||
|
||||
# Full bucket untouched; the geometry contract stays in extra.
|
||||
assert output.output.shape == (2, 3, 1, 832, 1216)
|
||||
assert (batch.width, batch.height) == (1216, 832)
|
||||
assert "crop_box" not in batch.extra[OUTPUT_GEOMETRY_EXTRA_KEY]
|
||||
|
||||
|
||||
def test_exact_size_performs_one_uniform_final_resample():
|
||||
geometry = build_hunyuan_image3_output_geometry(1000, 700, size_mode="exact_size")
|
||||
frames = torch.zeros(1, 3, 1, 832, 1216)
|
||||
|
||||
plan, metadata = _build_spatial_plan((1216, 832), geometry)
|
||||
output = apply_hunyuan_image3_spatial_plan(frames, plan)
|
||||
|
||||
assert plan.crop_box == (14, 0, 1203, 832)
|
||||
assert plan.resize_size == (1000, 700)
|
||||
assert output.shape == (1, 3, 1, 700, 1000)
|
||||
assert metadata["resampled"] is True
|
||||
|
||||
|
||||
def test_native_pad_retains_all_decoded_pixels():
|
||||
geometry = build_hunyuan_image3_output_geometry(1000, 700, strategy="native_pad")
|
||||
frames = torch.ones(1, 1, 832, 1216)
|
||||
|
||||
plan, metadata = _build_spatial_plan((1216, 832), geometry)
|
||||
padded = apply_hunyuan_image3_spatial_plan(frames, plan)
|
||||
|
||||
assert plan.padding == (2, 2, 11, 11)
|
||||
assert padded.shape == (1, 1, 854, 1220)
|
||||
assert torch.equal(padded[..., 11:843, 2:1218], frames)
|
||||
assert metadata["retained_pixel_fraction"] == 1.0
|
||||
|
||||
|
||||
def test_multi_output_request_keeps_native_bucket_size(monkeypatch):
|
||||
outputs = [
|
||||
SimpleNamespace(
|
||||
latents=torch.zeros(1, 1, 1, 1, 1),
|
||||
width=1216,
|
||||
height=832,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
stage = object.__new__(HunyuanImage3AR)
|
||||
monkeypatch.setattr(stage, "_expand_multi_output", lambda _batch: [object()] * 2)
|
||||
monkeypatch.setattr(stage, "_forward_batched", lambda _batches: outputs)
|
||||
batch = Req(
|
||||
sampling_params=SamplingParams(
|
||||
prompt="test",
|
||||
width=1000,
|
||||
height=700,
|
||||
num_outputs_per_prompt=2,
|
||||
)
|
||||
)
|
||||
|
||||
output = stage.forward(batch, SimpleNamespace())
|
||||
|
||||
assert output is batch
|
||||
assert output.latents.shape[0] == 2
|
||||
assert (output.width, output.height) == (1216, 832)
|
||||
|
||||
|
||||
def test_pipeline_config_delegates_condition_image_sizing_to_native_stage():
|
||||
config = HunyuanImage3PipelineConfig()
|
||||
reference = Image.new("RGB", (1000, 333))
|
||||
|
||||
assert config.calculate_condition_image_size(reference, 1280, 720) is None
|
||||
assert config.prepare_calculated_size(reference) is None
|
||||
|
||||
|
||||
def test_input_validation_does_not_apply_generic_32_pixel_edit_resize():
|
||||
reference = Image.new("RGB", (1000, 333))
|
||||
batch = Req(
|
||||
sampling_params=SamplingParams(prompt="edit", width=1008, height=336),
|
||||
condition_image=reference,
|
||||
)
|
||||
batch.extra["explicit_fields"] = ["width", "height"]
|
||||
server_args = SimpleNamespace(pipeline_config=HunyuanImage3PipelineConfig())
|
||||
|
||||
InputValidationStage().preprocess_condition_image(
|
||||
batch,
|
||||
server_args,
|
||||
condition_image_width=reference.width,
|
||||
condition_image_height=reference.height,
|
||||
)
|
||||
|
||||
assert batch.condition_image[0].size == (1000, 333)
|
||||
assert (batch.width, batch.height) == (1008, 336)
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Unit tests for HunyuanImage-3 RoPE image-info span/shape pairing.
|
||||
|
||||
The section shape stream follows sequence order -- each joint image
|
||||
contributes [vae, vit] dims and the tokenizer emits VAE tokens then ViT
|
||||
tokens inside every joint block -- while the cond slices arrive grouped by
|
||||
type. ``_build_rope_image_info`` must pair every span with its own shape
|
||||
even when consecutive cond images have different sizes.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3 import (
|
||||
ar_stage,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _tokenizer_output(vae_slices, vit_slices, gen_slices):
|
||||
return SimpleNamespace(
|
||||
cond_vae_image_slices=[vae_slices],
|
||||
cond_vit_image_slices=[vit_slices],
|
||||
gen_image_slices=[gen_slices],
|
||||
)
|
||||
|
||||
|
||||
def _joint_section(vae_hw, vit_hw):
|
||||
return {
|
||||
"type": "joint_image",
|
||||
"token_height": [vae_hw[0], vit_hw[0]],
|
||||
"token_width": [vae_hw[1], vit_hw[1]],
|
||||
}
|
||||
|
||||
|
||||
class TestHunyuanImage3RoPEPairing(CustomTestCase):
|
||||
def test_multi_image_different_sizes_pair_per_image(self):
|
||||
# Two cond images with different grids: image 0 = 24x32 vae / 14x14
|
||||
# vit, image 1 = 16x48 vae / 21x21 vit.
|
||||
vae0, vit0, vae1, vit1 = (24, 32), (14, 14), (16, 48), (21, 21)
|
||||
s_vae0, s_vit0 = slice(10, 20), slice(21, 30)
|
||||
s_vae1, s_vit1 = slice(40, 55), slice(56, 60)
|
||||
tok = _tokenizer_output(
|
||||
vae_slices=[s_vae0, s_vae1],
|
||||
vit_slices=[s_vit0, s_vit1],
|
||||
gen_slices=[],
|
||||
)
|
||||
sections = [_joint_section(vae0, vit0), _joint_section(vae1, vit1)]
|
||||
|
||||
info = ar_stage._build_rope_image_info(
|
||||
tokenizer_output=tok,
|
||||
batch_size=1,
|
||||
token_h=99,
|
||||
token_w=99,
|
||||
image_info=SimpleNamespace(token_height=32, token_width=32),
|
||||
sections=sections,
|
||||
)[0]
|
||||
|
||||
got = {s.start: shape for s, shape in info}
|
||||
self.assertEqual(
|
||||
got,
|
||||
{
|
||||
s_vae0.start: vae0,
|
||||
s_vit0.start: vit0,
|
||||
s_vae1.start: vae1,
|
||||
s_vit1.start: vit1,
|
||||
},
|
||||
)
|
||||
# Spans must stay in sequence order.
|
||||
spans = [s for s, _ in info]
|
||||
self.assertEqual(spans, sorted(spans, key=lambda s: s.start))
|
||||
|
||||
def test_single_image_pairing_unchanged(self):
|
||||
vae, vit = (24, 32), (14, 14)
|
||||
s_vae, s_vit = slice(10, 20), slice(21, 30)
|
||||
tok = _tokenizer_output(vae_slices=[s_vae], vit_slices=[s_vit], gen_slices=[])
|
||||
sections = [_joint_section(vae, vit)]
|
||||
|
||||
info = ar_stage._build_rope_image_info(
|
||||
tokenizer_output=tok,
|
||||
batch_size=1,
|
||||
token_h=99,
|
||||
token_w=99,
|
||||
image_info=SimpleNamespace(token_height=32, token_width=32),
|
||||
sections=sections,
|
||||
)[0]
|
||||
|
||||
self.assertEqual(info, [(s_vae, vae), (s_vit, vit)])
|
||||
|
||||
def test_missing_shapes_fall_back_to_gen_dims(self):
|
||||
# A cond span whose shape is absent from the section stream falls
|
||||
# back to the gen-image dims instead of consuming the wrong shape.
|
||||
vae, vit = (24, 32), (14, 14)
|
||||
s_vae, s_vit = slice(10, 20), slice(21, 30)
|
||||
tok = _tokenizer_output(vae_slices=[s_vae], vit_slices=[s_vit], gen_slices=[])
|
||||
sections = [] # no image sections -> empty shape stream
|
||||
|
||||
info = ar_stage._build_rope_image_info(
|
||||
tokenizer_output=tok,
|
||||
batch_size=1,
|
||||
token_h=99,
|
||||
token_w=99,
|
||||
image_info=SimpleNamespace(token_height=32, token_width=32),
|
||||
sections=sections,
|
||||
)[0]
|
||||
|
||||
self.assertEqual(info, [(s_vae, (99, 99)), (s_vit, (99, 99))])
|
||||
|
||||
def test_gen_images_take_their_own_shapes(self):
|
||||
gen_hw = (40, 24)
|
||||
s_gen = slice(70, 90)
|
||||
tok = _tokenizer_output(vae_slices=[], vit_slices=[], gen_slices=[s_gen])
|
||||
sections = [{"type": "gen_image", "token_height": 40, "token_width": 24}]
|
||||
|
||||
info = ar_stage._build_rope_image_info(
|
||||
tokenizer_output=tok,
|
||||
batch_size=1,
|
||||
token_h=99,
|
||||
token_w=99,
|
||||
image_info=SimpleNamespace(token_height=32, token_width=32),
|
||||
sections=sections,
|
||||
)[0]
|
||||
|
||||
self.assertEqual(info, [(s_gen, gen_hw)])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Unit tests for HunyuanImage-3 tensor-parallel input broadcasts."""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.hunyuan_image3 import (
|
||||
ar_stage,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class _FakeTensor:
|
||||
def __init__(self, shape):
|
||||
self.shape = shape
|
||||
|
||||
def contiguous(self):
|
||||
return self
|
||||
|
||||
def reshape(self, *_shape):
|
||||
return self
|
||||
|
||||
|
||||
class _FakeTPGroup:
|
||||
world_size = 2
|
||||
|
||||
def __init__(self):
|
||||
self.broadcast_calls = []
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
self.broadcast_calls.append((tensor, src))
|
||||
return tensor
|
||||
|
||||
|
||||
class TestHunyuanImage3TPBroadcast(CustomTestCase):
|
||||
def test_static_broadcast_happens_outside_denoising_loop(self):
|
||||
source = textwrap.dedent(
|
||||
inspect.getsource(ar_stage.HunyuanImage3AR._forward_batched)
|
||||
)
|
||||
function = ast.parse(source).body[0]
|
||||
static_broadcast_calls = [
|
||||
node
|
||||
for node in ast.walk(function)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "_broadcast_static_inputs"
|
||||
]
|
||||
self.assertEqual(len(static_broadcast_calls), 1)
|
||||
|
||||
denoising_loops = [
|
||||
node
|
||||
for node in ast.walk(function)
|
||||
if isinstance(node, ast.For)
|
||||
and any(
|
||||
isinstance(child, ast.Name) and child.id == "timesteps"
|
||||
for child in ast.walk(node.iter)
|
||||
)
|
||||
]
|
||||
self.assertEqual(len(denoising_loops), 1)
|
||||
self.assertLess(
|
||||
static_broadcast_calls[0].lineno,
|
||||
denoising_loops[0].lineno,
|
||||
"request-static inputs must be broadcast before the denoising loop",
|
||||
)
|
||||
|
||||
def test_static_inputs_are_broadcast_once(self):
|
||||
stage = ar_stage.HunyuanImage3AR.__new__(ar_stage.HunyuanImage3AR)
|
||||
tp_group = _FakeTPGroup()
|
||||
attention_mask = _FakeTensor((2, 1, 8, 8))
|
||||
cos = _FakeTensor((2, 8, 4))
|
||||
sin = _FakeTensor((2, 8, 4))
|
||||
|
||||
with (
|
||||
patch.object(ar_stage, "model_parallel_is_initialized", return_value=True),
|
||||
patch.object(ar_stage, "get_tp_group", return_value=tp_group),
|
||||
):
|
||||
result = stage._broadcast_static_inputs(attention_mask, (cos, sin))
|
||||
|
||||
self.assertEqual(result, (attention_mask, (cos, sin)))
|
||||
self.assertEqual(
|
||||
tp_group.broadcast_calls,
|
||||
[(attention_mask, 0), (cos, 0), (sin, 0)],
|
||||
)
|
||||
|
||||
def test_backbone_forward_only_broadcasts_dynamic_hidden_states(self):
|
||||
stage = ar_stage.HunyuanImage3AR.__new__(ar_stage.HunyuanImage3AR)
|
||||
tp_group = _FakeTPGroup()
|
||||
hidden_states = _FakeTensor((2, 3, 4))
|
||||
attention_mask = _FakeTensor((2, 1, 3, 3))
|
||||
cos = _FakeTensor((2, 3, 2))
|
||||
sin = _FakeTensor((2, 3, 2))
|
||||
output = MagicMock()
|
||||
output.shape = (6, 4)
|
||||
output.view.return_value = "reshaped-output"
|
||||
stage._cache_dit_adapter = None
|
||||
stage.ar_model = SimpleNamespace(forward_block=MagicMock(return_value=output))
|
||||
|
||||
with (
|
||||
patch.object(ar_stage, "model_parallel_is_initialized", return_value=True),
|
||||
patch.object(ar_stage, "get_tp_group", return_value=tp_group),
|
||||
):
|
||||
result = stage._backbone_forward(
|
||||
num_image_tokens=2,
|
||||
hidden_states=hidden_states,
|
||||
attention_mask=attention_mask,
|
||||
custom_pos_emb=(cos, sin),
|
||||
first_step=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result, "reshaped-output")
|
||||
self.assertEqual(tp_group.broadcast_calls, [(hidden_states, 0)])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
|
||||
from sglang.multimodal_gen.test.runner.pytest_runner import (
|
||||
_estimate_failed_test_time,
|
||||
_extract_short_test_summary,
|
||||
_is_retryable_failure,
|
||||
run_pytest,
|
||||
)
|
||||
@@ -60,6 +61,23 @@ def test_validation_failures_are_not_retryable(output):
|
||||
assert not _is_retryable_failure(output)
|
||||
|
||||
|
||||
def test_failure_summary_uses_outer_pytest_session():
|
||||
output = "\n".join(
|
||||
[
|
||||
"=== short test summary info ===",
|
||||
"FAILED nested/test_retry.py::test_slow - AssertionError: [performance]",
|
||||
"=== 1 failed ===",
|
||||
"nested failure was expected; outer test PASSED",
|
||||
"=== short test summary info ===",
|
||||
"FAILED test_parallel.py::test_destroy - AssertionError",
|
||||
"=== 1 failed, 10 passed ===",
|
||||
]
|
||||
)
|
||||
assert _extract_short_test_summary(output) == [
|
||||
"FAILED test_parallel.py::test_destroy - AssertionError"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output",
|
||||
[
|
||||
|
||||
@@ -4,13 +4,19 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Siglip2VisionConfig
|
||||
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.models.encoders import gemma_3
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
EncoderTensorParallelMixin,
|
||||
)
|
||||
from sglang.srt.models import siglip
|
||||
from sglang.multimodal_gen.runtime.models.encoders.hunyuan_image3 import (
|
||||
HunyuanImage3VisionModel,
|
||||
_PerImageSDPA,
|
||||
)
|
||||
from sglang.srt.models import siglip, siglip2
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def _vision_config():
|
||||
@@ -79,6 +85,119 @@ def test_gemma3_maps_hf_siglip_projection_name():
|
||||
assert mapped == "vision_tower.vision_model.encoder.layers.0.self_attn.proj.weight"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("replicated", [False, True])
|
||||
def test_siglip2_propagates_backend_and_parallelism(replicated):
|
||||
config = Siglip2VisionConfig(
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
num_attention_heads=2,
|
||||
num_hidden_layers=1,
|
||||
num_patches=16,
|
||||
)
|
||||
with (
|
||||
patch.object(siglip2, "VisionAttention", return_value=nn.Identity()) as attn,
|
||||
patch.object(
|
||||
siglip2, "ColumnParallelLinear", return_value=nn.Identity()
|
||||
) as fc1,
|
||||
patch.object(siglip2, "RowParallelLinear", return_value=nn.Identity()) as fc2,
|
||||
):
|
||||
kwargs = (
|
||||
{"qkv_backend": "sdpa", "use_data_parallel": True} if replicated else {}
|
||||
)
|
||||
siglip2.Siglip2Model(config, **kwargs)
|
||||
|
||||
assert attn.call_args.kwargs["qkv_backend"] == ("sdpa" if replicated else None)
|
||||
assert attn.call_args.kwargs["use_data_parallel"] is replicated
|
||||
for linear in (fc1, fc2):
|
||||
assert linear.call_args.kwargs["parallel_group"] == (
|
||||
"replicated" if replicated else "tp"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mask_ndim", [1, 2, 3])
|
||||
def test_hunyuan_siglip2_preserves_padded_layout(mask_ndim):
|
||||
# Exercise the processor boundary independently of device-specific kernels.
|
||||
model = HunyuanImage3VisionModel.__new__(HunyuanImage3VisionModel)
|
||||
nn.Module.__init__(model)
|
||||
batch = 1 if mask_ndim == 1 else 2
|
||||
pixels = torch.arange(batch * 8 * 3, dtype=torch.float32).reshape(batch, 8, 3)
|
||||
shapes = torch.tensor([[2, 3], [1, 4]])[:batch]
|
||||
mask = torch.arange(8)[None, :] < shapes.prod(dim=1)[:, None]
|
||||
input_mask = mask.reshape(-1) if mask_ndim == 1 else mask
|
||||
if mask_ndim == 3:
|
||||
input_mask = input_mask[:, None, :]
|
||||
packed = pixels[mask]
|
||||
|
||||
with patch.object(
|
||||
siglip2.Siglip2Model, "forward", return_value=packed[None]
|
||||
) as forward:
|
||||
output = model(pixels, input_mask, shapes)
|
||||
|
||||
torch.testing.assert_close(forward.call_args.kwargs["pixel_values_packed"], packed)
|
||||
assert forward.call_args.kwargs["spatial_shapes"].device.type == "cpu"
|
||||
assert forward.call_args.kwargs["cu_seqlens"].tolist() == (
|
||||
[0, 6] if batch == 1 else [0, 6, 10]
|
||||
)
|
||||
assert forward.call_args.kwargs["max_seqlen"] == 6
|
||||
torch.testing.assert_close(output[mask], packed)
|
||||
assert torch.count_nonzero(output[~mask]) == 0
|
||||
|
||||
|
||||
def test_hunyuan_siglip2_attention_does_not_mix_images():
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
q, k, v = [torch.randn(10, 2, 8, generator=generator) for _ in range(3)]
|
||||
attention = _PerImageSDPA()
|
||||
boundaries = torch.tensor([0, 6, 10], dtype=torch.int32)
|
||||
actual = attention(q, k, v, boundaries)
|
||||
expected = torch.cat(
|
||||
[
|
||||
torch.nn.functional.scaled_dot_product_attention(
|
||||
q[start:end].transpose(0, 1),
|
||||
k[start:end].transpose(0, 1),
|
||||
v[start:end].transpose(0, 1),
|
||||
).transpose(0, 1)
|
||||
for start, end in ((0, 6), (6, 10))
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
v[6:] += 100
|
||||
torch.testing.assert_close(attention(q, k, v, boundaries)[:6], actual[:6])
|
||||
|
||||
|
||||
def test_hunyuan_siglip2_loads_unfused_checkpoint_weights():
|
||||
with get_parallel().override(tp_size=1):
|
||||
model = HunyuanImage3VisionModel(
|
||||
dict(
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
num_attention_heads=2,
|
||||
num_hidden_layers=1,
|
||||
num_patches=16,
|
||||
patch_size=2,
|
||||
use_return_dict=True,
|
||||
)
|
||||
)
|
||||
checkpoint = {}
|
||||
expected = {}
|
||||
for name, param in model.named_parameters():
|
||||
value = torch.arange(param.numel(), dtype=param.dtype).reshape(param.shape)
|
||||
expected[name] = value
|
||||
checkpoint_name = name.removeprefix("vision_model.")
|
||||
if "attn.qkv_proj" in name:
|
||||
for projection, weight in zip(("q", "k", "v"), value.chunk(3)):
|
||||
checkpoint[
|
||||
checkpoint_name.replace("attn.qkv_proj", f"{projection}_proj")
|
||||
] = weight
|
||||
else:
|
||||
checkpoint[checkpoint_name.replace("attn.proj", "out_proj")] = value
|
||||
|
||||
loaded = model.load_weights(checkpoint.items())
|
||||
|
||||
assert loaded == set(expected)
|
||||
for name, param in model.named_parameters():
|
||||
torch.testing.assert_close(param, expected[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shard,normalized",
|
||||
[("q", 0), ("k", 1), ("v", 2), (0, "q"), (1, "k"), (2, "v"), ("0", 0), ("01", 1)],
|
||||
|
||||
@@ -176,6 +176,8 @@ class Siglip2Attention(nn.Module):
|
||||
config: Siglip2VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
qkv_backend: Optional[str] = None,
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -200,6 +202,8 @@ class Siglip2Attention(nn.Module):
|
||||
flatten_batch=True, # For variable-length sequence support
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
qkv_backend=qkv_backend,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -234,6 +238,7 @@ class Siglip2MLP(nn.Module):
|
||||
config: Siglip2VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -244,12 +249,14 @@ class Siglip2MLP(nn.Module):
|
||||
config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
parallel_group="replicated" if use_data_parallel else "tp",
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("fc2", prefix),
|
||||
parallel_group="replicated" if use_data_parallel else "tp",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -267,6 +274,8 @@ class Siglip2EncoderLayer(nn.Module):
|
||||
config: Siglip2VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
qkv_backend: Optional[str] = None,
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.embed_dim = config.hidden_size
|
||||
@@ -275,12 +284,15 @@ class Siglip2EncoderLayer(nn.Module):
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
qkv_backend=qkv_backend,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
self.mlp = Siglip2MLP(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -324,6 +336,8 @@ class Siglip2Encoder(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
num_hidden_layers_override: Optional[int] = None,
|
||||
prefix: str = "",
|
||||
qkv_backend: Optional[str] = None,
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -339,6 +353,8 @@ class Siglip2Encoder(nn.Module):
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{idx}", prefix),
|
||||
qkv_backend=qkv_backend,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
for idx in range(num_hidden_layers)
|
||||
]
|
||||
@@ -426,6 +442,8 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
num_hidden_layers_override: Optional[int] = None,
|
||||
require_post_norm: Optional[bool] = None,
|
||||
prefix: str = "",
|
||||
qkv_backend: Optional[str] = None,
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
embed_dim = config.hidden_size
|
||||
@@ -436,6 +454,8 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
prefix=add_prefix("encoder", prefix),
|
||||
qkv_backend=qkv_backend,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
if len(self.encoder.layers) > config.num_hidden_layers:
|
||||
@@ -513,6 +533,8 @@ class Siglip2Model(nn.Module):
|
||||
num_hidden_layers_override: Optional[int] = None,
|
||||
require_post_norm: Optional[bool] = None,
|
||||
prefix: str = "",
|
||||
qkv_backend: Optional[str] = None,
|
||||
use_data_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -522,6 +544,8 @@ class Siglip2Model(nn.Module):
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
require_post_norm=require_post_norm,
|
||||
prefix=add_prefix("vision_model", prefix),
|
||||
qkv_backend=qkv_backend,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user