[diffusion] model: support Anima Base v1.0 (#41011)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -43,15 +43,13 @@ def _is_diffusion_model_from_registry(model_path: str) -> bool:
|
||||
|
||||
|
||||
def _is_diffusers_model_dir(model_dir: str) -> bool:
|
||||
"""Check if a local directory contains a valid diffusers model_index.json."""
|
||||
config_path = os.path.join(model_dir, "model_index.json")
|
||||
if not os.path.exists(config_path):
|
||||
return False
|
||||
|
||||
with open(config_path) as f:
|
||||
config = json.load(f)
|
||||
|
||||
return "_diffusers_version" in config
|
||||
"""Check for a standard or modular Diffusers pipeline index."""
|
||||
for filename in ("model_index.json", "modular_model_index.json"):
|
||||
config_path = os.path.join(model_dir, filename)
|
||||
if os.path.isfile(config_path):
|
||||
with open(config_path) as f:
|
||||
return "_diffusers_version" in json.load(f)
|
||||
return False
|
||||
|
||||
|
||||
def _is_gated_diffusion_repo(repo_id: str) -> bool:
|
||||
|
||||
@@ -9,7 +9,7 @@ SGLang diffusion features an end-to-end unified pipeline for accelerating diffus
|
||||
## Key Features
|
||||
|
||||
SGLang Diffusion has the following features:
|
||||
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
||||
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
||||
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
|
||||
- Ease of use: OpenAI-compatible api, CLI, and python sdk support
|
||||
- Multi-platform support:
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.adapter.base import (
|
||||
AdapterArchConfig,
|
||||
AdapterConfig,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnimaTextConditionerArchConfig(AdapterArchConfig):
|
||||
source_dim: int = 1024
|
||||
target_dim: int = 1024
|
||||
model_dim: int = 1024
|
||||
num_layers: int = 6
|
||||
num_attention_heads: int = 16
|
||||
mlp_ratio: float = 4.0
|
||||
target_vocab_size: int = 32128
|
||||
use_self_attention: bool = True
|
||||
use_layer_norm: bool = False
|
||||
min_sequence_length: int = 512
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnimaTextConditionerConfig(AdapterConfig):
|
||||
arch_config: AnimaTextConditionerArchConfig = field(
|
||||
default_factory=AnimaTextConditionerArchConfig
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnimaArchConfig(DiTArchConfig):
|
||||
in_channels: int = 16
|
||||
out_channels: int = 16
|
||||
num_attention_heads: int = 16
|
||||
attention_head_dim: int = 128
|
||||
num_layers: int = 28
|
||||
mlp_ratio: float = 4.0
|
||||
text_embed_dim: int = 1024
|
||||
adaln_lora_dim: int = 256
|
||||
patch_size: tuple[int, int, int] = (1, 2, 2)
|
||||
max_size: tuple[int, int, int] = (128, 240, 240)
|
||||
rope_scale: tuple[float, float, float] = (1.0, 4.0, 4.0)
|
||||
concat_padding_mask: bool = True
|
||||
extra_pos_embed_type: str | None = None
|
||||
use_crossattn_projection: bool = False
|
||||
img_context_dim_in: int | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
self.hidden_size = self.num_attention_heads * self.attention_head_dim
|
||||
self.num_channels_latents = self.in_channels
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnimaDiTConfig(DiTConfig):
|
||||
arch_config: AnimaArchConfig = field(default_factory=AnimaArchConfig)
|
||||
prefix: str = "anima"
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig
|
||||
from sglang.multimodal_gen.configs.models.encoders.qwen3 import Qwen3TextConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ImagePipelineConfig,
|
||||
ModelTaskType,
|
||||
)
|
||||
|
||||
|
||||
def anima_text_output(outputs, inputs):
|
||||
return outputs.last_hidden_state * inputs.attention_mask.unsqueeze(-1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnimaPipelineConfig(ImagePipelineConfig):
|
||||
task_type: ModelTaskType = ModelTaskType.T2I
|
||||
native_only_components: tuple[str, ...] = ("transformer", "text_conditioner")
|
||||
dit_config: AnimaDiTConfig = field(default_factory=AnimaDiTConfig)
|
||||
vae_config: QwenImageVAEConfig = field(default_factory=QwenImageVAEConfig)
|
||||
text_encoder_configs: tuple = field(default_factory=lambda: (Qwen3TextConfig(),))
|
||||
text_encoder_precisions: tuple[str, ...] = ("bf16",)
|
||||
preprocess_text_funcs: tuple[Callable | None, ...] = (None,)
|
||||
postprocess_text_funcs: tuple[Callable, ...] = (anima_text_output,)
|
||||
should_use_guidance: bool = False
|
||||
enable_autocast: bool = False
|
||||
vae_precision: str = "bf16"
|
||||
|
||||
def tokenize_prompt(self, prompt, tokenizer, tok_kwargs):
|
||||
if not 1 <= tok_kwargs.get("max_length", 512) <= 4096:
|
||||
raise ValueError("Anima max_sequence_length must be between 1 and 4096")
|
||||
inputs = tokenizer(prompt, **{**tok_kwargs, "padding": "longest"})
|
||||
if inputs.input_ids.shape[1] == 0:
|
||||
inputs["input_ids"] = inputs.input_ids.new_zeros((len(prompt), 1))
|
||||
inputs["attention_mask"] = inputs.attention_mask.new_zeros((len(prompt), 1))
|
||||
return inputs
|
||||
|
||||
def prepare_sigmas(self, sigmas, num_inference_steps):
|
||||
return self._prepare_sigmas(sigmas, num_inference_steps)
|
||||
|
||||
def get_latent_dtype(self, prompt_dtype):
|
||||
return torch.float32
|
||||
|
||||
def expand_conditioning_to_sample_batch(self, batch):
|
||||
count = batch.num_outputs_per_prompt
|
||||
if count > 1:
|
||||
batch.prompt_embeds = [
|
||||
x.repeat_interleave(count, dim=0) for x in batch.prompt_embeds
|
||||
]
|
||||
if batch.do_classifier_free_guidance:
|
||||
batch.negative_prompt_embeds = [
|
||||
x.repeat_interleave(count, dim=0)
|
||||
for x in batch.negative_prompt_embeds
|
||||
]
|
||||
return batch
|
||||
|
||||
# the DiT shards patch tokens, leaving scheduler latents replicated
|
||||
def shard_latents_for_sp(self, batch, latents):
|
||||
return latents, False
|
||||
|
||||
def get_pos_prompt_embeds(self, batch):
|
||||
return batch.prompt_embeds[0]
|
||||
|
||||
def get_neg_prompt_embeds(self, batch):
|
||||
return batch.negative_prompt_embeds[0]
|
||||
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
config = self.vae_config.arch_config
|
||||
std = torch.tensor(config.latents_std, device=device, dtype=dtype)
|
||||
mean = torch.tensor(config.latents_mean, device=device, dtype=dtype)
|
||||
return std.reciprocal().view(1, -1, 1, 1, 1), mean.view(1, -1, 1, 1, 1)
|
||||
|
||||
def post_decoding(self, frames, server_args):
|
||||
return frames.squeeze(2)
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.anima import AnimaSamplingParams
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
register_configs(
|
||||
sampling_param_cls=AnimaSamplingParams,
|
||||
pipeline_config_cls=AnimaPipelineConfig,
|
||||
hf_model_paths=["circlestone-labs/Anima-Base-v1.0-Diffusers"],
|
||||
model_detectors=[lambda name: name.lower() == "animamodularpipeline"],
|
||||
)
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnimaSamplingParams(SamplingParams):
|
||||
height: int = 1024
|
||||
width: int = 1024
|
||||
num_frames: int = 1
|
||||
num_inference_steps: int = 30
|
||||
guidance_scale: float = 4.0
|
||||
negative_prompt: str = ""
|
||||
@@ -403,6 +403,10 @@ class CustomBlockAdapterSpec:
|
||||
|
||||
# Custom BlockAdapter metadata for models absent from cache-dit's registry.
|
||||
_CUSTOM_BLOCK_ADAPTER_SPECS: dict[str, CustomBlockAdapterSpec] = {
|
||||
"AnimaTransformer3DModel": CustomBlockAdapterSpec(
|
||||
blocks_attr="transformer_blocks",
|
||||
forward_pattern=ForwardPattern.Pattern_3,
|
||||
),
|
||||
"MingImageTransformer2DModel": CustomBlockAdapterSpec(
|
||||
blocks_attr="layers",
|
||||
forward_pattern=ForwardPattern.Pattern_3,
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
from sglang.multimodal_gen.configs.models.adapter.anima import (
|
||||
AnimaTextConditionerConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.adapter.ltx_2_connector import (
|
||||
LTX2ConnectorConfig,
|
||||
)
|
||||
@@ -10,9 +13,10 @@ from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader imp
|
||||
|
||||
|
||||
class AdapterLoader(PlainStateDictComponentLoader):
|
||||
component_names = ["connectors", "duration_head"]
|
||||
component_names = ["connectors", "duration_head", "text_conditioner"]
|
||||
|
||||
config_classes = {
|
||||
"text_conditioner": AnimaTextConditionerConfig,
|
||||
"connectors": LTX2ConnectorConfig,
|
||||
"duration_head": LTX2DurationHeadConfig,
|
||||
}
|
||||
|
||||
@@ -939,7 +939,7 @@ class AutoProcessorLoader(ComponentLoader):
|
||||
class TokenizerLoader(ComponentLoader):
|
||||
"""Loader for tokenizers."""
|
||||
|
||||
component_names = ["tokenizer", "text_tokenizer"]
|
||||
component_names = ["tokenizer", "text_tokenizer", "t5_tokenizer"]
|
||||
expected_library = "transformers"
|
||||
|
||||
def load_customized(
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Anima's learned T5-token queries over Qwen3 text states."""
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.adapter.anima import (
|
||||
AnimaTextConditionerConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention import LocalAttention
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
|
||||
|
||||
class AnimaConditionerAttention(nn.Module):
|
||||
def __init__(self, config, context_dim, cross_attention=False):
|
||||
super().__init__()
|
||||
dim = config.model_dim
|
||||
self.heads = config.num_attention_heads
|
||||
self.head_dim = dim // self.heads
|
||||
self.q_proj = nn.Linear(dim, dim, bias=False)
|
||||
self.k_proj = nn.Linear(context_dim, dim, bias=False)
|
||||
self.v_proj = nn.Linear(context_dim, dim, bias=False)
|
||||
self.o_proj = nn.Linear(dim, dim, bias=False)
|
||||
self.q_norm = nn.RMSNorm(self.head_dim, eps=1e-6)
|
||||
self.k_norm = nn.RMSNorm(self.head_dim, eps=1e-6)
|
||||
self.attn = LocalAttention(
|
||||
self.heads, self.head_dim, is_cross_attention=cross_attention
|
||||
)
|
||||
|
||||
def forward(self, x, context, mask, rope, context_rope):
|
||||
q = self.q_norm(self.q_proj(x).unflatten(-1, (self.heads, self.head_dim)))
|
||||
k = self.k_norm(self.k_proj(context).unflatten(-1, (self.heads, self.head_dim)))
|
||||
v = self.v_proj(context).unflatten(-1, (self.heads, self.head_dim))
|
||||
cos, sin = rope
|
||||
q1, q2 = q.chunk(2, -1)
|
||||
q = q * cos + torch.cat((-q2, q1), -1) * sin
|
||||
cos, sin = context_rope
|
||||
k1, k2 = k.chunk(2, -1)
|
||||
k = k * cos + torch.cat((-k2, k1), -1) * sin
|
||||
return self.o_proj(self.attn(q, k, v, attn_mask=mask).flatten(2))
|
||||
|
||||
|
||||
class AnimaConditionerBlock(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.use_self_attention = config.use_self_attention
|
||||
norm = nn.LayerNorm if config.use_layer_norm else nn.RMSNorm
|
||||
eps = 1e-5 if config.use_layer_norm else 1e-6
|
||||
dim = config.model_dim
|
||||
if self.use_self_attention:
|
||||
self.norm_self_attn = norm(dim, eps=eps)
|
||||
self.self_attn = AnimaConditionerAttention(config, dim)
|
||||
self.norm_cross_attn = norm(dim, eps=eps)
|
||||
self.cross_attn = AnimaConditionerAttention(
|
||||
config, config.source_dim, cross_attention=True
|
||||
)
|
||||
self.norm_mlp = norm(dim, eps=eps)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(dim, int(dim * config.mlp_ratio)),
|
||||
nn.GELU(),
|
||||
nn.Linear(int(dim * config.mlp_ratio), dim),
|
||||
)
|
||||
|
||||
def forward(self, x, source, target_mask, source_mask, rope, source_rope):
|
||||
if self.use_self_attention:
|
||||
normed = self.norm_self_attn(x)
|
||||
x = x + self.self_attn(normed, normed, target_mask, rope, rope)
|
||||
x = x + self.cross_attn(
|
||||
self.norm_cross_attn(x), source, source_mask, rope, source_rope
|
||||
)
|
||||
return x + self.mlp(self.norm_mlp(x))
|
||||
|
||||
|
||||
class AnimaTextConditioner(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = ["blocks"]
|
||||
|
||||
def __init__(self, config: AnimaTextConditionerConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embed = nn.Embedding(config.target_vocab_size, config.target_dim)
|
||||
self.in_proj = (
|
||||
nn.Linear(config.target_dim, config.model_dim)
|
||||
if config.target_dim != config.model_dim
|
||||
else nn.Identity()
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[AnimaConditionerBlock(config) for _ in range(config.num_layers)]
|
||||
)
|
||||
self.out_proj = nn.Linear(config.model_dim, config.target_dim)
|
||||
self.norm = nn.RMSNorm(config.target_dim, eps=1e-6)
|
||||
|
||||
def _rope(self, length, device, dtype):
|
||||
dim = self.config.model_dim // self.config.num_attention_heads
|
||||
inv = 1.0 / (10000.0 ** (torch.arange(0, dim, 2, device=device).float() / dim))
|
||||
freqs = torch.outer(torch.arange(length, device=device).float(), inv)
|
||||
freqs = torch.cat((freqs, freqs), dim=-1)[None, :, None, :]
|
||||
return freqs.cos().to(dtype), freqs.sin().to(dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
source_hidden_states,
|
||||
target_input_ids,
|
||||
target_attention_mask,
|
||||
source_attention_mask,
|
||||
):
|
||||
x = self.in_proj(self.embed(target_input_ids).to(source_hidden_states.dtype))
|
||||
rope = self._rope(x.shape[1], x.device, x.dtype)
|
||||
source_rope = self._rope(source_hidden_states.shape[1], x.device, x.dtype)
|
||||
# additive -inf preserves SDPA's zero output for fully masked rows
|
||||
target_mask = torch.zeros_like(target_attention_mask, dtype=x.dtype)
|
||||
target_mask.masked_fill_(target_attention_mask == 0, float("-inf"))
|
||||
source_mask = torch.zeros_like(source_attention_mask, dtype=x.dtype)
|
||||
source_mask.masked_fill_(source_attention_mask == 0, float("-inf"))
|
||||
for block in self.blocks:
|
||||
x = block(
|
||||
x, source_hidden_states, target_mask, source_mask, rope, source_rope
|
||||
)
|
||||
x = self.norm(self.out_proj(x)) * target_attention_mask.unsqueeze(-1).to(
|
||||
x.dtype
|
||||
)
|
||||
return F.pad(x, (0, 0, 0, max(0, self.config.min_sequence_length - x.shape[1])))
|
||||
|
||||
|
||||
EntryClass = AnimaTextConditioner
|
||||
@@ -0,0 +1,329 @@
|
||||
# Copyright 2025 The NVIDIA Team and The HuggingFace Team. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Native Cosmos Predict2 image transformer used by Anima."""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
from diffusers.models.embeddings import Timesteps
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
|
||||
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
|
||||
gather_seq,
|
||||
shard_like,
|
||||
shard_seq,
|
||||
tail_attn_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention import USPAttention
|
||||
from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
|
||||
|
||||
|
||||
def _is_anima_block(name, module):
|
||||
return is_module_list_entry_in(name, ("transformer_blocks",))
|
||||
|
||||
|
||||
class AnimaPatchEmbed(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.patch_size = config.patch_size
|
||||
channels = config.in_channels + int(config.concat_padding_mask)
|
||||
self.proj = ReplicatedLinear(
|
||||
channels * math.prod(config.patch_size), config.hidden_size, bias=False
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
b, c, t, h, w = x.shape
|
||||
pt, ph, pw = self.patch_size
|
||||
x = x.reshape(b, c, t // pt, pt, h // ph, ph, w // pw, pw)
|
||||
x = x.permute(0, 2, 4, 6, 1, 3, 5, 7).flatten(4).flatten(1, 3)
|
||||
return self.proj(x)[0]
|
||||
|
||||
|
||||
class AnimaRotaryEmbedding(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
d = config.attention_head_dim
|
||||
self.dims = (d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6))
|
||||
self.scales = config.rope_scale
|
||||
self.patch_size = config.patch_size
|
||||
|
||||
def forward(self, x):
|
||||
shape = [size // patch for size, patch in zip(x.shape[-3:], self.patch_size)]
|
||||
freqs = []
|
||||
for axis, (dim, scale) in enumerate(zip(self.dims, self.scales)):
|
||||
theta = 10000.0 * scale ** (dim / (dim - 2))
|
||||
inv = 1.0 / (
|
||||
theta ** (torch.arange(0, dim, 2, device=x.device).float() / dim)
|
||||
)
|
||||
angles = torch.outer(
|
||||
torch.arange(shape[axis], device=x.device).float(), inv
|
||||
)
|
||||
view = [1, 1, 1, dim // 2]
|
||||
view[axis] = shape[axis]
|
||||
freqs.append(angles.view(view).expand(*shape, dim // 2))
|
||||
angles = torch.cat(freqs * 2, dim=-1).flatten(0, 2)
|
||||
return angles.cos(), angles.sin()
|
||||
|
||||
|
||||
class AnimaTimeEmbedding(nn.Module):
|
||||
def __init__(self, hidden_size):
|
||||
super().__init__()
|
||||
self.time_proj = Timesteps(
|
||||
hidden_size, flip_sin_to_cos=True, downscale_freq_shift=0
|
||||
)
|
||||
self.t_embedder = nn.Module()
|
||||
self.t_embedder.linear_1 = ReplicatedLinear(
|
||||
hidden_size, hidden_size, bias=False
|
||||
)
|
||||
self.t_embedder.linear_2 = ReplicatedLinear(
|
||||
hidden_size, 3 * hidden_size, bias=False
|
||||
)
|
||||
self.norm = RMSNorm(hidden_size, eps=1e-6)
|
||||
|
||||
def forward(self, timestep, dtype):
|
||||
x = self.time_proj(timestep).to(dtype)
|
||||
emb = self.t_embedder.linear_1(x)[0]
|
||||
emb = self.t_embedder.linear_2(torch.nn.functional.silu(emb))[0]
|
||||
return emb, self.norm(x)
|
||||
|
||||
|
||||
class AnimaAdaLayerNorm(nn.Module):
|
||||
def __init__(self, hidden_size, rank, gated=True):
|
||||
super().__init__()
|
||||
self.gated = gated
|
||||
self.norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
|
||||
self.linear_1 = ReplicatedLinear(hidden_size, rank, bias=False)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
rank, (3 if gated else 2) * hidden_size, bias=False
|
||||
)
|
||||
|
||||
def forward(self, x, embedded_timestep, temb):
|
||||
modulation = self.linear_1(torch.nn.functional.silu(embedded_timestep))[0]
|
||||
modulation = self.linear_2(modulation)[0]
|
||||
modulation = modulation + temb[..., : modulation.shape[-1]]
|
||||
values = modulation.unsqueeze(1).chunk(3 if self.gated else 2, dim=-1)
|
||||
out = self.norm(x) * (1 + values[1]) + values[0]
|
||||
return (out, values[2]) if self.gated else out
|
||||
|
||||
|
||||
class AnimaAttention(nn.Module):
|
||||
def __init__(self, config, cross_attention, quant_config, prefix):
|
||||
super().__init__()
|
||||
self.heads = config.num_attention_heads // get_tp_world_size()
|
||||
self.head_dim = config.attention_head_dim
|
||||
size = config.hidden_size
|
||||
context_size = config.text_embed_dim if cross_attention else size
|
||||
self.to_q = ColumnParallelLinear(
|
||||
size, size, bias=False, quant_config=quant_config, prefix=f"{prefix}.to_q"
|
||||
)
|
||||
self.to_k = ColumnParallelLinear(
|
||||
context_size,
|
||||
size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.to_k",
|
||||
)
|
||||
self.to_v = ColumnParallelLinear(
|
||||
context_size,
|
||||
size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.to_v",
|
||||
)
|
||||
self.to_out = nn.ModuleList(
|
||||
[
|
||||
RowParallelLinear(
|
||||
size,
|
||||
size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.to_out.0",
|
||||
)
|
||||
]
|
||||
)
|
||||
self.norm_q = RMSNorm(self.head_dim, eps=1e-6)
|
||||
self.norm_k = RMSNorm(self.head_dim, eps=1e-6)
|
||||
self.attn = USPAttention(
|
||||
self.heads,
|
||||
self.head_dim,
|
||||
is_cross_attention=cross_attention,
|
||||
skip_sequence_parallel=cross_attention,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def forward(self, x, context=None, rope=None, attn_mask_meta=None):
|
||||
context = x if context is None else context
|
||||
q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
k = self.to_k(context)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
v = self.to_v(context)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
q, k = self.norm_q(q), self.norm_k(k)
|
||||
if rope is not None:
|
||||
cos, sin = (r[None, :, None, :] for r in rope)
|
||||
# Cosmos uses split-half RoPE in fp32, then rounds once
|
||||
q1, q2 = q.chunk(2, dim=-1)
|
||||
k1, k2 = k.chunk(2, dim=-1)
|
||||
q = (q.float() * cos + torch.cat((-q2, q1), -1).float() * sin).to(q.dtype)
|
||||
k = (k.float() * cos + torch.cat((-k2, k1), -1).float() * sin).to(k.dtype)
|
||||
out = self.attn(q, k, v, attn_mask_meta=attn_mask_meta).flatten(2)
|
||||
return self.to_out[0](out)[0]
|
||||
|
||||
|
||||
class AnimaFeedForward(nn.Module):
|
||||
def __init__(self, config, quant_config, prefix):
|
||||
super().__init__()
|
||||
size = config.hidden_size
|
||||
inner = int(size * config.mlp_ratio)
|
||||
projection = nn.Module()
|
||||
projection.proj = ColumnParallelLinear(
|
||||
size,
|
||||
inner,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.net.0.proj",
|
||||
)
|
||||
self.net = nn.ModuleList(
|
||||
[
|
||||
projection,
|
||||
nn.Identity(),
|
||||
RowParallelLinear(
|
||||
inner,
|
||||
size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.net.2",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x = torch.nn.functional.gelu(self.net[0].proj(x)[0])
|
||||
return self.net[2](x)[0]
|
||||
|
||||
|
||||
class AnimaTransformerBlock(nn.Module):
|
||||
def __init__(self, config, quant_config, prefix):
|
||||
super().__init__()
|
||||
size, rank = config.hidden_size, config.adaln_lora_dim
|
||||
self.norm1 = AnimaAdaLayerNorm(size, rank)
|
||||
self.attn1 = AnimaAttention(config, False, quant_config, f"{prefix}.attn1")
|
||||
self.norm2 = AnimaAdaLayerNorm(size, rank)
|
||||
self.attn2 = AnimaAttention(config, True, quant_config, f"{prefix}.attn2")
|
||||
self.norm3 = AnimaAdaLayerNorm(size, rank)
|
||||
self.ff = AnimaFeedForward(config, quant_config, f"{prefix}.ff")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
embedded_timestep,
|
||||
temb,
|
||||
rope,
|
||||
attn_mask_meta=None,
|
||||
):
|
||||
x, gate = self.norm1(hidden_states, embedded_timestep, temb)
|
||||
hidden_states = hidden_states + gate * self.attn1(
|
||||
x, rope=rope, attn_mask_meta=attn_mask_meta
|
||||
)
|
||||
x, gate = self.norm2(hidden_states, embedded_timestep, temb)
|
||||
hidden_states = hidden_states + gate * self.attn2(
|
||||
x, context=encoder_hidden_states
|
||||
)
|
||||
x, gate = self.norm3(hidden_states, embedded_timestep, temb)
|
||||
return hidden_states + gate * self.ff(x)
|
||||
|
||||
|
||||
class AnimaTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
_aliases = ["CosmosTransformer3DModel"]
|
||||
_fsdp_shard_conditions = [_is_anima_block]
|
||||
_compile_conditions = [_is_anima_block]
|
||||
layer_names = ["transformer_blocks"]
|
||||
param_names_mapping = {}
|
||||
reverse_param_names_mapping = {}
|
||||
|
||||
def __init__(self, config: AnimaDiTConfig, hf_config, quant_config=None, **kwargs):
|
||||
super().__init__(config=config, hf_config=hf_config)
|
||||
arch = self.config
|
||||
if (
|
||||
arch.extra_pos_embed_type
|
||||
or arch.use_crossattn_projection
|
||||
or arch.img_context_dim_in
|
||||
):
|
||||
raise ValueError(
|
||||
"Only Anima's Cosmos image transformer architecture is supported"
|
||||
)
|
||||
if arch.num_attention_heads % get_tp_world_size():
|
||||
raise ValueError("Anima attention heads must be divisible by TP size")
|
||||
self.hidden_size = arch.hidden_size
|
||||
self.num_attention_heads = arch.num_attention_heads
|
||||
self.num_channels_latents = arch.in_channels
|
||||
self.patch_embed = AnimaPatchEmbed(arch)
|
||||
self.rope = AnimaRotaryEmbedding(arch)
|
||||
self.time_embed = AnimaTimeEmbedding(arch.hidden_size)
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
AnimaTransformerBlock(arch, quant_config, f"transformer_blocks.{i}")
|
||||
for i in range(arch.num_layers)
|
||||
]
|
||||
)
|
||||
self.norm_out = AnimaAdaLayerNorm(
|
||||
arch.hidden_size, arch.adaln_lora_dim, gated=False
|
||||
)
|
||||
self.proj_out = ReplicatedLinear(
|
||||
arch.hidden_size, math.prod(arch.patch_size) * arch.out_channels, bias=False
|
||||
)
|
||||
self.__post_init__()
|
||||
|
||||
def forward(self, hidden_states, encoder_hidden_states, timestep, **kwargs):
|
||||
encoder_hidden_states = encoder_hidden_states.to(hidden_states.dtype)
|
||||
b, _, t, h, w = hidden_states.shape
|
||||
pt, ph, pw = self.config.patch_size
|
||||
if t % pt or h % ph or w % pw:
|
||||
raise ValueError(
|
||||
"Anima latent dimensions must be divisible by the patch size"
|
||||
)
|
||||
rope = self.rope(hidden_states)
|
||||
if self.config.concat_padding_mask:
|
||||
hidden_states = torch.cat(
|
||||
[hidden_states, hidden_states.new_zeros(b, 1, t, h, w)], dim=1
|
||||
)
|
||||
hidden_states = self.patch_embed(hidden_states)
|
||||
hidden_states, shard = shard_seq(hidden_states)
|
||||
rope = tuple(shard_like(r, shard, dim=0) for r in rope)
|
||||
attn_mask_meta = tail_attn_meta(shard, b, hidden_states.device)
|
||||
temb, embedded_timestep = self.time_embed(
|
||||
timestep.to(hidden_states.dtype) / 1000, hidden_states.dtype
|
||||
)
|
||||
for block in self.transformer_blocks:
|
||||
hidden_states = block(
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
embedded_timestep,
|
||||
temb,
|
||||
rope,
|
||||
attn_mask_meta,
|
||||
)
|
||||
hidden_states = self.norm_out(hidden_states, embedded_timestep, temb)
|
||||
hidden_states = self.proj_out(hidden_states)[0]
|
||||
hidden_states = gather_seq(hidden_states, shard.orig_len)
|
||||
# checkpoint output ordering is (ph, pw, pt, c), unlike the input patches
|
||||
hidden_states = hidden_states.reshape(
|
||||
b, t // pt, h // ph, w // pw, ph, pw, pt, -1
|
||||
)
|
||||
return hidden_states.permute(0, 7, 1, 6, 2, 4, 3, 5).reshape(
|
||||
b, self.config.out_channels, t, h, w
|
||||
)
|
||||
|
||||
|
||||
EntryClass = AnimaTransformer3DModel
|
||||
@@ -215,6 +215,10 @@ class Qwen3Attention(nn.Module):
|
||||
k_item = k[batch_index : batch_index + 1]
|
||||
v_item = v[batch_index : batch_index + 1]
|
||||
|
||||
if valid_len == 0:
|
||||
outputs.append(torch.zeros_like(q_item))
|
||||
continue
|
||||
|
||||
real_output = self.attn(
|
||||
q_item[:, :valid_len],
|
||||
k_item[:, :valid_len],
|
||||
@@ -323,6 +327,8 @@ class Qwen3ForCausalLM(TextEncoder):
|
||||
- FSDP sharding for CPU offload
|
||||
"""
|
||||
|
||||
_aliases = ["Qwen3Model"]
|
||||
|
||||
def __init__(self, config: Qwen3TextConfig) -> None:
|
||||
super().__init__(config)
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
|
||||
InputValidationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.anima import (
|
||||
AnimaTextConditioningStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
|
||||
|
||||
class AnimaPipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
pipeline_name = "AnimaModularPipeline"
|
||||
_required_config_modules = [
|
||||
"text_encoder",
|
||||
"tokenizer",
|
||||
"t5_tokenizer",
|
||||
"text_conditioner",
|
||||
"vae",
|
||||
"transformer",
|
||||
"scheduler",
|
||||
]
|
||||
|
||||
def create_pipeline_stages(self, server_args):
|
||||
self.add_stage(InputValidationStage())
|
||||
self.add_stage(
|
||||
TextEncodingStage(
|
||||
text_encoders=[self.get_module("text_encoder")],
|
||||
tokenizers=[self.get_module("tokenizer")],
|
||||
)
|
||||
)
|
||||
self.add_stage(
|
||||
AnimaTextConditioningStage(
|
||||
self.get_module("text_conditioner"), self.get_module("t5_tokenizer")
|
||||
)
|
||||
)
|
||||
self.add_standard_timestep_preparation_stage()
|
||||
self.add_standard_latent_preparation_stage()
|
||||
self.add_standard_denoising_stage()
|
||||
self.add_standard_decoding_stage()
|
||||
|
||||
|
||||
EntryClass = AnimaPipeline
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.condition_encoding import (
|
||||
ConditionEncodingStage,
|
||||
)
|
||||
|
||||
|
||||
class AnimaTextConditioningStage(ConditionEncodingStage):
|
||||
def __init__(self, conditioner, tokenizer):
|
||||
super().__init__()
|
||||
self.conditioner = conditioner
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def component_uses(self, server_args, stage_name=None):
|
||||
return [
|
||||
ComponentUse(self._component_stage_name(stage_name), "text_conditioner")
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch, server_args):
|
||||
prompts = batch.prompt if isinstance(batch.prompt, list) else [batch.prompt]
|
||||
max_length = batch.max_sequence_length or 512
|
||||
if not 1 <= max_length <= 4096:
|
||||
raise ValueError("Anima max_sequence_length must be between 1 and 4096")
|
||||
with self.use_declared_component(
|
||||
component_name="text_conditioner", module=self.conditioner
|
||||
) as conditioner:
|
||||
self.conditioner = conditioner
|
||||
batch.prompt_embeds = [
|
||||
self._condition(
|
||||
conditioner,
|
||||
prompts,
|
||||
batch.prompt_embeds[0],
|
||||
batch.prompt_attention_mask[0],
|
||||
max_length,
|
||||
)
|
||||
]
|
||||
if batch.do_classifier_free_guidance:
|
||||
negative = [batch.negative_prompt] * len(prompts)
|
||||
batch.negative_prompt_embeds = [
|
||||
self._condition(
|
||||
conditioner,
|
||||
negative,
|
||||
batch.negative_prompt_embeds[0],
|
||||
batch.negative_attention_mask[0],
|
||||
max_length,
|
||||
)
|
||||
]
|
||||
# Cosmos attends to conditioner padding; additional BCG padding is not valid
|
||||
batch.prompt_attention_mask = None
|
||||
batch.negative_attention_mask = None
|
||||
batch.prompt_embeds_mask = None
|
||||
batch.negative_prompt_embeds_mask = None
|
||||
batch.prompt_seq_lens = [
|
||||
[batch.prompt_embeds[0].shape[1]] * batch.prompt_embeds[0].shape[0]
|
||||
]
|
||||
if batch.do_classifier_free_guidance:
|
||||
batch.negative_prompt_seq_lens = [
|
||||
[batch.negative_prompt_embeds[0].shape[1]]
|
||||
* batch.negative_prompt_embeds[0].shape[0]
|
||||
]
|
||||
return batch
|
||||
|
||||
def _condition(self, conditioner, prompts, embeds, source_mask, max_length):
|
||||
tokens = self.tokenizer(
|
||||
prompts,
|
||||
padding="longest",
|
||||
truncation=True,
|
||||
max_length=max_length,
|
||||
return_tensors="pt",
|
||||
).to(embeds.device)
|
||||
embeds = embeds.to(dtype=next(conditioner.parameters()).dtype)
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
return conditioner(
|
||||
embeds, tokens.input_ids, tokens.attention_mask, source_mask
|
||||
)
|
||||
@@ -190,7 +190,9 @@ DEFAULT_BCG_TEXT_BUCKETS = (64, 128, 256, 512, 1024)
|
||||
|
||||
BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset(
|
||||
{
|
||||
"anima-base-v1.0-diffusers",
|
||||
"black-forest-labs/flux.1-dev",
|
||||
"circlestone-labs/anima-base-v1.0-diffusers",
|
||||
"comfy-org/ideogram-4",
|
||||
"efficient-large-model/sana1.5_1.6b_1024px_diffusers",
|
||||
"efficient-large-model/sana-video_2b_480p_diffusers",
|
||||
@@ -237,6 +239,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset(
|
||||
|
||||
BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS = frozenset(
|
||||
{
|
||||
"AnimaPipelineConfig",
|
||||
"FluxPipelineConfig",
|
||||
"GlmImagePipelineConfig",
|
||||
"Ideogram4PipelineConfig",
|
||||
@@ -793,7 +796,7 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"[Diffusion BCG] disabled for %s: only FLUX.1-dev, Ideogram-4, "
|
||||
"[Diffusion BCG] disabled for %s: only Anima Base v1.0, FLUX.1-dev, Ideogram-4, "
|
||||
"jdopensource/JoyAI-Echo, Lightricks/LTX-2, LongCat-Image, "
|
||||
"MiniMax-H3, Qwen/Qwen-Image, Qwen/Qwen-Image-2512, "
|
||||
"Qwen/Qwen-Image-2.1, SANA1.5, "
|
||||
|
||||
@@ -149,13 +149,12 @@ def _is_weight_bearing_diffusers_component(key: str, value: Any) -> bool:
|
||||
|
||||
|
||||
def _get_declared_weight_component_dirs(model_path: str) -> list[str]:
|
||||
model_index_path = os.path.join(model_path, "model_index.json")
|
||||
model_index_path = _local_model_index_path(model_path)
|
||||
if not os.path.exists(model_index_path):
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(model_index_path) as f:
|
||||
model_index = json.load(f)
|
||||
model_index = _read_model_index(model_index_path)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to read model_index.json at %s: %s", model_index_path, exc
|
||||
@@ -374,13 +373,12 @@ def _ci_validate_diffusers_model(model_path: str) -> tuple[bool, bool]:
|
||||
|
||||
def _verify_diffusers_model_complete(path: str) -> bool:
|
||||
"""Check if a diffusers model directory has all required component subdirectories."""
|
||||
config_path = os.path.join(path, "model_index.json")
|
||||
config_path = _local_model_index_path(path)
|
||||
if not os.path.exists(config_path):
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(config_path) as config_file:
|
||||
model_index = json.load(config_file)
|
||||
model_index = _read_model_index(config_path)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read model_index.json at %s: %s", config_path, exc)
|
||||
return False
|
||||
@@ -683,6 +681,29 @@ def maybe_download_lora(
|
||||
return target
|
||||
|
||||
|
||||
def _local_model_index_path(model_path: str) -> str:
|
||||
standard = os.path.join(model_path, "model_index.json")
|
||||
if os.path.isfile(standard):
|
||||
return standard
|
||||
return os.path.join(model_path, "modular_model_index.json")
|
||||
|
||||
|
||||
def _read_model_index(path: str) -> dict[str, Any]:
|
||||
with open(path) as f:
|
||||
config = json.load(f)
|
||||
if os.path.basename(path) == "modular_model_index.json":
|
||||
config.pop("_blocks_class_name", None)
|
||||
for name, spec in config.items():
|
||||
if isinstance(spec, list) and len(spec) == 3 and isinstance(spec[2], dict):
|
||||
# native loaders consume the component directories in this snapshot
|
||||
if spec[2].get("subfolder", name) != name:
|
||||
raise ValueError(
|
||||
f"Modular component {name!r} must use its own subfolder"
|
||||
)
|
||||
config[name] = spec[:2]
|
||||
return config
|
||||
|
||||
|
||||
def verify_model_config_and_directory(model_path: str) -> dict[str, Any]:
|
||||
"""
|
||||
Verify that the model directory contains a valid diffusers configuration.
|
||||
@@ -695,16 +716,16 @@ def verify_model_config_and_directory(model_path: str) -> dict[str, Any]:
|
||||
"""
|
||||
|
||||
# Check for model_index.json which is required for diffusers models
|
||||
config_path = os.path.join(model_path, "model_index.json")
|
||||
config_path = _local_model_index_path(model_path)
|
||||
if not os.path.exists(config_path):
|
||||
raise ValueError(
|
||||
f"Model directory {model_path} does not contain model_index.json. "
|
||||
f"Model directory {model_path} does not contain model_index.json "
|
||||
"or modular_model_index.json. "
|
||||
"Only HuggingFace diffusers format is supported."
|
||||
)
|
||||
|
||||
# Load the config
|
||||
with open(config_path) as f:
|
||||
config = json.load(f)
|
||||
config = _read_model_index(config_path)
|
||||
|
||||
# Verify diffusers version exists
|
||||
if "_diffusers_version" not in config:
|
||||
@@ -745,26 +766,37 @@ def verify_model_config_and_directory(model_path: str) -> dict[str, Any]:
|
||||
return cast(dict[str, Any], config)
|
||||
|
||||
|
||||
def _resolve_remote_repo_model_index_path(model_name_or_path: str) -> str:
|
||||
def _resolve_remote_repo_model_index_path(
|
||||
model_name_or_path: str, filename: str = "model_index.json"
|
||||
) -> str:
|
||||
"""Return a local path to a remote repo's ``model_index.json``"""
|
||||
try:
|
||||
# Cache-aware: no local_dir, so the selected Hub reuses its cache and
|
||||
# revalidates the remote file when online.
|
||||
return hf_hub_download(repo_id=model_name_or_path, filename="model_index.json")
|
||||
return hf_hub_download(repo_id=model_name_or_path, filename=filename)
|
||||
except EntryNotFoundError:
|
||||
# Repo exists but has no model_index.json (single-model repo); let the
|
||||
# caller fall through to the single-model path.
|
||||
if filename == "model_index.json":
|
||||
return _resolve_remote_repo_model_index_path(
|
||||
model_name_or_path, "modular_model_index.json"
|
||||
)
|
||||
raise
|
||||
except Exception as online_err:
|
||||
cached_path = None
|
||||
if not envs.SGLANG_USE_MODELSCOPE.get():
|
||||
from huggingface_hub import try_to_load_from_cache
|
||||
|
||||
cached = try_to_load_from_cache(
|
||||
repo_id=model_name_or_path, filename="model_index.json"
|
||||
filenames = (
|
||||
(filename, "modular_model_index.json")
|
||||
if filename == "model_index.json"
|
||||
else (filename,)
|
||||
)
|
||||
if isinstance(cached, str) and os.path.exists(cached):
|
||||
cached_path = cached
|
||||
for candidate in filenames:
|
||||
cached = try_to_load_from_cache(
|
||||
repo_id=model_name_or_path, filename=candidate
|
||||
)
|
||||
if isinstance(cached, str) and os.path.exists(cached):
|
||||
cached_path = cached
|
||||
break
|
||||
if cached_path is not None:
|
||||
logger.warning(
|
||||
"Could not fetch model_index.json for '%s' from the Hugging Face "
|
||||
@@ -815,8 +847,7 @@ def maybe_download_model_index(model_name_or_path: str) -> dict[str, Any]:
|
||||
model_index_path = _resolve_remote_repo_model_index_path(model_name_or_path)
|
||||
|
||||
# Load the model_index.json
|
||||
with open(model_index_path) as f:
|
||||
config: dict[str, Any] = json.load(f)
|
||||
config = _read_model_index(model_index_path)
|
||||
|
||||
# Verify it has the required fields
|
||||
if "_class_name" not in config:
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Opt-in HTTP validation: SGLANG_ANIMA_TEST_MODEL=<HF id or local checkpoint>."""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from openai import OpenAI
|
||||
from PIL import Image
|
||||
|
||||
from sglang.multimodal_gen.test.server.test_server_utils import ServerManager
|
||||
from sglang.multimodal_gen.test.test_utils import get_dynamic_server_port
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("SGLANG_ANIMA_TEST_MODEL"),
|
||||
reason="set SGLANG_ANIMA_TEST_MODEL for full-checkpoint HTTP tests",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
manager = ServerManager(
|
||||
model=os.environ["SGLANG_ANIMA_TEST_MODEL"],
|
||||
port=get_dynamic_server_port(),
|
||||
extra_args="--num-gpus 1 --performance-mode speed --attention-backend fa",
|
||||
)
|
||||
context = manager.start()
|
||||
try:
|
||||
yield context
|
||||
finally:
|
||||
context.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size,steps,outputs", [(512, 4, 2), (1024, 30, 1)])
|
||||
def test_repeated_generation(server, size, steps, outputs):
|
||||
with OpenAI(
|
||||
api_key="EMPTY", base_url=f"http://127.0.0.1:{server.port}/v1"
|
||||
) as client:
|
||||
requests = []
|
||||
for _ in range(2):
|
||||
result = client.images.generate(
|
||||
model=server.model,
|
||||
prompt="masterpiece, best quality, safe, watercolor landscape, a quiet seaside village at sunset",
|
||||
size=f"{size}x{size}",
|
||||
n=outputs,
|
||||
response_format="b64_json",
|
||||
extra_body={
|
||||
"seed": 42,
|
||||
"num_inference_steps": steps,
|
||||
"guidance_scale": 4.0,
|
||||
"negative_prompt": "",
|
||||
"generator_device": "cpu",
|
||||
"output_format": "png",
|
||||
},
|
||||
)
|
||||
assert len(result.data) == outputs
|
||||
images = []
|
||||
for item in result.data:
|
||||
with Image.open(io.BytesIO(base64.b64decode(item.b64_json))) as image:
|
||||
assert image.size == (size, size)
|
||||
pixels = np.asarray(image.convert("RGB"))
|
||||
assert pixels.std() > 5, "degenerate image"
|
||||
images.append(pixels)
|
||||
requests.append(images)
|
||||
for first, second in zip(*requests):
|
||||
np.testing.assert_array_equal(first, second)
|
||||
if outputs > 1:
|
||||
assert not np.array_equal(requests[0][0], requests[0][1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import json
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers.models.transformers.transformer_cosmos import CosmosRotaryPosEmbed
|
||||
from transformers import BatchEncoding
|
||||
|
||||
from sglang.cli.utils import get_is_diffusion_model
|
||||
from sglang.multimodal_gen.configs.models.dits.anima import AnimaArchConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.anima import AnimaPipelineConfig
|
||||
from sglang.multimodal_gen.configs.sample.anima import AnimaSamplingParams
|
||||
from sglang.multimodal_gen.registry import get_model_info
|
||||
from sglang.multimodal_gen.runtime.breakable_cuda_graph.prompt_padding import (
|
||||
pad_masked_prompt_kwargs,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.anima import AnimaRotaryEmbedding
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen3 import Qwen3Attention
|
||||
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
|
||||
from sglang.multimodal_gen.runtime.pipelines.anima_pipeline import AnimaPipeline
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.anima import (
|
||||
AnimaTextConditioningStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils import hf_diffusers_utils
|
||||
|
||||
|
||||
def test_registry_recognizes_official_and_local_anima(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
lambda _: {"_class_name": "AnimaModularPipeline"},
|
||||
)
|
||||
get_model_info.cache_clear()
|
||||
for model in ("circlestone-labs/Anima-Base-v1.0-Diffusers", "/models/custom-anima"):
|
||||
info = get_model_info(model)
|
||||
assert info.pipeline_cls is AnimaPipeline
|
||||
assert info.pipeline_config_cls is AnimaPipelineConfig
|
||||
assert info.sampling_param_cls is AnimaSamplingParams
|
||||
get_model_info.cache_clear()
|
||||
assert (
|
||||
ModelRegistry.resolve_model_cls("Qwen3Model")[0].__name__ == "Qwen3ForCausalLM"
|
||||
)
|
||||
assert (
|
||||
ModelRegistry.resolve_model_cls("AnimaTextConditioner")[0].__name__
|
||||
== "AnimaTextConditioner"
|
||||
)
|
||||
|
||||
|
||||
def test_empty_prompt_keeps_a_masked_qwen_token():
|
||||
def tokenizer(prompts, **kwargs):
|
||||
assert kwargs["padding"] == "longest"
|
||||
return BatchEncoding(
|
||||
{
|
||||
"input_ids": torch.empty(len(prompts), 0, dtype=torch.long),
|
||||
"attention_mask": torch.empty(len(prompts), 0, dtype=torch.long),
|
||||
}
|
||||
)
|
||||
|
||||
inputs = AnimaPipelineConfig().tokenize_prompt([""], tokenizer, {})
|
||||
assert inputs.input_ids.shape == (1, 1)
|
||||
assert not inputs.attention_mask.any()
|
||||
|
||||
|
||||
def test_breakable_cuda_graph_stays_enabled_for_anima():
|
||||
args = SimpleNamespace(
|
||||
model_id="circlestone-labs/Anima-Base-v1.0-Diffusers",
|
||||
model_path="/models/Anima-Base-v1.0-Diffusers",
|
||||
enable_breakable_cuda_graph=True,
|
||||
pipeline_config=AnimaPipelineConfig(),
|
||||
warmup_resolutions=["512x512"],
|
||||
)
|
||||
args._is_breakable_cuda_graph_supported_model = lambda: (
|
||||
ServerArgs._is_breakable_cuda_graph_supported_model(args)
|
||||
)
|
||||
ServerArgs._adjust_breakable_cuda_graph_support(args)
|
||||
assert args.enable_breakable_cuda_graph
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [512, 600])
|
||||
def test_conditioning_does_not_allow_extra_bcg_padding(length):
|
||||
embeds = torch.randn(1, length, 8)
|
||||
mask = torch.ones(1, length, dtype=torch.bool)
|
||||
batch = SimpleNamespace(
|
||||
prompt="landscape",
|
||||
negative_prompt="",
|
||||
max_sequence_length=1024,
|
||||
prompt_embeds=[embeds],
|
||||
negative_prompt_embeds=[embeds],
|
||||
prompt_attention_mask=[mask],
|
||||
negative_attention_mask=[mask],
|
||||
prompt_embeds_mask=[mask],
|
||||
negative_prompt_embeds_mask=[mask],
|
||||
do_classifier_free_guidance=True,
|
||||
)
|
||||
stage = SimpleNamespace(
|
||||
conditioner=None,
|
||||
use_declared_component=lambda **kwargs: nullcontext(None),
|
||||
_condition=lambda *args: embeds,
|
||||
)
|
||||
AnimaTextConditioningStage.forward(stage, batch, None)
|
||||
for masks in (batch.prompt_embeds_mask, batch.negative_prompt_embeds_mask):
|
||||
kwargs = {"encoder_hidden_states": embeds, "encoder_hidden_states_mask": masks}
|
||||
assert pad_masked_prompt_kwargs(kwargs, (1024,)) is kwargs
|
||||
assert batch.prompt_seq_lens == batch.negative_prompt_seq_lens == [[length]]
|
||||
|
||||
|
||||
def test_qwen_all_masked_row_never_calls_attention_with_empty_kv():
|
||||
def attention(*args):
|
||||
raise AssertionError("all-masked rows have no valid KV")
|
||||
|
||||
module = SimpleNamespace(attn=attention)
|
||||
q = torch.randn(1, 1, 2, 8)
|
||||
out = Qwen3Attention._masked_causal_attention(module, q, q, q, (0,))
|
||||
torch.testing.assert_close(out, torch.zeros_like(q))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("height,width", [(4, 6), (16, 16), (10, 14)])
|
||||
def test_rope_matches_cosmos(height, width):
|
||||
arch = AnimaArchConfig()
|
||||
x = torch.zeros(1, 16, 1, height, width)
|
||||
actual = AnimaRotaryEmbedding(arch)(x)
|
||||
reference = CosmosRotaryPosEmbed(
|
||||
arch.attention_head_dim,
|
||||
max_size=arch.max_size,
|
||||
patch_size=arch.patch_size,
|
||||
rope_scale=arch.rope_scale,
|
||||
)(x)
|
||||
for a, b in zip(actual, reference):
|
||||
torch.testing.assert_close(a, b, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_latents_and_conditioning_preserve_sample_batch():
|
||||
config = AnimaPipelineConfig()
|
||||
config.vae_config.post_init()
|
||||
batch = SimpleNamespace(
|
||||
height=512,
|
||||
width=768,
|
||||
num_outputs_per_prompt=2,
|
||||
prompt_embeds=[torch.randn(2, 512, 8)],
|
||||
negative_prompt_embeds=[torch.randn(2, 512, 8)],
|
||||
do_classifier_free_guidance=True,
|
||||
)
|
||||
original = batch.prompt_embeds[0].clone()
|
||||
assert config.prepare_latent_shape(batch, 4, 1) == (4, 16, 1, 64, 96)
|
||||
assert config.get_latent_dtype(torch.bfloat16) == torch.float32
|
||||
config.expand_conditioning_to_sample_batch(batch)
|
||||
torch.testing.assert_close(batch.prompt_embeds[0], original.repeat_interleave(2, 0))
|
||||
assert batch.negative_prompt_embeds[0].shape == (4, 512, 8)
|
||||
latents = torch.randn(4, 16, 1, 64, 96)
|
||||
value, sharded = config.shard_latents_for_sp(batch, latents)
|
||||
assert value is latents and not sharded
|
||||
|
||||
|
||||
def test_modular_index_uses_existing_component_loaders(tmp_path, monkeypatch):
|
||||
index = {
|
||||
"_class_name": "AnimaModularPipeline",
|
||||
"_blocks_class_name": "AnimaAutoBlocks",
|
||||
"_diffusers_version": "0.39.0.dev0",
|
||||
"transformer": [
|
||||
"diffusers",
|
||||
"CosmosTransformer3DModel",
|
||||
{"subfolder": "transformer"},
|
||||
],
|
||||
"t5_tokenizer": ["transformers", "T5Tokenizer", {"subfolder": "t5_tokenizer"}],
|
||||
}
|
||||
path = tmp_path / "modular_model_index.json"
|
||||
path.write_text(json.dumps(index))
|
||||
assert get_is_diffusion_model(str(tmp_path))
|
||||
(tmp_path / "transformer").mkdir()
|
||||
(tmp_path / "transformer" / "diffusion_pytorch_model.safetensors").touch()
|
||||
(tmp_path / "t5_tokenizer").mkdir()
|
||||
config = hf_diffusers_utils.verify_model_config_and_directory(str(tmp_path))
|
||||
assert config["transformer"] == index["transformer"][:2]
|
||||
assert "_blocks_class_name" not in config
|
||||
assert hf_diffusers_utils._verify_diffusers_model_complete(str(tmp_path))
|
||||
calls = []
|
||||
|
||||
def download(repo_id, filename):
|
||||
calls.append(filename)
|
||||
if filename != "modular_model_index.json":
|
||||
raise hf_diffusers_utils.EntryNotFoundError("not found")
|
||||
return str(path)
|
||||
|
||||
monkeypatch.setattr(hf_diffusers_utils, "hf_hub_download", download)
|
||||
remote = hf_diffusers_utils.maybe_download_model_index("test/anima")
|
||||
assert remote["transformer"] == config["transformer"]
|
||||
assert calls[-2:] == ["model_index.json", "modular_model_index.json"]
|
||||
(tmp_path / "model_index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"_class_name": "Legacy",
|
||||
"_diffusers_version": "0.37.0",
|
||||
"transformer": index["transformer"][:2],
|
||||
}
|
||||
)
|
||||
)
|
||||
assert (
|
||||
hf_diffusers_utils.verify_model_config_and_directory(str(tmp_path))[
|
||||
"_class_name"
|
||||
]
|
||||
== "Legacy"
|
||||
)
|
||||
|
||||
|
||||
def test_modular_index_works_from_offline_cache(tmp_path, monkeypatch):
|
||||
path = tmp_path / "modular_model_index.json"
|
||||
path.write_text("{}")
|
||||
|
||||
def offline(**kwargs):
|
||||
raise hf_diffusers_utils.RequestsConnectionError("offline")
|
||||
|
||||
monkeypatch.setattr(hf_diffusers_utils, "hf_hub_download", offline)
|
||||
monkeypatch.setattr(
|
||||
"huggingface_hub.try_to_load_from_cache",
|
||||
lambda repo_id, filename: (
|
||||
str(path) if filename == "modular_model_index.json" else None
|
||||
),
|
||||
)
|
||||
assert hf_diffusers_utils._resolve_remote_repo_model_index_path(
|
||||
"test/anima"
|
||||
) == str(path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Small, checkpoint-free numerical contracts against the Cosmos reference."""
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers.models.transformers.transformer_cosmos import CosmosTransformer3DModel
|
||||
|
||||
from sglang.multimodal_gen.configs.models.adapter.anima import (
|
||||
AnimaTextConditionerArchConfig,
|
||||
AnimaTextConditionerConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.anima import AnimaPipelineConfig
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
cleanup_dist_env_and_memory,
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.adapter.anima import AnimaTextConditioner
|
||||
from sglang.multimodal_gen.runtime.models.dits.anima import AnimaTransformer3DModel
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs, server_args
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def cleanup_owned_process_group():
|
||||
initialized = torch.distributed.is_initialized()
|
||||
yield
|
||||
if not initialized:
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("batch_size", [1, 2])
|
||||
@torch.no_grad()
|
||||
def test_transformer_matches_cosmos(dtype, batch_size, monkeypatch):
|
||||
kwargs = dict(
|
||||
in_channels=4,
|
||||
out_channels=4,
|
||||
num_attention_heads=4,
|
||||
attention_head_dim=32,
|
||||
num_layers=2,
|
||||
mlp_ratio=2,
|
||||
text_embed_dim=32,
|
||||
adaln_lora_dim=16,
|
||||
patch_size=(1, 2, 2),
|
||||
max_size=(128, 240, 240),
|
||||
rope_scale=(1.0, 4.0, 4.0),
|
||||
concat_padding_mask=True,
|
||||
extra_pos_embed_type=None,
|
||||
use_crossattn_projection=False,
|
||||
)
|
||||
config = AnimaDiTConfig()
|
||||
config.update_model_arch(kwargs)
|
||||
args = ServerArgs(
|
||||
model_path="circlestone-labs/Anima-Base-v1.0-Diffusers",
|
||||
num_gpus=1,
|
||||
pipeline_config=AnimaPipelineConfig(dit_config=config),
|
||||
attention_backend="torch_sdpa",
|
||||
)
|
||||
monkeypatch.setattr(server_args, "_global_server_args", args)
|
||||
ensure_distributed_env_defaults()
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
torch.manual_seed(42)
|
||||
reference = CosmosTransformer3DModel(**kwargs).cuda().to(dtype).eval()
|
||||
model = AnimaTransformer3DModel(config, kwargs).cuda().to(dtype).eval()
|
||||
model.load_state_dict(reference.state_dict(), strict=True)
|
||||
x = torch.randn(batch_size, 4, 1, 8, 12, device="cuda", dtype=dtype)
|
||||
context = torch.randn(batch_size, 7, 32, device="cuda", dtype=dtype)
|
||||
t = torch.linspace(600, 900, batch_size, device="cuda")
|
||||
padding = torch.zeros(1, 1, 8, 12, device="cuda", dtype=dtype)
|
||||
expected = reference(x, t.to(dtype) / 1000, context, padding_mask=padding).sample
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
actual = model(x, context, t)
|
||||
tolerance = 2e-5 if dtype == torch.float32 else 2e-2
|
||||
torch.testing.assert_close(actual, expected, atol=tolerance, rtol=tolerance)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_conditioner_masks_source_and_zero_pads_output(monkeypatch):
|
||||
args = ServerArgs(
|
||||
model_path="circlestone-labs/Anima-Base-v1.0-Diffusers",
|
||||
num_gpus=1,
|
||||
pipeline_config=AnimaPipelineConfig(),
|
||||
attention_backend="torch_sdpa",
|
||||
)
|
||||
monkeypatch.setattr(server_args, "_global_server_args", args)
|
||||
config = AnimaTextConditionerConfig(
|
||||
arch_config=AnimaTextConditionerArchConfig(
|
||||
source_dim=32,
|
||||
target_dim=32,
|
||||
model_dim=32,
|
||||
num_attention_heads=2,
|
||||
num_layers=2,
|
||||
target_vocab_size=16,
|
||||
)
|
||||
)
|
||||
model = AnimaTextConditioner(config).cuda().bfloat16().eval()
|
||||
source = torch.randn(2, 4, 32, device="cuda", dtype=torch.bfloat16)
|
||||
ids = torch.tensor([[1, 2, 0], [1, 0, 0]], device="cuda")
|
||||
target_mask = ids != 0
|
||||
source_mask = torch.tensor([[1, 1, 0, 0], [0, 0, 0, 0]], device="cuda")
|
||||
changed = source.clone()
|
||||
changed[source_mask == 0] += 10
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
actual = model(source, ids, target_mask, source_mask)
|
||||
expected = model(changed, ids, target_mask, source_mask)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
assert actual.shape == (2, 512, 32)
|
||||
assert torch.isfinite(actual).all()
|
||||
assert not actual[:, 3:].any()
|
||||
assert not actual[:, :3][~target_mask].any()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
@@ -301,14 +301,22 @@ class TestBuildCustomBlockAdapter(unittest.TestCase):
|
||||
def test_builds_adapter_for_registered_class(self):
|
||||
module = _import_module_with_stub()
|
||||
blocks = ["block_0", "block_1"]
|
||||
transformer = _make_transformer("ErnieImageTransformer2DModel", blocks)
|
||||
for class_name, blocks_attr in (
|
||||
("ErnieImageTransformer2DModel", "layers"),
|
||||
("MingImageTransformer2DModel", "layers"),
|
||||
("AnimaTransformer3DModel", "transformer_blocks"),
|
||||
):
|
||||
with self.subTest(class_name=class_name):
|
||||
transformer = type(class_name, (), {blocks_attr: blocks})()
|
||||
adapter = module._build_custom_block_adapter(
|
||||
transformer, has_separate_cfg=True
|
||||
)
|
||||
|
||||
adapter = module._build_custom_block_adapter(transformer, has_separate_cfg=True)
|
||||
|
||||
self.assertIsNotNone(adapter)
|
||||
self.assertEqual(adapter.blocks, blocks)
|
||||
self.assertEqual(adapter.forward_pattern, "Pattern_3")
|
||||
self.assertTrue(adapter.has_separate_cfg)
|
||||
self.assertIsNotNone(adapter)
|
||||
self.assertIs(adapter.transformer, transformer)
|
||||
self.assertIs(adapter.blocks, blocks)
|
||||
self.assertEqual(adapter.forward_pattern, "Pattern_3")
|
||||
self.assertTrue(adapter.has_separate_cfg)
|
||||
|
||||
def test_returns_none_for_unknown_class(self):
|
||||
module = _import_module_with_stub()
|
||||
|
||||
Reference in New Issue
Block a user