[diffusion] model: support Ming-Image Design and Design-Layer (#41067)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-09-25 14:44:37 +08:00
committed by GitHub
co-authored by Mick Qian
parent 9c8340a2e5
commit 0fb699ac03
33 changed files with 2350 additions and 62 deletions
@@ -752,9 +752,11 @@ def clear_b_tma_desc_cache() -> None:
def _get_b_tma_desc_cached(B: torch.Tensor, block_n: int, block_k: int):
"""
Cache TensorDescriptor for constant weight B.
Keyed by storage ptr + shape/stride/dtype + tile shape.
Keyed by tensor identity + storage ptr + shape/stride/dtype + tile shape.
"""
key = (
# offload can rebind a Parameter while another tensor reuses its address
id(B),
int(B.data_ptr()),
tuple(B.shape),
tuple(B.stride()),
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.zimage import (
ZImageArchConfig,
ZImageDitConfig,
)
@dataclass
class MingImageArchConfig(ZImageArchConfig):
axes_lens: tuple[int, ...] = (20480, 512, 512)
alignment_padding_mode: str | None = None
multi_frame_output: bool | None = None
@dataclass
class MingImageDitConfig(ZImageDitConfig):
arch_config: MingImageArchConfig = field(default_factory=MingImageArchConfig)
prefix: str = "ming_image"
def update_model_arch(self, config):
config = dict(config)
config["num_layers"] = config.pop("n_layers", self.num_layers)
config["num_attention_heads"] = config.pop("n_heads", self.num_attention_heads)
if type(config.get("multi_frame_output")) is not bool or (
config.get("alignment_padding_mode"),
config.get("multi_frame_output"),
) not in (
("zero_masked", False),
("learned", True),
):
raise ValueError(
"Ming-Image requires an explicit, valid checkpoint padding/output contract"
)
super().update_model_arch(config)
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.encoders.base import (
TextEncoderArchConfig,
TextEncoderConfig,
)
@dataclass
class MingImageEncoderArchConfig(TextEncoderArchConfig):
architectures: list[str] = field(default_factory=lambda: ["MingImageEncoder"])
hidden_size: int = 2048
num_attention_heads: int = 16
intermediate_size: int = 5120
llm_config: dict = field(default_factory=dict)
vision_config: dict = field(default_factory=dict)
connector_config: dict = field(default_factory=dict)
projection_config: dict = field(default_factory=dict)
@dataclass
class MingImageEncoderConfig(TextEncoderConfig):
arch_config: MingImageEncoderArchConfig = field(
default_factory=MingImageEncoderArchConfig
)
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
import torch
from sglang.multimodal_gen.configs.models.dits.ming_image import MingImageDitConfig
from sglang.multimodal_gen.configs.models.encoders.ming_image import (
MingImageEncoderConfig,
)
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ImagePipelineConfig,
ModelTaskType,
)
@dataclass
class MingImagePipelineConfig(ImagePipelineConfig):
task_type: ModelTaskType = ModelTaskType.TI2I
dit_config: MingImageDitConfig = field(default_factory=MingImageDitConfig)
vae_config: QwenImageVAEConfig = field(default_factory=QwenImageVAEConfig)
text_encoder_configs: tuple = field(
default_factory=lambda: (MingImageEncoderConfig(),)
)
text_encoder_precisions: tuple[str, ...] = ("bf16",)
native_only_components: tuple[str, ...] = ("text_encoder", "transformer", "vae")
vae_precision: str = "bf16"
vae_tiling: bool = False
vae_sp: bool = False
enable_autocast: bool = False
generator_device: str = "cpu"
def supports_dynamic_batching(self):
# Reference/query sequences have different lengths across requests.
return False
def supports_sequential_multi_output_inference(self):
return True
def get_latent_dtype(self, prompt_dtype):
return torch.float32
def prepare_latent_shape(self, batch, batch_size, num_frames):
# Draw frames before channels, as in the official CPU RNG stream.
return (
batch_size,
batch.extra["ming_frames"],
16,
batch.height // 8,
batch.width // 8,
)
def maybe_pack_latents(self, latents, batch_size, batch):
return latents.transpose(1, 2).contiguous()
def shard_latents_for_sp(self, batch, latents):
# Ming shards patchified tokens inside the DiT, not spatial pixels here.
return latents, False
def gather_latents_for_sp(self, latents, batch=None):
return latents
def gather_noise_pred_for_sp(self, batch, noise_pred):
return noise_pred
def post_denoising_loop(self, latents, batch):
return latents
def get_pos_prompt_embeds(self, batch):
return batch.prompt_embeds[0]
def get_neg_prompt_embeds(self, batch):
return batch.negative_prompt_embeds[0]
def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype):
return {
"direct_embeddings": batch.extra["ming_direct"],
"reference_latents": batch.extra.get("ming_reference_latents"),
}
def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype):
return {
"direct_embeddings": torch.zeros_like(batch.extra["ming_direct"]),
"reference_latents": batch.extra.get("ming_reference_latents"),
}
def get_classifier_free_guidance_scale(self, batch, guidance_scale):
# Official Ming uses cond + cfg * (cond - uncond).
return guidance_scale + 1.0
@dataclass
class MingImageLayerPipelineConfig(MingImagePipelineConfig):
task_type: ModelTaskType = ModelTaskType.I2I
def register():
from sglang.multimodal_gen.configs.sample.ming_image import (
MingImageLayerSamplingParams,
MingImageSamplingParams,
)
from sglang.multimodal_gen.registry import register_configs
register_configs(
sampling_param_cls=MingImageLayerSamplingParams,
pipeline_config_cls=MingImageLayerPipelineConfig,
hf_model_paths=["inclusionAI/Ming-Image-0.1-Design-Layer"],
model_detectors=[lambda model: "ming-image-0.1-design-layer" in model.lower()],
)
register_configs(
sampling_param_cls=MingImageSamplingParams,
pipeline_config_cls=MingImagePipelineConfig,
hf_model_paths=["inclusionAI/Ming-Image-0.1-Design"],
model_detectors=[
lambda model: (
"ming-image-0.1-design" in model.lower()
and "design-layer" not in model.lower()
)
],
)
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
import re
from dataclasses import dataclass
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
@dataclass
class MingImageSamplingParams(SamplingParams):
height: int = 2048
width: int = 2048
num_inference_steps: int = 12
guidance_scale: float = 1.0
negative_prompt: str = ""
generator_device: str = "cpu"
@classmethod
def default_image_output_format(cls):
return "png"
@dataclass
class MingImageLayerSamplingParams(MingImageSamplingParams):
height: int = 1024
width: int = 1024
guidance_scale: float = 2.0
prompt: str = ""
num_layers: int = 4
def build_request_extra(self):
extra = super().build_request_extra()
if type(self.num_layers) is not int or self.num_layers < 1:
raise ValueError("num_layers must be a positive integer")
match = re.search(
r"(?:into\s+(\d+)\s+layers|number of layers:\s*(\d+))",
self.prompt,
re.IGNORECASE,
)
count = (
int(next(value for value in match.groups() if value))
if match
else self.num_layers
)
if count < 1:
raise ValueError("The prompt must request at least one layer")
extra["ming_num_layers"] = count
return extra
@classmethod
def image_request_extra_fields(cls):
return frozenset({"num_layers"})
@property
def num_samples_per_request(self):
return self.build_request_extra()["ming_num_layers"]
+1
View File
@@ -169,6 +169,7 @@ _MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = []
# aliases next to the resolver that consumes them so CLI detection and
# pipeline selection cannot drift apart
KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
"ming-image-0.1-design": "MingImagePipeline",
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
"minimax/minimax-h3": "MiniMaxH3Pipeline",
"fastvideo/fastvideo-fasth3-4-step-preview-v1-vsa-datafree": "FastH3Pipeline",
@@ -0,0 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
"""Keep Ming's trained padding registers and RoPE offsets unchanged."""
from sglang.multimodal_gen.runtime.breakable_cuda_graph import prompt_padding
def is_ming_image(current_model, call_kwargs):
return prompt_padding.transformer_class_name_matches(current_model, "MingImage")
def preserve_ming_prompt(call_kwargs, current_model, buckets):
# Changing caption length also shifts every image's first RoPE coordinate.
# Capture exact signatures; uncaptured prompt lengths run eager.
return call_kwargs
prompt_padding.register_prompt_padder(is_ming_image, preserve_ming_prompt)
@@ -302,6 +302,7 @@ def _ensure_model_padders_registered() -> None:
from sglang.multimodal_gen.runtime.breakable_cuda_graph.model_padders import ( # noqa: F401
ideogram,
longcat_image,
ming_image,
minimax_h3,
qwen_image,
sana_video,
@@ -403,6 +403,10 @@ class CustomBlockAdapterSpec:
# Custom BlockAdapter metadata for models absent from cache-dit's registry.
_CUSTOM_BLOCK_ADAPTER_SPECS: dict[str, CustomBlockAdapterSpec] = {
"MingImageTransformer2DModel": CustomBlockAdapterSpec(
blocks_attr="layers",
forward_pattern=ForwardPattern.Pattern_3,
),
"QwenImage21Transformer2DModel": CustomBlockAdapterSpec(
blocks_attr="transformer_blocks",
forward_pattern=ForwardPattern.Pattern_3,
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import torch
from flash_attn import flash_attn_func
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionBackend,
@@ -10,9 +11,6 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend i
AttentionMetadata,
AttentionMetadataBuilder,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import (
flash_attn_func,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -68,10 +66,6 @@ class FlashAttention2Impl(AttentionImpl):
q=query, # type: ignore[no-untyped-call]
k=key,
v=value,
cu_seqlens_q=None,
cu_seqlens_k=None,
max_seqlen_q=None,
max_seqlen_k=None,
softmax_scale=self.softmax_scale,
causal=self.causal,
)
@@ -0,0 +1,61 @@
# SPDX-License-Identifier: Apache-2.0
from pathlib import Path
from transformers import PreTrainedTokenizerFast
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
ComponentLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import (
TextEncoderLoader,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import (
checkpoint_weights_iterator,
)
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
get_diffusers_component_config,
)
class MingImageEncoderLoader(TextEncoderLoader):
component_names = []
def load_customized(self, component_model_path, server_args, component_name):
root = Path(component_model_path)
config = server_args.pipeline_config.text_encoder_configs[0]
mllm = get_diffusers_component_config(component_path=str(root / "mllm"))
config.arch_config.llm_config = mllm["llm_config"]
config.arch_config.vision_config = mllm["vision_config"]
config.arch_config.connector_config = get_diffusers_component_config(
component_path=str(root / "connector")
)
config.arch_config.projection_config = get_diffusers_component_config(
component_path=str(root / "mlp")
)
server_args.model_paths[component_name] = component_model_path
return self.load_model(
component_model_path,
config,
server_args,
dtype=server_args.pipeline_config.text_encoder_precisions[0],
component_name=component_name,
)
def _get_all_weights(self, model, model_path, to_cpu):
for component in ("mllm", "connector", "mlp"):
for name, weight in checkpoint_weights_iterator(
str(Path(model_path) / component), to_cpu=to_cpu
):
yield (
(f"connector.{name}" if component == "connector" else name),
weight,
)
class MingImageTokenizerLoader(ComponentLoader):
expected_library = "transformers"
def load_customized(self, component_model_path, server_args, component_name):
return PreTrainedTokenizerFast.from_pretrained(
component_model_path, padding_side="right"
)
@@ -249,10 +249,18 @@ class ComponentResidencyManager:
no declared ``ComponentUse`` would otherwise be accepted but never
moved to the device before a forward pass.
"""
if not isinstance(self.server_args, ServerArgs):
if (
not isinstance(self.server_args, ServerArgs)
or not self.server_args.component_residency
):
return
declared_components = {use.component_name for use in self._ordered_uses}
# sequential multi-output execution runs subsets of the full pipeline
declared_components = {
use.component_name
for name, stage in self.pipeline._stage_name_mapping.items()
for use in stage.component_uses(self.server_args, name)
}
unmanaged_components = sorted(
component_name
for component_name, module in self.pipeline.modules.items()
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from inclusionAI/Ming-Image.
import math
import torch
from torch import nn
from torch.nn import functional as F
from sglang.multimodal_gen.runtime.distributed import get_sp_world_size
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
gather_seq,
shard_like,
shard_seq,
tail_attn_meta,
)
from sglang.multimodal_gen.runtime.models.dits.zimage import ZImageTransformer2DModel
from sglang.srt.layers.layernorm import RMSNorm
class MingRMSNorm(RMSNorm):
def __init__(self, dim, eps=1e-5):
# unlike Z-Image, Ming's reference accumulates the variance in fp32
super().__init__(dim, eps=eps, cast_x_before_out_mul=True, force_native=True)
class MingSiluAndMul(nn.Module):
def forward(self, x):
gate, up = x.chunk(2, dim=-1)
return F.silu(gate) * up
class MingImageTransformer2DModel(ZImageTransformer2DModel):
"""Ming-Image's joint DiT, sharing Z-Image blocks and TP/SP attention."""
norm_cls = MingRMSNorm
_aliases = ["DiffusionTransformer"]
def __init__(self, config, hf_config, quant_config=None):
super().__init__(config, hf_config, quant_config)
self.learned_padding = config.alignment_padding_mode == "learned"
if not self.learned_padding:
del self.x_pad_token, self.cap_pad_token
self.register_buffer(
"x_pad_token", torch.zeros(1, self.dim, device="cpu"), persistent=False
)
self.register_buffer(
"cap_pad_token",
torch.zeros(1, self.dim, device="cpu"),
persistent=False,
)
self.layer_names = ["noise_refiner", "context_refiner", "layers"]
for layer in (*self.noise_refiner, *self.context_refiner, *self.layers):
layer.feed_forward.act = MingSiluAndMul()
def forward(
self,
hidden_states,
encoder_hidden_states,
timestep,
direct_embeddings,
reference_latents=None,
**kwargs,
):
x = hidden_states
batch_size, channels, output_frames, height, width = x.shape
if reference_latents is not None:
x = torch.cat((x, reference_latents), 2)
frames = x.shape[2]
x = x.reshape(batch_size, channels, frames, height // 2, 2, width // 2, 2)
x = x.permute(0, 2, 3, 5, 4, 6, 1).flatten(1, 3).flatten(2)
x_length = x.shape[1]
sp_size = get_sp_world_size()
x_target = math.ceil(x_length / 32) * 32
x = F.pad(x, (0, 0, 0, x_target - x_length))
x, _ = self.all_x_embedder["2-1"](x)
x = self._replace_padding_with_token_mask(
x,
(torch.arange(x_target, device=x.device) < x_length).unsqueeze(0),
self.x_pad_token,
)
cap, _ = self.cap_embedder(encoder_hidden_states)
cap = torch.cat((cap, direct_embeddings), 1)
cap_length = cap.shape[1]
cap_target = math.ceil(cap_length / 32) * 32
cap = F.pad(cap, (0, 0, 0, cap_target - cap_length))
cap = self._replace_padding_with_token_mask(
cap,
(torch.arange(cap_target, device=x.device) < cap_length).unsqueeze(0),
self.cap_pad_token,
)
cap_ids = self.create_coordinate_grid(
(cap_target, 1, 1), (1, 0, 0), x.device
).flatten(0, 2)
x_ids = self.create_coordinate_grid(
(frames, height // 2, width // 2), (cap_target + 1, 0, 0), x.device
).flatten(0, 2)
x_ids = F.pad(x_ids, (0, 0, 0, x_target - x_length))
x_valid = x_target if self.learned_padding else x_length
cap_valid = cap_target if self.learned_padding else cap_length
x_freqs, cap_freqs = self.rotary_emb(x_ids), self.rotary_emb(cap_ids)
x_mask, x_meta = self._get_attn_mask_and_meta(
"_ming_x_mask", [x_valid] * batch_size, x_target, x.device
)
cap_mask, cap_meta = self._get_attn_mask_and_meta(
"_ming_cap_mask", [cap_valid] * batch_size, cap_target, x.device
)
adaln = self.t_embedder(((1000.0 - timestep) / 1000.0) * self.t_scale).to(
x.dtype
)
for layer in self.noise_refiner:
x = layer(
x,
x_freqs,
adaln,
attn_mask=x_mask,
attn_mask_meta=x_meta,
skip_sequence_parallel_override=True,
)
for layer in self.context_refiner:
cap = layer(cap, cap_freqs, attn_mask=cap_mask, attn_mask_meta=cap_meta)
if sp_size > 1:
# masked alignment rows never contribute; learned registers must stay
x, cap = x[:, :x_valid], cap[:, :cap_valid]
x_freqs = tuple(freq[:x_valid] for freq in x_freqs)
cap_freqs = tuple(freq[:cap_valid] for freq in cap_freqs)
unified = torch.cat((x, cap), 1)
freqs = tuple(
torch.cat((image_freq, text_freq), -2)
for image_freq, text_freq in zip(x_freqs, cap_freqs)
)
mask, meta = self._get_joint_attn_mask_and_meta(
[x_valid] * batch_size,
x_target,
[cap_valid] * batch_size,
cap_target,
x.device,
)
if sp_size > 1:
# shard the complete joint sequence, with only global-tail padding
unified, shard = shard_seq(unified)
freqs = tuple(shard_like(freq, shard, dim=0) for freq in freqs)
mask, meta = None, tail_attn_meta(shard, batch_size, x.device)
for layer in self.layers:
unified = layer(
unified,
freqs,
adaln,
attn_mask=mask,
attn_mask_meta=meta,
)
if sp_size == 1:
unified = unified[:, :x_target]
output = self.all_final_layer["2-1"](unified, adaln)
if sp_size > 1:
output = gather_seq(output, shard.orig_len)
output = output[:, :x_length].reshape(
batch_size, frames, height // 2, width // 2, 2, 2, channels
)
output = output.permute(0, 6, 1, 2, 4, 3, 5).reshape(
batch_size, channels, frames, height, width
)
return -output[:, :, :output_frames]
EntryClass = MingImageTransformer2DModel
@@ -245,6 +245,7 @@ class ZImageAttention(nn.Module):
eps: float = 1e-6,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
norm_cls: type[nn.Module] = ZImageRMSNorm,
) -> None:
super().__init__()
self.dim = dim
@@ -252,7 +253,9 @@ class ZImageAttention(nn.Module):
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.qk_norm = qk_norm
self.enable_zimage_qk_fusion = quant_config is None
self.enable_zimage_qk_fusion = (
quant_config is None and norm_cls is ZImageRMSNorm
)
tp_size = get_tp_world_size()
assert num_heads % tp_size == 0, (
@@ -303,8 +306,8 @@ class ZImageAttention(nn.Module):
)
if self.qk_norm:
self.norm_q = ZImageRMSNorm(self.head_dim, eps=eps)
self.norm_k = ZImageRMSNorm(self.head_dim, eps=eps)
self.norm_q = norm_cls(self.head_dim, eps=eps)
self.norm_k = norm_cls(self.head_dim, eps=eps)
else:
self.norm_q = None
self.norm_k = None
@@ -519,13 +522,16 @@ class ZImageTransformerBlock(nn.Module):
modulation=True,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
norm_cls: type[nn.Module] = ZImageRMSNorm,
):
super().__init__()
self.dim = dim
self.head_dim = dim // n_heads
self.layer_id = layer_id
self.modulation = modulation
self.enable_zimage_native_norm_fusion = quant_config is None
self.enable_zimage_native_norm_fusion = (
quant_config is None and norm_cls is ZImageRMSNorm
)
self.attention = ZImageAttention(
dim=dim,
@@ -535,6 +541,7 @@ class ZImageTransformerBlock(nn.Module):
eps=1e-5,
quant_config=quant_config,
prefix=f"{prefix}.attention",
norm_cls=norm_cls,
)
if not modulation:
# Context refiner runs on fully replicated caption tokens only.
@@ -574,11 +581,11 @@ class ZImageTransformerBlock(nn.Module):
prefix=f"{prefix}.feed_forward",
)
self.attention_norm1 = ZImageRMSNorm(dim, eps=norm_eps)
self.ffn_norm1 = ZImageRMSNorm(dim, eps=norm_eps)
self.attention_norm1 = norm_cls(dim, eps=norm_eps)
self.ffn_norm1 = norm_cls(dim, eps=norm_eps)
self.attention_norm2 = ZImageRMSNorm(dim, eps=norm_eps)
self.ffn_norm2 = ZImageRMSNorm(dim, eps=norm_eps)
self.attention_norm2 = norm_cls(dim, eps=norm_eps)
self.ffn_norm2 = norm_cls(dim, eps=norm_eps)
if modulation:
self.adaLN_modulation = nn.Sequential(
@@ -763,6 +770,7 @@ class RopeEmbedder:
class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
norm_cls = ZImageRMSNorm
_supports_gradient_checkpointing = True
_no_split_modules = ["ZImageTransformerBlock"]
_fsdp_shard_conditions = [is_zimage_layer]
@@ -856,6 +864,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
modulation=True,
quant_config=quant_config,
prefix=f"noise_refiner.{layer_id}",
norm_cls=self.norm_cls,
)
for layer_id in range(arch_config.n_refiner_layers)
]
@@ -872,6 +881,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
modulation=False,
quant_config=quant_config,
prefix=f"context_refiner.{layer_id}",
norm_cls=self.norm_cls,
)
for layer_id in range(arch_config.n_refiner_layers)
]
@@ -881,7 +891,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
)
self.cap_embedder = nn.Sequential(
ZImageRMSNorm(arch_config.cap_feat_dim, eps=arch_config.norm_eps),
self.norm_cls(arch_config.cap_feat_dim, eps=arch_config.norm_eps),
ReplicatedLinear(arch_config.cap_feat_dim, self.dim, bias=True),
)
@@ -899,6 +909,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
arch_config.qk_norm,
quant_config=quant_config,
prefix=f"layers.{layer_id}",
norm_cls=self.norm_cls,
)
for layer_id in range(arch_config.num_layers)
]
@@ -0,0 +1,472 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from inclusionAI/Ming-Image (Apache-2.0 modeling code).
"""Ming's Bailing multimodal encoder and non-causal query connector."""
from collections import defaultdict
from types import SimpleNamespace
import torch
from torch import nn
from torch.nn import functional as F
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.runtime.distributed import (
get_tp_group,
get_tp_rank,
get_tp_world_size,
)
from sglang.multimodal_gen.runtime.layers.attention import LocalAttention
from sglang.multimodal_gen.runtime.layers.linear import (
LinearBase,
QKVParallelLinear,
RowParallelLinear,
)
from sglang.multimodal_gen.runtime.layers.moe import LingBotVideoGroupedExperts
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl_vision import (
Qwen2_5VLVisionTransformer,
)
from sglang.multimodal_gen.runtime.models.encoders.qwen3 import Qwen3MLP
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_experts
from sglang.srt.layers.moe.topk import StandardTopKOutput
def ming_position_ids(
token_ids: list[int], grids: list[tuple[int, int, int]], image_token: int
):
"""CPU-side centered video-RoPE positions, including learned image queries."""
positions = []
cursor = 0
offset = 0
for frames, height, width in grids:
start = token_ids.index(image_token, cursor)
text_len = start - cursor
positions.append(torch.arange(offset, offset + text_len).expand(3, -1))
h, w = height // 2, width // 2
t = torch.arange(frames).repeat_interleave(h * w) + offset + text_len
rows = torch.arange(h).repeat_interleave(w).repeat(frames) - (h - 1) // 2
cols = torch.arange(w).repeat(frames * h) - (w - 1) // 2
positions.append(torch.stack((t, rows + t, cols + t)))
offset += text_len + frames
cursor = start + frames * h * w
positions.append(
torch.arange(offset, offset + len(token_ids) - cursor).expand(3, -1)
)
return torch.cat(positions, dim=1).unsqueeze(1)
class MingAttention(nn.Module):
def __init__(self, config, *, connector=False):
super().__init__()
self.head_dim = config.hidden_size // config.num_attention_heads
self.num_heads = config.num_attention_heads // get_tp_world_size()
self.num_kv_heads = max(1, config.num_key_value_heads // get_tp_world_size())
self.qkv_proj = QKVParallelLinear(
config.hidden_size,
self.head_dim,
config.num_attention_heads,
config.num_key_value_heads,
bias=connector,
)
self.o_proj = RowParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
input_is_parallel=True,
)
self.q_norm = (
nn.Identity()
if connector
else RMSNorm(
self.head_dim,
eps=config.rms_norm_eps,
cast_x_before_out_mul=True,
force_native=True,
)
)
self.k_norm = (
nn.Identity()
if connector
else RMSNorm(
self.head_dim,
eps=config.rms_norm_eps,
cast_x_before_out_mul=True,
force_native=True,
)
)
self.attn = LocalAttention(
self.num_heads, self.head_dim, self.num_kv_heads, causal=not connector
)
self.rotary_dim = (
self.head_dim
if connector
else int(self.head_dim * config.partial_rotary_factor)
)
self.rope_theta = config.rope_theta
self.connector = connector
def forward(self, x, positions):
qkv, _ = self.qkv_proj(x)
q, k, v = qkv.split(
[
self.num_heads * self.head_dim,
self.num_kv_heads * self.head_dim,
self.num_kv_heads * self.head_dim,
],
-1,
)
q = self.q_norm(q.unflatten(-1, (self.num_heads, self.head_dim)))
k = self.k_norm(k.unflatten(-1, (self.num_kv_heads, self.head_dim)))
v = v.unflatten(-1, (self.num_kv_heads, self.head_dim))
inv = 1.0 / self.rope_theta ** (
torch.arange(0, self.rotary_dim, 2, device=x.device, dtype=torch.float32)
/ self.rotary_dim
)
# the public BF16 inference path casts RoPE buffers before evaluation
inv = inv.to(x.dtype).float()
with torch.autocast(x.device.type, enabled=False):
angles = positions.float().unsqueeze(-1) * inv
angles = torch.cat((angles, angles), -1)
if not self.connector:
# the spatial frequencies alternate H/W; temporal frequencies are last
axes = ([1, 2] * 12 + [0] * 8) * 2
angles = torch.stack(
[angles[axis, ..., i] for i, axis in enumerate(axes)], -1
)
cos, sin = (
angles.cos().to(x.dtype).unsqueeze(-2),
angles.sin().to(x.dtype).unsqueeze(-2),
)
def rotate(value):
rot, tail = value[..., : self.rotary_dim], value[..., self.rotary_dim :]
half = self.rotary_dim // 2
rotated = torch.cat((-rot[..., half:], rot[..., :half]), -1)
return torch.cat((rot * cos + rotated * sin, tail), -1)
output = self.attn(rotate(q), rotate(k), v).flatten(-2)
return self.o_proj(output)[0]
class MingRouter(nn.Module):
def __init__(self, config):
super().__init__()
self.weight = nn.Parameter(torch.empty(config.num_experts, config.hidden_size))
self.expert_bias = nn.Parameter(
torch.zeros(config.num_experts), requires_grad=False
)
def forward(self, x):
return F.linear(x.float(), self.weight.float()).sigmoid()
class MingMLP(Qwen3MLP):
def forward(self, x):
gate, up = self.gate_up_proj(x)[0].chunk(2, dim=-1)
# preserve the checkpoint's BF16 rounding between SiLU and multiplication
return self.down_proj(F.silu(gate) * up)[0]
class MingExperts(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.gate = MingRouter(config)
self.image_gate = MingRouter(config)
self.intermediate_size = config.moe_intermediate_size // get_tp_world_size()
self.experts = LingBotVideoGroupedExperts(
config.num_experts, config.hidden_size, self.intermediate_size
)
self.shared_experts = MingMLP(
config.hidden_size,
config.moe_intermediate_size * config.num_shared_experts,
"silu",
)
self.runner_config = MoeRunnerConfig(
num_experts=config.num_experts,
num_local_experts=config.num_experts,
hidden_size=config.hidden_size,
intermediate_size_per_partition=self.intermediate_size,
top_k=config.num_experts_per_tok,
activation="silu_rounded",
is_gated=True,
inplace=False,
no_combine=True,
apply_router_weight_on_input=False,
routed_scaling_factor=None,
gate_up_interleaved=False,
)
def forward(self, x, image_mask):
tokens = x.flatten(0, 1)
mask = image_mask.reshape(-1, 1)
scores = torch.where(mask, self.image_gate(tokens), self.gate(tokens))
choice = scores + torch.where(
mask, self.image_gate.expert_bias, self.gate.expert_bias
)
grouped = choice.unflatten(-1, (self.config.n_group, -1))
groups = (
grouped.topk(2, dim=-1)
.values.sum(-1)
.topk(self.config.topk_group, dim=-1, sorted=False)
.indices
)
group_mask = torch.zeros_like(grouped[..., 0], dtype=torch.bool).scatter_(
1, groups, True
)
choice = choice.masked_fill(
~group_mask.unsqueeze(-1).expand_as(grouped).flatten(1), float("-inf")
)
ids = choice.topk(self.config.num_experts_per_tok, dim=-1, sorted=False).indices
weights = scores.gather(1, ids)
weights = (
weights
/ (weights.sum(-1, keepdim=True) + 1e-20)
* self.config.routed_scaling_factor
)
topk = StandardTopKOutput(
weights.float(), ids.int(), torch.empty(0, device=x.device)
)
out = fused_experts(
tokens.contiguous(),
self.experts.w13_weight,
self.experts.w2,
topk,
self.runner_config,
)
if get_tp_world_size() > 1:
out = get_tp_group().all_reduce(out)
out = (out.float() * weights.unsqueeze(-1)).sum(1).to(x.dtype)
return out.view_as(x) + self.shared_experts(x)
class MingEncoderBlock(nn.Module):
def __init__(self, config, layer_idx, *, connector=False):
super().__init__()
self.attention = MingAttention(config, connector=connector)
self.is_moe = not connector and layer_idx >= config.first_k_dense_replace
self.mlp = (
MingExperts(config)
if self.is_moe
else MingMLP(config.hidden_size, config.intermediate_size, "silu")
)
self.input_layernorm = RMSNorm(
config.hidden_size,
eps=config.rms_norm_eps,
cast_x_before_out_mul=True,
force_native=True,
)
self.post_attention_layernorm = RMSNorm(
config.hidden_size,
eps=config.rms_norm_eps,
cast_x_before_out_mul=True,
force_native=True,
)
def forward(self, x, positions, image_mask=None):
x = x + self.attention(self.input_layernorm(x), positions)
normalized = self.post_attention_layernorm(x)
return x + (
self.mlp(normalized, image_mask) if self.is_moe else self.mlp(normalized)
)
class MingImageEncoder(TextEncoder):
@property
def dtype(self):
return self.word_embeddings.weight.dtype
def __init__(self, config):
super().__init__(config)
llm = SimpleNamespace(**config.llm_config)
connector = SimpleNamespace(**config.connector_config)
projection = config.projection_config
self.word_embeddings = VocabParallelEmbedding(llm.vocab_size, llm.hidden_size)
self.layers = nn.ModuleList(
MingEncoderBlock(llm, i) for i in range(llm.num_hidden_layers)
)
self.norm = RMSNorm(
llm.hidden_size,
eps=llm.rms_norm_eps,
cast_x_before_out_mul=True,
force_native=True,
)
self.vision = Qwen2_5VLVisionTransformer(
SimpleNamespace(**config.vision_config)
)
# Ming casts vision RoPE buffers with the model, unlike Qwen2.5-VL
self.vision.rotary_pos_emb.inv_freq = self.vision.rotary_pos_emb.inv_freq.to(
self.vision.dtype
)
self.vision.merger.ln_q.cast_x_before_out_mul = False
for block in self.vision.blocks:
block.norm1.cast_x_before_out_mul = False
block.norm2.cast_x_before_out_mul = False
self.linear_proj = nn.Sequential(
nn.Linear(config.vision_config["out_hidden_size"], llm.hidden_size),
nn.GELU(),
nn.Linear(llm.hidden_size, llm.hidden_size),
)
self.connector = nn.ModuleList(
MingEncoderBlock(connector, i, connector=True)
for i in range(connector.num_hidden_layers)
)
self.connector_norm = RMSNorm(
connector.hidden_size,
eps=connector.rms_norm_eps,
cast_x_before_out_mul=True,
force_native=True,
)
self.proj_in = nn.Linear(llm.hidden_size, connector.hidden_size)
self.proj_out = nn.Linear(
connector.hidden_size, projection["diffusion_c_input_dim"]
)
self.selected_layers = tuple(projection["selected_hidden_states_layers"])
direct_dim = llm.hidden_size * len(self.selected_layers)
self.proj_directvlm = nn.Sequential(
RMSNorm(
direct_dim, eps=1e-5, cast_x_before_out_mul=True, force_native=True
),
nn.Linear(direct_dim, projection["diffusion_inner_dim"]),
)
self.query_tokens_dict = nn.ParameterDict(
{
f"{scale}x{scale}": nn.Parameter(
torch.empty(scale * scale, llm.hidden_size)
)
for scale in projection["img_gen_scales"]
}
)
self.image_token = llm.image_patch_token
self.layer_names = ["layers", "vision.blocks", "connector"]
def forward(
self, input_ids, position_ids, pixel_values=None, image_grid_thw=None, **kwargs
):
x = self.word_embeddings(input_ids)
queries = torch.cat(list(self.query_tokens_dict.values()), 0)
query_count = queries.shape[0]
image_mask = input_ids == self.image_token
image_embeddings = queries
if pixel_values is not None:
vision_features = self.vision(pixel_values, image_grid_thw)
with torch.autocast(x.device.type, enabled=False):
image_embeddings = F.normalize(
self.linear_proj(vision_features), dim=-1
)
image_embeddings = torch.cat((image_embeddings, queries), 0)
x = x.masked_scatter(image_mask.unsqueeze(-1), image_embeddings.to(x.dtype))
selected = []
for i, layer in enumerate(self.layers):
if i in self.selected_layers:
selected.append(x)
x = layer(x, position_ids, image_mask)
x = self.norm(x)
if len(self.layers) in self.selected_layers:
selected.append(x)
prompt_length = input_ids.shape[1] - query_count - 2
direct = self.proj_directvlm(
torch.cat([hidden[:, :prompt_length] for hidden in selected], -1)
)
queries = self.proj_in(x[:, -query_count - 1 : -1])
positions = torch.arange(query_count, device=x.device).unsqueeze(0)
for layer in self.connector:
queries = layer(queries, positions)
queries = self.proj_out(self.connector_norm(queries))
if self.config.projection_config["connector_norm"]:
queries = F.normalize(queries, dim=-1)
return BaseEncoderOutput(last_hidden_state=queries, hidden_states=(direct,))
def should_materialize_checkpoint_weight(self, name):
return not (
name.endswith("lm_head.weight")
or ".audio_gate." in name
or "rotary_emb.inv_freq" in name
)
def load_weights(self, weights):
params = dict(self.named_parameters())
loaded = set()
pieces = defaultdict(set)
for name, weight in weights:
if not self.should_materialize_checkpoint_weight(name):
continue
name = name.removeprefix("model.model.")
name = name.replace(".attn.qkv.", ".attn.qkv_proj.")
name = name.replace("connector.model.layers.", "connector.").replace(
"connector.model.norm.", "connector_norm."
)
if name == "connector.model.embed_tokens.weight":
continue
name = (
name.replace(".self_attn.", ".attention.")
.replace(".query_key_value.", ".qkv_proj.")
.replace(".dense.", ".o_proj.")
)
shard = None
if ".experts." in name:
prefix, suffix = name.split(".experts.")
expert, projection, _ = suffix.split(".")
module = self.get_submodule(prefix)
width = module.intermediate_size
rank = get_tp_rank()
if projection == "down_proj":
target = f"{prefix}.experts.w2"
params[target].data[int(expert)].copy_(
weight[:, rank * width : (rank + 1) * width]
)
else:
target = f"{prefix}.experts.w13_weight"
offset = 0 if projection == "gate_proj" else width
params[target].data[int(expert), offset : offset + width].copy_(
weight[rank * width : (rank + 1) * width]
)
pieces[target].add((int(expert), projection))
expected = module.config.num_experts * (
1 if projection == "down_proj" else 2
)
if len(pieces[target]) == expected:
loaded.add(target)
continue
for source, target, shard_id in (
(".gate_proj.", ".gate_up_proj.", 0),
(".up_proj.", ".gate_up_proj.", 1),
(".q_proj.", ".qkv_proj.", "q"),
(".k_proj.", ".qkv_proj.", "k"),
(".v_proj.", ".qkv_proj.", "v"),
):
if source in name and not name.startswith("vision."):
name, shard = name.replace(source, target), shard_id
break
if name not in params:
raise ValueError(f"Unexpected Ming encoder checkpoint tensor: {name}")
param = params[name]
owner = self.get_submodule(name.rsplit(".", 1)[0])
if (
isinstance(owner, (LinearBase, VocabParallelEmbedding))
or name.startswith("vision.")
and "weight_loader" in param.__dict__
):
if shard is None:
param.weight_loader(param, weight)
else:
param.weight_loader(param, weight, shard)
else:
default_weight_loader(param, weight)
if shard is None:
loaded.add(name)
else:
pieces[name].add(shard)
expected = {"q", "k", "v"} if isinstance(shard, str) else {0, 1}
if pieces[name] == expected:
loaded.add(name)
return loaded
EntryClass = MingImageEncoder
@@ -969,6 +969,8 @@ class AutoencoderKLQwenImage(ParallelTiledVAE):
self.shift_factor = (
torch.tensor(
config.arch_config.latents_mean
if config.arch_config.latents_mean is not None
else [config.arch_config.shift_factor or 0.0] * latent_channels
)
.view(1, latent_channels, 1, 1, 1)
.to(cuda_device, dtype)
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.loader.component_loaders.ming_image import (
MingImageEncoderLoader,
MingImageTokenizerLoader,
)
from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
InputValidationStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import (
MingImageDecodingStage,
MingImageEncodingStage,
MingImageReferenceStage,
)
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import maybe_download_model
def prepare_mu(batch, server_args):
tokens = (batch.height // 16) * (batch.width // 16)
return "mu", 1.35 if tokens >= 4096 else 0.5 + (tokens - 256) * (1.15 - 0.5) / (
4096 - 256
)
class MingImagePipeline(LoRAPipeline, ComposedPipelineBase):
pipeline_name = "MingImagePipeline"
_required_config_modules = [
"text_encoder",
"tokenizer",
"transformer",
"vae",
"scheduler",
]
component_loaders = {
"text_encoder": MingImageEncoderLoader,
"tokenizer": MingImageTokenizerLoader,
}
def _load_config(self):
self.model_path = maybe_download_model(
self.model_path, revision=self.server_args.revision
)
return {
"_class_name": self.pipeline_name,
"_diffusers_version": "0.36.0",
"text_encoder": ["transformers", "MingImageEncoder"],
"tokenizer": ["transformers", "PreTrainedTokenizerFast"],
"transformer": ["diffusers", "DiffusionTransformer"],
"vae": ["diffusers", "AutoencoderKLQwenImage"],
"scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
}
def _resolve_component_path(self, server_args, module_name, load_module_name):
if module_name == "text_encoder":
load_module_name = ""
elif module_name == "tokenizer":
load_module_name = "mllm"
return super()._resolve_component_path(
server_args, module_name, load_module_name
)
def create_pipeline_stages(self, server_args):
self.add_stage(InputValidationStage())
self.get_module("scheduler").register_to_config(use_dynamic_shifting=True)
self.get_module("scheduler").sigma_min = 0.0
self.add_stage_factory(
RoleType.ENCODER,
lambda: MingImageEncodingStage(
[self.get_module("text_encoder")], [self.get_module("tokenizer")]
),
"text_encoding_stage",
)
self.add_stage_factory(
RoleType.ENCODER,
lambda: MingImageReferenceStage(self.get_module("vae")),
"reference_encoding_stage",
)
self.add_standard_latent_preparation_stage()
self.add_standard_timestep_preparation_stage(prepare_extra_kwargs=[prepare_mu])
self.add_standard_denoising_stage()
self.add_stage_factory(
RoleType.DECODER,
lambda: MingImageDecodingStage(self.get_module("vae"), pipeline=self),
"decoding_stage",
)
EntryClass = MingImagePipeline
@@ -90,13 +90,15 @@ class InputValidationStage(PipelineStage):
if num_outputs == 1:
return iter((batch,))
return iter(
expand_request_outputs(
batch,
reuse_parent_trace_ctx=True,
preserve_parent_metrics=True,
)
outputs = expand_request_outputs(
batch,
reuse_parent_trace_ctx=True,
preserve_parent_metrics=True,
)
# expansion resets generators after this stage has already validated them
for output in outputs:
self._generate_seeds(output, server_args)
return iter(outputs)
@staticmethod
def _calculate_dimensions_from_area(
@@ -0,0 +1,346 @@
# SPDX-License-Identifier: Apache-2.0
"""Native Ming conditioning and independent RGBA layer decoding."""
import math
import numpy as np
import torch
from PIL import Image
from torchvision.transforms.functional import pil_to_tensor, to_tensor
from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import (
Qwen2VLImageProcessorPil,
smart_resize,
)
from sglang.multimodal_gen.configs.pipeline_configs.ming_image import (
MingImageLayerPipelineConfig,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.encoders.ming_image import ming_position_ids
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
from sglang.multimodal_gen.runtime.utils.vision import load_image
class MingImageEncodingStage(TextEncodingStage):
def __init__(self, text_encoders, tokenizers):
super().__init__(text_encoders, tokenizers)
self.image_processor = Qwen2VLImageProcessorPil.from_pretrained(
tokenizers[0].name_or_path
)
@torch.no_grad()
def forward(self, batch, server_args):
if batch.batch_size != 1:
raise ValueError(
"Ming conditioning expects one prompt per expanded request"
)
if batch.negative_prompt and str(batch.negative_prompt).strip():
raise ValueError(
"Ming uses zero unconditional embeddings, not a negative prompt"
)
layered = isinstance(server_args.pipeline_config, MingImageLayerPipelineConfig)
expected_contract = ("learned", True) if layered else ("zero_masked", False)
arch = server_args.pipeline_config.dit_config.arch_config
if (arch.alignment_padding_mode, arch.multi_frame_output) != expected_contract:
raise ValueError(
"Ming checkpoint padding/output contract does not match the selected pipeline"
)
count = batch.extra.get("ming_num_layers", 1)
batch.extra["ming_frames"] = count + 1 if layered else 1
prompt = batch.prompt if isinstance(batch.prompt, str) else batch.prompt[0]
if layered and not prompt.strip():
prompt = f"Decompose this image into {count} layers."
image_paths = batch.image_path
if not isinstance(image_paths, list):
image_paths = [image_paths] if image_paths else []
if len(image_paths) > 1 or (layered and not image_paths):
raise ValueError(
"Ming accepts one reference image; layer decomposition requires it"
)
device = get_local_torch_device()
encoder = self.text_encoders[0]
tokenizer = self.tokenizers[0]
inputs = {}
grids = []
image_text = ""
if image_paths:
image = load_image(
image_paths[0], convert_method=lambda value: value.convert("RGBA")
)
# Reference VAE pixels and VLM pixels use separate official transforms.
batch.extra["ming_reference_image"] = image
h, w = ming_reference_size(
image.height, image.width, min(batch.height, batch.width)
)
batch.height, batch.width = h, w
ratio = max(h / image.height, w / image.width)
batch.extra["ming_output_size"] = (
int(image.width * ratio),
int(image.height * ratio),
)
h0, w0 = smart_resize(
image.height, image.width, min_pixels=56 * 56, max_pixels=1024 * 28 * 28
)
processed = self.image_processor(
images=image.convert("RGB").resize((w0, h0)), return_tensors="pt"
)
grids = [tuple(grid) for grid in processed.image_grid_thw.tolist()]
patches = math.prod(grids[0]) // 4
image_text = "<image>" + "<imagePatch>" * patches + "</image>\n"
inputs = {
"pixel_values": processed.pixel_values.to(
device=device, dtype=encoder.dtype
),
"image_grid_thw": processed.image_grid_thw,
}
if batch.height % 16 or batch.width % 16:
raise ValueError("Ming height and width must be divisible by 16")
# Fixed training template, not the tokenizer's generic chat template.
system = "<role>SYSTEM</role>\u4f60\u662f\u4e00\u4e2a\u53cb\u597d\u7684AI\u52a9\u624b\u3002\n\ndetailed thinking off"
text = (
system
+ tokenizer.eos_token
+ "<role>HUMAN</role>"
+ image_text
+ prompt
+ tokenizer.eos_token
+ "<role>ASSISTANT</role>"
)
ids = tokenizer(text, add_special_tokens=False)["input_ids"]
query_count = sum(
scale * scale
for scale in encoder.config.projection_config["img_gen_scales"]
)
ids += (
[tokenizer.convert_tokens_to_ids("<image>")]
+ [encoder.image_token] * query_count
+ [tokenizer.convert_tokens_to_ids("</image>")]
)
grids.append((1, 2, query_count * 2))
inputs.update(
input_ids=torch.tensor([ids], device=device),
position_ids=ming_position_ids(ids, grids, encoder.image_token).to(device),
)
self._begin_text_encoder_use(0)
# the official MoE router runs under BF16 autocast, including its logits
with torch.autocast(device.type, dtype=encoder.dtype):
encoded = self._forward_text_encoder(encoder, inputs)
batch.prompt_embeds = [encoded.last_hidden_state]
batch.negative_prompt_embeds = [torch.zeros_like(encoded.last_hidden_state)]
batch.extra["ming_direct"] = encoded.hidden_states[0]
return batch
class MingImageReferenceStage(PipelineStage):
role_affinity = RoleType.ENCODER
def __init__(self, vae):
super().__init__()
self.vae = vae
def component_uses(self, server_args, stage_name=None):
return [
ComponentUse(
stage_name=self._component_stage_name(stage_name),
component_name="vae",
target_dtype=resolve_precision(
server_args, "vae", precision_attr="vae_precision"
),
)
]
@torch.no_grad()
def forward(self, batch, server_args):
image = batch.extra.pop("ming_reference_image", None)
if image is None:
return batch
pixels = to_tensor(
image.resize((batch.width, batch.height), Image.Resampling.BILINEAR)
)
pixels = ((pixels - 0.5) * 2).unsqueeze(0).unsqueeze(2)
with self.use_declared_component(component_name="vae", module=self.vae) as vae:
if server_args.pipeline_config.vae_tiling:
vae.enable_tiling()
pixels = pixels.to(
device=get_local_torch_device(), dtype=next(vae.parameters()).dtype
)
batch.extra["ming_reference_latents"] = vae.encode(pixels).mode() * 8.0064
return batch
class MingImageDecodingStage(DecodingStage):
def scale_and_shift(self, latents, server_args):
return latents / 8.0064
def decode(self, latents, server_args, *, vae_dtype):
# Rounding before scaling is part of the official BF16 decode contract.
latents = latents.to(vae_dtype)
batch, channels, frames, height, width = latents.shape
flat = latents.transpose(1, 2).reshape(
batch * frames, channels, 1, height, width
)
decoded = super().decode(flat, server_args, vae_dtype=vae_dtype)
return decoded.reshape(batch, frames, 4, *decoded.shape[-2:]).transpose(1, 2)
def forward(self, batch, server_args):
output = super().forward(batch, server_args)
images = output.output
if isinstance(server_args.pipeline_config, MingImageLayerPipelineConfig):
images = images[:, :, 1:]
images = images.transpose(1, 2).flatten(0, 1)
size = batch.extra.get("ming_output_size")
if size is not None and images.shape[-2:] != size[::-1]:
# Match the public CLI's uint8 conversion before LANCZOS resizing.
pixels = (
(images.float().permute(0, 2, 3, 1).cpu().numpy() * 255)
.round()
.astype(np.uint8)
)
images = (
torch.stack(
[
pil_to_tensor(
Image.fromarray(pixel).resize(
size, Image.Resampling.LANCZOS
)
)
for pixel in pixels
]
).float()
/ 255
)
output.output = images
return output
def ming_reference_size(height, width, resolution):
"""The published 512/1024 aspect buckets; keys preserve official rounding."""
buckets = {
512: {
0.25: (256, 1024),
0.26: (256, 992),
0.27: (256, 960),
0.28: (256, 928),
0.32: (288, 896),
0.33: (288, 864),
0.35: (288, 832),
0.4: (320, 800),
0.42: (320, 768),
0.48: (352, 736),
0.5: (352, 704),
0.52: (352, 672),
0.5455: (384, 704),
0.57: (384, 672),
0.6: (384, 640),
0.65: (416, 640),
0.68: (416, 608),
0.72: (416, 576),
0.78: (448, 576),
0.82: (448, 544),
0.88: (480, 544),
0.94: (480, 512),
1.0: (512, 512),
1.07: (512, 480),
1.13: (544, 480),
1.21: (544, 448),
1.29: (576, 448),
1.38: (576, 416),
1.46: (608, 416),
1.5385: (640, 416),
1.67: (640, 384),
1.75: (672, 384),
1.8333: (704, 384),
2.0: (704, 352),
2.09: (736, 352),
2.4: (768, 320),
2.5: (800, 320),
2.89: (832, 288),
3.0: (864, 288),
3.11: (896, 288),
3.62: (928, 256),
3.75: (960, 256),
3.88: (992, 256),
4.0: (1024, 256),
},
1024: {
0.25: (512, 2048),
0.26: (512, 1984),
0.27: (512, 1920),
0.28: (512, 1856),
0.32: (576, 1792),
0.33: (576, 1728),
0.35: (576, 1664),
0.4: (640, 1600),
0.42: (640, 1536),
0.48: (704, 1472),
0.5: (704, 1408),
0.52: (704, 1344),
0.5581: (768, 1376),
0.5625: (720, 1280),
0.5647: (768, 1360),
0.57: (768, 1344),
0.6: (768, 1280),
0.622: (816, 1312),
0.625: (800, 1280),
0.65: (832, 1280),
0.6582: (832, 1264),
0.6667: (832, 1248),
0.6709: (848, 1264),
0.68: (832, 1216),
0.7013: (864, 1232),
0.72: (832, 1152),
0.7467: (896, 1200),
0.75: (864, 1152),
0.7568: (896, 1184),
0.78: (896, 1152),
0.8: (896, 1120),
0.8056: (928, 1152),
0.82: (896, 1088),
0.88: (960, 1088),
0.94: (960, 1024),
0.9846: (1024, 1040),
1.0: (1024, 1024),
1.07: (1024, 960),
1.13: (1088, 960),
1.21: (1088, 896),
1.2414: (1152, 928),
1.25: (1120, 896),
1.2807: (1168, 912),
1.29: (1152, 896),
1.3333: (1152, 864),
1.3393: (1200, 896),
1.38: (1152, 832),
1.46: (1216, 832),
1.4906: (1264, 848),
1.5: (1248, 832),
1.6: (1280, 800),
1.67: (1280, 768),
1.75: (1344, 768),
1.7708: (1360, 768),
1.7778: (1280, 720),
2.0: (1408, 704),
2.09: (1472, 704),
2.4: (1536, 640),
2.5: (1600, 640),
2.89: (1664, 576),
3.0: (1728, 576),
3.11: (1792, 576),
3.62: (1856, 512),
3.75: (1920, 512),
3.88: (1984, 512),
4.0: (2048, 512),
},
}
resolution = min(buckets, key=lambda value: (abs(value - resolution), value))
table = buckets[resolution]
return table[min(table, key=lambda ratio: abs(ratio - height / width))]
@@ -217,6 +217,10 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset(
"ltx-2.3",
"minimax-h3",
"minimaxai/minimax-h3",
"inclusionai/ming-image-0.1-design",
"inclusionai/ming-image-0.1-design-layer",
"ming-image-0.1-design",
"ming-image-0.1-design-layer",
"qwen/qwen-image",
"qwen/qwen-image-2512",
"qwen/qwen-image-2.1",
@@ -242,6 +246,8 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS = frozenset(
"LongCatImagePipelineConfig",
"LongCatImageEditPipelineConfig",
"MiniMaxH3PipelineConfig",
"MingImagePipelineConfig",
"MingImageLayerPipelineConfig",
"QwenImagePipelineConfig",
"QwenImage21PipelineConfig",
"SanaPipelineConfig",
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
"""Opt-in HTTP regression tests for both public Ming checkpoints.
Set SGLANG_TEST_MING_IMAGE=1 on a GPU host with sufficient device/host memory.
"""
import base64
import io
import os
import numpy as np
import pytest
import requests
from PIL import Image, ImageDraw
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
diffusion_server,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
DiffusionSamplingParams,
DiffusionServerArgs,
DiffusionTestCase,
)
pytestmark = pytest.mark.skipif(
os.environ.get("SGLANG_TEST_MING_IMAGE") != "1",
reason="opt-in full-checkpoint Ming-Image test",
)
@pytest.fixture(
params=[
("Design", False, "off", False),
("Design-Layer", False, "off", False),
("Design-Layer", True, "off", False),
("Design", False, "request", False),
("Design-Layer", False, "request", False),
("Design-Layer", False, "off", True),
],
ids=[
"design-cold",
"layer-cold",
"layer-tiled",
"design-warm",
"layer-warm",
"layer-offload",
],
)
def case(request):
checkpoint, tiled, warmup, offload = request.param
extras = [
"--performance-mode speed",
f"--warmup-mode {warmup}",
f"--vae-tiling {str(tiled).lower()}",
]
if offload:
extras.append(
"--component-residency text_encoder=layerwise-offload transformer=layerwise-offload"
)
return DiffusionTestCase(
f"ming_image_{checkpoint.lower()}_{'offload' if offload else 'tiled' if tiled else 'full'}_{warmup}",
DiffusionServerArgs(
model_path=f"inclusionAI/Ming-Image-0.1-{checkpoint}",
modality="image",
extras=extras,
),
DiffusionSamplingParams(output_size="512x512"),
)
def test_repeated_generation_and_editing(diffusion_server, case):
url = f"http://localhost:{diffusion_server.port}/v1/images"
layered = case.server_args.model_path.endswith("-Layer")
reference = Image.new("RGBA", (512, 512), "white")
ImageDraw.Draw(reference).rectangle((120, 120, 390, 390), fill="red")
buffer = io.BytesIO()
reference.save(buffer, format="PNG")
modes = ["edit"] if layered else ["generation", "edit"]
for mode in modes:
payload = {
"prompt": (
"Decompose this image into 2 layers."
if layered
else "Change the red square to blue."
if mode == "edit"
else "A red ceramic teapot on a white background."
),
"size": "512x512",
"num_inference_steps": 12,
"seed": 42,
"output_format": "png",
"response_format": "b64_json",
}
if layered:
payload["num_layers"] = 2
previous = {}
for count in (1, 1, 2, 2):
payload["n"] = count
if mode == "edit":
response = requests.post(
f"{url}/edits",
data=payload,
files={"image": ("input.png", buffer.getvalue(), "image/png")},
timeout=300,
)
else:
response = requests.post(
f"{url}/generations", json=payload, timeout=300
)
assert response.ok, response.text
data = response.json()["data"]
assert len(data) == count * (2 if layered else 1)
images = []
for item in data:
with Image.open(
io.BytesIO(base64.b64decode(item["b64_json"]))
) as image:
assert image.mode == "RGBA"
assert image.size == (512, 512)
pixels = np.array(image)
images.append(pixels)
# a decomposed background layer can legitimately be a solid color
assert max(pixels[..., :3].std() for pixels in images) > 5
if count in previous:
for actual, expected in zip(images, previous[count], strict=True):
np.testing.assert_array_equal(actual, expected)
if count == 2:
for actual, expected in zip(images, previous[1]):
np.testing.assert_array_equal(actual, expected)
previous[count] = images
@@ -617,6 +617,38 @@ def test_declared_component_use_admits_explicit_component_offload():
manager.begin_request([stage], SimpleNamespace(is_warmup=False), server_args)
@pytest.mark.parametrize(
"mode", ["component-offload", "snapshot-offload", "layerwise-offload"]
)
def test_partial_execution_validates_the_full_pipeline(mode):
validation = _Stage()
encoding = _Stage(ComponentUse("encode", "text_encoder"))
denoising = _Stage(ComponentUse("denoise", "transformer"))
pipeline = SimpleNamespace(
modules={
name: torch.nn.Linear(2, 2) for name in ("text_encoder", "transformer")
},
_stage_name_mapping={
"validate": validation,
"encode": encoding,
"denoise": denoising,
},
component_residency_strategies={},
)
server_args = _server_args_with_component_offload("transformer")
server_args.component_residency = dict.fromkeys(pipeline.modules, mode)
manager = ComponentResidencyManager(pipeline, server_args)
manager.refresh_pipeline(pipeline)
for stages in ([validation], [encoding, denoising]):
manager.begin_request(stages, SimpleNamespace(is_warmup=False), server_args)
assert manager.state.stages == stages
assert manager._ordered_uses == tuple(
use for stage in stages for use in stage.component_uses(server_args)
)
manager.finish_request()
def test_single_component_stage_is_prepared_at_stage_entry():
module = torch.nn.Linear(2, 2)
use = ComponentUse("stage", "text_encoder")
@@ -0,0 +1,253 @@
# SPDX-License-Identifier: Apache-2.0
import sys
from types import SimpleNamespace
import pytest
import torch
from torch.nn import functional as F
from sglang.multimodal_gen.configs.models.dits.ming_image import MingImageDitConfig
from sglang.multimodal_gen.configs.pipeline_configs.ming_image import (
MingImageLayerPipelineConfig,
MingImagePipelineConfig,
)
from sglang.multimodal_gen.configs.sample.ming_image import (
MingImageLayerSamplingParams,
MingImageSamplingParams,
)
from sglang.multimodal_gen.registry import get_model_info
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
ComponentLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.ming_image import (
MingImageEncoderLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import (
TextEncoderLoader,
)
from sglang.multimodal_gen.runtime.models.dits.ming_image import (
MingImageTransformer2DModel,
MingRMSNorm,
MingSiluAndMul,
)
from sglang.multimodal_gen.runtime.models.encoders.ming_image import ming_position_ids
from sglang.multimodal_gen.runtime.pipelines.ming_image_pipeline import (
MingImagePipeline,
prepare_mu,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import (
MingImageReferenceStage,
ming_reference_size,
)
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_experts
from sglang.srt.layers.moe.topk import StandardTopKOutput
from sglang.srt.runtime_context import get_context
@pytest.mark.parametrize(
"checkpoint,config_cls,sampling_cls",
[
("Design", MingImagePipelineConfig, MingImageSamplingParams),
("Design-Layer", MingImageLayerPipelineConfig, MingImageLayerSamplingParams),
],
)
@pytest.mark.parametrize(
"path_template",
[
"inclusionAI/Ming-Image-0.1-{}",
"/models/Ming-Image-0.1-{}",
"/cache/models--inclusionAI--Ming-Image-0.1-{}/snapshots/revision",
],
)
def test_registry_resolves_both_checkpoints(
checkpoint, config_cls, sampling_cls, path_template, monkeypatch
):
monkeypatch.setattr(
"sglang.multimodal_gen.registry.maybe_download_model_index",
lambda _: pytest.fail("Ming checkpoints do not have a model_index.json"),
)
info = get_model_info(path_template.format(checkpoint), backend="sglang")
assert info.pipeline_cls is MingImagePipeline
assert info.pipeline_config_cls is config_cls
assert info.sampling_param_cls is sampling_cls
@pytest.mark.parametrize("mode,multi", [("zero_masked", False), ("learned", True)])
def test_checkpoint_contract(mode, multi):
config = MingImageDitConfig()
config.update_model_arch(
{
"alignment_padding_mode": mode,
"multi_frame_output": multi,
"n_layers": 30,
"n_heads": 30,
}
)
assert config.num_layers == 30
assert config.num_attention_heads == 30
assert config.alignment_padding_mode == mode
@pytest.mark.parametrize(
"metadata",
[
{},
{"alignment_padding_mode": "learned"},
{"alignment_padding_mode": "zero_masked", "multi_frame_output": True},
{"alignment_padding_mode": "learned", "multi_frame_output": False},
],
)
def test_checkpoint_contract_rejects_ambiguous_padding(metadata):
with pytest.raises(ValueError):
MingImageDitConfig().update_model_arch(metadata)
def test_encoder_loader_is_pipeline_local():
assert MingImageEncoderLoader.component_names == []
assert (
type(ComponentLoader.for_component_type("text_encoder", "transformers"))
is TextEncoderLoader
)
@pytest.mark.parametrize(
"prompt,count",
[("", 4), ("Decompose this image into 2 layers.", 2), ("Number of layers: 3", 3)],
)
def test_layer_count_request_and_output_contract(prompt, count):
params = MingImageLayerSamplingParams(prompt=prompt)
assert params.build_request_extra()["ming_num_layers"] == count
assert params.num_samples_per_request == count
assert "num_layers" in params.image_request_extra_fields()
assert params.default_image_output_format() == "png"
@pytest.mark.parametrize("count", [0, -1, True, 1.5, "2"])
def test_layer_count_rejects_empty_output(count):
with pytest.raises(ValueError, match="positive integer"):
MingImageLayerSamplingParams(num_layers=count).build_request_extra()
def test_cpu_noise_draw_preserves_official_frame_order():
config = MingImagePipelineConfig()
config.check_pipeline_config()
assert not config.vae_tiling
assert not config.vae_sp
assert config.supports_sequential_multi_output_inference()
batch = SimpleNamespace(extra={"ming_frames": 3}, height=32, width=48)
shape = config.prepare_latent_shape(batch, 1, 1)
actual = torch.randn(shape, generator=torch.Generator().manual_seed(42))
actual = config.maybe_pack_latents(actual, 1, batch)
expected = (
torch.randn((3, 16, 4, 6), generator=torch.Generator().manual_seed(42))
.transpose(0, 1)
.unsqueeze(0)
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
assert config.get_latent_dtype(torch.bfloat16) == torch.float32
assert config.shard_latents_for_sp(batch, actual) == (actual, False)
assert config.get_classifier_free_guidance_scale(batch, 2.0) == 3.0
def test_centered_video_rope_with_reference_and_query_tokens():
ids = [10, 11] + [99] * 4 + [12, 13] + [99] * 3 + [14]
positions = ming_position_ids(ids, [(1, 4, 4), (1, 2, 6)], 99)[:, 0]
assert positions.shape == (3, len(ids))
torch.testing.assert_close(positions[:, :2], torch.tensor([[0, 1]]).expand(3, -1))
torch.testing.assert_close(
positions[:, 2:6], torch.tensor([[2, 2, 2, 2], [2, 2, 3, 3], [2, 3, 2, 3]])
)
torch.testing.assert_close(
positions[:, 8:11], torch.tensor([[5, 5, 5], [5, 5, 5], [4, 5, 6]])
)
assert positions[:, -1].tolist() == [6, 6, 6]
@pytest.mark.parametrize(
"height,width,resolution,expected",
[
(512, 512, 512, (512, 512)),
(720, 1280, 1024, (720, 1280)),
(1280, 720, 1024, (1280, 720)),
(100, 400, 512, (256, 1024)),
],
)
def test_official_reference_buckets(height, width, resolution, expected):
assert ming_reference_size(height, width, resolution) == expected
@pytest.mark.parametrize(
"override,expected", [({}, torch.bfloat16), ({"vae": "fp32"}, torch.float32)]
)
def test_reference_vae_declares_precision_before_first_encode(override, expected):
args = SimpleNamespace(
component_precisions=override, pipeline_config=MingImagePipelineConfig()
)
(use,) = MingImageReferenceStage(torch.nn.Identity()).component_uses(args)
assert use.component_name == "vae"
assert use.target_dtype == expected
def test_ming_norm_and_swiglu_preserve_reference_rounding():
torch.manual_seed(9)
x = torch.randn(3, 128, dtype=torch.bfloat16)
norm = MingRMSNorm(128).to(dtype=x.dtype)
with torch.no_grad():
norm.weight.copy_(torch.randn_like(norm.weight))
expected = (
x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-5)
).to(x.dtype) * norm.weight
torch.testing.assert_close(norm(x), expected, rtol=0, atol=0)
gate, up = x.chunk(2, -1)
torch.testing.assert_close(MingSiluAndMul()(x), F.silu(gate) * up, rtol=0, atol=0)
def test_padding_replacement_keeps_valid_rows_and_registers():
x = torch.randn(2, 5, 8)
token = torch.randn(1, 8)
mask = (torch.arange(5) < 3).unsqueeze(0)
output = MingImageTransformer2DModel._replace_padding_with_token_mask(
x, mask, token
)
torch.testing.assert_close(output[:, :3], x[:, :3], rtol=0, atol=0)
torch.testing.assert_close(output[:, 3:], token.expand(2, 2, 8), rtol=0, atol=0)
assert output.data_ptr() != x.data_ptr()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_moe_preserves_bf16_activation_before_fp32_combine():
torch.manual_seed(7)
x = torch.randn(16, 128, dtype=torch.bfloat16, device="cuda")
w13 = torch.randn(4, 256, 128, dtype=x.dtype, device=x.device) * 0.02
w2 = torch.randn(4, 128, 128, dtype=x.dtype, device=x.device) * 0.02
weights, ids = torch.randn(16, 4, device=x.device).softmax(-1).topk(2)
config = MoeRunnerConfig(
activation="silu_rounded",
is_gated=True,
no_combine=True,
inplace=False,
top_k=2,
)
with get_context().override_server_args(model_path="dummy"):
actual = fused_experts(
x, w13, w2, StandardTopKOutput(weights, ids.int(), None), config
)
expected = torch.empty_like(actual)
for expert in range(4):
rows, slots = torch.where(ids == expert)
gate, up = F.linear(x[rows], w13[expert]).chunk(2, -1)
expected[rows, slots] = F.linear(F.silu(gate) * up, w2[expert])
torch.testing.assert_close(actual, expected, rtol=0, atol=2e-4)
combined = (actual.float() * weights.unsqueeze(-1)).sum(1).to(x.dtype)
reference = (expected.float() * weights.unsqueeze(-1)).sum(1).to(x.dtype)
torch.testing.assert_close(combined, reference, rtol=0, atol=2e-4)
@pytest.mark.parametrize("size,mu", [(256, 0.5), (1024, 1.35), (2048, 1.35)])
def test_flow_shift_uses_spatial_tokens_not_layer_count(size, mu):
assert prepare_mu(SimpleNamespace(height=size, width=size), None) == ("mu", mu)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, *sys.argv[1:]]))
@@ -144,6 +144,7 @@ class TestMultiOutputGrouping(unittest.TestCase):
output_file_name="image.png",
num_outputs_per_prompt=2,
seed=[100, 101],
generator_device="cpu",
)
)
return req
@@ -182,6 +183,14 @@ class TestMultiOutputGrouping(unittest.TestCase):
for req in sequential_outputs
)
)
self.assertEqual(
[req.generator[0].initial_seed() for req in sequential_outputs],
[100, 101],
)
self.assertEqual([req.seeds for req in sequential_outputs], [[100], [101]])
self.assertIsNot(
sequential_outputs[0].generator[0], sequential_outputs[1].generator[0]
)
def test_split_batched_latents_uses_original_batched_tensor(self):
stage = LatentPreparationStage.__new__(LatentPreparationStage)
@@ -62,6 +62,7 @@ if _is_cuda:
gelu_and_mul,
gelu_tanh_and_mul,
silu_and_mul,
silu_and_mul_with_activation_rounding,
)
elif _is_cpu and _is_cpu_amx_available:
pass
@@ -673,6 +674,14 @@ def _fused_moe_kernel_sequence(
if fuse_swiglu_interleaved:
# silu(gate) * up was already applied by the up-GEMM epilogue.
pass
elif activation == "silu_rounded" and is_gated:
if _is_cuda:
silu_and_mul_with_activation_rounding(
intermediate_cache1.view(-1, N), intermediate_cache2
)
else:
gate, up = intermediate_cache1.view(-1, N).chunk(2, dim=-1)
intermediate_cache2.copy_(F.silu(gate) * up)
elif activation == "silu" and is_gated:
# - gemm1_alpha != None: GPT-OSS-style swiglu(alpha, limit)
# - gemm1_alpha == None and gemm1_limit != None: silu+clamp+mul(limit-only)