From 0fb699ac03e28ce6dda8a7ac6835e95a12ce410d Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 25 Sep 2026 14:44:37 +0800 Subject: [PATCH] [diffusion] model: support Ming-Image Design and Design-Layer (#41067) Co-authored-by: Mick Qian --- .../diffusion/inclusionAI/Ming-Image.mdx | 115 +++++ docs/cookbook/diffusion/intro.mdx | 6 + docs/docs.json | 6 + .../sglang-diffusion/compatibility_matrix.mdx | 31 ++ .../configs/inclusionAI/ming-image.jsx | 234 +++++++++ .../snippets/diffusion/comfyui-support.jsx | 73 ++- docs/src/snippets/diffusion/model-catalog.jsx | 5 + .../ops/moe/fused_moe_triton_kernels.py | 4 +- .../configs/models/dits/ming_image.py | 36 ++ .../configs/models/encoders/ming_image.py | 26 + .../configs/pipeline_configs/ming_image.py | 120 +++++ .../configs/sample/ming_image.py | 55 ++ python/sglang/multimodal_gen/registry.py | 1 + .../model_padders/ming_image.py | 17 + .../breakable_cuda_graph/prompt_padding.py | 1 + .../runtime/cache/cache_dit_integration.py | 4 + .../layers/attention/backends/flash_attn_2.py | 8 +- .../loader/component_loaders/ming_image.py | 61 +++ .../memory_managers/component_manager.py | 12 +- .../runtime/models/dits/ming_image.py | 166 ++++++ .../runtime/models/dits/zimage.py | 29 +- .../runtime/models/encoders/ming_image.py | 472 ++++++++++++++++++ .../models/vaes/autoencoder_kl_qwenimage.py | 2 + .../runtime/pipelines/ming_image_pipeline.py | 92 ++++ .../pipelines_core/stages/input_validation.py | 14 +- .../model_specific_stages/ming_image.py | 346 +++++++++++++ .../runtime/server_args/server_args.py | 6 + .../test/server/test_server_ming_image.py | 130 +++++ .../test/unit/test_component_residency.py | 32 ++ .../test/unit/test_ming_image.py | 253 ++++++++++ .../test/unit/test_multi_output_grouping.py | 9 + .../moe/moe_runner/triton_utils/fused_moe.py | 9 + .../kernels/ops/moe/test_moe_tma_cache.py | 37 ++ 33 files changed, 2350 insertions(+), 62 deletions(-) create mode 100644 docs/cookbook/diffusion/inclusionAI/Ming-Image.mdx create mode 100644 docs/src/snippets/configs/inclusionAI/ming-image.jsx create mode 100644 python/sglang/multimodal_gen/configs/models/dits/ming_image.py create mode 100644 python/sglang/multimodal_gen/configs/models/encoders/ming_image.py create mode 100644 python/sglang/multimodal_gen/configs/pipeline_configs/ming_image.py create mode 100644 python/sglang/multimodal_gen/configs/sample/ming_image.py create mode 100644 python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ming_image.py create mode 100644 python/sglang/multimodal_gen/runtime/loader/component_loaders/ming_image.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/ming_image.py create mode 100644 python/sglang/multimodal_gen/runtime/models/encoders/ming_image.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines/ming_image_pipeline.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py create mode 100644 python/sglang/multimodal_gen/test/server/test_server_ming_image.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_ming_image.py create mode 100644 test/registered/kernels/ops/moe/test_moe_tma_cache.py diff --git a/docs/cookbook/diffusion/inclusionAI/Ming-Image.mdx b/docs/cookbook/diffusion/inclusionAI/Ming-Image.mdx new file mode 100644 index 0000000000..216d96af41 --- /dev/null +++ b/docs/cookbook/diffusion/inclusionAI/Ming-Image.mdx @@ -0,0 +1,115 @@ +--- +title: Ming-Image +tag: NEW +description: "Deploy Ming-Image Design and Design-Layer with SGLang Diffusion for text-to-image, RGBA image editing, and ordered transparent layer decomposition." +--- + +import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx'; +import { Deployment } from '/src/snippets/_deployment.jsx'; +import { config } from '/src/snippets/configs/inclusionAI/ming-image.jsx'; + + + +## 1. Quick start + +Install the dependencies, then this integration from its source checkout: + +```bash Installation +uv pip install "sglang[diffusion]" --prerelease=allow +uv pip install -e "python[diffusion]" --no-deps +``` + +Choose **Design** for generation or editing and **Design-Layer** for decomposition. +The two checkpoints require separate servers. Start with one H200 on Linux/CUDA; +the picker marks unverified hardware and feature combinations explicitly. + + + +Requests return base64 PNGs, preserving all four RGBA channels. For decomposition, +each returned image is one ordered layer, not an independent generation. The +internal composite frame is omitted from the response. Save all entries in +`data`, rather than only the first image. +`n` selects independent generations (or sets of layers), processed sequentially. + +## 2. Model capabilities + +[Ming-Image Design](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design) +generates images from text and edits a single reference image. It targets design +assets, layouts, and transparent RGBA output. Use the prompt prefix +`RGBA, 4-channel, transparent background` to request transparency; RGBA output +alone does not guarantee a transparent background. + +[Design-Layer](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design-Layer) +decomposes one reference into ordered transparent layers. Supply `num_layers`, +or specify a count with `into N layers` or `Number of layers: N` in the prompt; +the prompt count takes precedence. Defaults are 12 steps with guidance 1 for +Design and guidance 2 for Design-Layer. The model uses zero unconditional +embeddings and does not support custom negative prompts. + +Both checkpoints use the native Bailing multimodal MoE encoder, query connector, +joint DiT, and four-channel Qwen-style VAE. No remote modeling code is executed. +The original checkpoint directory layout is accepted without conversion. + +## 3. Offline generation + +```bash Design +sglang generate --model-path inclusionAI/Ming-Image-0.1-Design \ + --prompt "A minimalist exhibition poster with a red geometric sculpture" \ + --height 1024 --width 1024 --save-output +``` + +```bash Layer decomposition +sglang generate --model-path inclusionAI/Ming-Image-0.1-Design-Layer \ + --image-path /path/to/input.png --num-layers 4 --save-output +``` + +Design defaults to 2048 by 2048; 1024 by 1024 reduces latency and activation +memory. Decomposition defaults to the official 1024-resolution aspect buckets; +use `--height 512 --width 512` for the 512 buckets. Reference-image requests +select the closest official bucket and restore the input aspect ratio in the +returned PNGs. Multiple reference images are not supported. + +For a renamed local checkpoint directory, also provide +`--model-id inclusionAI/Ming-Image-0.1-Design` or the corresponding Layer ID. + +## 4. Runtime features + +DiT parallelism reuses the native TP and sequence-parallel attention layers. +The DiT has 30 attention heads: TP multiplied by Ulysses must divide 30. +Encoder folding has its own head constraints; two-way folding is compatible +with the 16-head language tower, 16-head vision tower, and 12-head connector. +Do not infer encoder compatibility from the DiT's head count. + +Use the standard component residency controls for `transformer`, `text_encoder`, +and `vae`. Layerwise offload trades host memory and transfer time for lower VRAM. +The default eager path does not use approximate caching or quantized attention. +Cache-DiT and SageAttention require workload-specific quality validation. +VAE tiling is disabled by default to match the official inference path. Enable +`--vae-tiling true` when decode memory is constrained; tiling can change pixels. + +Breakable CUDA graphs preserve Ming's checkpoint-specific padding and RoPE +offsets. They capture exact prompt shapes; an uncaptured shape runs eagerly. +Requests with different prompts are not merged into a dynamic batch. A fixed +seed does not imply bit-identical output across attention backends, GPU types, +or parallel topologies. + +Full-checkpoint validation covers generation, editing, and layer decomposition +on H200, including repeated HTTP requests before and after warmup. Tested +resolutions include 2048-square generation and four-layer 1024-square +decomposition. See the [compatibility notes](/docs/sglang-diffusion/compatibility_matrix#ming-image-integration-status) +for the tested feature scope. Other GPU families and quantized checkpoints +have not been validated for this model. + +See [performance optimization](/docs/sglang-diffusion/performance-optimization) +for shared runtime controls and the +[official repository](https://github.com/inclusionAI/Ming-Image) for model examples. + +## 5. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + + +Use the generic image node for Design generation. The ComfyUI integration has +not been validated for ordered multi-layer output; use the HTTP API above for +Design-Layer decomposition. diff --git a/docs/cookbook/diffusion/intro.mdx b/docs/cookbook/diffusion/intro.mdx index 840af5b873..48067a89fd 100644 --- a/docs/cookbook/diffusion/intro.mdx +++ b/docs/cookbook/diffusion/intro.mdx @@ -49,6 +49,12 @@ Image models generate one image request as a bounded denoising job, usually with href="/cookbook/diffusion/Z-Image/Z-Image-Turbo" img="/cards/logos/zimage.png" /> + s.mode !== "text", + disableReason: + "Reference-image requests use the 512 or 1024 bucket families.", + }, + ], + }, + { + id: "layers", + title: "Layers", + scope: "request", + kind: "number", + min: 1, + max: 8, + default: 4, + description: + "Number of returned layers for Design-Layer; ignored by Design requests.", + options: [], + }, + ], + commandBuilder: { + defaultSelection: { + hw: "h200", + nodes: 1, + gpus_per_node: 1, + topology_mode: "auto", + tp_size: 1, + ulysses_degree: 1, + ring_degree: 1, + }, + resource: { + limits: { nodes: { min: 1, max: 1 }, gpus_per_node: { min: 1, max: 2 } }, + verifiedRecipes: [], + autoTopology: (s) => ({ + tp_size: 1, + ulysses_degree: Number(s.gpus_per_node), + ring_degree: 1, + }), + validateTopology: (s, topology) => { + const errors = []; + const { tp_size: tp, ulysses_degree: sp, ring_degree: ring } = topology; + if (Number(s.nodes) !== 1) + errors.push("This picker covers a single node."); + if (![tp, sp, ring].every((n) => Number.isInteger(n) && n >= 1)) + errors.push("Parallel degrees must be positive integers."); + if (Number(s.nodes) * Number(s.gpus_per_node) !== tp * sp * ring) + errors.push("GPU count must equal TP times Ulysses times Ring."); + if (30 % (tp * sp)) + errors.push("TP times Ulysses must divide the 30 DiT heads."); + if (tp > 2) + errors.push( + "This recipe covers encoder tensor parallelism up to two GPUs.", + ); + if (ring > 1 && s.attention === "sdpa") + errors.push("Ring requires FlashAttention."); + return errors; + }, + }, + resolveDeployment: (s) => { + const resource = config.commandBuilder.resource; + const topology = + s.topology_mode === "manual" + ? { + tp_size: Number(s.tp_size), + ulysses_degree: Number(s.ulysses_degree), + ring_degree: Number(s.ring_degree), + } + : resource.autoTopology(s); + const errors = resource.validateTopology(s, topology); + const world = Number(s.nodes) * Number(s.gpus_per_node); + const verifiedServe = + s.hw === "h200" && + world === 1 && + s.placement === "resident" && + s.attention === "platform"; + const verifiedRequest = + verifiedServe && + s.resolution === "512" && + (s.mode !== "layer" || Number(s.layers) === 2); + const model = + s.mode === "layer" + ? "inclusionAI/Ming-Image-0.1-Design-Layer" + : "inclusionAI/Ming-Image-0.1-Design"; + const flags = [`--model-path ${model}`]; + if (world > 1) + flags.push( + `--num-gpus ${world}`, + `--tp-size ${topology.tp_size}`, + `--ulysses-degree ${topology.ulysses_degree}`, + ); + if (topology.ring_degree > 1) + flags.push(`--ring-degree ${topology.ring_degree}`); + flags.push("--host {{HOST_IP}}", "--port {{PORT}}"); + return { + match: { hw: s.hw }, + nnodes: Number(s.nodes), + verified: verifiedRequest, + flags, + builder: { + topology, + topologySummary: `TP ${topology.tp_size}, Ulysses ${topology.ulysses_degree}, Ring ${topology.ring_degree}`, + errors, + warnings: verifiedRequest + ? [] + : ["This exact HTTP recipe has not completed verification."], + verification: { + serve: errors.length + ? "error" + : verifiedServe + ? "verified" + : "unverified", + request: errors.length + ? "error" + : verifiedRequest + ? "verified" + : "unverified", + }, + }, + }; + }, + }, + modelNames: { default: "inclusionAI/Ming-Image-0.1-Design" }, + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + INPUT_IMAGE: { + target: "curl", + label: "Reference PNG on the client", + default: "/path/to/input.png", + }, + }, + curl: (s) => { + const body = { + prompt: + s.mode === "layer" + ? `Decompose this image into ${Number(s.layers)} layers.` + : s.mode === "edit" + ? "Change the teapot to blue, preserving its shape." + : "RGBA, 4-channel, transparent background. A red enamel teapot, product photography.", + size: `${s.resolution}x${s.resolution}`, + output_format: "png", + response_format: "b64_json", + }; + if (s.mode === "text") + return `curl -sS --fail-with-body http://{{CURL_HOST}}:{{CURL_PORT}}/v1/images/generations \\ + -H 'Content-Type: application/json' \\ + -d '${JSON.stringify(body, null, 2)}'`; + if (s.mode === "layer") body.num_layers = Number(s.layers); + const fields = Object.entries(body).map( + ([key, value]) => ` --form-string '${key}=${value}'`, + ); + fields.push(' -F "image=@{{INPUT_IMAGE}};type=image/png"'); + return `curl -sS --fail-with-body http://{{CURL_HOST}}:{{CURL_PORT}}/v1/images/edits \\ +${fields.join(" \\\n")}`; + }, + runModes: () => ["python"], + showPlaygroundLink: false, + cells: [], +}; diff --git a/docs/src/snippets/diffusion/comfyui-support.jsx b/docs/src/snippets/diffusion/comfyui-support.jsx index a2540f4c81..367c644ebb 100644 --- a/docs/src/snippets/diffusion/comfyui-support.jsx +++ b/docs/src/snippets/diffusion/comfyui-support.jsx @@ -5,44 +5,43 @@ // (python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py); // a model absent from that dict has no integrated mode. -const PLUGIN_PATH = "python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion"; -const PLUGIN_URL = - "https://github.com/sgl-project/sglang/tree/main/" + PLUGIN_PATH; - -const MODELS = { - flux: { - serverNode: "SGLDiffusion Generate Image", - integratedKey: "flux", - workflow: "flux_sgld_sp.json", - }, - "z-image": { - serverNode: "SGLDiffusion Generate Image", - integratedKey: "lumina2", - workflow: "z-image_sgld.json", - }, - "qwen-image": { - serverNode: "SGLDiffusion Generate Image", - integratedKey: "qwen_image", - workflow: "qwen_image_sgld.json", - }, - "qwen-image-edit": { - serverNode: "SGLDiffusion Generate Image", - integratedKey: "qwen_image_edit", - note: "Image editing through the integrated path is experimental.", - }, - "minimax-h3": { - serverNode: "SGLDiffusion Generate H3", - verified: true, - integratedBlockedBecause: - "H3 denoises a packed video-and-audio sequence in one pass and routes " + - "conditioning by task, while ComfyUI's KSampler drives a single latent " + - "tensor and has no audio branch", - }, - image: { serverNode: "SGLDiffusion Generate Image" }, - video: { serverNode: "SGLDiffusion Generate Video" }, -}; - export const ComfyUISupport = ({ model = "video", note }) => { + const PLUGIN_PATH = "python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion"; + const PLUGIN_URL = + "https://github.com/sgl-project/sglang/tree/main/" + PLUGIN_PATH; + + const MODELS = { + flux: { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "flux", + workflow: "flux_sgld_sp.json", + }, + "z-image": { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "lumina2", + workflow: "z-image_sgld.json", + }, + "qwen-image": { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "qwen_image", + workflow: "qwen_image_sgld.json", + }, + "qwen-image-edit": { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "qwen_image_edit", + note: "Image editing through the integrated path is experimental.", + }, + "minimax-h3": { + serverNode: "SGLDiffusion Generate H3", + verified: true, + integratedBlockedBecause: + "H3 denoises a packed video-and-audio sequence in one pass and routes " + + "conditioning by task, while ComfyUI's KSampler drives a single latent " + + "tensor and has no audio branch", + }, + image: { serverNode: "SGLDiffusion Generate Image" }, + video: { serverNode: "SGLDiffusion Generate Video" }, + }; const spec = MODELS[model] || MODELS.video; const extraNote = note || spec.note; diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx index 7a4f44f5dc..5c34eb710c 100644 --- a/docs/src/snippets/diffusion/model-catalog.jsx +++ b/docs/src/snippets/diffusion/model-catalog.jsx @@ -3,6 +3,11 @@ export const DiffusionModelCatalog = ({ category }) => { const MODEL_CATALOG = { image: [ + { + name: "Ming-Image", + modelIds: ["inclusionAI/Ming-Image-0.1-Design", "inclusionAI/Ming-Image-0.1-Design-Layer"], + cookbook: "/cookbook/diffusion/inclusionAI/Ming-Image", + }, { name: "FLUX", modelIds: [ diff --git a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py index c232a4a60c..73f722a16d 100644 --- a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py +++ b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py @@ -752,9 +752,11 @@ def clear_b_tma_desc_cache() -> None: def _get_b_tma_desc_cached(B: torch.Tensor, block_n: int, block_k: int): """ Cache TensorDescriptor for constant weight B. - Keyed by storage ptr + shape/stride/dtype + tile shape. + Keyed by tensor identity + storage ptr + shape/stride/dtype + tile shape. """ key = ( + # offload can rebind a Parameter while another tensor reuses its address + id(B), int(B.data_ptr()), tuple(B.shape), tuple(B.stride()), diff --git a/python/sglang/multimodal_gen/configs/models/dits/ming_image.py b/python/sglang/multimodal_gen/configs/models/dits/ming_image.py new file mode 100644 index 0000000000..a511e7c0dc --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/dits/ming_image.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.zimage import ( + ZImageArchConfig, + ZImageDitConfig, +) + + +@dataclass +class MingImageArchConfig(ZImageArchConfig): + axes_lens: tuple[int, ...] = (20480, 512, 512) + alignment_padding_mode: str | None = None + multi_frame_output: bool | None = None + + +@dataclass +class MingImageDitConfig(ZImageDitConfig): + arch_config: MingImageArchConfig = field(default_factory=MingImageArchConfig) + prefix: str = "ming_image" + + def update_model_arch(self, config): + config = dict(config) + config["num_layers"] = config.pop("n_layers", self.num_layers) + config["num_attention_heads"] = config.pop("n_heads", self.num_attention_heads) + if type(config.get("multi_frame_output")) is not bool or ( + config.get("alignment_padding_mode"), + config.get("multi_frame_output"), + ) not in ( + ("zero_masked", False), + ("learned", True), + ): + raise ValueError( + "Ming-Image requires an explicit, valid checkpoint padding/output contract" + ) + super().update_model_arch(config) diff --git a/python/sglang/multimodal_gen/configs/models/encoders/ming_image.py b/python/sglang/multimodal_gen/configs/models/encoders/ming_image.py new file mode 100644 index 0000000000..bd137fdec9 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/encoders/ming_image.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.encoders.base import ( + TextEncoderArchConfig, + TextEncoderConfig, +) + + +@dataclass +class MingImageEncoderArchConfig(TextEncoderArchConfig): + architectures: list[str] = field(default_factory=lambda: ["MingImageEncoder"]) + hidden_size: int = 2048 + num_attention_heads: int = 16 + intermediate_size: int = 5120 + llm_config: dict = field(default_factory=dict) + vision_config: dict = field(default_factory=dict) + connector_config: dict = field(default_factory=dict) + projection_config: dict = field(default_factory=dict) + + +@dataclass +class MingImageEncoderConfig(TextEncoderConfig): + arch_config: MingImageEncoderArchConfig = field( + default_factory=MingImageEncoderArchConfig + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ming_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ming_image.py new file mode 100644 index 0000000000..ffab88619f --- /dev/null +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ming_image.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +import torch + +from sglang.multimodal_gen.configs.models.dits.ming_image import MingImageDitConfig +from sglang.multimodal_gen.configs.models.encoders.ming_image import ( + MingImageEncoderConfig, +) +from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ImagePipelineConfig, + ModelTaskType, +) + + +@dataclass +class MingImagePipelineConfig(ImagePipelineConfig): + task_type: ModelTaskType = ModelTaskType.TI2I + dit_config: MingImageDitConfig = field(default_factory=MingImageDitConfig) + vae_config: QwenImageVAEConfig = field(default_factory=QwenImageVAEConfig) + text_encoder_configs: tuple = field( + default_factory=lambda: (MingImageEncoderConfig(),) + ) + text_encoder_precisions: tuple[str, ...] = ("bf16",) + native_only_components: tuple[str, ...] = ("text_encoder", "transformer", "vae") + vae_precision: str = "bf16" + vae_tiling: bool = False + vae_sp: bool = False + enable_autocast: bool = False + generator_device: str = "cpu" + + def supports_dynamic_batching(self): + # Reference/query sequences have different lengths across requests. + return False + + def supports_sequential_multi_output_inference(self): + return True + + def get_latent_dtype(self, prompt_dtype): + return torch.float32 + + def prepare_latent_shape(self, batch, batch_size, num_frames): + # Draw frames before channels, as in the official CPU RNG stream. + return ( + batch_size, + batch.extra["ming_frames"], + 16, + batch.height // 8, + batch.width // 8, + ) + + def maybe_pack_latents(self, latents, batch_size, batch): + return latents.transpose(1, 2).contiguous() + + def shard_latents_for_sp(self, batch, latents): + # Ming shards patchified tokens inside the DiT, not spatial pixels here. + return latents, False + + def gather_latents_for_sp(self, latents, batch=None): + return latents + + def gather_noise_pred_for_sp(self, batch, noise_pred): + return noise_pred + + def post_denoising_loop(self, latents, batch): + return latents + + def get_pos_prompt_embeds(self, batch): + return batch.prompt_embeds[0] + + def get_neg_prompt_embeds(self, batch): + return batch.negative_prompt_embeds[0] + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return { + "direct_embeddings": batch.extra["ming_direct"], + "reference_latents": batch.extra.get("ming_reference_latents"), + } + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + return { + "direct_embeddings": torch.zeros_like(batch.extra["ming_direct"]), + "reference_latents": batch.extra.get("ming_reference_latents"), + } + + def get_classifier_free_guidance_scale(self, batch, guidance_scale): + # Official Ming uses cond + cfg * (cond - uncond). + return guidance_scale + 1.0 + + +@dataclass +class MingImageLayerPipelineConfig(MingImagePipelineConfig): + task_type: ModelTaskType = ModelTaskType.I2I + + +def register(): + from sglang.multimodal_gen.configs.sample.ming_image import ( + MingImageLayerSamplingParams, + MingImageSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=MingImageLayerSamplingParams, + pipeline_config_cls=MingImageLayerPipelineConfig, + hf_model_paths=["inclusionAI/Ming-Image-0.1-Design-Layer"], + model_detectors=[lambda model: "ming-image-0.1-design-layer" in model.lower()], + ) + register_configs( + sampling_param_cls=MingImageSamplingParams, + pipeline_config_cls=MingImagePipelineConfig, + hf_model_paths=["inclusionAI/Ming-Image-0.1-Design"], + model_detectors=[ + lambda model: ( + "ming-image-0.1-design" in model.lower() + and "design-layer" not in model.lower() + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/sample/ming_image.py b/python/sglang/multimodal_gen/configs/sample/ming_image.py new file mode 100644 index 0000000000..ec734f2585 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/sample/ming_image.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +import re +from dataclasses import dataclass + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + + +@dataclass +class MingImageSamplingParams(SamplingParams): + height: int = 2048 + width: int = 2048 + num_inference_steps: int = 12 + guidance_scale: float = 1.0 + negative_prompt: str = "" + generator_device: str = "cpu" + + @classmethod + def default_image_output_format(cls): + return "png" + + +@dataclass +class MingImageLayerSamplingParams(MingImageSamplingParams): + height: int = 1024 + width: int = 1024 + guidance_scale: float = 2.0 + prompt: str = "" + num_layers: int = 4 + + def build_request_extra(self): + extra = super().build_request_extra() + if type(self.num_layers) is not int or self.num_layers < 1: + raise ValueError("num_layers must be a positive integer") + match = re.search( + r"(?:into\s+(\d+)\s+layers|number of layers:\s*(\d+))", + self.prompt, + re.IGNORECASE, + ) + count = ( + int(next(value for value in match.groups() if value)) + if match + else self.num_layers + ) + if count < 1: + raise ValueError("The prompt must request at least one layer") + extra["ming_num_layers"] = count + return extra + + @classmethod + def image_request_extra_fields(cls): + return frozenset({"num_layers"}) + + @property + def num_samples_per_request(self): + return self.build_request_extra()["ming_num_layers"] diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index 40abb10ce2..cdb3af8422 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -169,6 +169,7 @@ _MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = [] # aliases next to the resolver that consumes them so CLI detection and # pipeline selection cannot drift apart KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = { + "ming-image-0.1-design": "MingImagePipeline", "minimaxai/minimax-h3": "MiniMaxH3Pipeline", "minimax/minimax-h3": "MiniMaxH3Pipeline", "fastvideo/fastvideo-fasth3-4-step-preview-v1-vsa-datafree": "FastH3Pipeline", diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ming_image.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ming_image.py new file mode 100644 index 0000000000..83d42fb75d --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/ming_image.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Keep Ming's trained padding registers and RoPE offsets unchanged.""" + +from sglang.multimodal_gen.runtime.breakable_cuda_graph import prompt_padding + + +def is_ming_image(current_model, call_kwargs): + return prompt_padding.transformer_class_name_matches(current_model, "MingImage") + + +def preserve_ming_prompt(call_kwargs, current_model, buckets): + # Changing caption length also shifts every image's first RoPE coordinate. + # Capture exact signatures; uncaptured prompt lengths run eager. + return call_kwargs + + +prompt_padding.register_prompt_padder(is_ming_image, preserve_ming_prompt) diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py index b9d5105530..e856bcc0c5 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py @@ -302,6 +302,7 @@ def _ensure_model_padders_registered() -> None: from sglang.multimodal_gen.runtime.breakable_cuda_graph.model_padders import ( # noqa: F401 ideogram, longcat_image, + ming_image, minimax_h3, qwen_image, sana_video, diff --git a/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py b/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py index 6232e01d8d..d200e6f868 100644 --- a/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py +++ b/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py @@ -403,6 +403,10 @@ class CustomBlockAdapterSpec: # Custom BlockAdapter metadata for models absent from cache-dit's registry. _CUSTOM_BLOCK_ADAPTER_SPECS: dict[str, CustomBlockAdapterSpec] = { + "MingImageTransformer2DModel": CustomBlockAdapterSpec( + blocks_attr="layers", + forward_pattern=ForwardPattern.Pattern_3, + ), "QwenImage21Transformer2DModel": CustomBlockAdapterSpec( blocks_attr="transformer_blocks", forward_pattern=ForwardPattern.Pattern_3, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py index 7b6b8df435..81b92a39d5 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import torch +from flash_attn import flash_attn_func from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( AttentionBackend, @@ -10,9 +11,6 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend i AttentionMetadata, AttentionMetadataBuilder, ) -from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( - flash_attn_func, -) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -68,10 +66,6 @@ class FlashAttention2Impl(AttentionImpl): q=query, # type: ignore[no-untyped-call] k=key, v=value, - cu_seqlens_q=None, - cu_seqlens_k=None, - max_seqlen_q=None, - max_seqlen_k=None, softmax_scale=self.softmax_scale, causal=self.causal, ) diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/ming_image.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/ming_image.py new file mode 100644 index 0000000000..2723a2c7ad --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/ming_image.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + +from transformers import PreTrainedTokenizerFast + +from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( + ComponentLoader, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import ( + TextEncoderLoader, +) +from sglang.multimodal_gen.runtime.loader.weight_utils import ( + checkpoint_weights_iterator, +) +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + get_diffusers_component_config, +) + + +class MingImageEncoderLoader(TextEncoderLoader): + component_names = [] + + def load_customized(self, component_model_path, server_args, component_name): + root = Path(component_model_path) + config = server_args.pipeline_config.text_encoder_configs[0] + mllm = get_diffusers_component_config(component_path=str(root / "mllm")) + config.arch_config.llm_config = mllm["llm_config"] + config.arch_config.vision_config = mllm["vision_config"] + config.arch_config.connector_config = get_diffusers_component_config( + component_path=str(root / "connector") + ) + config.arch_config.projection_config = get_diffusers_component_config( + component_path=str(root / "mlp") + ) + server_args.model_paths[component_name] = component_model_path + return self.load_model( + component_model_path, + config, + server_args, + dtype=server_args.pipeline_config.text_encoder_precisions[0], + component_name=component_name, + ) + + def _get_all_weights(self, model, model_path, to_cpu): + for component in ("mllm", "connector", "mlp"): + for name, weight in checkpoint_weights_iterator( + str(Path(model_path) / component), to_cpu=to_cpu + ): + yield ( + (f"connector.{name}" if component == "connector" else name), + weight, + ) + + +class MingImageTokenizerLoader(ComponentLoader): + expected_library = "transformers" + + def load_customized(self, component_model_path, server_args, component_name): + return PreTrainedTokenizerFast.from_pretrained( + component_model_path, padding_side="right" + ) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index 82990a6c97..450a3ca746 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -249,10 +249,18 @@ class ComponentResidencyManager: no declared ``ComponentUse`` would otherwise be accepted but never moved to the device before a forward pass. """ - if not isinstance(self.server_args, ServerArgs): + if ( + not isinstance(self.server_args, ServerArgs) + or not self.server_args.component_residency + ): return - declared_components = {use.component_name for use in self._ordered_uses} + # sequential multi-output execution runs subsets of the full pipeline + declared_components = { + use.component_name + for name, stage in self.pipeline._stage_name_mapping.items() + for use in stage.component_uses(self.server_args, name) + } unmanaged_components = sorted( component_name for component_name, module in self.pipeline.modules.items() diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py b/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py new file mode 100644 index 0000000000..9bd4446cd8 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from inclusionAI/Ming-Image. +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from sglang.multimodal_gen.runtime.distributed import get_sp_world_size +from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import ( + gather_seq, + shard_like, + shard_seq, + tail_attn_meta, +) +from sglang.multimodal_gen.runtime.models.dits.zimage import ZImageTransformer2DModel +from sglang.srt.layers.layernorm import RMSNorm + + +class MingRMSNorm(RMSNorm): + def __init__(self, dim, eps=1e-5): + # unlike Z-Image, Ming's reference accumulates the variance in fp32 + super().__init__(dim, eps=eps, cast_x_before_out_mul=True, force_native=True) + + +class MingSiluAndMul(nn.Module): + def forward(self, x): + gate, up = x.chunk(2, dim=-1) + return F.silu(gate) * up + + +class MingImageTransformer2DModel(ZImageTransformer2DModel): + """Ming-Image's joint DiT, sharing Z-Image blocks and TP/SP attention.""" + + norm_cls = MingRMSNorm + _aliases = ["DiffusionTransformer"] + + def __init__(self, config, hf_config, quant_config=None): + super().__init__(config, hf_config, quant_config) + self.learned_padding = config.alignment_padding_mode == "learned" + if not self.learned_padding: + del self.x_pad_token, self.cap_pad_token + self.register_buffer( + "x_pad_token", torch.zeros(1, self.dim, device="cpu"), persistent=False + ) + self.register_buffer( + "cap_pad_token", + torch.zeros(1, self.dim, device="cpu"), + persistent=False, + ) + self.layer_names = ["noise_refiner", "context_refiner", "layers"] + for layer in (*self.noise_refiner, *self.context_refiner, *self.layers): + layer.feed_forward.act = MingSiluAndMul() + + def forward( + self, + hidden_states, + encoder_hidden_states, + timestep, + direct_embeddings, + reference_latents=None, + **kwargs, + ): + x = hidden_states + batch_size, channels, output_frames, height, width = x.shape + if reference_latents is not None: + x = torch.cat((x, reference_latents), 2) + frames = x.shape[2] + x = x.reshape(batch_size, channels, frames, height // 2, 2, width // 2, 2) + x = x.permute(0, 2, 3, 5, 4, 6, 1).flatten(1, 3).flatten(2) + x_length = x.shape[1] + sp_size = get_sp_world_size() + x_target = math.ceil(x_length / 32) * 32 + x = F.pad(x, (0, 0, 0, x_target - x_length)) + x, _ = self.all_x_embedder["2-1"](x) + x = self._replace_padding_with_token_mask( + x, + (torch.arange(x_target, device=x.device) < x_length).unsqueeze(0), + self.x_pad_token, + ) + cap, _ = self.cap_embedder(encoder_hidden_states) + cap = torch.cat((cap, direct_embeddings), 1) + cap_length = cap.shape[1] + cap_target = math.ceil(cap_length / 32) * 32 + cap = F.pad(cap, (0, 0, 0, cap_target - cap_length)) + cap = self._replace_padding_with_token_mask( + cap, + (torch.arange(cap_target, device=x.device) < cap_length).unsqueeze(0), + self.cap_pad_token, + ) + + cap_ids = self.create_coordinate_grid( + (cap_target, 1, 1), (1, 0, 0), x.device + ).flatten(0, 2) + x_ids = self.create_coordinate_grid( + (frames, height // 2, width // 2), (cap_target + 1, 0, 0), x.device + ).flatten(0, 2) + x_ids = F.pad(x_ids, (0, 0, 0, x_target - x_length)) + x_valid = x_target if self.learned_padding else x_length + cap_valid = cap_target if self.learned_padding else cap_length + x_freqs, cap_freqs = self.rotary_emb(x_ids), self.rotary_emb(cap_ids) + x_mask, x_meta = self._get_attn_mask_and_meta( + "_ming_x_mask", [x_valid] * batch_size, x_target, x.device + ) + cap_mask, cap_meta = self._get_attn_mask_and_meta( + "_ming_cap_mask", [cap_valid] * batch_size, cap_target, x.device + ) + adaln = self.t_embedder(((1000.0 - timestep) / 1000.0) * self.t_scale).to( + x.dtype + ) + for layer in self.noise_refiner: + x = layer( + x, + x_freqs, + adaln, + attn_mask=x_mask, + attn_mask_meta=x_meta, + skip_sequence_parallel_override=True, + ) + for layer in self.context_refiner: + cap = layer(cap, cap_freqs, attn_mask=cap_mask, attn_mask_meta=cap_meta) + if sp_size > 1: + # masked alignment rows never contribute; learned registers must stay + x, cap = x[:, :x_valid], cap[:, :cap_valid] + x_freqs = tuple(freq[:x_valid] for freq in x_freqs) + cap_freqs = tuple(freq[:cap_valid] for freq in cap_freqs) + unified = torch.cat((x, cap), 1) + freqs = tuple( + torch.cat((image_freq, text_freq), -2) + for image_freq, text_freq in zip(x_freqs, cap_freqs) + ) + mask, meta = self._get_joint_attn_mask_and_meta( + [x_valid] * batch_size, + x_target, + [cap_valid] * batch_size, + cap_target, + x.device, + ) + if sp_size > 1: + # shard the complete joint sequence, with only global-tail padding + unified, shard = shard_seq(unified) + freqs = tuple(shard_like(freq, shard, dim=0) for freq in freqs) + mask, meta = None, tail_attn_meta(shard, batch_size, x.device) + for layer in self.layers: + unified = layer( + unified, + freqs, + adaln, + attn_mask=mask, + attn_mask_meta=meta, + ) + if sp_size == 1: + unified = unified[:, :x_target] + output = self.all_final_layer["2-1"](unified, adaln) + if sp_size > 1: + output = gather_seq(output, shard.orig_len) + output = output[:, :x_length].reshape( + batch_size, frames, height // 2, width // 2, 2, 2, channels + ) + output = output.permute(0, 6, 1, 2, 4, 3, 5).reshape( + batch_size, channels, frames, height, width + ) + return -output[:, :, :output_frames] + + +EntryClass = MingImageTransformer2DModel diff --git a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py index 3519b25833..d976e6e141 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py @@ -245,6 +245,7 @@ class ZImageAttention(nn.Module): eps: float = 1e-6, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + norm_cls: type[nn.Module] = ZImageRMSNorm, ) -> None: super().__init__() self.dim = dim @@ -252,7 +253,9 @@ class ZImageAttention(nn.Module): self.num_heads = num_heads self.num_kv_heads = num_kv_heads self.qk_norm = qk_norm - self.enable_zimage_qk_fusion = quant_config is None + self.enable_zimage_qk_fusion = ( + quant_config is None and norm_cls is ZImageRMSNorm + ) tp_size = get_tp_world_size() assert num_heads % tp_size == 0, ( @@ -303,8 +306,8 @@ class ZImageAttention(nn.Module): ) if self.qk_norm: - self.norm_q = ZImageRMSNorm(self.head_dim, eps=eps) - self.norm_k = ZImageRMSNorm(self.head_dim, eps=eps) + self.norm_q = norm_cls(self.head_dim, eps=eps) + self.norm_k = norm_cls(self.head_dim, eps=eps) else: self.norm_q = None self.norm_k = None @@ -519,13 +522,16 @@ class ZImageTransformerBlock(nn.Module): modulation=True, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + norm_cls: type[nn.Module] = ZImageRMSNorm, ): super().__init__() self.dim = dim self.head_dim = dim // n_heads self.layer_id = layer_id self.modulation = modulation - self.enable_zimage_native_norm_fusion = quant_config is None + self.enable_zimage_native_norm_fusion = ( + quant_config is None and norm_cls is ZImageRMSNorm + ) self.attention = ZImageAttention( dim=dim, @@ -535,6 +541,7 @@ class ZImageTransformerBlock(nn.Module): eps=1e-5, quant_config=quant_config, prefix=f"{prefix}.attention", + norm_cls=norm_cls, ) if not modulation: # Context refiner runs on fully replicated caption tokens only. @@ -574,11 +581,11 @@ class ZImageTransformerBlock(nn.Module): prefix=f"{prefix}.feed_forward", ) - self.attention_norm1 = ZImageRMSNorm(dim, eps=norm_eps) - self.ffn_norm1 = ZImageRMSNorm(dim, eps=norm_eps) + self.attention_norm1 = norm_cls(dim, eps=norm_eps) + self.ffn_norm1 = norm_cls(dim, eps=norm_eps) - self.attention_norm2 = ZImageRMSNorm(dim, eps=norm_eps) - self.ffn_norm2 = ZImageRMSNorm(dim, eps=norm_eps) + self.attention_norm2 = norm_cls(dim, eps=norm_eps) + self.ffn_norm2 = norm_cls(dim, eps=norm_eps) if modulation: self.adaLN_modulation = nn.Sequential( @@ -763,6 +770,7 @@ class RopeEmbedder: class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): + norm_cls = ZImageRMSNorm _supports_gradient_checkpointing = True _no_split_modules = ["ZImageTransformerBlock"] _fsdp_shard_conditions = [is_zimage_layer] @@ -856,6 +864,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): modulation=True, quant_config=quant_config, prefix=f"noise_refiner.{layer_id}", + norm_cls=self.norm_cls, ) for layer_id in range(arch_config.n_refiner_layers) ] @@ -872,6 +881,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): modulation=False, quant_config=quant_config, prefix=f"context_refiner.{layer_id}", + norm_cls=self.norm_cls, ) for layer_id in range(arch_config.n_refiner_layers) ] @@ -881,7 +891,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): ) self.cap_embedder = nn.Sequential( - ZImageRMSNorm(arch_config.cap_feat_dim, eps=arch_config.norm_eps), + self.norm_cls(arch_config.cap_feat_dim, eps=arch_config.norm_eps), ReplicatedLinear(arch_config.cap_feat_dim, self.dim, bias=True), ) @@ -899,6 +909,7 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): arch_config.qk_norm, quant_config=quant_config, prefix=f"layers.{layer_id}", + norm_cls=self.norm_cls, ) for layer_id in range(arch_config.num_layers) ] diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/ming_image.py b/python/sglang/multimodal_gen/runtime/models/encoders/ming_image.py new file mode 100644 index 0000000000..c6ebcb3d27 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/encoders/ming_image.py @@ -0,0 +1,472 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from inclusionAI/Ming-Image (Apache-2.0 modeling code). +"""Ming's Bailing multimodal encoder and non-causal query connector.""" + +from collections import defaultdict +from types import SimpleNamespace + +import torch +from torch import nn +from torch.nn import functional as F + +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput +from sglang.multimodal_gen.runtime.distributed import ( + get_tp_group, + get_tp_rank, + get_tp_world_size, +) +from sglang.multimodal_gen.runtime.layers.attention import LocalAttention +from sglang.multimodal_gen.runtime.layers.linear import ( + LinearBase, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.moe import LingBotVideoGroupedExperts +from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) +from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader +from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder +from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl_vision import ( + Qwen2_5VLVisionTransformer, +) +from sglang.multimodal_gen.runtime.models.encoders.qwen3 import Qwen3MLP +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_experts +from sglang.srt.layers.moe.topk import StandardTopKOutput + + +def ming_position_ids( + token_ids: list[int], grids: list[tuple[int, int, int]], image_token: int +): + """CPU-side centered video-RoPE positions, including learned image queries.""" + positions = [] + cursor = 0 + offset = 0 + for frames, height, width in grids: + start = token_ids.index(image_token, cursor) + text_len = start - cursor + positions.append(torch.arange(offset, offset + text_len).expand(3, -1)) + h, w = height // 2, width // 2 + t = torch.arange(frames).repeat_interleave(h * w) + offset + text_len + rows = torch.arange(h).repeat_interleave(w).repeat(frames) - (h - 1) // 2 + cols = torch.arange(w).repeat(frames * h) - (w - 1) // 2 + positions.append(torch.stack((t, rows + t, cols + t))) + offset += text_len + frames + cursor = start + frames * h * w + positions.append( + torch.arange(offset, offset + len(token_ids) - cursor).expand(3, -1) + ) + return torch.cat(positions, dim=1).unsqueeze(1) + + +class MingAttention(nn.Module): + def __init__(self, config, *, connector=False): + super().__init__() + self.head_dim = config.hidden_size // config.num_attention_heads + self.num_heads = config.num_attention_heads // get_tp_world_size() + self.num_kv_heads = max(1, config.num_key_value_heads // get_tp_world_size()) + self.qkv_proj = QKVParallelLinear( + config.hidden_size, + self.head_dim, + config.num_attention_heads, + config.num_key_value_heads, + bias=connector, + ) + self.o_proj = RowParallelLinear( + config.hidden_size, + config.hidden_size, + bias=False, + input_is_parallel=True, + ) + self.q_norm = ( + nn.Identity() + if connector + else RMSNorm( + self.head_dim, + eps=config.rms_norm_eps, + cast_x_before_out_mul=True, + force_native=True, + ) + ) + self.k_norm = ( + nn.Identity() + if connector + else RMSNorm( + self.head_dim, + eps=config.rms_norm_eps, + cast_x_before_out_mul=True, + force_native=True, + ) + ) + self.attn = LocalAttention( + self.num_heads, self.head_dim, self.num_kv_heads, causal=not connector + ) + self.rotary_dim = ( + self.head_dim + if connector + else int(self.head_dim * config.partial_rotary_factor) + ) + self.rope_theta = config.rope_theta + self.connector = connector + + def forward(self, x, positions): + qkv, _ = self.qkv_proj(x) + q, k, v = qkv.split( + [ + self.num_heads * self.head_dim, + self.num_kv_heads * self.head_dim, + self.num_kv_heads * self.head_dim, + ], + -1, + ) + q = self.q_norm(q.unflatten(-1, (self.num_heads, self.head_dim))) + k = self.k_norm(k.unflatten(-1, (self.num_kv_heads, self.head_dim))) + v = v.unflatten(-1, (self.num_kv_heads, self.head_dim)) + inv = 1.0 / self.rope_theta ** ( + torch.arange(0, self.rotary_dim, 2, device=x.device, dtype=torch.float32) + / self.rotary_dim + ) + # the public BF16 inference path casts RoPE buffers before evaluation + inv = inv.to(x.dtype).float() + with torch.autocast(x.device.type, enabled=False): + angles = positions.float().unsqueeze(-1) * inv + angles = torch.cat((angles, angles), -1) + if not self.connector: + # the spatial frequencies alternate H/W; temporal frequencies are last + axes = ([1, 2] * 12 + [0] * 8) * 2 + angles = torch.stack( + [angles[axis, ..., i] for i, axis in enumerate(axes)], -1 + ) + cos, sin = ( + angles.cos().to(x.dtype).unsqueeze(-2), + angles.sin().to(x.dtype).unsqueeze(-2), + ) + + def rotate(value): + rot, tail = value[..., : self.rotary_dim], value[..., self.rotary_dim :] + half = self.rotary_dim // 2 + rotated = torch.cat((-rot[..., half:], rot[..., :half]), -1) + return torch.cat((rot * cos + rotated * sin, tail), -1) + + output = self.attn(rotate(q), rotate(k), v).flatten(-2) + return self.o_proj(output)[0] + + +class MingRouter(nn.Module): + def __init__(self, config): + super().__init__() + self.weight = nn.Parameter(torch.empty(config.num_experts, config.hidden_size)) + self.expert_bias = nn.Parameter( + torch.zeros(config.num_experts), requires_grad=False + ) + + def forward(self, x): + return F.linear(x.float(), self.weight.float()).sigmoid() + + +class MingMLP(Qwen3MLP): + def forward(self, x): + gate, up = self.gate_up_proj(x)[0].chunk(2, dim=-1) + # preserve the checkpoint's BF16 rounding between SiLU and multiplication + return self.down_proj(F.silu(gate) * up)[0] + + +class MingExperts(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.gate = MingRouter(config) + self.image_gate = MingRouter(config) + self.intermediate_size = config.moe_intermediate_size // get_tp_world_size() + self.experts = LingBotVideoGroupedExperts( + config.num_experts, config.hidden_size, self.intermediate_size + ) + self.shared_experts = MingMLP( + config.hidden_size, + config.moe_intermediate_size * config.num_shared_experts, + "silu", + ) + self.runner_config = MoeRunnerConfig( + num_experts=config.num_experts, + num_local_experts=config.num_experts, + hidden_size=config.hidden_size, + intermediate_size_per_partition=self.intermediate_size, + top_k=config.num_experts_per_tok, + activation="silu_rounded", + is_gated=True, + inplace=False, + no_combine=True, + apply_router_weight_on_input=False, + routed_scaling_factor=None, + gate_up_interleaved=False, + ) + + def forward(self, x, image_mask): + tokens = x.flatten(0, 1) + mask = image_mask.reshape(-1, 1) + scores = torch.where(mask, self.image_gate(tokens), self.gate(tokens)) + choice = scores + torch.where( + mask, self.image_gate.expert_bias, self.gate.expert_bias + ) + grouped = choice.unflatten(-1, (self.config.n_group, -1)) + groups = ( + grouped.topk(2, dim=-1) + .values.sum(-1) + .topk(self.config.topk_group, dim=-1, sorted=False) + .indices + ) + group_mask = torch.zeros_like(grouped[..., 0], dtype=torch.bool).scatter_( + 1, groups, True + ) + choice = choice.masked_fill( + ~group_mask.unsqueeze(-1).expand_as(grouped).flatten(1), float("-inf") + ) + ids = choice.topk(self.config.num_experts_per_tok, dim=-1, sorted=False).indices + weights = scores.gather(1, ids) + weights = ( + weights + / (weights.sum(-1, keepdim=True) + 1e-20) + * self.config.routed_scaling_factor + ) + topk = StandardTopKOutput( + weights.float(), ids.int(), torch.empty(0, device=x.device) + ) + out = fused_experts( + tokens.contiguous(), + self.experts.w13_weight, + self.experts.w2, + topk, + self.runner_config, + ) + if get_tp_world_size() > 1: + out = get_tp_group().all_reduce(out) + out = (out.float() * weights.unsqueeze(-1)).sum(1).to(x.dtype) + return out.view_as(x) + self.shared_experts(x) + + +class MingEncoderBlock(nn.Module): + def __init__(self, config, layer_idx, *, connector=False): + super().__init__() + self.attention = MingAttention(config, connector=connector) + self.is_moe = not connector and layer_idx >= config.first_k_dense_replace + self.mlp = ( + MingExperts(config) + if self.is_moe + else MingMLP(config.hidden_size, config.intermediate_size, "silu") + ) + self.input_layernorm = RMSNorm( + config.hidden_size, + eps=config.rms_norm_eps, + cast_x_before_out_mul=True, + force_native=True, + ) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, + eps=config.rms_norm_eps, + cast_x_before_out_mul=True, + force_native=True, + ) + + def forward(self, x, positions, image_mask=None): + x = x + self.attention(self.input_layernorm(x), positions) + normalized = self.post_attention_layernorm(x) + return x + ( + self.mlp(normalized, image_mask) if self.is_moe else self.mlp(normalized) + ) + + +class MingImageEncoder(TextEncoder): + @property + def dtype(self): + return self.word_embeddings.weight.dtype + + def __init__(self, config): + super().__init__(config) + llm = SimpleNamespace(**config.llm_config) + connector = SimpleNamespace(**config.connector_config) + projection = config.projection_config + self.word_embeddings = VocabParallelEmbedding(llm.vocab_size, llm.hidden_size) + self.layers = nn.ModuleList( + MingEncoderBlock(llm, i) for i in range(llm.num_hidden_layers) + ) + self.norm = RMSNorm( + llm.hidden_size, + eps=llm.rms_norm_eps, + cast_x_before_out_mul=True, + force_native=True, + ) + self.vision = Qwen2_5VLVisionTransformer( + SimpleNamespace(**config.vision_config) + ) + # Ming casts vision RoPE buffers with the model, unlike Qwen2.5-VL + self.vision.rotary_pos_emb.inv_freq = self.vision.rotary_pos_emb.inv_freq.to( + self.vision.dtype + ) + self.vision.merger.ln_q.cast_x_before_out_mul = False + for block in self.vision.blocks: + block.norm1.cast_x_before_out_mul = False + block.norm2.cast_x_before_out_mul = False + self.linear_proj = nn.Sequential( + nn.Linear(config.vision_config["out_hidden_size"], llm.hidden_size), + nn.GELU(), + nn.Linear(llm.hidden_size, llm.hidden_size), + ) + self.connector = nn.ModuleList( + MingEncoderBlock(connector, i, connector=True) + for i in range(connector.num_hidden_layers) + ) + self.connector_norm = RMSNorm( + connector.hidden_size, + eps=connector.rms_norm_eps, + cast_x_before_out_mul=True, + force_native=True, + ) + self.proj_in = nn.Linear(llm.hidden_size, connector.hidden_size) + self.proj_out = nn.Linear( + connector.hidden_size, projection["diffusion_c_input_dim"] + ) + self.selected_layers = tuple(projection["selected_hidden_states_layers"]) + direct_dim = llm.hidden_size * len(self.selected_layers) + self.proj_directvlm = nn.Sequential( + RMSNorm( + direct_dim, eps=1e-5, cast_x_before_out_mul=True, force_native=True + ), + nn.Linear(direct_dim, projection["diffusion_inner_dim"]), + ) + self.query_tokens_dict = nn.ParameterDict( + { + f"{scale}x{scale}": nn.Parameter( + torch.empty(scale * scale, llm.hidden_size) + ) + for scale in projection["img_gen_scales"] + } + ) + self.image_token = llm.image_patch_token + self.layer_names = ["layers", "vision.blocks", "connector"] + + def forward( + self, input_ids, position_ids, pixel_values=None, image_grid_thw=None, **kwargs + ): + x = self.word_embeddings(input_ids) + queries = torch.cat(list(self.query_tokens_dict.values()), 0) + query_count = queries.shape[0] + image_mask = input_ids == self.image_token + image_embeddings = queries + if pixel_values is not None: + vision_features = self.vision(pixel_values, image_grid_thw) + with torch.autocast(x.device.type, enabled=False): + image_embeddings = F.normalize( + self.linear_proj(vision_features), dim=-1 + ) + image_embeddings = torch.cat((image_embeddings, queries), 0) + x = x.masked_scatter(image_mask.unsqueeze(-1), image_embeddings.to(x.dtype)) + selected = [] + for i, layer in enumerate(self.layers): + if i in self.selected_layers: + selected.append(x) + x = layer(x, position_ids, image_mask) + x = self.norm(x) + if len(self.layers) in self.selected_layers: + selected.append(x) + prompt_length = input_ids.shape[1] - query_count - 2 + direct = self.proj_directvlm( + torch.cat([hidden[:, :prompt_length] for hidden in selected], -1) + ) + queries = self.proj_in(x[:, -query_count - 1 : -1]) + positions = torch.arange(query_count, device=x.device).unsqueeze(0) + for layer in self.connector: + queries = layer(queries, positions) + queries = self.proj_out(self.connector_norm(queries)) + if self.config.projection_config["connector_norm"]: + queries = F.normalize(queries, dim=-1) + return BaseEncoderOutput(last_hidden_state=queries, hidden_states=(direct,)) + + def should_materialize_checkpoint_weight(self, name): + return not ( + name.endswith("lm_head.weight") + or ".audio_gate." in name + or "rotary_emb.inv_freq" in name + ) + + def load_weights(self, weights): + params = dict(self.named_parameters()) + loaded = set() + pieces = defaultdict(set) + for name, weight in weights: + if not self.should_materialize_checkpoint_weight(name): + continue + name = name.removeprefix("model.model.") + name = name.replace(".attn.qkv.", ".attn.qkv_proj.") + name = name.replace("connector.model.layers.", "connector.").replace( + "connector.model.norm.", "connector_norm." + ) + if name == "connector.model.embed_tokens.weight": + continue + name = ( + name.replace(".self_attn.", ".attention.") + .replace(".query_key_value.", ".qkv_proj.") + .replace(".dense.", ".o_proj.") + ) + shard = None + if ".experts." in name: + prefix, suffix = name.split(".experts.") + expert, projection, _ = suffix.split(".") + module = self.get_submodule(prefix) + width = module.intermediate_size + rank = get_tp_rank() + if projection == "down_proj": + target = f"{prefix}.experts.w2" + params[target].data[int(expert)].copy_( + weight[:, rank * width : (rank + 1) * width] + ) + else: + target = f"{prefix}.experts.w13_weight" + offset = 0 if projection == "gate_proj" else width + params[target].data[int(expert), offset : offset + width].copy_( + weight[rank * width : (rank + 1) * width] + ) + pieces[target].add((int(expert), projection)) + expected = module.config.num_experts * ( + 1 if projection == "down_proj" else 2 + ) + if len(pieces[target]) == expected: + loaded.add(target) + continue + for source, target, shard_id in ( + (".gate_proj.", ".gate_up_proj.", 0), + (".up_proj.", ".gate_up_proj.", 1), + (".q_proj.", ".qkv_proj.", "q"), + (".k_proj.", ".qkv_proj.", "k"), + (".v_proj.", ".qkv_proj.", "v"), + ): + if source in name and not name.startswith("vision."): + name, shard = name.replace(source, target), shard_id + break + if name not in params: + raise ValueError(f"Unexpected Ming encoder checkpoint tensor: {name}") + param = params[name] + owner = self.get_submodule(name.rsplit(".", 1)[0]) + if ( + isinstance(owner, (LinearBase, VocabParallelEmbedding)) + or name.startswith("vision.") + and "weight_loader" in param.__dict__ + ): + if shard is None: + param.weight_loader(param, weight) + else: + param.weight_loader(param, weight, shard) + else: + default_weight_loader(param, weight) + if shard is None: + loaded.add(name) + else: + pieces[name].add(shard) + expected = {"q", "k", "v"} if isinstance(shard, str) else {0, 1} + if pieces[name] == expected: + loaded.add(name) + return loaded + + +EntryClass = MingImageEncoder diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py index c106319f23..3aab61538c 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py @@ -969,6 +969,8 @@ class AutoencoderKLQwenImage(ParallelTiledVAE): self.shift_factor = ( torch.tensor( config.arch_config.latents_mean + if config.arch_config.latents_mean is not None + else [config.arch_config.shift_factor or 0.0] * latent_channels ) .view(1, latent_channels, 1, 1, 1) .to(cuda_device, dtype) diff --git a/python/sglang/multimodal_gen/runtime/pipelines/ming_image_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/ming_image_pipeline.py new file mode 100644 index 0000000000..d8fdfe0e1f --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines/ming_image_pipeline.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.loader.component_loaders.ming_image import ( + MingImageEncoderLoader, + MingImageTokenizerLoader, +) +from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import ( + InputValidationStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import ( + MingImageDecodingStage, + MingImageEncodingStage, + MingImageReferenceStage, +) +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import maybe_download_model + + +def prepare_mu(batch, server_args): + tokens = (batch.height // 16) * (batch.width // 16) + return "mu", 1.35 if tokens >= 4096 else 0.5 + (tokens - 256) * (1.15 - 0.5) / ( + 4096 - 256 + ) + + +class MingImagePipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "MingImagePipeline" + _required_config_modules = [ + "text_encoder", + "tokenizer", + "transformer", + "vae", + "scheduler", + ] + component_loaders = { + "text_encoder": MingImageEncoderLoader, + "tokenizer": MingImageTokenizerLoader, + } + + def _load_config(self): + self.model_path = maybe_download_model( + self.model_path, revision=self.server_args.revision + ) + return { + "_class_name": self.pipeline_name, + "_diffusers_version": "0.36.0", + "text_encoder": ["transformers", "MingImageEncoder"], + "tokenizer": ["transformers", "PreTrainedTokenizerFast"], + "transformer": ["diffusers", "DiffusionTransformer"], + "vae": ["diffusers", "AutoencoderKLQwenImage"], + "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"], + } + + def _resolve_component_path(self, server_args, module_name, load_module_name): + if module_name == "text_encoder": + load_module_name = "" + elif module_name == "tokenizer": + load_module_name = "mllm" + return super()._resolve_component_path( + server_args, module_name, load_module_name + ) + + def create_pipeline_stages(self, server_args): + self.add_stage(InputValidationStage()) + self.get_module("scheduler").register_to_config(use_dynamic_shifting=True) + self.get_module("scheduler").sigma_min = 0.0 + self.add_stage_factory( + RoleType.ENCODER, + lambda: MingImageEncodingStage( + [self.get_module("text_encoder")], [self.get_module("tokenizer")] + ), + "text_encoding_stage", + ) + self.add_stage_factory( + RoleType.ENCODER, + lambda: MingImageReferenceStage(self.get_module("vae")), + "reference_encoding_stage", + ) + self.add_standard_latent_preparation_stage() + self.add_standard_timestep_preparation_stage(prepare_extra_kwargs=[prepare_mu]) + self.add_standard_denoising_stage() + self.add_stage_factory( + RoleType.DECODER, + lambda: MingImageDecodingStage(self.get_module("vae"), pipeline=self), + "decoding_stage", + ) + + +EntryClass = MingImagePipeline diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py index 88ed7bd39d..955cdeded9 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py @@ -90,13 +90,15 @@ class InputValidationStage(PipelineStage): if num_outputs == 1: return iter((batch,)) - return iter( - expand_request_outputs( - batch, - reuse_parent_trace_ctx=True, - preserve_parent_metrics=True, - ) + outputs = expand_request_outputs( + batch, + reuse_parent_trace_ctx=True, + preserve_parent_metrics=True, ) + # expansion resets generators after this stage has already validated them + for output in outputs: + self._generate_seeds(output, server_args) + return iter(outputs) @staticmethod def _calculate_dimensions_from_area( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py new file mode 100644 index 0000000000..5f7157ead6 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py @@ -0,0 +1,346 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native Ming conditioning and independent RGBA layer decoding.""" + +import math + +import numpy as np +import torch +from PIL import Image +from torchvision.transforms.functional import pil_to_tensor, to_tensor +from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import ( + Qwen2VLImageProcessorPil, + smart_resize, +) + +from sglang.multimodal_gen.configs.pipeline_configs.ming_image import ( + MingImageLayerPipelineConfig, +) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.models.encoders.ming_image import ming_position_ids +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.utils.precision import resolve_precision +from sglang.multimodal_gen.runtime.utils.vision import load_image + + +class MingImageEncodingStage(TextEncodingStage): + def __init__(self, text_encoders, tokenizers): + super().__init__(text_encoders, tokenizers) + self.image_processor = Qwen2VLImageProcessorPil.from_pretrained( + tokenizers[0].name_or_path + ) + + @torch.no_grad() + def forward(self, batch, server_args): + if batch.batch_size != 1: + raise ValueError( + "Ming conditioning expects one prompt per expanded request" + ) + if batch.negative_prompt and str(batch.negative_prompt).strip(): + raise ValueError( + "Ming uses zero unconditional embeddings, not a negative prompt" + ) + layered = isinstance(server_args.pipeline_config, MingImageLayerPipelineConfig) + expected_contract = ("learned", True) if layered else ("zero_masked", False) + arch = server_args.pipeline_config.dit_config.arch_config + if (arch.alignment_padding_mode, arch.multi_frame_output) != expected_contract: + raise ValueError( + "Ming checkpoint padding/output contract does not match the selected pipeline" + ) + count = batch.extra.get("ming_num_layers", 1) + batch.extra["ming_frames"] = count + 1 if layered else 1 + prompt = batch.prompt if isinstance(batch.prompt, str) else batch.prompt[0] + if layered and not prompt.strip(): + prompt = f"Decompose this image into {count} layers." + + image_paths = batch.image_path + if not isinstance(image_paths, list): + image_paths = [image_paths] if image_paths else [] + if len(image_paths) > 1 or (layered and not image_paths): + raise ValueError( + "Ming accepts one reference image; layer decomposition requires it" + ) + device = get_local_torch_device() + encoder = self.text_encoders[0] + tokenizer = self.tokenizers[0] + inputs = {} + grids = [] + image_text = "" + if image_paths: + image = load_image( + image_paths[0], convert_method=lambda value: value.convert("RGBA") + ) + # Reference VAE pixels and VLM pixels use separate official transforms. + batch.extra["ming_reference_image"] = image + h, w = ming_reference_size( + image.height, image.width, min(batch.height, batch.width) + ) + batch.height, batch.width = h, w + ratio = max(h / image.height, w / image.width) + batch.extra["ming_output_size"] = ( + int(image.width * ratio), + int(image.height * ratio), + ) + h0, w0 = smart_resize( + image.height, image.width, min_pixels=56 * 56, max_pixels=1024 * 28 * 28 + ) + processed = self.image_processor( + images=image.convert("RGB").resize((w0, h0)), return_tensors="pt" + ) + grids = [tuple(grid) for grid in processed.image_grid_thw.tolist()] + patches = math.prod(grids[0]) // 4 + image_text = "" + "" * patches + "\n" + inputs = { + "pixel_values": processed.pixel_values.to( + device=device, dtype=encoder.dtype + ), + "image_grid_thw": processed.image_grid_thw, + } + if batch.height % 16 or batch.width % 16: + raise ValueError("Ming height and width must be divisible by 16") + # Fixed training template, not the tokenizer's generic chat template. + system = "SYSTEM\u4f60\u662f\u4e00\u4e2a\u53cb\u597d\u7684AI\u52a9\u624b\u3002\n\ndetailed thinking off" + text = ( + system + + tokenizer.eos_token + + "HUMAN" + + image_text + + prompt + + tokenizer.eos_token + + "ASSISTANT" + ) + ids = tokenizer(text, add_special_tokens=False)["input_ids"] + query_count = sum( + scale * scale + for scale in encoder.config.projection_config["img_gen_scales"] + ) + ids += ( + [tokenizer.convert_tokens_to_ids("")] + + [encoder.image_token] * query_count + + [tokenizer.convert_tokens_to_ids("")] + ) + grids.append((1, 2, query_count * 2)) + inputs.update( + input_ids=torch.tensor([ids], device=device), + position_ids=ming_position_ids(ids, grids, encoder.image_token).to(device), + ) + self._begin_text_encoder_use(0) + # the official MoE router runs under BF16 autocast, including its logits + with torch.autocast(device.type, dtype=encoder.dtype): + encoded = self._forward_text_encoder(encoder, inputs) + batch.prompt_embeds = [encoded.last_hidden_state] + batch.negative_prompt_embeds = [torch.zeros_like(encoded.last_hidden_state)] + batch.extra["ming_direct"] = encoded.hidden_states[0] + return batch + + +class MingImageReferenceStage(PipelineStage): + role_affinity = RoleType.ENCODER + + def __init__(self, vae): + super().__init__() + self.vae = vae + + def component_uses(self, server_args, stage_name=None): + return [ + ComponentUse( + stage_name=self._component_stage_name(stage_name), + component_name="vae", + target_dtype=resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ), + ) + ] + + @torch.no_grad() + def forward(self, batch, server_args): + image = batch.extra.pop("ming_reference_image", None) + if image is None: + return batch + pixels = to_tensor( + image.resize((batch.width, batch.height), Image.Resampling.BILINEAR) + ) + pixels = ((pixels - 0.5) * 2).unsqueeze(0).unsqueeze(2) + with self.use_declared_component(component_name="vae", module=self.vae) as vae: + if server_args.pipeline_config.vae_tiling: + vae.enable_tiling() + pixels = pixels.to( + device=get_local_torch_device(), dtype=next(vae.parameters()).dtype + ) + batch.extra["ming_reference_latents"] = vae.encode(pixels).mode() * 8.0064 + return batch + + +class MingImageDecodingStage(DecodingStage): + def scale_and_shift(self, latents, server_args): + return latents / 8.0064 + + def decode(self, latents, server_args, *, vae_dtype): + # Rounding before scaling is part of the official BF16 decode contract. + latents = latents.to(vae_dtype) + batch, channels, frames, height, width = latents.shape + flat = latents.transpose(1, 2).reshape( + batch * frames, channels, 1, height, width + ) + decoded = super().decode(flat, server_args, vae_dtype=vae_dtype) + return decoded.reshape(batch, frames, 4, *decoded.shape[-2:]).transpose(1, 2) + + def forward(self, batch, server_args): + output = super().forward(batch, server_args) + images = output.output + if isinstance(server_args.pipeline_config, MingImageLayerPipelineConfig): + images = images[:, :, 1:] + images = images.transpose(1, 2).flatten(0, 1) + size = batch.extra.get("ming_output_size") + if size is not None and images.shape[-2:] != size[::-1]: + # Match the public CLI's uint8 conversion before LANCZOS resizing. + pixels = ( + (images.float().permute(0, 2, 3, 1).cpu().numpy() * 255) + .round() + .astype(np.uint8) + ) + images = ( + torch.stack( + [ + pil_to_tensor( + Image.fromarray(pixel).resize( + size, Image.Resampling.LANCZOS + ) + ) + for pixel in pixels + ] + ).float() + / 255 + ) + output.output = images + return output + + +def ming_reference_size(height, width, resolution): + """The published 512/1024 aspect buckets; keys preserve official rounding.""" + buckets = { + 512: { + 0.25: (256, 1024), + 0.26: (256, 992), + 0.27: (256, 960), + 0.28: (256, 928), + 0.32: (288, 896), + 0.33: (288, 864), + 0.35: (288, 832), + 0.4: (320, 800), + 0.42: (320, 768), + 0.48: (352, 736), + 0.5: (352, 704), + 0.52: (352, 672), + 0.5455: (384, 704), + 0.57: (384, 672), + 0.6: (384, 640), + 0.65: (416, 640), + 0.68: (416, 608), + 0.72: (416, 576), + 0.78: (448, 576), + 0.82: (448, 544), + 0.88: (480, 544), + 0.94: (480, 512), + 1.0: (512, 512), + 1.07: (512, 480), + 1.13: (544, 480), + 1.21: (544, 448), + 1.29: (576, 448), + 1.38: (576, 416), + 1.46: (608, 416), + 1.5385: (640, 416), + 1.67: (640, 384), + 1.75: (672, 384), + 1.8333: (704, 384), + 2.0: (704, 352), + 2.09: (736, 352), + 2.4: (768, 320), + 2.5: (800, 320), + 2.89: (832, 288), + 3.0: (864, 288), + 3.11: (896, 288), + 3.62: (928, 256), + 3.75: (960, 256), + 3.88: (992, 256), + 4.0: (1024, 256), + }, + 1024: { + 0.25: (512, 2048), + 0.26: (512, 1984), + 0.27: (512, 1920), + 0.28: (512, 1856), + 0.32: (576, 1792), + 0.33: (576, 1728), + 0.35: (576, 1664), + 0.4: (640, 1600), + 0.42: (640, 1536), + 0.48: (704, 1472), + 0.5: (704, 1408), + 0.52: (704, 1344), + 0.5581: (768, 1376), + 0.5625: (720, 1280), + 0.5647: (768, 1360), + 0.57: (768, 1344), + 0.6: (768, 1280), + 0.622: (816, 1312), + 0.625: (800, 1280), + 0.65: (832, 1280), + 0.6582: (832, 1264), + 0.6667: (832, 1248), + 0.6709: (848, 1264), + 0.68: (832, 1216), + 0.7013: (864, 1232), + 0.72: (832, 1152), + 0.7467: (896, 1200), + 0.75: (864, 1152), + 0.7568: (896, 1184), + 0.78: (896, 1152), + 0.8: (896, 1120), + 0.8056: (928, 1152), + 0.82: (896, 1088), + 0.88: (960, 1088), + 0.94: (960, 1024), + 0.9846: (1024, 1040), + 1.0: (1024, 1024), + 1.07: (1024, 960), + 1.13: (1088, 960), + 1.21: (1088, 896), + 1.2414: (1152, 928), + 1.25: (1120, 896), + 1.2807: (1168, 912), + 1.29: (1152, 896), + 1.3333: (1152, 864), + 1.3393: (1200, 896), + 1.38: (1152, 832), + 1.46: (1216, 832), + 1.4906: (1264, 848), + 1.5: (1248, 832), + 1.6: (1280, 800), + 1.67: (1280, 768), + 1.75: (1344, 768), + 1.7708: (1360, 768), + 1.7778: (1280, 720), + 2.0: (1408, 704), + 2.09: (1472, 704), + 2.4: (1536, 640), + 2.5: (1600, 640), + 2.89: (1664, 576), + 3.0: (1728, 576), + 3.11: (1792, 576), + 3.62: (1856, 512), + 3.75: (1920, 512), + 3.88: (1984, 512), + 4.0: (2048, 512), + }, + } + resolution = min(buckets, key=lambda value: (abs(value - resolution), value)) + table = buckets[resolution] + return table[min(table, key=lambda ratio: abs(ratio - height / width))] diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 6dce4718fd..bbb49c8690 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -217,6 +217,10 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset( "ltx-2.3", "minimax-h3", "minimaxai/minimax-h3", + "inclusionai/ming-image-0.1-design", + "inclusionai/ming-image-0.1-design-layer", + "ming-image-0.1-design", + "ming-image-0.1-design-layer", "qwen/qwen-image", "qwen/qwen-image-2512", "qwen/qwen-image-2.1", @@ -242,6 +246,8 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS = frozenset( "LongCatImagePipelineConfig", "LongCatImageEditPipelineConfig", "MiniMaxH3PipelineConfig", + "MingImagePipelineConfig", + "MingImageLayerPipelineConfig", "QwenImagePipelineConfig", "QwenImage21PipelineConfig", "SanaPipelineConfig", diff --git a/python/sglang/multimodal_gen/test/server/test_server_ming_image.py b/python/sglang/multimodal_gen/test/server/test_server_ming_image.py new file mode 100644 index 0000000000..fc64086108 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_ming_image.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in HTTP regression tests for both public Ming checkpoints. + +Set SGLANG_TEST_MING_IMAGE=1 on a GPU host with sufficient device/host memory. +""" + +import base64 +import io +import os + +import numpy as np +import pytest +import requests +from PIL import Image, ImageDraw + +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + DiffusionSamplingParams, + DiffusionServerArgs, + DiffusionTestCase, +) + +pytestmark = pytest.mark.skipif( + os.environ.get("SGLANG_TEST_MING_IMAGE") != "1", + reason="opt-in full-checkpoint Ming-Image test", +) + + +@pytest.fixture( + params=[ + ("Design", False, "off", False), + ("Design-Layer", False, "off", False), + ("Design-Layer", True, "off", False), + ("Design", False, "request", False), + ("Design-Layer", False, "request", False), + ("Design-Layer", False, "off", True), + ], + ids=[ + "design-cold", + "layer-cold", + "layer-tiled", + "design-warm", + "layer-warm", + "layer-offload", + ], +) +def case(request): + checkpoint, tiled, warmup, offload = request.param + extras = [ + "--performance-mode speed", + f"--warmup-mode {warmup}", + f"--vae-tiling {str(tiled).lower()}", + ] + if offload: + extras.append( + "--component-residency text_encoder=layerwise-offload transformer=layerwise-offload" + ) + return DiffusionTestCase( + f"ming_image_{checkpoint.lower()}_{'offload' if offload else 'tiled' if tiled else 'full'}_{warmup}", + DiffusionServerArgs( + model_path=f"inclusionAI/Ming-Image-0.1-{checkpoint}", + modality="image", + extras=extras, + ), + DiffusionSamplingParams(output_size="512x512"), + ) + + +def test_repeated_generation_and_editing(diffusion_server, case): + url = f"http://localhost:{diffusion_server.port}/v1/images" + layered = case.server_args.model_path.endswith("-Layer") + reference = Image.new("RGBA", (512, 512), "white") + ImageDraw.Draw(reference).rectangle((120, 120, 390, 390), fill="red") + buffer = io.BytesIO() + reference.save(buffer, format="PNG") + modes = ["edit"] if layered else ["generation", "edit"] + for mode in modes: + payload = { + "prompt": ( + "Decompose this image into 2 layers." + if layered + else "Change the red square to blue." + if mode == "edit" + else "A red ceramic teapot on a white background." + ), + "size": "512x512", + "num_inference_steps": 12, + "seed": 42, + "output_format": "png", + "response_format": "b64_json", + } + if layered: + payload["num_layers"] = 2 + previous = {} + for count in (1, 1, 2, 2): + payload["n"] = count + if mode == "edit": + response = requests.post( + f"{url}/edits", + data=payload, + files={"image": ("input.png", buffer.getvalue(), "image/png")}, + timeout=300, + ) + else: + response = requests.post( + f"{url}/generations", json=payload, timeout=300 + ) + assert response.ok, response.text + data = response.json()["data"] + assert len(data) == count * (2 if layered else 1) + images = [] + for item in data: + with Image.open( + io.BytesIO(base64.b64decode(item["b64_json"])) + ) as image: + assert image.mode == "RGBA" + assert image.size == (512, 512) + pixels = np.array(image) + images.append(pixels) + # a decomposed background layer can legitimately be a solid color + assert max(pixels[..., :3].std() for pixels in images) > 5 + if count in previous: + for actual, expected in zip(images, previous[count], strict=True): + np.testing.assert_array_equal(actual, expected) + if count == 2: + for actual, expected in zip(images, previous[1]): + np.testing.assert_array_equal(actual, expected) + previous[count] = images diff --git a/python/sglang/multimodal_gen/test/unit/test_component_residency.py b/python/sglang/multimodal_gen/test/unit/test_component_residency.py index 3af44ea18c..3ea25d49af 100644 --- a/python/sglang/multimodal_gen/test/unit/test_component_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_component_residency.py @@ -617,6 +617,38 @@ def test_declared_component_use_admits_explicit_component_offload(): manager.begin_request([stage], SimpleNamespace(is_warmup=False), server_args) +@pytest.mark.parametrize( + "mode", ["component-offload", "snapshot-offload", "layerwise-offload"] +) +def test_partial_execution_validates_the_full_pipeline(mode): + validation = _Stage() + encoding = _Stage(ComponentUse("encode", "text_encoder")) + denoising = _Stage(ComponentUse("denoise", "transformer")) + pipeline = SimpleNamespace( + modules={ + name: torch.nn.Linear(2, 2) for name in ("text_encoder", "transformer") + }, + _stage_name_mapping={ + "validate": validation, + "encode": encoding, + "denoise": denoising, + }, + component_residency_strategies={}, + ) + server_args = _server_args_with_component_offload("transformer") + server_args.component_residency = dict.fromkeys(pipeline.modules, mode) + manager = ComponentResidencyManager(pipeline, server_args) + manager.refresh_pipeline(pipeline) + + for stages in ([validation], [encoding, denoising]): + manager.begin_request(stages, SimpleNamespace(is_warmup=False), server_args) + assert manager.state.stages == stages + assert manager._ordered_uses == tuple( + use for stage in stages for use in stage.component_uses(server_args) + ) + manager.finish_request() + + def test_single_component_stage_is_prepared_at_stage_entry(): module = torch.nn.Linear(2, 2) use = ComponentUse("stage", "text_encoder") diff --git a/python/sglang/multimodal_gen/test/unit/test_ming_image.py b/python/sglang/multimodal_gen/test/unit/test_ming_image.py new file mode 100644 index 0000000000..4a0f85fc80 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ming_image.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from types import SimpleNamespace + +import pytest +import torch +from torch.nn import functional as F + +from sglang.multimodal_gen.configs.models.dits.ming_image import MingImageDitConfig +from sglang.multimodal_gen.configs.pipeline_configs.ming_image import ( + MingImageLayerPipelineConfig, + MingImagePipelineConfig, +) +from sglang.multimodal_gen.configs.sample.ming_image import ( + MingImageLayerSamplingParams, + MingImageSamplingParams, +) +from sglang.multimodal_gen.registry import get_model_info +from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( + ComponentLoader, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.ming_image import ( + MingImageEncoderLoader, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import ( + TextEncoderLoader, +) +from sglang.multimodal_gen.runtime.models.dits.ming_image import ( + MingImageTransformer2DModel, + MingRMSNorm, + MingSiluAndMul, +) +from sglang.multimodal_gen.runtime.models.encoders.ming_image import ming_position_ids +from sglang.multimodal_gen.runtime.pipelines.ming_image_pipeline import ( + MingImagePipeline, + prepare_mu, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import ( + MingImageReferenceStage, + ming_reference_size, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_experts +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.runtime_context import get_context + + +@pytest.mark.parametrize( + "checkpoint,config_cls,sampling_cls", + [ + ("Design", MingImagePipelineConfig, MingImageSamplingParams), + ("Design-Layer", MingImageLayerPipelineConfig, MingImageLayerSamplingParams), + ], +) +@pytest.mark.parametrize( + "path_template", + [ + "inclusionAI/Ming-Image-0.1-{}", + "/models/Ming-Image-0.1-{}", + "/cache/models--inclusionAI--Ming-Image-0.1-{}/snapshots/revision", + ], +) +def test_registry_resolves_both_checkpoints( + checkpoint, config_cls, sampling_cls, path_template, monkeypatch +): + monkeypatch.setattr( + "sglang.multimodal_gen.registry.maybe_download_model_index", + lambda _: pytest.fail("Ming checkpoints do not have a model_index.json"), + ) + info = get_model_info(path_template.format(checkpoint), backend="sglang") + assert info.pipeline_cls is MingImagePipeline + assert info.pipeline_config_cls is config_cls + assert info.sampling_param_cls is sampling_cls + + +@pytest.mark.parametrize("mode,multi", [("zero_masked", False), ("learned", True)]) +def test_checkpoint_contract(mode, multi): + config = MingImageDitConfig() + config.update_model_arch( + { + "alignment_padding_mode": mode, + "multi_frame_output": multi, + "n_layers": 30, + "n_heads": 30, + } + ) + assert config.num_layers == 30 + assert config.num_attention_heads == 30 + assert config.alignment_padding_mode == mode + + +@pytest.mark.parametrize( + "metadata", + [ + {}, + {"alignment_padding_mode": "learned"}, + {"alignment_padding_mode": "zero_masked", "multi_frame_output": True}, + {"alignment_padding_mode": "learned", "multi_frame_output": False}, + ], +) +def test_checkpoint_contract_rejects_ambiguous_padding(metadata): + with pytest.raises(ValueError): + MingImageDitConfig().update_model_arch(metadata) + + +def test_encoder_loader_is_pipeline_local(): + assert MingImageEncoderLoader.component_names == [] + assert ( + type(ComponentLoader.for_component_type("text_encoder", "transformers")) + is TextEncoderLoader + ) + + +@pytest.mark.parametrize( + "prompt,count", + [("", 4), ("Decompose this image into 2 layers.", 2), ("Number of layers: 3", 3)], +) +def test_layer_count_request_and_output_contract(prompt, count): + params = MingImageLayerSamplingParams(prompt=prompt) + assert params.build_request_extra()["ming_num_layers"] == count + assert params.num_samples_per_request == count + assert "num_layers" in params.image_request_extra_fields() + assert params.default_image_output_format() == "png" + + +@pytest.mark.parametrize("count", [0, -1, True, 1.5, "2"]) +def test_layer_count_rejects_empty_output(count): + with pytest.raises(ValueError, match="positive integer"): + MingImageLayerSamplingParams(num_layers=count).build_request_extra() + + +def test_cpu_noise_draw_preserves_official_frame_order(): + config = MingImagePipelineConfig() + config.check_pipeline_config() + assert not config.vae_tiling + assert not config.vae_sp + assert config.supports_sequential_multi_output_inference() + batch = SimpleNamespace(extra={"ming_frames": 3}, height=32, width=48) + shape = config.prepare_latent_shape(batch, 1, 1) + actual = torch.randn(shape, generator=torch.Generator().manual_seed(42)) + actual = config.maybe_pack_latents(actual, 1, batch) + expected = ( + torch.randn((3, 16, 4, 6), generator=torch.Generator().manual_seed(42)) + .transpose(0, 1) + .unsqueeze(0) + ) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert config.get_latent_dtype(torch.bfloat16) == torch.float32 + assert config.shard_latents_for_sp(batch, actual) == (actual, False) + assert config.get_classifier_free_guidance_scale(batch, 2.0) == 3.0 + + +def test_centered_video_rope_with_reference_and_query_tokens(): + ids = [10, 11] + [99] * 4 + [12, 13] + [99] * 3 + [14] + positions = ming_position_ids(ids, [(1, 4, 4), (1, 2, 6)], 99)[:, 0] + assert positions.shape == (3, len(ids)) + torch.testing.assert_close(positions[:, :2], torch.tensor([[0, 1]]).expand(3, -1)) + torch.testing.assert_close( + positions[:, 2:6], torch.tensor([[2, 2, 2, 2], [2, 2, 3, 3], [2, 3, 2, 3]]) + ) + torch.testing.assert_close( + positions[:, 8:11], torch.tensor([[5, 5, 5], [5, 5, 5], [4, 5, 6]]) + ) + assert positions[:, -1].tolist() == [6, 6, 6] + + +@pytest.mark.parametrize( + "height,width,resolution,expected", + [ + (512, 512, 512, (512, 512)), + (720, 1280, 1024, (720, 1280)), + (1280, 720, 1024, (1280, 720)), + (100, 400, 512, (256, 1024)), + ], +) +def test_official_reference_buckets(height, width, resolution, expected): + assert ming_reference_size(height, width, resolution) == expected + + +@pytest.mark.parametrize( + "override,expected", [({}, torch.bfloat16), ({"vae": "fp32"}, torch.float32)] +) +def test_reference_vae_declares_precision_before_first_encode(override, expected): + args = SimpleNamespace( + component_precisions=override, pipeline_config=MingImagePipelineConfig() + ) + (use,) = MingImageReferenceStage(torch.nn.Identity()).component_uses(args) + assert use.component_name == "vae" + assert use.target_dtype == expected + + +def test_ming_norm_and_swiglu_preserve_reference_rounding(): + torch.manual_seed(9) + x = torch.randn(3, 128, dtype=torch.bfloat16) + norm = MingRMSNorm(128).to(dtype=x.dtype) + with torch.no_grad(): + norm.weight.copy_(torch.randn_like(norm.weight)) + expected = ( + x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-5) + ).to(x.dtype) * norm.weight + torch.testing.assert_close(norm(x), expected, rtol=0, atol=0) + gate, up = x.chunk(2, -1) + torch.testing.assert_close(MingSiluAndMul()(x), F.silu(gate) * up, rtol=0, atol=0) + + +def test_padding_replacement_keeps_valid_rows_and_registers(): + x = torch.randn(2, 5, 8) + token = torch.randn(1, 8) + mask = (torch.arange(5) < 3).unsqueeze(0) + output = MingImageTransformer2DModel._replace_padding_with_token_mask( + x, mask, token + ) + torch.testing.assert_close(output[:, :3], x[:, :3], rtol=0, atol=0) + torch.testing.assert_close(output[:, 3:], token.expand(2, 2, 8), rtol=0, atol=0) + assert output.data_ptr() != x.data_ptr() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_moe_preserves_bf16_activation_before_fp32_combine(): + torch.manual_seed(7) + x = torch.randn(16, 128, dtype=torch.bfloat16, device="cuda") + w13 = torch.randn(4, 256, 128, dtype=x.dtype, device=x.device) * 0.02 + w2 = torch.randn(4, 128, 128, dtype=x.dtype, device=x.device) * 0.02 + weights, ids = torch.randn(16, 4, device=x.device).softmax(-1).topk(2) + config = MoeRunnerConfig( + activation="silu_rounded", + is_gated=True, + no_combine=True, + inplace=False, + top_k=2, + ) + with get_context().override_server_args(model_path="dummy"): + actual = fused_experts( + x, w13, w2, StandardTopKOutput(weights, ids.int(), None), config + ) + expected = torch.empty_like(actual) + for expert in range(4): + rows, slots = torch.where(ids == expert) + gate, up = F.linear(x[rows], w13[expert]).chunk(2, -1) + expected[rows, slots] = F.linear(F.silu(gate) * up, w2[expert]) + torch.testing.assert_close(actual, expected, rtol=0, atol=2e-4) + combined = (actual.float() * weights.unsqueeze(-1)).sum(1).to(x.dtype) + reference = (expected.float() * weights.unsqueeze(-1)).sum(1).to(x.dtype) + torch.testing.assert_close(combined, reference, rtol=0, atol=2e-4) + + +@pytest.mark.parametrize("size,mu", [(256, 0.5), (1024, 1.35), (2048, 1.35)]) +def test_flow_shift_uses_spatial_tokens_not_layer_count(size, mu): + assert prepare_mu(SimpleNamespace(height=size, width=size), None) == ("mu", mu) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, *sys.argv[1:]])) diff --git a/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py b/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py index 4900d0d9c3..1c6233cfe0 100644 --- a/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py +++ b/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py @@ -144,6 +144,7 @@ class TestMultiOutputGrouping(unittest.TestCase): output_file_name="image.png", num_outputs_per_prompt=2, seed=[100, 101], + generator_device="cpu", ) ) return req @@ -182,6 +183,14 @@ class TestMultiOutputGrouping(unittest.TestCase): for req in sequential_outputs ) ) + self.assertEqual( + [req.generator[0].initial_seed() for req in sequential_outputs], + [100, 101], + ) + self.assertEqual([req.seeds for req in sequential_outputs], [[100], [101]]) + self.assertIsNot( + sequential_outputs[0].generator[0], sequential_outputs[1].generator[0] + ) def test_split_batched_latents_uses_original_batched_tensor(self): stage = LatentPreparationStage.__new__(LatentPreparationStage) diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index f6cd8f2805..cdc2a84dd7 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -62,6 +62,7 @@ if _is_cuda: gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, + silu_and_mul_with_activation_rounding, ) elif _is_cpu and _is_cpu_amx_available: pass @@ -673,6 +674,14 @@ def _fused_moe_kernel_sequence( if fuse_swiglu_interleaved: # silu(gate) * up was already applied by the up-GEMM epilogue. pass + elif activation == "silu_rounded" and is_gated: + if _is_cuda: + silu_and_mul_with_activation_rounding( + intermediate_cache1.view(-1, N), intermediate_cache2 + ) + else: + gate, up = intermediate_cache1.view(-1, N).chunk(2, dim=-1) + intermediate_cache2.copy_(F.silu(gate) * up) elif activation == "silu" and is_gated: # - gemm1_alpha != None: GPT-OSS-style swiglu(alpha, limit) # - gemm1_alpha == None and gemm1_limit != None: silu+clamp+mul(limit-only) diff --git a/test/registered/kernels/ops/moe/test_moe_tma_cache.py b/test/registered/kernels/ops/moe/test_moe_tma_cache.py new file mode 100644 index 0000000000..fd89c08155 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_moe_tma_cache.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys + +import pytest +import torch + +from sglang.kernels.ops.moe.fused_moe_triton_kernels import ( + _get_b_tma_desc_cached, + clear_b_tma_desc_cache, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_descriptor_does_not_follow_a_rebound_parameter(): + storage = torch.empty(2, 64, 64, dtype=torch.bfloat16, device="cuda") + first = torch.nn.Parameter(storage) + second = torch.nn.Parameter(storage) + clear_b_tma_desc_cache() + try: + old = _get_b_tma_desc_cached(first, 32, 32) + assert _get_b_tma_desc_cached(first, 32, 32) is old + # offload rebinds the first parameter; another layer can reuse its address + first.data = torch.empty(0, dtype=storage.dtype, device=storage.device) + current = _get_b_tma_desc_cached(second, 32, 32) + assert current.base.data_ptr() == second.data_ptr() + assert tuple(current.base.shape) == tuple(second.shape) + assert _get_b_tma_desc_cached(second, 32, 32) is current + finally: + clear_b_tma_desc_cache() + + +if __name__ == "__main__": + args = ["-x" if arg == "-f" else arg for arg in sys.argv[1:]] + sys.exit(pytest.main([__file__, *args]))